From a504bde3a7462b54fafd5cfc2f52e58d0f3218e1 Mon Sep 17 00:00:00 2001 From: Vas Crabb Date: Thu, 4 May 2023 02:41:16 +1000 Subject: 3rdparty/lzma: Updated to LZMA SDK version 22.01 --- 3rdparty/lzma/Asm/arm64/7zAsm.S | 181 ++ 3rdparty/lzma/Asm/arm64/LzmaDecOpt.S | 1487 ++++++++++ 3rdparty/lzma/Asm/x86/7zAsm.asm | 202 +- 3rdparty/lzma/Asm/x86/7zCrcOpt.asm | 57 +- 3rdparty/lzma/Asm/x86/AesOpt.asm | 851 ++++-- 3rdparty/lzma/Asm/x86/LzFindOpt.asm | 513 ++++ 3rdparty/lzma/Asm/x86/LzmaDecOpt.asm | 1303 +++++++++ 3rdparty/lzma/Asm/x86/Sha256Opt.asm | 275 ++ 3rdparty/lzma/Asm/x86/XzCrc64Opt.asm | 108 +- 3rdparty/lzma/C/7z.h | 14 +- 3rdparty/lzma/C/7zAlloc.c | 12 +- 3rdparty/lzma/C/7zAlloc.h | 20 +- 3rdparty/lzma/C/7zArcIn.c | 142 +- 3rdparty/lzma/C/7zBuf.c | 12 +- 3rdparty/lzma/C/7zBuf.h | 10 +- 3rdparty/lzma/C/7zBuf2.c | 17 +- 3rdparty/lzma/C/7zCrc.c | 210 +- 3rdparty/lzma/C/7zCrcOpt.c | 64 +- 3rdparty/lzma/C/7zDec.c | 105 +- 3rdparty/lzma/C/7zFile.c | 276 +- 3rdparty/lzma/C/7zFile.h | 20 +- 3rdparty/lzma/C/7zStream.c | 113 +- 3rdparty/lzma/C/7zTypes.h | 383 ++- 3rdparty/lzma/C/7zVersion.h | 22 +- 3rdparty/lzma/C/7zip_gcc_c.mak | 343 +++ 3rdparty/lzma/C/Aes.c | 196 +- 3rdparty/lzma/C/Aes.h | 26 +- 3rdparty/lzma/C/AesOpt.c | 796 ++++- 3rdparty/lzma/C/Alloc.c | 419 ++- 3rdparty/lzma/C/Alloc.h | 29 +- 3rdparty/lzma/C/Bcj2.c | 21 +- 3rdparty/lzma/C/Bcj2Enc.c | 17 +- 3rdparty/lzma/C/Bra.c | 279 +- 3rdparty/lzma/C/Bra86.c | 6 +- 3rdparty/lzma/C/BraIA64.c | 82 +- 3rdparty/lzma/C/Compiler.h | 13 +- 3rdparty/lzma/C/CpuArch.c | 304 +- 3rdparty/lzma/C/CpuArch.h | 302 +- 3rdparty/lzma/C/Delta.c | 167 +- 3rdparty/lzma/C/DllSecur.c | 35 +- 3rdparty/lzma/C/DllSecur.h | 5 +- 3rdparty/lzma/C/LzFind.c | 1512 +++++++--- 3rdparty/lzma/C/LzFind.h | 49 +- 3rdparty/lzma/C/LzFindMt.c | 1293 ++++++--- 3rdparty/lzma/C/LzFindMt.h | 58 +- 3rdparty/lzma/C/LzFindOpt.c | 578 ++++ 3rdparty/lzma/C/LzHash.h | 67 +- 3rdparty/lzma/C/Lzma2Dec.c | 357 ++- 3rdparty/lzma/C/Lzma2Dec.h | 66 +- 3rdparty/lzma/C/Lzma2DecMt.c | 1090 +++++++ 3rdparty/lzma/C/Lzma2DecMt.h | 79 + 3rdparty/lzma/C/Lzma2Enc.c | 681 +++-- 3rdparty/lzma/C/Lzma2Enc.h | 45 +- 3rdparty/lzma/C/Lzma86Enc.c | 10 +- 3rdparty/lzma/C/LzmaDec.c | 777 +++-- 3rdparty/lzma/C/LzmaDec.h | 49 +- 3rdparty/lzma/C/LzmaEnc.c | 2971 ++++++++++++------- 3rdparty/lzma/C/LzmaEnc.h | 50 +- 3rdparty/lzma/C/LzmaLib.h | 25 +- 3rdparty/lzma/C/MtCoder.c | 714 +++-- 3rdparty/lzma/C/MtCoder.h | 149 +- 3rdparty/lzma/C/MtDec.c | 1139 ++++++++ 3rdparty/lzma/C/MtDec.h | 202 ++ 3rdparty/lzma/C/Ppmd.h | 146 +- 3rdparty/lzma/C/Ppmd7.c | 900 ++++-- 3rdparty/lzma/C/Ppmd7.h | 179 +- 3rdparty/lzma/C/Ppmd7Dec.c | 328 ++- 3rdparty/lzma/C/Ppmd7Enc.c | 318 +- 3rdparty/lzma/C/Sha256.c | 452 ++- 3rdparty/lzma/C/Sha256.h | 62 +- 3rdparty/lzma/C/Sha256Opt.c | 373 +++ 3rdparty/lzma/C/Threads.c | 483 +++- 3rdparty/lzma/C/Threads.h | 185 +- 3rdparty/lzma/C/Util/7z/7zMain.c | 431 ++- 3rdparty/lzma/C/Util/7z/makefile.gcc | 109 +- 3rdparty/lzma/C/Util/Lzma/LzmaUtil.c | 88 +- 3rdparty/lzma/C/Util/Lzma/LzmaUtil.dsp | 12 +- 3rdparty/lzma/C/Util/Lzma/makefile | 2 + 3rdparty/lzma/C/Util/Lzma/makefile.gcc | 59 +- 3rdparty/lzma/C/Util/LzmaLib/LzmaLib.dsp | 4 + 3rdparty/lzma/C/Util/LzmaLib/makefile | 2 + 3rdparty/lzma/C/Util/SfxSetup/SfxSetup.c | 42 +- 3rdparty/lzma/C/Util/SfxSetup/makefile | 1 + 3rdparty/lzma/C/Util/SfxSetup/makefile_con | 1 + 3rdparty/lzma/C/Xz.c | 20 +- 3rdparty/lzma/C/Xz.h | 302 +- 3rdparty/lzma/C/XzCrc64.c | 22 +- 3rdparty/lzma/C/XzCrc64Opt.c | 36 +- 3rdparty/lzma/C/XzDec.c | 2344 +++++++++++++-- 3rdparty/lzma/C/XzEnc.c | 1230 ++++++-- 3rdparty/lzma/C/XzEnc.h | 27 +- 3rdparty/lzma/C/XzIn.c | 84 +- 3rdparty/lzma/C/var_clang.mak | 11 + 3rdparty/lzma/C/var_clang_arm64.mak | 11 + 3rdparty/lzma/C/var_clang_x64.mak | 11 + 3rdparty/lzma/C/var_clang_x86.mak | 11 + 3rdparty/lzma/C/var_gcc.mak | 12 + 3rdparty/lzma/C/var_gcc_arm64.mak | 12 + 3rdparty/lzma/C/var_gcc_x64.mak | 10 + 3rdparty/lzma/C/var_gcc_x86.mak | 10 + 3rdparty/lzma/C/var_mac_arm64.mak | 11 + 3rdparty/lzma/C/var_mac_x64.mak | 11 + 3rdparty/lzma/C/warn_clang.mak | 37 + 3rdparty/lzma/C/warn_clang_mac.mak | 37 + 3rdparty/lzma/C/warn_gcc.mak | 51 + 3rdparty/lzma/CPP/7zip/7zip_gcc.mak | 1259 ++++++++ 3rdparty/lzma/CPP/7zip/Aes.mak | 5 +- .../lzma/CPP/7zip/Archive/7z/7zCompressionMode.h | 21 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.cpp | 116 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.h | 6 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.cpp | 51 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.h | 2 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zExtract.cpp | 38 +- .../lzma/CPP/7zip/Archive/7z/7zFolderInStream.cpp | 138 +- .../lzma/CPP/7zip/Archive/7z/7zFolderInStream.h | 31 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.cpp | 121 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.h | 58 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zHandlerOut.cpp | 283 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zHeader.h | 5 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.cpp | 239 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.h | 28 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zItem.h | 51 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.cpp | 179 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.h | 32 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zRegister.cpp | 10 +- 3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.cpp | 324 ++- 3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.h | 16 +- 3rdparty/lzma/CPP/7zip/Archive/ArchiveExports.cpp | 8 +- .../lzma/CPP/7zip/Archive/Common/CoderMixer2.cpp | 186 +- .../lzma/CPP/7zip/Archive/Common/CoderMixer2.h | 50 +- .../lzma/CPP/7zip/Archive/Common/HandlerOut.cpp | 278 +- 3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.h | 121 +- .../lzma/CPP/7zip/Archive/Common/ItemNameUtils.cpp | 129 +- .../lzma/CPP/7zip/Archive/Common/ItemNameUtils.h | 29 +- .../lzma/CPP/7zip/Archive/Common/MultiStream.cpp | 6 +- 3rdparty/lzma/CPP/7zip/Archive/DllExports2.cpp | 59 +- 3rdparty/lzma/CPP/7zip/Archive/IArchive.h | 145 +- 3rdparty/lzma/CPP/7zip/Archive/LzmaHandler.cpp | 139 +- 3rdparty/lzma/CPP/7zip/Archive/SplitHandler.cpp | 13 +- 3rdparty/lzma/CPP/7zip/Archive/XzHandler.cpp | 1108 +++++-- 3rdparty/lzma/CPP/7zip/Archive/XzHandler.h | 54 - 3rdparty/lzma/CPP/7zip/Asm.mak | 2 +- 3rdparty/lzma/CPP/7zip/Bundles/Alone7z/Alone.dsp | 236 +- 3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile | 22 +- .../lzma/CPP/7zip/Bundles/Alone7z/makefile.gcc | 278 ++ 3rdparty/lzma/CPP/7zip/Bundles/Alone7z/resource.rc | 4 + .../CPP/7zip/Bundles/Format7zExtractR/makefile | 3 + 3rdparty/lzma/CPP/7zip/Bundles/Format7zR/makefile | 5 + .../lzma/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp | 104 +- 3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp | 33 + 3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile | 14 +- .../lzma/CPP/7zip/Bundles/LzmaCon/makefile.gcc | 268 +- 3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SFXCon.dsp | 43 + 3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SfxCon.cpp | 91 +- 3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile | 14 +- 3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile.gcc | 213 ++ .../7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp | 6 +- .../CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp | 6 +- .../lzma/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp | 34 + .../lzma/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp | 56 +- 3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/makefile | 7 + 3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SFXWin.dsp | 51 + 3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SfxWin.cpp | 10 +- 3rdparty/lzma/CPP/7zip/Bundles/SFXWin/makefile | 10 +- 3rdparty/lzma/CPP/7zip/Common/CWrappers.cpp | 249 +- 3rdparty/lzma/CPP/7zip/Common/CWrappers.h | 94 +- 3rdparty/lzma/CPP/7zip/Common/CreateCoder.cpp | 128 +- 3rdparty/lzma/CPP/7zip/Common/CreateCoder.h | 29 +- .../lzma/CPP/7zip/Common/FilePathAutoRename.cpp | 13 +- 3rdparty/lzma/CPP/7zip/Common/FileStreams.cpp | 442 ++- 3rdparty/lzma/CPP/7zip/Common/FileStreams.h | 83 +- 3rdparty/lzma/CPP/7zip/Common/FilterCoder.cpp | 64 +- 3rdparty/lzma/CPP/7zip/Common/FilterCoder.h | 43 +- 3rdparty/lzma/CPP/7zip/Common/InBuffer.cpp | 35 +- 3rdparty/lzma/CPP/7zip/Common/InBuffer.h | 25 +- 3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.cpp | 106 +- 3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.h | 30 +- 3rdparty/lzma/CPP/7zip/Common/LimitedStreams.cpp | 88 +- 3rdparty/lzma/CPP/7zip/Common/LimitedStreams.h | 29 +- 3rdparty/lzma/CPP/7zip/Common/MethodProps.cpp | 362 ++- 3rdparty/lzma/CPP/7zip/Common/MethodProps.h | 225 +- 3rdparty/lzma/CPP/7zip/Common/OffsetStream.cpp | 4 +- 3rdparty/lzma/CPP/7zip/Common/OutBuffer.h | 7 +- 3rdparty/lzma/CPP/7zip/Common/PropId.cpp | 9 +- 3rdparty/lzma/CPP/7zip/Common/RegisterArc.h | 24 +- 3rdparty/lzma/CPP/7zip/Common/RegisterCodec.h | 20 +- 3rdparty/lzma/CPP/7zip/Common/StreamBinder.cpp | 74 +- 3rdparty/lzma/CPP/7zip/Common/StreamBinder.h | 56 +- 3rdparty/lzma/CPP/7zip/Common/StreamObjects.cpp | 56 +- 3rdparty/lzma/CPP/7zip/Common/StreamObjects.h | 15 +- 3rdparty/lzma/CPP/7zip/Common/UniqBlocks.cpp | 8 +- 3rdparty/lzma/CPP/7zip/Common/UniqBlocks.h | 21 +- 3rdparty/lzma/CPP/7zip/Common/VirtThread.cpp | 15 +- 3rdparty/lzma/CPP/7zip/Common/VirtThread.h | 4 +- 3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.cpp | 26 +- 3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.h | 5 +- 3rdparty/lzma/CPP/7zip/Compress/Bcj2Register.cpp | 2 +- 3rdparty/lzma/CPP/7zip/Compress/CodecExports.cpp | 27 +- 3rdparty/lzma/CPP/7zip/Compress/CopyCoder.cpp | 55 +- 3rdparty/lzma/CPP/7zip/Compress/CopyCoder.h | 5 +- 3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.cpp | 391 ++- 3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.h | 66 +- 3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.cpp | 71 +- 3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.h | 8 +- 3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.cpp | 304 +- 3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.h | 50 +- 3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.cpp | 271 +- 3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.h | 10 +- 3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.cpp | 99 +- 3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.h | 43 +- 3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.cpp | 73 +- 3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.h | 1 - 3rdparty/lzma/CPP/7zip/Compress/XzDecoder.cpp | 150 + 3rdparty/lzma/CPP/7zip/Compress/XzDecoder.h | 93 + 3rdparty/lzma/CPP/7zip/Compress/XzEncoder.cpp | 245 ++ 3rdparty/lzma/CPP/7zip/Compress/XzEncoder.h | 46 + 3rdparty/lzma/CPP/7zip/Crc.mak | 2 +- 3rdparty/lzma/CPP/7zip/Crc64.mak | 2 +- 3rdparty/lzma/CPP/7zip/Crypto/7zAes.cpp | 63 +- 3rdparty/lzma/CPP/7zip/Crypto/7zAes.h | 11 + 3rdparty/lzma/CPP/7zip/Crypto/7zAesRegister.cpp | 6 +- 3rdparty/lzma/CPP/7zip/Crypto/MyAes.cpp | 163 +- 3rdparty/lzma/CPP/7zip/Crypto/MyAes.h | 42 +- 3rdparty/lzma/CPP/7zip/Crypto/MyAesReg.cpp | 22 +- 3rdparty/lzma/CPP/7zip/Crypto/RandGen.cpp | 139 +- 3rdparty/lzma/CPP/7zip/Crypto/RandGen.h | 19 + 3rdparty/lzma/CPP/7zip/GuiCommon.rc | 3 + 3rdparty/lzma/CPP/7zip/Guid.txt | 19 +- 3rdparty/lzma/CPP/7zip/ICoder.h | 98 +- 3rdparty/lzma/CPP/7zip/IPassword.h | 30 + 3rdparty/lzma/CPP/7zip/IStream.h | 16 + 3rdparty/lzma/CPP/7zip/LzFindOpt.mak | 7 + 3rdparty/lzma/CPP/7zip/LzmaDec.mak | 7 + 3rdparty/lzma/CPP/7zip/LzmaDec_gcc.mak | 14 + 3rdparty/lzma/CPP/7zip/PropID.h | 49 + 3rdparty/lzma/CPP/7zip/Sha256.mak | 13 + 3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.cpp | 472 ++- 3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.dsp | 2 +- 3rdparty/lzma/CPP/7zip/UI/Client7z/makefile | 2 +- 3rdparty/lzma/CPP/7zip/UI/Client7z/makefile.gcc | 69 + 3rdparty/lzma/CPP/7zip/UI/Client7z/resource.rc | 2 +- .../lzma/CPP/7zip/UI/Common/ArchiveCommandLine.cpp | 993 ++++--- .../lzma/CPP/7zip/UI/Common/ArchiveCommandLine.h | 55 +- .../CPP/7zip/UI/Common/ArchiveExtractCallback.cpp | 2286 ++++++++++----- .../CPP/7zip/UI/Common/ArchiveExtractCallback.h | 289 +- 3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.cpp | 89 +- 3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.h | 5 +- .../CPP/7zip/UI/Common/ArchiveOpenCallback.cpp | 40 +- .../lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.h | 17 +- 3rdparty/lzma/CPP/7zip/UI/Common/Bench.cpp | 3055 +++++++++++++++----- 3rdparty/lzma/CPP/7zip/UI/Common/Bench.h | 81 +- 3rdparty/lzma/CPP/7zip/UI/Common/DefaultName.cpp | 2 +- 3rdparty/lzma/CPP/7zip/UI/Common/DirItem.h | 307 +- 3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.cpp | 1086 +++++-- 3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.h | 25 +- 3rdparty/lzma/CPP/7zip/UI/Common/Extract.cpp | 167 +- 3rdparty/lzma/CPP/7zip/UI/Common/Extract.h | 19 +- 3rdparty/lzma/CPP/7zip/UI/Common/ExtractMode.h | 10 + .../lzma/CPP/7zip/UI/Common/ExtractingFilePath.cpp | 111 +- .../lzma/CPP/7zip/UI/Common/ExtractingFilePath.h | 18 +- 3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.cpp | 1853 +++++++++++- 3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.h | 249 +- .../lzma/CPP/7zip/UI/Common/IFileExtractCallback.h | 4 +- 3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.cpp | 393 ++- 3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.h | 104 +- 3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.cpp | 566 ++-- 3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.h | 91 +- 3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.cpp | 319 +- 3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.h | 4 +- 3rdparty/lzma/CPP/7zip/UI/Common/SetProperties.cpp | 29 +- 3rdparty/lzma/CPP/7zip/UI/Common/Update.cpp | 725 +++-- 3rdparty/lzma/CPP/7zip/UI/Common/Update.h | 25 +- .../lzma/CPP/7zip/UI/Common/UpdateCallback.cpp | 372 ++- 3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.h | 53 +- 3rdparty/lzma/CPP/7zip/UI/Common/UpdatePair.cpp | 156 +- 3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.cpp | 8 +- 3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.h | 8 +- 3rdparty/lzma/CPP/7zip/UI/Common/WorkDir.cpp | 11 +- 3rdparty/lzma/CPP/7zip/UI/Common/ZipRegistry.h | 87 +- 3rdparty/lzma/CPP/7zip/UI/Console/Console.mak | 9 +- 3rdparty/lzma/CPP/7zip/UI/Console/Console.manifest | 13 + 3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.cpp | 61 +- 3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.h | 22 +- .../CPP/7zip/UI/Console/ExtractCallbackConsole.cpp | 193 +- .../CPP/7zip/UI/Console/ExtractCallbackConsole.h | 24 + 3rdparty/lzma/CPP/7zip/UI/Console/HashCon.cpp | 227 +- 3rdparty/lzma/CPP/7zip/UI/Console/HashCon.h | 29 +- 3rdparty/lzma/CPP/7zip/UI/Console/List.cpp | 258 +- 3rdparty/lzma/CPP/7zip/UI/Console/List.h | 23 +- 3rdparty/lzma/CPP/7zip/UI/Console/Main.cpp | 546 +++- 3rdparty/lzma/CPP/7zip/UI/Console/MainAr.cpp | 28 +- .../CPP/7zip/UI/Console/OpenCallbackConsole.cpp | 12 +- .../lzma/CPP/7zip/UI/Console/OpenCallbackConsole.h | 9 +- .../lzma/CPP/7zip/UI/Console/PercentPrinter.cpp | 13 +- .../CPP/7zip/UI/Console/UpdateCallbackConsole.cpp | 263 +- .../CPP/7zip/UI/Console/UpdateCallbackConsole.h | 22 +- .../lzma/CPP/7zip/UI/Console/UserInputUtils.cpp | 46 +- 3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.h | 7 +- 3rdparty/lzma/CPP/7zip/UI/Console/makefile | 10 +- 3rdparty/lzma/CPP/7zip/UI/Console/makefile.gcc | 185 ++ 3rdparty/lzma/CPP/7zip/UI/Console/resource.rc | 6 +- 3rdparty/lzma/CPP/7zip/UI/Explorer/MyMessages.cpp | 2 +- .../lzma/CPP/7zip/UI/FileManager/BrowseDialog.cpp | 51 +- .../CPP/7zip/UI/FileManager/ExtractCallback.cpp | 72 +- .../lzma/CPP/7zip/UI/FileManager/ExtractCallback.h | 15 +- .../CPP/7zip/UI/FileManager/OverwriteDialog.cpp | 35 +- .../lzma/CPP/7zip/UI/FileManager/OverwriteDialog.h | 22 +- .../CPP/7zip/UI/FileManager/ProgressDialog.cpp | 9 +- .../lzma/CPP/7zip/UI/FileManager/ProgressDialog.h | 2 +- .../CPP/7zip/UI/FileManager/ProgressDialog2.cpp | 229 +- .../lzma/CPP/7zip/UI/FileManager/ProgressDialog2.h | 76 +- .../CPP/7zip/UI/FileManager/ProgressDialog2Res.h | 3 +- .../CPP/7zip/UI/FileManager/ProgressDialog2a.rc | 11 +- .../lzma/CPP/7zip/UI/FileManager/PropertyNameRes.h | 7 + .../lzma/CPP/7zip/UI/FileManager/SysIconUtils.cpp | 16 +- .../lzma/CPP/7zip/UI/FileManager/SysIconUtils.h | 2 +- 3rdparty/lzma/CPP/7zip/UI/FileManager/resource.h | 7 + 3rdparty/lzma/CPP/7zip/UI/GUI/Extract.rc | 8 +- 3rdparty/lzma/CPP/7zip/UI/GUI/ExtractDialog.cpp | 15 +- 3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.cpp | 104 +- 3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.h | 1 + 3rdparty/lzma/CPP/7zip/UI/GUI/HashGUI.h | 13 +- 3rdparty/lzma/CPP/7zip/cmpl_clang.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_clang_arm64.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_clang_x64.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_clang_x86.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_gcc.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_gcc_arm64.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_gcc_x64.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_gcc_x86.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_mac_arm64.mak | 3 + 3rdparty/lzma/CPP/7zip/cmpl_mac_x64.mak | 3 + 3rdparty/lzma/CPP/7zip/var_clang.mak | 11 + 3rdparty/lzma/CPP/7zip/var_clang_arm64.mak | 11 + 3rdparty/lzma/CPP/7zip/var_clang_x64.mak | 12 + 3rdparty/lzma/CPP/7zip/var_clang_x86.mak | 12 + 3rdparty/lzma/CPP/7zip/var_gcc.mak | 12 + 3rdparty/lzma/CPP/7zip/var_gcc_arm64.mak | 12 + 3rdparty/lzma/CPP/7zip/var_gcc_x64.mak | 10 + 3rdparty/lzma/CPP/7zip/var_gcc_x86.mak | 11 + 3rdparty/lzma/CPP/7zip/var_mac_arm64.mak | 11 + 3rdparty/lzma/CPP/7zip/var_mac_x64.mak | 11 + 3rdparty/lzma/CPP/7zip/warn_clang.mak | 37 + 3rdparty/lzma/CPP/7zip/warn_clang_mac.mak | 39 + 3rdparty/lzma/CPP/7zip/warn_gcc.mak | 57 + 3rdparty/lzma/CPP/Build.mak | 137 +- 3rdparty/lzma/CPP/Common/CRC.cpp | 2 +- 3rdparty/lzma/CPP/Common/C_FileIO.cpp | 91 +- 3rdparty/lzma/CPP/Common/C_FileIO.h | 47 - 3rdparty/lzma/CPP/Common/ComTry.h | 6 +- 3rdparty/lzma/CPP/Common/CommandLineParser.cpp | 60 +- 3rdparty/lzma/CPP/Common/CommandLineParser.h | 12 +- 3rdparty/lzma/CPP/Common/Common.h | 46 +- 3rdparty/lzma/CPP/Common/CrcReg.cpp | 31 +- 3rdparty/lzma/CPP/Common/Defs.h | 1 + 3rdparty/lzma/CPP/Common/DynLimBuf.cpp | 93 + 3rdparty/lzma/CPP/Common/DynLimBuf.h | 41 + 3rdparty/lzma/CPP/Common/DynamicBuffer.h | 2 +- 3rdparty/lzma/CPP/Common/IntToString.cpp | 80 +- 3rdparty/lzma/CPP/Common/IntToString.h | 14 +- 3rdparty/lzma/CPP/Common/Lang.h | 2 +- 3rdparty/lzma/CPP/Common/ListFileUtils.cpp | 73 +- 3rdparty/lzma/CPP/Common/ListFileUtils.h | 6 +- 3rdparty/lzma/CPP/Common/LzFindPrepare.cpp | 7 + 3rdparty/lzma/CPP/Common/MyBuffer.h | 43 +- 3rdparty/lzma/CPP/Common/MyBuffer2.h | 139 + 3rdparty/lzma/CPP/Common/MyCom.h | 59 +- 3rdparty/lzma/CPP/Common/MyGuidDef.h | 3 + 3rdparty/lzma/CPP/Common/MyInitGuid.h | 6 +- 3rdparty/lzma/CPP/Common/MyLinux.h | 33 + 3rdparty/lzma/CPP/Common/MyString.cpp | 511 +++- 3rdparty/lzma/CPP/Common/MyString.h | 347 ++- 3rdparty/lzma/CPP/Common/MyTypes.h | 17 + 3rdparty/lzma/CPP/Common/MyVector.h | 241 +- 3rdparty/lzma/CPP/Common/MyWindows.cpp | 163 +- 3rdparty/lzma/CPP/Common/MyWindows.h | 94 +- 3rdparty/lzma/CPP/Common/NewHandler.cpp | 21 +- 3rdparty/lzma/CPP/Common/NewHandler.h | 51 +- 3rdparty/lzma/CPP/Common/Sha256Prepare.cpp | 7 + 3rdparty/lzma/CPP/Common/Sha256Reg.cpp | 42 +- 3rdparty/lzma/CPP/Common/StdInStream.cpp | 79 +- 3rdparty/lzma/CPP/Common/StdInStream.h | 23 +- 3rdparty/lzma/CPP/Common/StdOutStream.cpp | 94 +- 3rdparty/lzma/CPP/Common/StdOutStream.h | 22 +- 3rdparty/lzma/CPP/Common/StringConvert.cpp | 474 ++- 3rdparty/lzma/CPP/Common/StringConvert.h | 145 +- 3rdparty/lzma/CPP/Common/StringToInt.cpp | 29 +- 3rdparty/lzma/CPP/Common/StringToInt.h | 1 + 3rdparty/lzma/CPP/Common/TextConfig.cpp | 8 +- 3rdparty/lzma/CPP/Common/TextConfig.h | 4 +- 3rdparty/lzma/CPP/Common/UTFConvert.cpp | 711 ++++- 3rdparty/lzma/CPP/Common/UTFConvert.h | 378 ++- 3rdparty/lzma/CPP/Common/Wildcard.cpp | 282 +- 3rdparty/lzma/CPP/Common/Wildcard.h | 128 +- 3rdparty/lzma/CPP/Common/XzCrc64Init.cpp | 7 + 3rdparty/lzma/CPP/Windows/Clipboard.cpp | 130 + 3rdparty/lzma/CPP/Windows/Clipboard.h | 28 + 3rdparty/lzma/CPP/Windows/CommonDialog.cpp | 23 + 3rdparty/lzma/CPP/Windows/Control/ComboBox.cpp | 4 +- 3rdparty/lzma/CPP/Windows/Control/ComboBox.h | 26 +- 3rdparty/lzma/CPP/Windows/Control/Dialog.cpp | 238 +- 3rdparty/lzma/CPP/Windows/Control/Dialog.h | 25 +- 3rdparty/lzma/CPP/Windows/Control/ImageList.h | 2 +- 3rdparty/lzma/CPP/Windows/Control/ListView.cpp | 12 +- 3rdparty/lzma/CPP/Windows/Control/ListView.h | 5 +- 3rdparty/lzma/CPP/Windows/Control/ProgressBar.h | 2 +- 3rdparty/lzma/CPP/Windows/Control/PropertyPage.cpp | 2 +- 3rdparty/lzma/CPP/Windows/Control/PropertyPage.h | 2 +- 3rdparty/lzma/CPP/Windows/DLL.cpp | 91 +- 3rdparty/lzma/CPP/Windows/DLL.h | 32 +- 3rdparty/lzma/CPP/Windows/Defs.h | 3 +- 3rdparty/lzma/CPP/Windows/ErrorMsg.cpp | 75 +- 3rdparty/lzma/CPP/Windows/ErrorMsg.h | 1 + 3rdparty/lzma/CPP/Windows/FileDir.cpp | 516 +++- 3rdparty/lzma/CPP/Windows/FileDir.h | 37 +- 3rdparty/lzma/CPP/Windows/FileFind.cpp | 670 ++++- 3rdparty/lzma/CPP/Windows/FileFind.h | 213 +- 3rdparty/lzma/CPP/Windows/FileIO.cpp | 515 +++- 3rdparty/lzma/CPP/Windows/FileIO.h | 208 +- 3rdparty/lzma/CPP/Windows/FileLink.cpp | 257 +- 3rdparty/lzma/CPP/Windows/FileName.cpp | 104 +- 3rdparty/lzma/CPP/Windows/FileName.h | 10 +- 3rdparty/lzma/CPP/Windows/FileSystem.cpp | 8 +- 3rdparty/lzma/CPP/Windows/FileSystem.h | 4 + 3rdparty/lzma/CPP/Windows/Handle.h | 4 +- 3rdparty/lzma/CPP/Windows/MemoryGlobal.cpp | 36 + 3rdparty/lzma/CPP/Windows/MemoryGlobal.h | 55 + 3rdparty/lzma/CPP/Windows/MemoryLock.cpp | 42 +- 3rdparty/lzma/CPP/Windows/MemoryLock.h | 2 + 3rdparty/lzma/CPP/Windows/NtCheck.h | 2 + 3rdparty/lzma/CPP/Windows/PropVariant.cpp | 158 +- 3rdparty/lzma/CPP/Windows/PropVariant.h | 73 +- 3rdparty/lzma/CPP/Windows/PropVariantConv.cpp | 131 +- 3rdparty/lzma/CPP/Windows/PropVariantConv.h | 14 +- 3rdparty/lzma/CPP/Windows/Registry.cpp | 56 +- 3rdparty/lzma/CPP/Windows/ResourceString.cpp | 8 +- 3rdparty/lzma/CPP/Windows/SecurityUtils.cpp | 17 +- 3rdparty/lzma/CPP/Windows/SecurityUtils.h | 83 +- 3rdparty/lzma/CPP/Windows/Shell.cpp | 51 +- 3rdparty/lzma/CPP/Windows/Shell.h | 4 +- 3rdparty/lzma/CPP/Windows/Synchronization.cpp | 53 + 3rdparty/lzma/CPP/Windows/Synchronization.h | 257 +- 3rdparty/lzma/CPP/Windows/System.cpp | 142 +- 3rdparty/lzma/CPP/Windows/System.h | 112 + 3rdparty/lzma/CPP/Windows/SystemInfo.cpp | 917 ++++++ 3rdparty/lzma/CPP/Windows/SystemInfo.h | 18 + 3rdparty/lzma/CPP/Windows/Thread.h | 14 +- 3rdparty/lzma/CPP/Windows/TimeUtils.cpp | 251 +- 3rdparty/lzma/CPP/Windows/TimeUtils.h | 134 +- 3rdparty/lzma/CPP/Windows/Window.cpp | 8 +- 3rdparty/lzma/CPP/Windows/Window.h | 4 +- .../CS/7zip/Compress/LzmaAlone/LzmaAlone.csproj | 90 - 3rdparty/lzma/DOC/7zFormat.txt | 6 +- 3rdparty/lzma/DOC/Methods.txt | 22 +- 3rdparty/lzma/DOC/lzma-history.txt | 157 + 3rdparty/lzma/DOC/lzma-sdk.txt | 65 +- 3rdparty/lzma/DOC/lzma.txt | 8 +- scripts/genie.lua | 5 - scripts/src/3rdparty.lua | 31 +- src/lib/util/chdcodec.cpp | 16 +- src/lib/util/un7z.cpp | 29 +- 461 files changed, 56861 insertions(+), 13458 deletions(-) create mode 100644 3rdparty/lzma/Asm/arm64/7zAsm.S create mode 100644 3rdparty/lzma/Asm/arm64/LzmaDecOpt.S create mode 100644 3rdparty/lzma/Asm/x86/LzFindOpt.asm create mode 100644 3rdparty/lzma/Asm/x86/LzmaDecOpt.asm create mode 100644 3rdparty/lzma/Asm/x86/Sha256Opt.asm create mode 100644 3rdparty/lzma/C/7zip_gcc_c.mak create mode 100644 3rdparty/lzma/C/LzFindOpt.c create mode 100644 3rdparty/lzma/C/Lzma2DecMt.c create mode 100644 3rdparty/lzma/C/Lzma2DecMt.h create mode 100644 3rdparty/lzma/C/MtDec.c create mode 100644 3rdparty/lzma/C/MtDec.h create mode 100644 3rdparty/lzma/C/Sha256Opt.c create mode 100644 3rdparty/lzma/C/var_clang.mak create mode 100644 3rdparty/lzma/C/var_clang_arm64.mak create mode 100644 3rdparty/lzma/C/var_clang_x64.mak create mode 100644 3rdparty/lzma/C/var_clang_x86.mak create mode 100644 3rdparty/lzma/C/var_gcc.mak create mode 100644 3rdparty/lzma/C/var_gcc_arm64.mak create mode 100644 3rdparty/lzma/C/var_gcc_x64.mak create mode 100644 3rdparty/lzma/C/var_gcc_x86.mak create mode 100644 3rdparty/lzma/C/var_mac_arm64.mak create mode 100644 3rdparty/lzma/C/var_mac_x64.mak create mode 100644 3rdparty/lzma/C/warn_clang.mak create mode 100644 3rdparty/lzma/C/warn_clang_mac.mak create mode 100644 3rdparty/lzma/C/warn_gcc.mak create mode 100644 3rdparty/lzma/CPP/7zip/7zip_gcc.mak create mode 100644 3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile.gcc create mode 100644 3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile.gcc create mode 100644 3rdparty/lzma/CPP/7zip/Compress/XzDecoder.cpp create mode 100644 3rdparty/lzma/CPP/7zip/Compress/XzDecoder.h create mode 100644 3rdparty/lzma/CPP/7zip/Compress/XzEncoder.cpp create mode 100644 3rdparty/lzma/CPP/7zip/Compress/XzEncoder.h create mode 100644 3rdparty/lzma/CPP/7zip/LzFindOpt.mak create mode 100644 3rdparty/lzma/CPP/7zip/LzmaDec.mak create mode 100644 3rdparty/lzma/CPP/7zip/LzmaDec_gcc.mak create mode 100644 3rdparty/lzma/CPP/7zip/Sha256.mak create mode 100644 3rdparty/lzma/CPP/7zip/UI/Client7z/makefile.gcc create mode 100644 3rdparty/lzma/CPP/7zip/UI/Console/Console.manifest create mode 100644 3rdparty/lzma/CPP/7zip/UI/Console/makefile.gcc create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_clang.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_clang_arm64.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_clang_x64.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_clang_x86.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_gcc.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_gcc_arm64.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_gcc_x64.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_gcc_x86.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_mac_arm64.mak create mode 100644 3rdparty/lzma/CPP/7zip/cmpl_mac_x64.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_clang.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_clang_arm64.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_clang_x64.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_clang_x86.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_gcc.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_gcc_arm64.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_gcc_x64.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_gcc_x86.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_mac_arm64.mak create mode 100644 3rdparty/lzma/CPP/7zip/var_mac_x64.mak create mode 100644 3rdparty/lzma/CPP/7zip/warn_clang.mak create mode 100644 3rdparty/lzma/CPP/7zip/warn_clang_mac.mak create mode 100644 3rdparty/lzma/CPP/7zip/warn_gcc.mak create mode 100644 3rdparty/lzma/CPP/Common/DynLimBuf.cpp create mode 100644 3rdparty/lzma/CPP/Common/DynLimBuf.h create mode 100644 3rdparty/lzma/CPP/Common/LzFindPrepare.cpp create mode 100644 3rdparty/lzma/CPP/Common/MyBuffer2.h create mode 100644 3rdparty/lzma/CPP/Common/Sha256Prepare.cpp create mode 100644 3rdparty/lzma/CPP/Common/XzCrc64Init.cpp create mode 100644 3rdparty/lzma/CPP/Windows/Clipboard.cpp create mode 100644 3rdparty/lzma/CPP/Windows/Clipboard.h create mode 100644 3rdparty/lzma/CPP/Windows/MemoryGlobal.cpp create mode 100644 3rdparty/lzma/CPP/Windows/MemoryGlobal.h create mode 100644 3rdparty/lzma/CPP/Windows/SystemInfo.cpp create mode 100644 3rdparty/lzma/CPP/Windows/SystemInfo.h delete mode 100644 3rdparty/lzma/CS/7zip/Compress/LzmaAlone/LzmaAlone.csproj diff --git a/3rdparty/lzma/Asm/arm64/7zAsm.S b/3rdparty/lzma/Asm/arm64/7zAsm.S new file mode 100644 index 00000000000..12e950b4c57 --- /dev/null +++ b/3rdparty/lzma/Asm/arm64/7zAsm.S @@ -0,0 +1,181 @@ +// 7zAsm.S -- ASM macros for arm64 +// 2021-04-25 : Igor Pavlov : Public domain + +#define r0 x0 +#define r1 x1 +#define r2 x2 +#define r3 x3 +#define r4 x4 +#define r5 x5 +#define r6 x6 +#define r7 x7 +#define r8 x8 +#define r9 x9 +#define r10 x10 +#define r11 x11 +#define r12 x12 +#define r13 x13 +#define r14 x14 +#define r15 x15 +#define r16 x16 +#define r17 x17 +#define r18 x18 +#define r19 x19 +#define r20 x20 +#define r21 x21 +#define r22 x22 +#define r23 x23 +#define r24 x24 +#define r25 x25 +#define r26 x26 +#define r27 x27 +#define r28 x28 +#define r29 x29 +#define r30 x30 + +#define REG_ABI_PARAM_0 r0 +#define REG_ABI_PARAM_1 r1 +#define REG_ABI_PARAM_2 r2 + + +.macro p2_add reg:req, param:req + add \reg, \reg, \param +.endm + +.macro p2_sub reg:req, param:req + sub \reg, \reg, \param +.endm + +.macro p2_sub_s reg:req, param:req + subs \reg, \reg, \param +.endm + +.macro p2_and reg:req, param:req + and \reg, \reg, \param +.endm + +.macro xor reg:req, param:req + eor \reg, \reg, \param +.endm + +.macro or reg:req, param:req + orr \reg, \reg, \param +.endm + +.macro shl reg:req, param:req + lsl \reg, \reg, \param +.endm + +.macro shr reg:req, param:req + lsr \reg, \reg, \param +.endm + +.macro sar reg:req, param:req + asr \reg, \reg, \param +.endm + +.macro p1_neg reg:req + neg \reg, \reg +.endm + +.macro dec reg:req + sub \reg, \reg, 1 +.endm + +.macro dec_s reg:req + subs \reg, \reg, 1 +.endm + +.macro inc reg:req + add \reg, \reg, 1 +.endm + +.macro inc_s reg:req + adds \reg, \reg, 1 +.endm + + +.macro imul reg:req, param:req + mul \reg, \reg, \param +.endm + +/* +arm64 and arm use reverted c flag after subs/cmp instructions: + arm64-arm : x86 + b.lo / b.cc : jb / jc + b.hs / b.cs : jae / jnc +*/ + +.macro jmp lab:req + b \lab +.endm + +.macro je lab:req + b.eq \lab +.endm + +.macro jz lab:req + b.eq \lab +.endm + +.macro jnz lab:req + b.ne \lab +.endm + +.macro jne lab:req + b.ne \lab +.endm + +.macro jb lab:req + b.lo \lab +.endm + +.macro jbe lab:req + b.ls \lab +.endm + +.macro ja lab:req + b.hi \lab +.endm + +.macro jae lab:req + b.hs \lab +.endm + + +.macro cmove dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, eq +.endm + +.macro cmovne dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, ne +.endm + +.macro cmovs dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, mi +.endm + +.macro cmovns dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, pl +.endm + +.macro cmovb dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, lo +.endm + +.macro cmovae dest:req, srcTrue:req + csel \dest, \srcTrue, \dest, hs +.endm + + +.macro MY_ALIGN_16 macro + .p2align 4,, (1 << 4) - 1 +.endm + +.macro MY_ALIGN_32 macro + .p2align 5,, (1 << 5) - 1 +.endm + +.macro MY_ALIGN_64 macro + .p2align 6,, (1 << 6) - 1 +.endm diff --git a/3rdparty/lzma/Asm/arm64/LzmaDecOpt.S b/3rdparty/lzma/Asm/arm64/LzmaDecOpt.S new file mode 100644 index 00000000000..10dc4735486 --- /dev/null +++ b/3rdparty/lzma/Asm/arm64/LzmaDecOpt.S @@ -0,0 +1,1487 @@ +// LzmaDecOpt.S -- ARM64-ASM version of LzmaDec_DecodeReal_3() function +// 2021-04-25 : Igor Pavlov : Public domain + +/* +; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() +; function for check at link time. +; That code is tightly coupled with LzmaDec_TryDummy() +; and with another functions in LzmaDec.c file. +; CLzmaDec structure, (probs) array layout, input and output of +; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). +*/ + + +#include "7zAsm.S" + + // .arch armv8-a + // .file "LzmaDecOpt.c" + .text + .align 2 + .p2align 4,,15 +#ifdef __APPLE__ + .globl _LzmaDec_DecodeReal_3 +#else + .global LzmaDec_DecodeReal_3 +#endif + // .type LzmaDec_DecodeReal_3, %function + +// #define _LZMA_SIZE_OPT 1 + +#define LZMA_USE_4BYTES_FILL 1 +// #define LZMA_USE_2BYTES_COPY 1 +// #define LZMA_USE_CMOV_LZ_WRAP 1 +// #define _LZMA_PROB32 1 + +#define MY_ALIGN_FOR_ENTRY MY_ALIGN_32 +#define MY_ALIGN_FOR_LOOP MY_ALIGN_32 +#define MY_ALIGN_FOR_LOOP_16 MY_ALIGN_16 + +#ifdef _LZMA_PROB32 + .equ PSHIFT , 2 + .macro PLOAD dest:req, mem:req + ldr \dest, [\mem] + .endm + .macro PLOAD_PREINDEXED dest:req, mem:req, offset:req + ldr \dest, [\mem, \offset]! + .endm + .macro PLOAD_2 dest:req, mem1:req, mem2:req + ldr \dest, [\mem1, \mem2] + .endm + .macro PLOAD_LSL dest:req, mem1:req, mem2:req + ldr \dest, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE src:req, mem:req + str \src, [\mem] + .endm + .macro PSTORE_2 src:req, mem1:req, mem2:req + str \src, [\mem1, \mem2] + .endm + .macro PSTORE_LSL src:req, mem1:req, mem2:req + str \src, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE_LSL_M1 src:req, mem1:req, mem2:req, temp_reg:req + // you must check that temp_reg is free register when macro is used + add \temp_reg, \mem1, \mem2 + str \src, [\temp_reg, \mem2] + .endm +#else + // .equ PSHIFT , 1 + #define PSHIFT 1 + .macro PLOAD dest:req, mem:req + ldrh \dest, [\mem] + .endm + .macro PLOAD_PREINDEXED dest:req, mem:req, offset:req + ldrh \dest, [\mem, \offset]! + .endm + .macro PLOAD_2 dest:req, mem1:req, mem2:req + ldrh \dest, [\mem1, \mem2] + .endm + .macro PLOAD_LSL dest:req, mem1:req, mem2:req + ldrh \dest, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE src:req, mem:req + strh \src, [\mem] + .endm + .macro PSTORE_2 src:req, mem1:req, mem2:req + strh \src, [\mem1, \mem2] + .endm + .macro PSTORE_LSL src:req, mem1:req, mem2:req + strh \src, [\mem1, \mem2, lsl #PSHIFT] + .endm + .macro PSTORE_LSL_M1 src:req, mem1:req, mem2:req, temp_reg:req + strh \src, [\mem1, \mem2] + .endm +#endif + +.equ PMULT , (1 << PSHIFT) +.equ PMULT_2 , (2 << PSHIFT) + +.equ kMatchSpecLen_Error_Data , (1 << 9) + +# x7 t0 : NORM_CALC : prob2 (IF_BIT_1) +# x6 t1 : NORM_CALC : probs_state +# x8 t2 : (LITM) temp : (TREE) temp +# x4 t3 : (LITM) bit : (TREE) temp : UPDATE_0/UPDATE_0 temp +# x10 t4 : (LITM) offs : (TREE) probs_PMULT : numBits +# x9 t5 : (LITM) match : sym2 (ShortDist) +# x1 t6 : (LITM) litm_prob : (TREE) prob_reg : pbPos +# x2 t7 : (LITM) prm : probBranch : cnt +# x3 sym : dist +# x12 len +# x0 range +# x5 cod + + +#define range w0 + +// t6 +#define pbPos w1 +#define pbPos_R r1 +#define prob_reg w1 +#define litm_prob prob_reg + +// t7 +#define probBranch w2 +#define cnt w2 +#define cnt_R r2 +#define prm r2 + +#define sym w3 +#define sym_R r3 +#define dist sym + +#define t3 w4 +#define bit w4 +#define bit_R r4 +#define update_temp_reg r4 + +#define cod w5 + +#define t1 w6 +#define t1_R r6 +#define probs_state t1_R + +#define t0 w7 +#define t0_R r7 +#define prob2 t0 + +#define t2 w8 +#define t2_R r8 + +// t5 +#define match w9 +#define sym2 w9 +#define sym2_R r9 + +#define t4 w10 +#define t4_R r10 + +#define offs w10 +#define offs_R r10 + +#define probs r11 + +#define len w12 +#define len_R x12 + +#define state w13 +#define state_R r13 + +#define dicPos r14 +#define buf r15 +#define bufLimit r16 +#define dicBufSize r17 + +#define limit r19 +#define rep0 w20 +#define rep0_R r20 +#define rep1 w21 +#define rep2 w22 +#define rep3 w23 +#define dic r24 +#define probs_IsMatch r25 +#define probs_Spec r26 +#define checkDicSize w27 +#define processedPos w28 +#define pbMask w29 +#define lc2_lpMask w30 + + +.equ kNumBitModelTotalBits , 11 +.equ kBitModelTotal , (1 << kNumBitModelTotalBits) +.equ kNumMoveBits , 5 +.equ kBitModelOffset , (kBitModelTotal - (1 << kNumMoveBits) + 1) + +.macro NORM_2 macro + ldrb t0, [buf], 1 + shl range, 8 + orr cod, t0, cod, lsl 8 + /* + mov t0, cod + ldrb cod, [buf], 1 + shl range, 8 + bfi cod, t0, #8, #24 + */ +.endm + +.macro TEST_HIGH_BYTE_range macro + tst range, 0xFF000000 +.endm + +.macro NORM macro + TEST_HIGH_BYTE_range + jnz 1f + NORM_2 +1: +.endm + + +# ---------- Branch MACROS ---------- + +.macro UPDATE_0__0 + sub prob2, probBranch, kBitModelOffset +.endm + +.macro UPDATE_0__1 + sub probBranch, probBranch, prob2, asr #(kNumMoveBits) +.endm + +.macro UPDATE_0__2 probsArray:req, probOffset:req, probDisp:req + .if \probDisp == 0 + PSTORE_2 probBranch, \probsArray, \probOffset + .elseif \probOffset == 0 + PSTORE_2 probBranch, \probsArray, \probDisp * PMULT + .else + .error "unsupported" + // add update_temp_reg, \probsArray, \probOffset + PSTORE_2 probBranch, update_temp_reg, \probDisp * PMULT + .endif +.endm + +.macro UPDATE_0 probsArray:req, probOffset:req, probDisp:req + UPDATE_0__0 + UPDATE_0__1 + UPDATE_0__2 \probsArray, \probOffset, \probDisp +.endm + + +.macro UPDATE_1 probsArray:req, probOffset:req, probDisp:req + // sub cod, cod, prob2 + // sub range, range, prob2 + p2_sub cod, range + sub range, prob2, range + sub prob2, probBranch, probBranch, lsr #(kNumMoveBits) + .if \probDisp == 0 + PSTORE_2 prob2, \probsArray, \probOffset + .elseif \probOffset == 0 + PSTORE_2 prob2, \probsArray, \probDisp * PMULT + .else + .error "unsupported" + // add update_temp_reg, \probsArray, \probOffset + PSTORE_2 prob2, update_temp_reg, \probDisp * PMULT + .endif +.endm + + +.macro CMP_COD_BASE + NORM + // lsr prob2, range, kNumBitModelTotalBits + // imul prob2, probBranch + // cmp cod, prob2 + mov prob2, range + shr range, kNumBitModelTotalBits + imul range, probBranch + cmp cod, range +.endm + +.macro CMP_COD_1 probsArray:req + PLOAD probBranch, \probsArray + CMP_COD_BASE +.endm + +.macro CMP_COD_3 probsArray:req, probOffset:req, probDisp:req + .if \probDisp == 0 + PLOAD_2 probBranch, \probsArray, \probOffset + .elseif \probOffset == 0 + PLOAD_2 probBranch, \probsArray, \probDisp * PMULT + .else + .error "unsupported" + add update_temp_reg, \probsArray, \probOffset + PLOAD_2 probBranch, update_temp_reg, \probDisp * PMULT + .endif + CMP_COD_BASE +.endm + + +.macro IF_BIT_1_NOUP probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD_3 \probsArray, \probOffset, \probDisp + jae \toLabel +.endm + + +.macro IF_BIT_1 probsArray:req, probOffset:req, probDisp:req, toLabel:req + IF_BIT_1_NOUP \probsArray, \probOffset, \probDisp, \toLabel + UPDATE_0 \probsArray, \probOffset, \probDisp +.endm + + +.macro IF_BIT_0_NOUP probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD_3 \probsArray, \probOffset, \probDisp + jb \toLabel +.endm + +.macro IF_BIT_0_NOUP_1 probsArray:req, toLabel:req + CMP_COD_1 \probsArray + jb \toLabel +.endm + + +# ---------- CMOV MACROS ---------- + +.macro NORM_LSR + NORM + lsr t0, range, #kNumBitModelTotalBits +.endm + +.macro COD_RANGE_SUB + subs t1, cod, t0 + p2_sub range, t0 +.endm + +.macro RANGE_IMUL prob:req + imul t0, \prob +.endm + +.macro NORM_CALC prob:req + NORM_LSR + RANGE_IMUL \prob + COD_RANGE_SUB +.endm + +.macro CMOV_range + cmovb range, t0 +.endm + +.macro CMOV_code + cmovae cod, t1 +.endm + +.macro CMOV_code_Model_Pre prob:req + sub t0, \prob, kBitModelOffset + CMOV_code + cmovae t0, \prob +.endm + + +.macro PUP_BASE_2 prob:req, dest_reg:req + # only sar works for both 16/32 bit prob modes + sub \dest_reg, \prob, \dest_reg, asr #(kNumMoveBits) +.endm + +.macro PUP prob:req, probPtr:req, mem2:req + PUP_BASE_2 \prob, t0 + PSTORE_2 t0, \probPtr, \mem2 +.endm + + + +#define probs_PMULT t4_R + +.macro BIT_01 + add probs_PMULT, probs, PMULT +.endm + + +.macro BIT_0_R prob:req + PLOAD_2 \prob, probs, 1 * PMULT + NORM_LSR + sub t3, \prob, kBitModelOffset + RANGE_IMUL \prob + PLOAD_2 t2, probs, 1 * PMULT_2 + COD_RANGE_SUB + CMOV_range + cmovae t3, \prob + PLOAD_2 t0, probs, 1 * PMULT_2 + PMULT + PUP_BASE_2 \prob, t3 + csel \prob, t2, t0, lo + CMOV_code + mov sym, 2 + PSTORE_2 t3, probs, 1 * PMULT + adc sym, sym, wzr + BIT_01 +.endm + +.macro BIT_1_R prob:req + NORM_LSR + p2_add sym, sym + sub t3, \prob, kBitModelOffset + RANGE_IMUL \prob + PLOAD_LSL t2, probs, sym_R + COD_RANGE_SUB + CMOV_range + cmovae t3, \prob + PLOAD_LSL t0, probs_PMULT, sym_R + PUP_BASE_2 \prob, t3 + csel \prob, t2, t0, lo + CMOV_code + PSTORE_LSL_M1 t3, probs, sym_R, t2_R + adc sym, sym, wzr +.endm + + +.macro BIT_2_R prob:req + NORM_LSR + p2_add sym, sym + sub t3, \prob, kBitModelOffset + RANGE_IMUL \prob + COD_RANGE_SUB + CMOV_range + cmovae t3, \prob + CMOV_code + PUP_BASE_2 \prob, t3 + PSTORE_LSL_M1 t3, probs, sym_R, t2_R + adc sym, sym, wzr +.endm + + +# ---------- MATCHED LITERAL ---------- + +.macro LITM_0 macro + shl match, (PSHIFT + 1) + and bit, match, 256 * PMULT + add prm, probs, 256 * PMULT + 1 * PMULT + p2_add match, match + p2_add prm, bit_R + eor offs, bit, 256 * PMULT + PLOAD litm_prob, prm + + NORM_LSR + sub t2, litm_prob, kBitModelOffset + RANGE_IMUL litm_prob + COD_RANGE_SUB + cmovae offs, bit + CMOV_range + and bit, match, offs + cmovae t2, litm_prob + CMOV_code + mov sym, 2 + PUP_BASE_2 litm_prob, t2 + PSTORE t2, prm + add prm, probs, offs_R + adc sym, sym, wzr +.endm + +.macro LITM macro + p2_add prm, bit_R + xor offs, bit + PLOAD_LSL litm_prob, prm, sym_R + + NORM_LSR + p2_add match, match + sub t2, litm_prob, kBitModelOffset + RANGE_IMUL litm_prob + COD_RANGE_SUB + cmovae offs, bit + CMOV_range + and bit, match, offs + cmovae t2, litm_prob + CMOV_code + PUP_BASE_2 litm_prob, t2 + PSTORE_LSL t2, prm, sym_R + add prm, probs, offs_R + adc sym, sym, sym +.endm + + +.macro LITM_2 macro + p2_add prm, bit_R + PLOAD_LSL litm_prob, prm, sym_R + + NORM_LSR + sub t2, litm_prob, kBitModelOffset + RANGE_IMUL litm_prob + COD_RANGE_SUB + CMOV_range + cmovae t2, litm_prob + CMOV_code + PUP_BASE_2 litm_prob, t2 + PSTORE_LSL t2, prm, sym_R + adc sym, sym, sym +.endm + + +# ---------- REVERSE BITS ---------- + +.macro REV_0 prob:req + NORM_CALC \prob + CMOV_range + PLOAD t2, sym2_R + PLOAD_2 t3, probs, 3 * PMULT + CMOV_code_Model_Pre \prob + add t1_R, probs, 3 * PMULT + cmovae sym2_R, t1_R + PUP \prob, probs, 1 * PMULT + csel \prob, t2, t3, lo +.endm + + +.macro REV_1 prob:req, step:req + NORM_LSR + PLOAD_PREINDEXED t2, sym2_R, (\step * PMULT) + RANGE_IMUL \prob + COD_RANGE_SUB + CMOV_range + PLOAD_2 t3, sym2_R, (\step * PMULT) + sub t0, \prob, kBitModelOffset + CMOV_code + add t1_R, sym2_R, \step * PMULT + cmovae t0, \prob + cmovae sym2_R, t1_R + PUP_BASE_2 \prob, t0 + csel \prob, t2, t3, lo + PSTORE_2 t0, t1_R, 0 - \step * PMULT_2 +.endm + + +.macro REV_2 prob:req, step:req + sub t1_R, sym2_R, probs + NORM_LSR + orr sym, sym, t1, lsr #PSHIFT + RANGE_IMUL \prob + COD_RANGE_SUB + sub t2, sym, \step + CMOV_range + cmovb sym, t2 + CMOV_code_Model_Pre \prob + PUP \prob, sym2_R, 0 +.endm + + +.macro REV_1_VAR prob:req + PLOAD \prob, sym_R + mov probs, sym_R + p2_add sym_R, sym2_R + NORM_LSR + add t2_R, sym_R, sym2_R + RANGE_IMUL \prob + COD_RANGE_SUB + cmovae sym_R, t2_R + CMOV_range + CMOV_code_Model_Pre \prob + p2_add sym2, sym2 + PUP \prob, probs, 0 +.endm + + +.macro add_big dest:req, src:req, param:req + .if (\param) < (1 << 12) + add \dest, \src, \param + .else + #ifndef _LZMA_PROB32 + .error "unexpcted add_big expansion" + #endif + add \dest, \src, (\param) / 2 + add \dest, \dest, (\param) - (\param) / 2 + .endif +.endm + +.macro sub_big dest:req, src:req, param:req + .if (\param) < (1 << 12) + sub \dest, \src, \param + .else + #ifndef _LZMA_PROB32 + .error "unexpcted sub_big expansion" + #endif + sub \dest, \src, (\param) / 2 + sub \dest, \dest, (\param) - (\param) / 2 + .endif +.endm + + +.macro SET_probs offset:req + // add_big probs, probs_Spec, (\offset) * PMULT + add probs, probs_IsMatch, ((\offset) - IsMatch) * PMULT +.endm + + +.macro LIT_PROBS + add sym, sym, processedPos, lsl 8 + inc processedPos + UPDATE_0__0 + shl sym, lc2_lpMask + SET_probs Literal + p2_and sym, lc2_lpMask + // p2_add probs_state, pbPos_R + p2_add probs, sym_R + UPDATE_0__1 + add probs, probs, sym_R, lsl 1 + UPDATE_0__2 probs_state, pbPos_R, 0 +.endm + + + +.equ kNumPosBitsMax , 4 +.equ kNumPosStatesMax , (1 << kNumPosBitsMax) + +.equ kLenNumLowBits , 3 +.equ kLenNumLowSymbols , (1 << kLenNumLowBits) +.equ kLenNumHighBits , 8 +.equ kLenNumHighSymbols , (1 << kLenNumHighBits) +.equ kNumLenProbs , (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +.equ LenLow , 0 +.equ LenChoice , LenLow +.equ LenChoice2 , (LenLow + kLenNumLowSymbols) +.equ LenHigh , (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +.equ kNumStates , 12 +.equ kNumStates2 , 16 +.equ kNumLitStates , 7 + +.equ kStartPosModelIndex , 4 +.equ kEndPosModelIndex , 14 +.equ kNumFullDistances , (1 << (kEndPosModelIndex >> 1)) + +.equ kNumPosSlotBits , 6 +.equ kNumLenToPosStates , 4 + +.equ kNumAlignBits , 4 +.equ kAlignTableSize , (1 << kNumAlignBits) + +.equ kMatchMinLen , 2 +.equ kMatchSpecLenStart , (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +// .equ kStartOffset , 1408 +.equ kStartOffset , 0 +.equ SpecPos , (-kStartOffset) +.equ IsRep0Long , (SpecPos + kNumFullDistances) +.equ RepLenCoder , (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +.equ LenCoder , (RepLenCoder + kNumLenProbs) +.equ IsMatch , (LenCoder + kNumLenProbs) +.equ kAlign , (IsMatch + (kNumStates2 << kNumPosBitsMax)) +.equ IsRep , (kAlign + kAlignTableSize) +.equ IsRepG0 , (IsRep + kNumStates) +.equ IsRepG1 , (IsRepG0 + kNumStates) +.equ IsRepG2 , (IsRepG1 + kNumStates) +.equ PosSlot , (IsRepG2 + kNumStates) +.equ Literal , (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +.equ NUM_BASE_PROBS , (Literal + kStartOffset) + +.if kStartOffset != 0 // && IsMatch != 0 + .error "Stop_Compiling_Bad_StartOffset" +.endif + +.if NUM_BASE_PROBS != 1984 + .error "Stop_Compiling_Bad_LZMA_PROBS" +.endif + +.equ offset_lc , 0 +.equ offset_lp , 1 +.equ offset_pb , 2 +.equ offset_dicSize , 4 +.equ offset_probs , 4 + offset_dicSize +.equ offset_probs_1664 , 8 + offset_probs +.equ offset_dic , 8 + offset_probs_1664 +.equ offset_dicBufSize , 8 + offset_dic +.equ offset_dicPos , 8 + offset_dicBufSize +.equ offset_buf , 8 + offset_dicPos +.equ offset_range , 8 + offset_buf +.equ offset_code , 4 + offset_range +.equ offset_processedPos , 4 + offset_code +.equ offset_checkDicSize , 4 + offset_processedPos +.equ offset_rep0 , 4 + offset_checkDicSize +.equ offset_rep1 , 4 + offset_rep0 +.equ offset_rep2 , 4 + offset_rep1 +.equ offset_rep3 , 4 + offset_rep2 +.equ offset_state , 4 + offset_rep3 +.equ offset_remainLen , 4 + offset_state +.equ offset_TOTAL_SIZE , 4 + offset_remainLen + +.if offset_TOTAL_SIZE != 96 + .error "Incorrect offset_TOTAL_SIZE" +.endif + + +.macro IsMatchBranch_Pre + # prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + and pbPos, pbMask, processedPos, lsl #(kLenNumLowBits + 1 + PSHIFT) + add probs_state, probs_IsMatch, state_R +.endm + + +/* +.macro IsMatchBranch + IsMatchBranch_Pre + IF_BIT_1 probs_state, pbPos_R, (IsMatch - IsMatch), IsMatch_label +.endm +*/ + +.macro CheckLimits + cmp buf, bufLimit + jae fin_OK + cmp dicPos, limit + jae fin_OK +.endm + +#define CheckLimits_lit CheckLimits +/* +.macro CheckLimits_lit + cmp buf, bufLimit + jae fin_OK_lit + cmp dicPos, limit + jae fin_OK_lit +.endm +*/ + + +#define PARAM_lzma REG_ABI_PARAM_0 +#define PARAM_limit REG_ABI_PARAM_1 +#define PARAM_bufLimit REG_ABI_PARAM_2 + + +.macro LOAD_LZMA_VAR reg:req, struct_offs:req + ldr \reg, [PARAM_lzma, \struct_offs] +.endm + +.macro LOAD_LZMA_BYTE reg:req, struct_offs:req + ldrb \reg, [PARAM_lzma, \struct_offs] +.endm + +.macro LOAD_LZMA_PAIR reg0:req, reg1:req, struct_offs:req + ldp \reg0, \reg1, [PARAM_lzma, \struct_offs] +.endm + + +LzmaDec_DecodeReal_3: +_LzmaDec_DecodeReal_3: +/* +.LFB0: + .cfi_startproc +*/ + + stp x19, x20, [sp, -128]! + stp x21, x22, [sp, 16] + stp x23, x24, [sp, 32] + stp x25, x26, [sp, 48] + stp x27, x28, [sp, 64] + stp x29, x30, [sp, 80] + + str PARAM_lzma, [sp, 120] + + mov bufLimit, PARAM_bufLimit + mov limit, PARAM_limit + + LOAD_LZMA_PAIR dic, dicBufSize, offset_dic + LOAD_LZMA_PAIR dicPos, buf, offset_dicPos + LOAD_LZMA_PAIR rep0, rep1, offset_rep0 + LOAD_LZMA_PAIR rep2, rep3, offset_rep2 + + mov t0, 1 << (kLenNumLowBits + 1 + PSHIFT) + LOAD_LZMA_BYTE pbMask, offset_pb + p2_add limit, dic + mov len, wzr // we can set it in all requiread branches instead + lsl pbMask, t0, pbMask + p2_add dicPos, dic + p2_sub pbMask, t0 + + LOAD_LZMA_BYTE lc2_lpMask, offset_lc + mov t0, 256 << PSHIFT + LOAD_LZMA_BYTE t1, offset_lp + p2_add t1, lc2_lpMask + p2_sub lc2_lpMask, (256 << PSHIFT) - PSHIFT + shl t0, t1 + p2_add lc2_lpMask, t0 + + LOAD_LZMA_VAR probs_Spec, offset_probs + LOAD_LZMA_VAR checkDicSize, offset_checkDicSize + LOAD_LZMA_VAR processedPos, offset_processedPos + LOAD_LZMA_VAR state, offset_state + // range is r0 : this load must be last don't move + LOAD_LZMA_PAIR range, cod, offset_range + mov sym, wzr + shl state, PSHIFT + + add_big probs_IsMatch, probs_Spec, ((IsMatch - SpecPos) << PSHIFT) + + // if (processedPos != 0 || checkDicSize != 0) + orr t0, checkDicSize, processedPos + cbz t0, 1f + add t0_R, dicBufSize, dic + cmp dicPos, dic + cmovne t0_R, dicPos + ldrb sym, [t0_R, -1] +1: + IsMatchBranch_Pre + cmp state, 4 * PMULT + jb lit_end + cmp state, kNumLitStates * PMULT + jb lit_matched_end + jmp lz_end + + + +#define BIT_0 BIT_0_R prob_reg +#define BIT_1 BIT_1_R prob_reg +#define BIT_2 BIT_2_R prob_reg + +# ---------- LITERAL ---------- +MY_ALIGN_64 +lit_start: + mov state, wzr +lit_start_2: + LIT_PROBS + + #ifdef _LZMA_SIZE_OPT + + PLOAD_2 prob_reg, probs, 1 * PMULT + mov sym, 1 + BIT_01 +MY_ALIGN_FOR_LOOP +lit_loop: + BIT_1 + tbz sym, 7, lit_loop + + #else + + BIT_0 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + + #endif + + BIT_2 + IsMatchBranch_Pre + strb sym, [dicPos], 1 + p2_and sym, 255 + + CheckLimits_lit +lit_end: + IF_BIT_0_NOUP probs_state, pbPos_R, (IsMatch - IsMatch), lit_start + + # jmp IsMatch_label + + +#define FLAG_STATE_BITS (4 + PSHIFT) + +# ---------- MATCHES ---------- +# MY_ALIGN_FOR_ENTRY +IsMatch_label: + UPDATE_1 probs_state, pbPos_R, (IsMatch - IsMatch) + IF_BIT_1 probs_state, 0, (IsRep - IsMatch), IsRep_label + + SET_probs LenCoder + or state, (1 << FLAG_STATE_BITS) + +# ---------- LEN DECODE ---------- +len_decode: + mov len, 8 - kMatchMinLen + IF_BIT_0_NOUP_1 probs, len_mid_0 + UPDATE_1 probs, 0, 0 + p2_add probs, (1 << (kLenNumLowBits + PSHIFT)) + mov len, 0 - kMatchMinLen + IF_BIT_0_NOUP_1 probs, len_mid_0 + UPDATE_1 probs, 0, 0 + p2_add probs, LenHigh * PMULT - (1 << (kLenNumLowBits + PSHIFT)) + + #if 0 == 1 + BIT_0 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + #else + PLOAD_2 prob_reg, probs, 1 * PMULT + mov sym, 1 + BIT_01 +MY_ALIGN_FOR_LOOP +len8_loop: + BIT_1 + tbz sym, 6, len8_loop + #endif + + mov len, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - kMatchMinLen + jmp len_mid_2 + +MY_ALIGN_FOR_ENTRY +len_mid_0: + UPDATE_0 probs, 0, 0 + p2_add probs, pbPos_R + BIT_0 +len_mid_2: + BIT_1 + BIT_2 + sub len, sym, len + tbz state, FLAG_STATE_BITS, copy_match + +# ---------- DECODE DISTANCE ---------- + // probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + + mov t0, 3 + kMatchMinLen + cmp len, 3 + kMatchMinLen + cmovb t0, len + SET_probs PosSlot - (kMatchMinLen << (kNumPosSlotBits)) + add probs, probs, t0_R, lsl #(kNumPosSlotBits + PSHIFT) + + #ifdef _LZMA_SIZE_OPT + + PLOAD_2 prob_reg, probs, 1 * PMULT + mov sym, 1 + BIT_01 +MY_ALIGN_FOR_LOOP +slot_loop: + BIT_1 + tbz sym, 5, slot_loop + + #else + + BIT_0 + BIT_1 + BIT_1 + BIT_1 + BIT_1 + + #endif + + #define numBits t4 + mov numBits, sym + BIT_2 + // we need only low bits + p2_and sym, 3 + cmp numBits, 32 + kEndPosModelIndex / 2 + jb short_dist + + SET_probs kAlign + + # unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + p2_sub numBits, (32 + 1 + kNumAlignBits) + # distance = (2 | (distance & 1)); + or sym, 2 + PLOAD_2 prob_reg, probs, 1 * PMULT + add sym2_R, probs, 2 * PMULT + +# ---------- DIRECT DISTANCE ---------- + +.macro DIRECT_1 + shr range, 1 + subs t0, cod, range + p2_add sym, sym + // add t1, sym, 1 + csel cod, cod, t0, mi + csinc sym, sym, sym, mi + // csel sym, t1, sym, pl + // adc sym, sym, sym // not 100% compatible for "corruptued-allowed" LZMA streams + dec_s numBits + je direct_end +.endm + + #ifdef _LZMA_SIZE_OPT + + jmp direct_norm +MY_ALIGN_FOR_ENTRY +direct_loop: + DIRECT_1 +direct_norm: + TEST_HIGH_BYTE_range + jnz direct_loop + NORM_2 + jmp direct_loop + + #else + +.macro DIRECT_2 + TEST_HIGH_BYTE_range + jz direct_unroll + DIRECT_1 +.endm + + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + DIRECT_2 + +direct_unroll: + NORM_2 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + DIRECT_1 + jmp direct_unroll + + #endif + +MY_ALIGN_FOR_ENTRY +direct_end: + shl sym, kNumAlignBits + REV_0 prob_reg + REV_1 prob_reg, 2 + REV_1 prob_reg, 4 + REV_2 prob_reg, 8 + +decode_dist_end: + + // if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + + tst checkDicSize, checkDicSize + csel t0, processedPos, checkDicSize, eq + cmp sym, t0 + jae end_of_payload + // jmp end_of_payload # for debug + + mov rep3, rep2 + mov rep2, rep1 + mov rep1, rep0 + add rep0, sym, 1 + +.macro STATE_UPDATE_FOR_MATCH + // state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + // cmp state, (kNumStates + kNumLitStates) * PMULT + cmp state, kNumLitStates * PMULT + (1 << FLAG_STATE_BITS) + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 +.endm + STATE_UPDATE_FOR_MATCH + +# ---------- COPY MATCH ---------- +copy_match: + + // if ((rem = limit - dicPos) == 0) break // return SZ_ERROR_DATA; + subs cnt_R, limit, dicPos + // jz fin_dicPos_LIMIT + jz fin_OK + + // curLen = ((rem < len) ? (unsigned)rem : len); + cmp cnt_R, len_R + cmovae cnt, len + + sub t0_R, dicPos, dic + p2_add dicPos, cnt_R + p2_add processedPos, cnt + p2_sub len, cnt + + // pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + p2_sub_s t0_R, rep0_R + jae 1f + + cmn t0_R, cnt_R + p2_add t0_R, dicBufSize + ja copy_match_cross +1: +# ---------- COPY MATCH FAST ---------- + # t0_R : src_pos + p2_add t0_R, dic + ldrb sym, [t0_R] + p2_add t0_R, cnt_R + p1_neg cnt_R + +copy_common: + dec dicPos + + # dicPos : (ptr_to_last_dest_BYTE) + # t0_R : (src_lim) + # cnt_R : (-curLen) + + IsMatchBranch_Pre + + inc_s cnt_R + jz copy_end + + cmp rep0, 1 + je copy_match_0 + + #ifdef LZMA_USE_2BYTES_COPY + strb sym, [dicPos, cnt_R] + dec dicPos + # dicPos : (ptr_to_last_dest_16bitWORD) + p2_and cnt_R, -2 + ldrh sym, [t0_R, cnt_R] + adds cnt_R, cnt_R, 2 + jz 2f +MY_ALIGN_FOR_LOOP +1: + /* + strh sym, [dicPos, cnt_R] + ldrh sym, [t0_R, cnt_R] + adds cnt_R, cnt_R, 2 + jz 2f + */ + + strh sym, [dicPos, cnt_R] + ldrh sym, [t0_R, cnt_R] + adds cnt_R, cnt_R, 2 + jnz 1b +2: + + /* + // for universal little/big endian code, but slow + strh sym, [dicPos] + inc dicPos + ldrb sym, [t0_R, -1] + */ + + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + // we must improve big-endian detection for another compilers + // for big-endian we need to revert bytes + rev16 sym, sym + #endif + + // (sym) must represent as little-endian here: + strb sym, [dicPos], 1 + shr sym, 8 + + #else + +MY_ALIGN_FOR_LOOP +1: + strb sym, [dicPos, cnt_R] + ldrb sym, [t0_R, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + ldrb sym, [t0_R, cnt_R] + inc_s cnt_R + jnz 1b + #endif + +copy_end: +lz_end_match: + strb sym, [dicPos], 1 + + # IsMatchBranch_Pre + CheckLimits +lz_end: + IF_BIT_1_NOUP probs_state, pbPos_R, (IsMatch - IsMatch), IsMatch_label + + + +# ---------- LITERAL MATCHED ---------- + + LIT_PROBS + + // matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + + sub t0_R, dicPos, dic + p2_sub_s t0_R, rep0_R + + #ifdef LZMA_USE_CMOV_LZ_WRAP + add t1_R, t0_R, dicBufSize + cmovb t0_R, t1_R + #else + jae 1f + p2_add t0_R, dicBufSize +1: + #endif + + ldrb match, [dic, t0_R] + + // state -= (state < 10) ? 3 : 6; + sub sym, state, 6 * PMULT + cmp state, 10 * PMULT + p2_sub state, 3 * PMULT + cmovae state, sym + + #ifdef _LZMA_SIZE_OPT + + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov sym, 1 + and bit, match, offs + add prm, probs, offs_R + +MY_ALIGN_FOR_LOOP +litm_loop: + LITM + tbz sym, 8, litm_loop + + #else + + LITM_0 + LITM + LITM + LITM + LITM + LITM + LITM + LITM_2 + + #endif + + IsMatchBranch_Pre + strb sym, [dicPos], 1 + p2_and sym, 255 + + // mov len, wzr // LITM uses same regisetr (len / offs). So we clear it + CheckLimits_lit +lit_matched_end: + IF_BIT_1_NOUP probs_state, pbPos_R, (IsMatch - IsMatch), IsMatch_label + # IsMatchBranch + p2_sub state, 3 * PMULT + jmp lit_start_2 + + + +# ---------- REP 0 LITERAL ---------- +MY_ALIGN_FOR_ENTRY +IsRep0Short_label: + UPDATE_0 probs_state, pbPos_R, 0 + + // dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + sub t0_R, dicPos, dic + + // state = state < kNumLitStates ? 9 : 11; + or state, 1 * PMULT + + # the caller doesn't allow (dicPos >= limit) case for REP_SHORT + # so we don't need the following (dicPos == limit) check here: + # cmp dicPos, limit + # jae fin_dicPos_LIMIT_REP_SHORT + # // jmp fin_dicPos_LIMIT_REP_SHORT // for testing/debug puposes + + inc processedPos + + IsMatchBranch_Pre + + p2_sub_s t0_R, rep0_R + #ifdef LZMA_USE_CMOV_LZ_WRAP + add sym_R, t0_R, dicBufSize + cmovb t0_R, sym_R + #else + jae 1f + p2_add t0_R, dicBufSize +1: + #endif + + ldrb sym, [dic, t0_R] + // mov len, wzr + jmp lz_end_match + +MY_ALIGN_FOR_ENTRY +IsRep_label: + UPDATE_1 probs_state, 0, (IsRep - IsMatch) + + # The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. + # So we don't check it here. + + # mov t0, processedPos + # or t0, checkDicSize + # jz fin_ERROR_2 + + // state = state < kNumLitStates ? 8 : 11; + cmp state, kNumLitStates * PMULT + mov state, 8 * PMULT + mov probBranch, 11 * PMULT + cmovae state, probBranch + + SET_probs RepLenCoder + + IF_BIT_1 probs_state, 0, (IsRepG0 - IsMatch), IsRepG0_label + sub_big probs_state, probs_state, (IsMatch - IsRep0Long) << PSHIFT + IF_BIT_0_NOUP probs_state, pbPos_R, 0, IsRep0Short_label + UPDATE_1 probs_state, pbPos_R, 0 + jmp len_decode + +MY_ALIGN_FOR_ENTRY +IsRepG0_label: + UPDATE_1 probs_state, 0, (IsRepG0 - IsMatch) + IF_BIT_1 probs_state, 0, (IsRepG1 - IsMatch), IsRepG1_label + mov dist, rep1 + mov rep1, rep0 + mov rep0, dist + jmp len_decode + +# MY_ALIGN_FOR_ENTRY +IsRepG1_label: + UPDATE_1 probs_state, 0, (IsRepG1 - IsMatch) + IF_BIT_1 probs_state, 0, (IsRepG2 - IsMatch), IsRepG2_label + mov dist, rep2 + mov rep2, rep1 + mov rep1, rep0 + mov rep0, dist + jmp len_decode + +# MY_ALIGN_FOR_ENTRY +IsRepG2_label: + UPDATE_1 probs_state, 0, (IsRepG2 - IsMatch) + mov dist, rep3 + mov rep3, rep2 + mov rep2, rep1 + mov rep1, rep0 + mov rep0, dist + jmp len_decode + + + +# ---------- SPEC SHORT DISTANCE ---------- + +MY_ALIGN_FOR_ENTRY +short_dist: + p2_sub_s numBits, 32 + 1 + jbe decode_dist_end + or sym, 2 + shl sym, numBits + add sym_R, probs_Spec, sym_R, lsl #PSHIFT + p2_add sym_R, SpecPos * PMULT + 1 * PMULT + mov sym2, PMULT // # step +MY_ALIGN_FOR_LOOP +spec_loop: + REV_1_VAR prob_reg + dec_s numBits + jnz spec_loop + + p2_add sym2_R, probs_Spec + .if SpecPos != 0 + p2_add sym2_R, SpecPos * PMULT + .endif + p2_sub sym_R, sym2_R + shr sym, PSHIFT + + jmp decode_dist_end + + + +# ---------- COPY MATCH 0 ---------- +MY_ALIGN_FOR_ENTRY +copy_match_0: + #ifdef LZMA_USE_4BYTES_FILL + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + orr t3, sym, sym, lsl 8 + p2_and cnt_R, -4 + orr t3, t3, t3, lsl 16 +MY_ALIGN_FOR_LOOP_16 +1: + /* + str t3, [dicPos, cnt_R] + adds cnt_R, cnt_R, 4 + jz 2f + */ + + str t3, [dicPos, cnt_R] + adds cnt_R, cnt_R, 4 + jnz 1b +2: + // p2_and sym, 255 + #else + +MY_ALIGN_FOR_LOOP +1: + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jz copy_end + + strb sym, [dicPos, cnt_R] + inc_s cnt_R + jnz 1b + #endif + + jmp copy_end + + +# ---------- COPY MATCH CROSS ---------- +copy_match_cross: + # t0_R - src pos + # cnt_R - total copy len + + p1_neg cnt_R +1: + ldrb sym, [dic, t0_R] + inc t0_R + strb sym, [dicPos, cnt_R] + inc cnt_R + cmp t0_R, dicBufSize + jne 1b + + ldrb sym, [dic] + sub t0_R, dic, cnt_R + jmp copy_common + + + + +/* +fin_dicPos_LIMIT_REP_SHORT: + mov len, 1 + jmp fin_OK +*/ + +/* +fin_dicPos_LIMIT: + jmp fin_OK + # For more strict mode we can stop decoding with error + # mov sym, 1 + # jmp fin +*/ + +fin_ERROR_MATCH_DIST: + # rep0 = distance + 1; + p2_add len, kMatchSpecLen_Error_Data + mov rep3, rep2 + mov rep2, rep1 + mov rep1, rep0 + mov rep0, sym + STATE_UPDATE_FOR_MATCH + # jmp fin_OK + mov sym, 1 + jmp fin + +end_of_payload: + inc_s sym + jnz fin_ERROR_MATCH_DIST + + mov len, kMatchSpecLenStart + xor state, (1 << FLAG_STATE_BITS) + jmp fin_OK + +/* +fin_OK_lit: + mov len, wzr +*/ + +fin_OK: + mov sym, wzr + +fin: + NORM + + #define fin_lzma_reg t0_R + + .macro STORE_LZMA_VAR reg:req, struct_offs:req + str \reg, [fin_lzma_reg, \struct_offs] + .endm + + .macro STORE_LZMA_PAIR reg0:req, reg1:req, struct_offs:req + stp \reg0, \reg1, [fin_lzma_reg, \struct_offs] + .endm + + ldr fin_lzma_reg, [sp, 120] + p2_sub dicPos, dic + shr state, PSHIFT + + STORE_LZMA_PAIR dicPos, buf, offset_dicPos + STORE_LZMA_PAIR range, cod, offset_range + STORE_LZMA_VAR processedPos, offset_processedPos + STORE_LZMA_PAIR rep0, rep1, offset_rep0 + STORE_LZMA_PAIR rep2, rep3, offset_rep2 + STORE_LZMA_PAIR state, len, offset_state + + mov w0, sym + + ldp x29, x30, [sp, 80] + ldp x27, x28, [sp, 64] + ldp x25, x26, [sp, 48] + ldp x23, x24, [sp, 32] + ldp x21, x22, [sp, 16] + ldp x19, x20, [sp], 128 + + ret +/* + .cfi_endproc +.LFE0: + .size LzmaDec_DecodeReal_3, .-LzmaDec_DecodeReal_3 + .ident "TAG_LZMA" + .section .note.GNU-stack,"",@progbits +*/ diff --git a/3rdparty/lzma/Asm/x86/7zAsm.asm b/3rdparty/lzma/Asm/x86/7zAsm.asm index 3707e90c566..19c40dac440 100644 --- a/3rdparty/lzma/Asm/x86/7zAsm.asm +++ b/3rdparty/lzma/Asm/x86/7zAsm.asm @@ -1,5 +1,50 @@ ; 7zAsm.asm -- ASM macros -; 2012-12-30 : Igor Pavlov : Public domain +; 2022-05-16 : Igor Pavlov : Public domain + + +; UASM can require these changes +; OPTION FRAMEPRESERVEFLAGS:ON +; OPTION PROLOGUE:NONE +; OPTION EPILOGUE:NONE + +ifdef @wordsize +; @wordsize is defined only in JWASM and ASMC and is not defined in MASM +; @wordsize eq 8 for 64-bit x64 +; @wordsize eq 2 for 32-bit x86 +if @wordsize eq 8 + x64 equ 1 +endif +else +ifdef RAX + x64 equ 1 +endif +endif + + +ifdef x64 + IS_X64 equ 1 +else + IS_X64 equ 0 +endif + +ifdef ABI_LINUX + IS_LINUX equ 1 +else + IS_LINUX equ 0 +endif + +ifndef x64 +; Use ABI_CDECL for x86 (32-bit) only +; if ABI_CDECL is not defined, we use fastcall abi +ifdef ABI_CDECL + IS_CDECL equ 1 +else + IS_CDECL equ 0 +endif +endif + +OPTION PROLOGUE:NONE +OPTION EPILOGUE:NONE MY_ASM_START macro ifdef x64 @@ -14,8 +59,12 @@ endm MY_PROC macro name:req, numParams:req align 16 proc_numParams = numParams - ifdef x64 + if (IS_X64 gt 0) + proc_name equ name + elseif (IS_LINUX gt 0) proc_name equ name + elseif (IS_CDECL gt 0) + proc_name equ @CatStr(_,name) else proc_name equ @CatStr(@,name,@, %numParams * 4) endif @@ -23,18 +72,19 @@ MY_PROC macro name:req, numParams:req endm MY_ENDP macro - ifdef x64 - ret - else - if proc_numParams LT 3 - ret + if (IS_X64 gt 0) + ret + elseif (IS_CDECL gt 0) + ret + elseif (proc_numParams LT 3) + ret else - ret (proc_numParams - 2) * 4 + ret (proc_numParams - 2) * 4 endif - endif proc_name ENDP endm + ifdef x64 REG_SIZE equ 8 REG_LOGAR_SIZE equ 3 @@ -52,6 +102,15 @@ endif x6 equ ESI x7 equ EDI + x0_W equ AX + x1_W equ CX + x2_W equ DX + x3_W equ BX + + x5_W equ BP + x6_W equ SI + x7_W equ DI + x0_L equ AL x1_L equ CL x2_L equ DL @@ -63,6 +122,10 @@ endif x3_H equ BH ifdef x64 + x5_L equ BPL + x6_L equ SIL + x7_L equ DIL + r0 equ RAX r1 equ RCX r2 equ RDX @@ -90,6 +153,24 @@ else r7 equ x7 endif + +ifdef x64 +ifdef ABI_LINUX + +MY_PUSH_2_REGS macro + push r3 + push r5 +endm + +MY_POP_2_REGS macro + pop r5 + pop r3 +endm + +endif +endif + + MY_PUSH_4_REGS macro push r3 push r5 @@ -103,3 +184,106 @@ MY_POP_4_REGS macro pop r5 pop r3 endm + + +; for fastcall and for WIN-x64 +REG_PARAM_0_x equ x1 +REG_PARAM_0 equ r1 +REG_PARAM_1_x equ x2 +REG_PARAM_1 equ r2 + +ifndef x64 +; for x86-fastcall + +REG_ABI_PARAM_0_x equ REG_PARAM_0_x +REG_ABI_PARAM_0 equ REG_PARAM_0 +REG_ABI_PARAM_1_x equ REG_PARAM_1_x +REG_ABI_PARAM_1 equ REG_PARAM_1 + +else +; x64 + +if (IS_LINUX eq 0) + +; for WIN-x64: +REG_PARAM_2_x equ x8 +REG_PARAM_2 equ r8 +REG_PARAM_3 equ r9 + +REG_ABI_PARAM_0_x equ REG_PARAM_0_x +REG_ABI_PARAM_0 equ REG_PARAM_0 +REG_ABI_PARAM_1_x equ REG_PARAM_1_x +REG_ABI_PARAM_1 equ REG_PARAM_1 +REG_ABI_PARAM_2_x equ REG_PARAM_2_x +REG_ABI_PARAM_2 equ REG_PARAM_2 +REG_ABI_PARAM_3 equ REG_PARAM_3 + +else +; for LINUX-x64: +REG_LINUX_PARAM_0_x equ x7 +REG_LINUX_PARAM_0 equ r7 +REG_LINUX_PARAM_1_x equ x6 +REG_LINUX_PARAM_1 equ r6 +REG_LINUX_PARAM_2 equ r2 +REG_LINUX_PARAM_3 equ r1 +REG_LINUX_PARAM_4_x equ x8 +REG_LINUX_PARAM_4 equ r8 +REG_LINUX_PARAM_5 equ r9 + +REG_ABI_PARAM_0_x equ REG_LINUX_PARAM_0_x +REG_ABI_PARAM_0 equ REG_LINUX_PARAM_0 +REG_ABI_PARAM_1_x equ REG_LINUX_PARAM_1_x +REG_ABI_PARAM_1 equ REG_LINUX_PARAM_1 +REG_ABI_PARAM_2 equ REG_LINUX_PARAM_2 +REG_ABI_PARAM_3 equ REG_LINUX_PARAM_3 +REG_ABI_PARAM_4_x equ REG_LINUX_PARAM_4_x +REG_ABI_PARAM_4 equ REG_LINUX_PARAM_4 +REG_ABI_PARAM_5 equ REG_LINUX_PARAM_5 + +MY_ABI_LINUX_TO_WIN_2 macro + mov r2, r6 + mov r1, r7 +endm + +MY_ABI_LINUX_TO_WIN_3 macro + mov r8, r2 + mov r2, r6 + mov r1, r7 +endm + +MY_ABI_LINUX_TO_WIN_4 macro + mov r9, r1 + mov r8, r2 + mov r2, r6 + mov r1, r7 +endm + +endif ; IS_LINUX + + +MY_PUSH_PRESERVED_ABI_REGS macro + if (IS_LINUX gt 0) + MY_PUSH_2_REGS + else + MY_PUSH_4_REGS + endif + push r12 + push r13 + push r14 + push r15 +endm + + +MY_POP_PRESERVED_ABI_REGS macro + pop r15 + pop r14 + pop r13 + pop r12 + if (IS_LINUX gt 0) + MY_POP_2_REGS + else + MY_POP_4_REGS + endif +endm + +endif ; x64 diff --git a/3rdparty/lzma/Asm/x86/7zCrcOpt.asm b/3rdparty/lzma/Asm/x86/7zCrcOpt.asm index 724b9b2c40e..0fee2064e1f 100644 --- a/3rdparty/lzma/Asm/x86/7zCrcOpt.asm +++ b/3rdparty/lzma/Asm/x86/7zCrcOpt.asm @@ -1,5 +1,5 @@ ; 7zCrcOpt.asm -- CRC32 calculation : optimized version -; 2009-12-12 : Igor Pavlov : Public domain +; 2021-02-07 : Igor Pavlov : Public domain include 7zAsm.asm @@ -7,21 +7,28 @@ MY_ASM_START rD equ r2 rN equ r7 +rT equ r5 ifdef x64 num_VAR equ r8 table_VAR equ r9 else - data_size equ (REG_SIZE * 5) - crc_table equ (REG_SIZE + data_size) - num_VAR equ [r4 + data_size] - table_VAR equ [r4 + crc_table] + if (IS_CDECL gt 0) + crc_OFFS equ (REG_SIZE * 5) + data_OFFS equ (REG_SIZE + crc_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + else + size_OFFS equ (REG_SIZE * 5) + endif + table_OFFS equ (REG_SIZE + size_OFFS) + num_VAR equ [r4 + size_OFFS] + table_VAR equ [r4 + table_OFFS] endif -SRCDAT equ rN + rD + 4 * +SRCDAT equ rD + rN * 1 + 4 * CRC macro op:req, dest:req, src:req, t:req - op dest, DWORD PTR [r5 + src * 4 + 0400h * t] + op dest, DWORD PTR [rT + src * 4 + 0400h * t] endm CRC_XOR macro dest:req, src:req, t:req @@ -43,11 +50,33 @@ CRC1b macro endm MY_PROLOG macro crc_end:req - MY_PUSH_4_REGS + + ifdef x64 + if (IS_LINUX gt 0) + MY_PUSH_2_REGS + mov x0, REG_ABI_PARAM_0_x ; x0 = x7 + mov rT, REG_ABI_PARAM_3 ; r5 = r1 + mov rN, REG_ABI_PARAM_2 ; r7 = r2 + mov rD, REG_ABI_PARAM_1 ; r2 = r6 + else + MY_PUSH_4_REGS + mov x0, REG_ABI_PARAM_0_x ; x0 = x1 + mov rT, REG_ABI_PARAM_3 ; r5 = r9 + mov rN, REG_ABI_PARAM_2 ; r7 = r8 + ; mov rD, REG_ABI_PARAM_1 ; r2 = r2 + endif + else + MY_PUSH_4_REGS + if (IS_CDECL gt 0) + mov x0, [r4 + crc_OFFS] + mov rD, [r4 + data_OFFS] + else + mov x0, REG_ABI_PARAM_0_x + endif + mov rN, num_VAR + mov rT, table_VAR + endif - mov x0, x1 - mov rN, num_VAR - mov r5, table_VAR test rN, rN jz crc_end @@: @@ -77,7 +106,11 @@ MY_EPILOG macro crc_end:req CRC1b jmp crc_end @@: - MY_POP_4_REGS + if (IS_X64 gt 0) and (IS_LINUX gt 0) + MY_POP_2_REGS + else + MY_POP_4_REGS + endif endm MY_PROC CrcUpdateT8, 4 diff --git a/3rdparty/lzma/Asm/x86/AesOpt.asm b/3rdparty/lzma/Asm/x86/AesOpt.asm index 21466712994..84bf8977f88 100644 --- a/3rdparty/lzma/Asm/x86/AesOpt.asm +++ b/3rdparty/lzma/Asm/x86/AesOpt.asm @@ -1,237 +1,742 @@ -; AesOpt.asm -- Intel's AES. -; 2009-12-12 : Igor Pavlov : Public domain +; AesOpt.asm -- AES optimized code for x86 AES hardware instructions +; 2021-12-25 : Igor Pavlov : Public domain include 7zAsm.asm +ifdef __ASMC__ + use_vaes_256 equ 1 +else +ifdef ymm0 + use_vaes_256 equ 1 +endif +endif + + +ifdef use_vaes_256 + ECHO "++ VAES 256" +else + ECHO "-- NO VAES 256" +endif + +ifdef x64 + ECHO "x86-64" +else + ECHO "x86" +if (IS_CDECL gt 0) + ECHO "ABI : CDECL" +else + ECHO "ABI : no CDECL : FASTCALL" +endif +endif + +if (IS_LINUX gt 0) + ECHO "ABI : LINUX" +else + ECHO "ABI : WINDOWS" +endif + MY_ASM_START ifndef x64 + .686 .xmm endif + +; MY_ALIGN EQU ALIGN(64) +MY_ALIGN EQU + +SEG_ALIGN EQU MY_ALIGN + +MY_SEG_PROC macro name:req, numParams:req + ; seg_name equ @CatStr(_TEXT$, name) + ; seg_name SEGMENT SEG_ALIGN 'CODE' + MY_PROC name, numParams +endm + +MY_SEG_ENDP macro + ; seg_name ENDS +endm + + +NUM_AES_KEYS_MAX equ 15 + +; the number of push operators in function PROLOG +if (IS_LINUX eq 0) or (IS_X64 eq 0) +num_regs_push equ 2 +stack_param_offset equ (REG_SIZE * (1 + num_regs_push)) +endif + ifdef x64 - num equ r8 + num_param equ REG_ABI_PARAM_2 else - num equ [r4 + REG_SIZE * 4] + if (IS_CDECL gt 0) + ; size_t size + ; void * data + ; UInt32 * aes + ; ret-ip <- (r4) + aes_OFFS equ (stack_param_offset) + data_OFFS equ (REG_SIZE + aes_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + num_param equ [r4 + size_OFFS] + else + num_param equ [r4 + stack_param_offset] + endif endif -rD equ r2 -rN equ r0 +keys equ REG_PARAM_0 ; r1 +rD equ REG_PARAM_1 ; r2 +rN equ r0 -MY_PROLOG macro reg:req - ifdef x64 - movdqa [r4 + 8], xmm6 - movdqa [r4 + 8 + 16], xmm7 - endif +koffs_x equ x7 +koffs_r equ r7 + +ksize_x equ x6 +ksize_r equ r6 - push r3 - push r5 - push r6 +keys2 equ r3 - mov rN, num - mov x6, [r1 + 16] - shl x6, 5 +state equ xmm0 +key equ xmm0 +key_ymm equ ymm0 +key_ymm_n equ 0 - movdqa reg, [r1] - add r1, 32 +ifdef x64 + ways = 11 +else + ways = 4 +endif + +ways_start_reg equ 1 + +iv equ @CatStr(xmm, %(ways_start_reg + ways)) +iv_ymm equ @CatStr(ymm, %(ways_start_reg + ways)) + + +WOP macro op, op2 + i = 0 + rept ways + op @CatStr(xmm, %(ways_start_reg + i)), op2 + i = i + 1 + endm endm -MY_EPILOG macro - pop r6 - pop r5 - pop r3 +ifndef ABI_LINUX +ifdef x64 + +; we use 32 bytes of home space in stack in WIN64-x64 +NUM_HOME_MM_REGS equ (32 / 16) +; we preserve xmm registers starting from xmm6 in WIN64-x64 +MM_START_SAVE_REG equ 6 + +SAVE_XMM macro num_used_mm_regs:req + num_save_mm_regs = num_used_mm_regs - MM_START_SAVE_REG + if num_save_mm_regs GT 0 + num_save_mm_regs2 = num_save_mm_regs - NUM_HOME_MM_REGS + ; RSP is (16*x + 8) after entering the function in WIN64-x64 + stack_offset = 16 * num_save_mm_regs2 + (stack_param_offset mod 16) + + i = 0 + rept num_save_mm_regs + + if i eq NUM_HOME_MM_REGS + sub r4, stack_offset + endif + + if i lt NUM_HOME_MM_REGS + movdqa [r4 + stack_param_offset + i * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i)) + else + movdqa [r4 + (i - NUM_HOME_MM_REGS) * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i)) + endif + + i = i + 1 + endm + endif +endm + +RESTORE_XMM macro num_used_mm_regs:req + if num_save_mm_regs GT 0 + i = 0 + if num_save_mm_regs2 GT 0 + rept num_save_mm_regs2 + movdqa @CatStr(xmm, %(MM_START_SAVE_REG + NUM_HOME_MM_REGS + i)), [r4 + i * 16] + i = i + 1 + endm + add r4, stack_offset + endif + + num_low_regs = num_save_mm_regs - i + i = 0 + rept num_low_regs + movdqa @CatStr(xmm, %(MM_START_SAVE_REG + i)), [r4 + stack_param_offset + i * 16] + i = i + 1 + endm + endif +endm + +endif ; x64 +endif ; ABI_LINUX + + +MY_PROLOG macro num_used_mm_regs:req + ; num_regs_push: must be equal to the number of push operators + ; push r3 + ; push r5 + if (IS_LINUX eq 0) or (IS_X64 eq 0) + push r6 + push r7 + endif + + mov rN, num_param ; don't move it; num_param can use stack pointer (r4) + + if (IS_X64 eq 0) + if (IS_CDECL gt 0) + mov rD, [r4 + data_OFFS] + mov keys, [r4 + aes_OFFS] + endif + elseif (IS_LINUX gt 0) + MY_ABI_LINUX_TO_WIN_2 + endif + + + ifndef ABI_LINUX ifdef x64 - movdqa xmm6, [r4 + 8] - movdqa xmm7, [r4 + 8 + 16] + SAVE_XMM num_used_mm_regs endif + endif + + mov ksize_x, [keys + 16] + shl ksize_x, 5 +endm + +MY_EPILOG macro + ifndef ABI_LINUX + ifdef x64 + RESTORE_XMM num_save_mm_regs + endif + endif + + if (IS_LINUX eq 0) or (IS_X64 eq 0) + pop r7 + pop r6 + endif + ; pop r5 + ; pop r3 MY_ENDP endm -ways equ 4 -ways16 equ (ways * 16) -OP_W macro op, op2 +OP_KEY macro op:req, offs:req + op state, [keys + offs] +endm + + +WOP_KEY macro op:req, offs:req + movdqa key, [keys + offs] + WOP op, key +endm + + +; ---------- AES-CBC Decode ---------- + + +XOR_WITH_DATA macro reg, _ppp_ + pxor reg, [rD + i * 16] +endm + +WRITE_TO_DATA macro reg, _ppp_ + movdqa [rD + i * 16], reg +endm + + +; state0 equ @CatStr(xmm, %(ways_start_reg)) + +key0 equ @CatStr(xmm, %(ways_start_reg + ways + 1)) +key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1)) + +key_last equ @CatStr(xmm, %(ways_start_reg + ways + 2)) +key_last_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2)) +key_last_ymm_n equ (ways_start_reg + ways + 2) + +NUM_CBC_REGS equ (ways_start_reg + ways + 3) + + +MY_SEG_PROC AesCbc_Decode_HW, 3 + + AesCbc_Decode_HW_start:: + MY_PROLOG NUM_CBC_REGS + + AesCbc_Decode_HW_start_2:: + movdqa iv, [keys] + add keys, 32 + + movdqa key0, [keys + 1 * ksize_r] + movdqa key_last, [keys] + sub ksize_x, 16 + + jmp check2 + align 16 + nextBlocks2: + WOP movdqa, [rD + i * 16] + mov koffs_x, ksize_x + ; WOP_KEY pxor, ksize_r + 16 + WOP pxor, key0 + ; align 16 + @@: + WOP_KEY aesdec, 1 * koffs_r + sub koffs_r, 16 + jnz @B + ; WOP_KEY aesdeclast, 0 + WOP aesdeclast, key_last + + pxor @CatStr(xmm, %(ways_start_reg)), iv + i = 1 + rept ways - 1 + pxor @CatStr(xmm, %(ways_start_reg + i)), [rD + i * 16 - 16] + i = i + 1 + endm + movdqa iv, [rD + ways * 16 - 16] + WOP WRITE_TO_DATA + + add rD, ways * 16 + AesCbc_Decode_HW_start_3:: + check2: + sub rN, ways + jnc nextBlocks2 + add rN, ways + + sub ksize_x, 16 + + jmp check + nextBlock: + movdqa state, [rD] + mov koffs_x, ksize_x + ; OP_KEY pxor, 1 * ksize_r + 32 + pxor state, key0 + ; movdqa state0, [rD] + ; movdqa state, key0 + ; pxor state, state0 + @@: + OP_KEY aesdec, 1 * koffs_r + 16 + OP_KEY aesdec, 1 * koffs_r + sub koffs_r, 32 + jnz @B + OP_KEY aesdec, 16 + ; OP_KEY aesdeclast, 0 + aesdeclast state, key_last + + pxor state, iv + movdqa iv, [rD] + ; movdqa iv, state0 + movdqa [rD], state + + add rD, 16 + check: + sub rN, 1 + jnc nextBlock + + movdqa [keys - 32], iv +MY_EPILOG + + + + +; ---------- AVX ---------- + + +AVX__WOP_n macro op i = 0 rept ways - op @CatStr(xmm,%i), op2 - i = i + 1 + op (ways_start_reg + i) + i = i + 1 endm endm -LOAD_OP macro op:req, offs:req - op xmm0, [r1 + r3 offs] +AVX__WOP macro op + i = 0 + rept ways + op @CatStr(ymm, %(ways_start_reg + i)) + i = i + 1 + endm endm - -LOAD_OP_W macro op:req, offs:req - movdqa xmm7, [r1 + r3 offs] - OP_W op, xmm7 + + +AVX__WOP_KEY macro op:req, offs:req + vmovdqa key_ymm, ymmword ptr [keys2 + offs] + AVX__WOP_n op endm -; ---------- AES-CBC Decode ---------- +AVX__CBC_START macro reg + ; vpxor reg, key_ymm, ymmword ptr [rD + 32 * i] + vpxor reg, key0_ymm, ymmword ptr [rD + 32 * i] +endm -CBC_DEC_UPDATE macro reg, offs - pxor reg, xmm6 - movdqa xmm6, [rD + offs] - movdqa [rD + offs], reg -endm - -DECODE macro op:req - op aesdec, +16 - @@: - op aesdec, +0 - op aesdec, -16 - sub x3, 32 - jnz @B - op aesdeclast, +0 -endm - -MY_PROC AesCbc_Decode_Intel, 3 - MY_PROLOG xmm6 - - sub x6, 32 - - jmp check2 - - align 16 - nextBlocks2: - mov x3, x6 - OP_W movdqa, [rD + i * 16] - LOAD_OP_W pxor, +32 - DECODE LOAD_OP_W - OP_W CBC_DEC_UPDATE, i * 16 - add rD, ways16 - check2: - sub rN, ways - jnc nextBlocks2 - - add rN, ways - jmp check - - nextBlock: - mov x3, x6 - movdqa xmm1, [rD] - LOAD_OP movdqa, +32 - pxor xmm0, xmm1 - DECODE LOAD_OP - pxor xmm0, xmm6 - movdqa [rD], xmm0 - movdqa xmm6, xmm1 - add rD, 16 - check: - sub rN, 1 - jnc nextBlock - - movdqa [r1 - 32], xmm6 - MY_EPILOG +AVX__CBC_END macro reg + if i eq 0 + vpxor reg, reg, iv_ymm + else + vpxor reg, reg, ymmword ptr [rD + i * 32 - 16] + endif +endm -; ---------- AES-CBC Encode ---------- +AVX__WRITE_TO_DATA macro reg + vmovdqu ymmword ptr [rD + 32 * i], reg +endm + +AVX__XOR_WITH_DATA macro reg + vpxor reg, reg, ymmword ptr [rD + 32 * i] +endm -ENCODE macro op:req - op aesenc, -16 - @@: - op aesenc, +0 - op aesenc, +16 - add r3, 32 - jnz @B - op aesenclast, +0 +AVX__CTR_START macro reg + vpaddq iv_ymm, iv_ymm, one_ymm + ; vpxor reg, iv_ymm, key_ymm + vpxor reg, iv_ymm, key0_ymm endm -MY_PROC AesCbc_Encode_Intel, 3 - MY_PROLOG xmm0 - add r1, r6 - neg r6 - add r6, 32 +MY_VAES_INSTR_2 macro cmd, dest, a1, a2 + db 0c4H + db 2 + 040H + 020h * (1 - (a2) / 8) + 080h * (1 - (dest) / 8) + db 5 + 8 * ((not (a1)) and 15) + db cmd + db 0c0H + 8 * ((dest) and 7) + ((a2) and 7) +endm + +MY_VAES_INSTR macro cmd, dest, a + MY_VAES_INSTR_2 cmd, dest, dest, a +endm - jmp check_e +MY_vaesenc macro dest, a + MY_VAES_INSTR 0dcH, dest, a +endm +MY_vaesenclast macro dest, a + MY_VAES_INSTR 0ddH, dest, a +endm +MY_vaesdec macro dest, a + MY_VAES_INSTR 0deH, dest, a +endm +MY_vaesdeclast macro dest, a + MY_VAES_INSTR 0dfH, dest, a +endm - align 16 - nextBlock_e: - mov r3, r6 - pxor xmm0, [rD] - pxor xmm0, [r1 + r3 - 32] - ENCODE LOAD_OP - movdqa [rD], xmm0 - add rD, 16 - check_e: - sub rN, 1 - jnc nextBlock_e - movdqa [r1 + r6 - 64], xmm0 - MY_EPILOG +AVX__VAES_DEC macro reg + MY_vaesdec reg, key_ymm_n +endm +AVX__VAES_DEC_LAST_key_last macro reg + ; MY_vaesdeclast reg, key_ymm_n + MY_vaesdeclast reg, key_last_ymm_n +endm -; ---------- AES-CTR ---------- +AVX__VAES_ENC macro reg + MY_vaesenc reg, key_ymm_n +endm -XOR_UPD_1 macro reg, offs - pxor reg, [rD + offs] +AVX__VAES_ENC_LAST macro reg + MY_vaesenclast reg, key_ymm_n endm -XOR_UPD_2 macro reg, offs - movdqa [rD + offs], reg +AVX__vinserti128_TO_HIGH macro dest, src + vinserti128 dest, dest, src, 1 endm -MY_PROC AesCtr_Code_Intel, 3 - MY_PROLOG xmm6 - mov r5, r4 - shr r5, 4 - dec r5 - shl r5, 4 +MY_PROC AesCbc_Decode_HW_256, 3 + ifdef use_vaes_256 + MY_PROLOG NUM_CBC_REGS + + cmp rN, ways * 2 + jb AesCbc_Decode_HW_start_2 + + vmovdqa iv, xmmword ptr [keys] + add keys, 32 + + vbroadcasti128 key0_ymm, xmmword ptr [keys + 1 * ksize_r] + vbroadcasti128 key_last_ymm, xmmword ptr [keys] + sub ksize_x, 16 + mov koffs_x, ksize_x + add ksize_x, ksize_x + + AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 2) * 32) + push keys2 + sub r4, AVX_STACK_SUB + ; sub r4, 32 + ; sub r4, ksize_r + ; lea keys2, [r4 + 32] + mov keys2, r4 + and keys2, -32 + broad: + vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r] + vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm + sub koffs_r, 16 + ; jnc broad + jnz broad + + sub rN, ways * 2 + + align 16 + avx_cbcdec_nextBlock2: + mov koffs_x, ksize_x + ; AVX__WOP_KEY AVX__CBC_START, 1 * koffs_r + 32 + AVX__WOP AVX__CBC_START + @@: + AVX__WOP_KEY AVX__VAES_DEC, 1 * koffs_r + sub koffs_r, 32 + jnz @B + ; AVX__WOP_KEY AVX__VAES_DEC_LAST, 0 + AVX__WOP_n AVX__VAES_DEC_LAST_key_last + + AVX__vinserti128_TO_HIGH iv_ymm, xmmword ptr [rD] + AVX__WOP AVX__CBC_END + + vmovdqa iv, xmmword ptr [rD + ways * 32 - 16] + AVX__WOP AVX__WRITE_TO_DATA + + add rD, ways * 32 + sub rN, ways * 2 + jnc avx_cbcdec_nextBlock2 + add rN, ways * 2 + + shr ksize_x, 1 + + ; lea r4, [r4 + 1 * ksize_r + 32] + add r4, AVX_STACK_SUB + pop keys2 + + vzeroupper + jmp AesCbc_Decode_HW_start_3 + else + jmp AesCbc_Decode_HW_start + endif +MY_ENDP +MY_SEG_ENDP + + - mov DWORD PTR [r5], 1 - mov DWORD PTR [r5 + 4], 0 - mov DWORD PTR [r5 + 8], 0 - mov DWORD PTR [r5 + 12], 0 - add r1, r6 - neg r6 - add r6, 32 +; ---------- AES-CBC Encode ---------- + +e0 equ xmm1 - jmp check2_c +CENC_START_KEY equ 2 +CENC_NUM_REG_KEYS equ (3 * 2) +; last_key equ @CatStr(xmm, %(CENC_START_KEY + CENC_NUM_REG_KEYS)) - align 16 - nextBlocks2_c: - movdqa xmm7, [r5] +MY_SEG_PROC AesCbc_Encode_HW, 3 + MY_PROLOG (CENC_START_KEY + CENC_NUM_REG_KEYS + 0) + movdqa state, [keys] + add keys, 32 + i = 0 - rept ways - paddq xmm6, xmm7 - movdqa @CatStr(xmm,%i), xmm6 - i = i + 1 + rept CENC_NUM_REG_KEYS + movdqa @CatStr(xmm, %(CENC_START_KEY + i)), [keys + i * 16] + i = i + 1 + endm + + add keys, ksize_r + neg ksize_r + add ksize_r, (16 * CENC_NUM_REG_KEYS) + ; movdqa last_key, [keys] + jmp check_e + + align 16 + nextBlock_e: + movdqa e0, [rD] + mov koffs_r, ksize_r + pxor e0, @CatStr(xmm, %(CENC_START_KEY)) + pxor state, e0 + + i = 1 + rept (CENC_NUM_REG_KEYS - 1) + aesenc state, @CatStr(xmm, %(CENC_START_KEY + i)) + i = i + 1 endm - mov r3, r6 - LOAD_OP_W pxor, -32 - ENCODE LOAD_OP_W - OP_W XOR_UPD_1, i * 16 - OP_W XOR_UPD_2, i * 16 - add rD, ways16 - check2_c: - sub rN, ways - jnc nextBlocks2_c - - add rN, ways - jmp check_c - - nextBlock_c: - paddq xmm6, [r5] - mov r3, r6 - movdqa xmm0, [r1 + r3 - 32] - pxor xmm0, xmm6 - ENCODE LOAD_OP - XOR_UPD_1 xmm0, 0 - XOR_UPD_2 xmm0, 0 - add rD, 16 - check_c: - sub rN, 1 - jnc nextBlock_c - - movdqa [r1 + r6 - 64], xmm6 - MY_EPILOG + @@: + OP_KEY aesenc, 1 * koffs_r + OP_KEY aesenc, 1 * koffs_r + 16 + add koffs_r, 32 + jnz @B + OP_KEY aesenclast, 0 + ; aesenclast state, last_key + + movdqa [rD], state + add rD, 16 + check_e: + sub rN, 1 + jnc nextBlock_e + + ; movdqa [keys - 32], state + movdqa [keys + 1 * ksize_r - (16 * CENC_NUM_REG_KEYS) - 32], state +MY_EPILOG +MY_SEG_ENDP + + + +; ---------- AES-CTR ---------- + +ifdef x64 + ; ways = 11 +endif + + +one equ @CatStr(xmm, %(ways_start_reg + ways + 1)) +one_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1)) +key0 equ @CatStr(xmm, %(ways_start_reg + ways + 2)) +key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2)) +NUM_CTR_REGS equ (ways_start_reg + ways + 3) + +INIT_CTR macro reg, _ppp_ + paddq iv, one + movdqa reg, iv +endm + + +MY_SEG_PROC AesCtr_Code_HW, 3 + Ctr_start:: + MY_PROLOG NUM_CTR_REGS + + Ctr_start_2:: + movdqa iv, [keys] + add keys, 32 + movdqa key0, [keys] + + add keys, ksize_r + neg ksize_r + add ksize_r, 16 + + Ctr_start_3:: + mov koffs_x, 1 + movd one, koffs_x + jmp check2_c + + align 16 + nextBlocks2_c: + WOP INIT_CTR, 0 + mov koffs_r, ksize_r + ; WOP_KEY pxor, 1 * koffs_r -16 + WOP pxor, key0 + @@: + WOP_KEY aesenc, 1 * koffs_r + add koffs_r, 16 + jnz @B + WOP_KEY aesenclast, 0 + + WOP XOR_WITH_DATA + WOP WRITE_TO_DATA + add rD, ways * 16 + check2_c: + sub rN, ways + jnc nextBlocks2_c + add rN, ways + + sub keys, 16 + add ksize_r, 16 + + jmp check_c + + ; align 16 + nextBlock_c: + paddq iv, one + ; movdqa state, [keys + 1 * koffs_r - 16] + movdqa state, key0 + mov koffs_r, ksize_r + pxor state, iv + + @@: + OP_KEY aesenc, 1 * koffs_r + OP_KEY aesenc, 1 * koffs_r + 16 + add koffs_r, 32 + jnz @B + OP_KEY aesenc, 0 + OP_KEY aesenclast, 16 + + pxor state, [rD] + movdqa [rD], state + add rD, 16 + check_c: + sub rN, 1 + jnc nextBlock_c + + ; movdqa [keys - 32], iv + movdqa [keys + 1 * ksize_r - 16 - 32], iv +MY_EPILOG + + +MY_PROC AesCtr_Code_HW_256, 3 + ifdef use_vaes_256 + MY_PROLOG NUM_CTR_REGS + + cmp rN, ways * 2 + jb Ctr_start_2 + + vbroadcasti128 iv_ymm, xmmword ptr [keys] + add keys, 32 + vbroadcasti128 key0_ymm, xmmword ptr [keys] + mov koffs_x, 1 + vmovd one, koffs_x + vpsubq iv_ymm, iv_ymm, one_ymm + vpaddq one, one, one + AVX__vinserti128_TO_HIGH one_ymm, one + + add keys, ksize_r + sub ksize_x, 16 + neg ksize_r + mov koffs_r, ksize_r + add ksize_r, ksize_r + + AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 1) * 32) + push keys2 + lea keys2, [r4 - 32] + sub r4, AVX_STACK_SUB + and keys2, -32 + vbroadcasti128 key_ymm, xmmword ptr [keys] + vmovdqa ymmword ptr [keys2], key_ymm + @@: + vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r] + vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm + add koffs_r, 16 + jnz @B + + sub rN, ways * 2 + + align 16 + avx_ctr_nextBlock2: + mov koffs_r, ksize_r + AVX__WOP AVX__CTR_START + ; AVX__WOP_KEY AVX__CTR_START, 1 * koffs_r - 32 + @@: + AVX__WOP_KEY AVX__VAES_ENC, 1 * koffs_r + add koffs_r, 32 + jnz @B + AVX__WOP_KEY AVX__VAES_ENC_LAST, 0 + + AVX__WOP AVX__XOR_WITH_DATA + AVX__WOP AVX__WRITE_TO_DATA + + add rD, ways * 32 + sub rN, ways * 2 + jnc avx_ctr_nextBlock2 + add rN, ways * 2 + + vextracti128 iv, iv_ymm, 1 + sar ksize_r, 1 + + add r4, AVX_STACK_SUB + pop keys2 + + vzeroupper + jmp Ctr_start_3 + else + jmp Ctr_start + endif +MY_ENDP +MY_SEG_ENDP end diff --git a/3rdparty/lzma/Asm/x86/LzFindOpt.asm b/3rdparty/lzma/Asm/x86/LzFindOpt.asm new file mode 100644 index 00000000000..42e10bda2ee --- /dev/null +++ b/3rdparty/lzma/Asm/x86/LzFindOpt.asm @@ -0,0 +1,513 @@ +; LzFindOpt.asm -- ASM version of GetMatchesSpecN_2() function +; 2021-07-21: Igor Pavlov : Public domain +; + +ifndef x64 +; x64=1 +; .err +endif + +include 7zAsm.asm + +MY_ASM_START + +_TEXT$LZFINDOPT SEGMENT ALIGN(64) 'CODE' + +MY_ALIGN macro num:req + align num +endm + +MY_ALIGN_32 macro + MY_ALIGN 32 +endm + +MY_ALIGN_64 macro + MY_ALIGN 64 +endm + + +t0_L equ x0_L +t0_x equ x0 +t0 equ r0 +t1_x equ x3 +t1 equ r3 + +cp_x equ t1_x +cp_r equ t1 +m equ x5 +m_r equ r5 +len_x equ x6 +len equ r6 +diff_x equ x7 +diff equ r7 +len0 equ r10 +len1_x equ x11 +len1 equ r11 +maxLen_x equ x12 +maxLen equ r12 +d equ r13 +ptr0 equ r14 +ptr1 equ r15 + +d_lim equ m_r +cycSize equ len_x +hash_lim equ len0 +delta1_x equ len1_x +delta1_r equ len1 +delta_x equ maxLen_x +delta_r equ maxLen +hash equ ptr0 +src equ ptr1 + + + +if (IS_LINUX gt 0) + +; r1 r2 r8 r9 : win32 +; r7 r6 r2 r1 r8 r9 : linux + +lenLimit equ r8 +lenLimit_x equ x8 +; pos_r equ r2 +pos equ x2 +cur equ r1 +son equ r9 + +else + +lenLimit equ REG_ABI_PARAM_2 +lenLimit_x equ REG_ABI_PARAM_2_x +pos equ REG_ABI_PARAM_1_x +cur equ REG_ABI_PARAM_0 +son equ REG_ABI_PARAM_3 + +endif + + +if (IS_LINUX gt 0) + maxLen_OFFS equ (REG_SIZE * (6 + 1)) +else + cutValue_OFFS equ (REG_SIZE * (8 + 1 + 4)) + d_OFFS equ (REG_SIZE + cutValue_OFFS) + maxLen_OFFS equ (REG_SIZE + d_OFFS) +endif + hash_OFFS equ (REG_SIZE + maxLen_OFFS) + limit_OFFS equ (REG_SIZE + hash_OFFS) + size_OFFS equ (REG_SIZE + limit_OFFS) + cycPos_OFFS equ (REG_SIZE + size_OFFS) + cycSize_OFFS equ (REG_SIZE + cycPos_OFFS) + posRes_OFFS equ (REG_SIZE + cycSize_OFFS) + +if (IS_LINUX gt 0) +else + cutValue_PAR equ [r0 + cutValue_OFFS] + d_PAR equ [r0 + d_OFFS] +endif + maxLen_PAR equ [r0 + maxLen_OFFS] + hash_PAR equ [r0 + hash_OFFS] + limit_PAR equ [r0 + limit_OFFS] + size_PAR equ [r0 + size_OFFS] + cycPos_PAR equ [r0 + cycPos_OFFS] + cycSize_PAR equ [r0 + cycSize_OFFS] + posRes_PAR equ [r0 + posRes_OFFS] + + + cutValue_VAR equ DWORD PTR [r4 + 8 * 0] + cutValueCur_VAR equ DWORD PTR [r4 + 8 * 0 + 4] + cycPos_VAR equ DWORD PTR [r4 + 8 * 1 + 0] + cycSize_VAR equ DWORD PTR [r4 + 8 * 1 + 4] + hash_VAR equ QWORD PTR [r4 + 8 * 2] + limit_VAR equ QWORD PTR [r4 + 8 * 3] + size_VAR equ QWORD PTR [r4 + 8 * 4] + distances equ QWORD PTR [r4 + 8 * 5] + maxLen_VAR equ QWORD PTR [r4 + 8 * 6] + + Old_RSP equ QWORD PTR [r4 + 8 * 7] + LOCAL_SIZE equ 8 * 8 + +COPY_VAR_32 macro dest_var, src_var + mov x3, src_var + mov dest_var, x3 +endm + +COPY_VAR_64 macro dest_var, src_var + mov r3, src_var + mov dest_var, r3 +endm + + +; MY_ALIGN_64 +MY_PROC GetMatchesSpecN_2, 13 +MY_PUSH_PRESERVED_ABI_REGS + mov r0, RSP + lea r3, [r0 - LOCAL_SIZE] + and r3, -64 + mov RSP, r3 + mov Old_RSP, r0 + +if (IS_LINUX gt 0) + mov d, REG_ABI_PARAM_5 ; r13 = r9 + mov cutValue_VAR, REG_ABI_PARAM_4_x ; = r8 + mov son, REG_ABI_PARAM_3 ; r9 = r1 + mov r8, REG_ABI_PARAM_2 ; r8 = r2 + mov pos, REG_ABI_PARAM_1_x ; r2 = x6 + mov r1, REG_ABI_PARAM_0 ; r1 = r7 +else + COPY_VAR_32 cutValue_VAR, cutValue_PAR + mov d, d_PAR +endif + + COPY_VAR_64 limit_VAR, limit_PAR + + mov hash_lim, size_PAR + mov size_VAR, hash_lim + + mov cp_x, cycPos_PAR + mov hash, hash_PAR + + mov cycSize, cycSize_PAR + mov cycSize_VAR, cycSize + + ; we want cur in (rcx). So we change the cur and lenLimit variables + sub lenLimit, cur + neg lenLimit_x + inc lenLimit_x + + mov t0_x, maxLen_PAR + sub t0, lenLimit + mov maxLen_VAR, t0 + + jmp main_loop + +MY_ALIGN_64 +fill_empty: + ; ptr0 = *ptr1 = kEmptyHashValue; + mov QWORD PTR [ptr1], 0 + inc pos + inc cp_x + mov DWORD PTR [d - 4], 0 + cmp d, limit_VAR + jae fin + cmp hash, hash_lim + je fin + +; MY_ALIGN_64 +main_loop: + ; UInt32 delta = *hash++; + mov diff_x, [hash] ; delta + add hash, 4 + ; mov cycPos_VAR, cp_x + + inc cur + add d, 4 + mov m, pos + sub m, diff_x; ; matchPos + + ; CLzRef *ptr1 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + lea ptr1, [son + 8 * cp_r] + ; mov cycSize, cycSize_VAR + cmp pos, cycSize + jb directMode ; if (pos < cycSize_VAR) + + ; CYC MODE + + cmp diff_x, cycSize + jae fill_empty ; if (delta >= cycSize_VAR) + + xor t0_x, t0_x + mov cycPos_VAR, cp_x + sub cp_x, diff_x + ; jae prepare_for_tree_loop + ; add cp_x, cycSize + cmovb t0_x, cycSize + add cp_x, t0_x ; cp_x += (cycPos < delta ? cycSize : 0) + jmp prepare_for_tree_loop + + +directMode: + cmp diff_x, pos + je fill_empty ; if (delta == pos) + jae fin_error ; if (delta >= pos) + + mov cycPos_VAR, cp_x + mov cp_x, m + +prepare_for_tree_loop: + mov len0, lenLimit + mov hash_VAR, hash + ; CLzRef *ptr0 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2 + 1; + lea ptr0, [ptr1 + 4] + ; UInt32 *_distances = ++d; + mov distances, d + + neg len0 + mov len1, len0 + + mov t0_x, cutValue_VAR + mov maxLen, maxLen_VAR + mov cutValueCur_VAR, t0_x + +MY_ALIGN_32 +tree_loop: + neg diff + mov len, len0 + cmp len1, len0 + cmovb len, len1 ; len = (len1 < len0 ? len1 : len0); + add diff, cur + + mov t0_x, [son + cp_r * 8] ; prefetch + movzx t0_x, BYTE PTR [diff + 1 * len] + lea cp_r, [son + cp_r * 8] + cmp [cur + 1 * len], t0_L + je matched_1 + + jb left_0 + + mov [ptr1], m + mov m, [cp_r + 4] + lea ptr1, [cp_r + 4] + sub diff, cur ; FIX32 + jmp next_node + +MY_ALIGN_32 +left_0: + mov [ptr0], m + mov m, [cp_r] + mov ptr0, cp_r + sub diff, cur ; FIX32 + ; jmp next_node + +; ------------ NEXT NODE ------------ +; MY_ALIGN_32 +next_node: + mov cycSize, cycSize_VAR + dec cutValueCur_VAR + je finish_tree + + add diff_x, pos ; prev_match = pos + diff + cmp m, diff_x + jae fin_error ; if (new_match >= prev_match) + + mov diff_x, pos + sub diff_x, m ; delta = pos - new_match + cmp pos, cycSize + jae cyc_mode_2 ; if (pos >= cycSize) + + mov cp_x, m + test m, m + jne tree_loop ; if (m != 0) + +finish_tree: + ; ptr0 = *ptr1 = kEmptyHashValue; + mov DWORD PTR [ptr0], 0 + mov DWORD PTR [ptr1], 0 + + inc pos + + ; _distances[-1] = (UInt32)(d - _distances); + mov t0, distances + mov t1, d + sub t1, t0 + shr t1_x, 2 + mov [t0 - 4], t1_x + + cmp d, limit_VAR + jae fin ; if (d >= limit) + + mov cp_x, cycPos_VAR + mov hash, hash_VAR + mov hash_lim, size_VAR + inc cp_x + cmp hash, hash_lim + jne main_loop ; if (hash != size) + jmp fin + + +MY_ALIGN_32 +cyc_mode_2: + cmp diff_x, cycSize + jae finish_tree ; if (delta >= cycSize) + + mov cp_x, cycPos_VAR + xor t0_x, t0_x + sub cp_x, diff_x ; cp_x = cycPos - delta + cmovb t0_x, cycSize + add cp_x, t0_x ; cp_x += (cycPos < delta ? cycSize : 0) + jmp tree_loop + + +MY_ALIGN_32 +matched_1: + + inc len + ; cmp len_x, lenLimit_x + je short lenLimit_reach + movzx t0_x, BYTE PTR [diff + 1 * len] + cmp [cur + 1 * len], t0_L + jne mismatch + + +MY_ALIGN_32 +match_loop: + ; while (++len != lenLimit) (len[diff] != len[0]) ; + + inc len + ; cmp len_x, lenLimit_x + je short lenLimit_reach + movzx t0_x, BYTE PTR [diff + 1 * len] + cmp BYTE PTR [cur + 1 * len], t0_L + je match_loop + +mismatch: + jb left_2 + + mov [ptr1], m + mov m, [cp_r + 4] + lea ptr1, [cp_r + 4] + mov len1, len + + jmp max_update + +MY_ALIGN_32 +left_2: + mov [ptr0], m + mov m, [cp_r] + mov ptr0, cp_r + mov len0, len + +max_update: + sub diff, cur ; restore diff + + cmp maxLen, len + jae next_node + + mov maxLen, len + add len, lenLimit + mov [d], len_x + mov t0_x, diff_x + not t0_x + mov [d + 4], t0_x + add d, 8 + + jmp next_node + + + +MY_ALIGN_32 +lenLimit_reach: + + mov delta_r, cur + sub delta_r, diff + lea delta1_r, [delta_r - 1] + + mov t0_x, [cp_r] + mov [ptr1], t0_x + mov t0_x, [cp_r + 4] + mov [ptr0], t0_x + + mov [d], lenLimit_x + mov [d + 4], delta1_x + add d, 8 + + ; _distances[-1] = (UInt32)(d - _distances); + mov t0, distances + mov t1, d + sub t1, t0 + shr t1_x, 2 + mov [t0 - 4], t1_x + + mov hash, hash_VAR + mov hash_lim, size_VAR + + inc pos + mov cp_x, cycPos_VAR + inc cp_x + + mov d_lim, limit_VAR + mov cycSize, cycSize_VAR + ; if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + ; break; + cmp hash, hash_lim + je fin + cmp d, d_lim + jae fin + cmp delta_x, [hash] + jne main_loop + movzx t0_x, BYTE PTR [diff] + cmp [cur], t0_L + jne main_loop + + ; jmp main_loop ; bypass for debug + + mov cycPos_VAR, cp_x + shl len, 3 ; cycSize * 8 + sub diff, cur ; restore diff + xor t0_x, t0_x + cmp cp_x, delta_x ; cmp (cycPos_VAR, delta) + lea cp_r, [son + 8 * cp_r] ; dest + lea src, [cp_r + 8 * diff] + cmovb t0, len ; t0 = (cycPos_VAR < delta ? cycSize * 8 : 0) + add src, t0 + add len, son ; len = son + cycSize * 8 + + +MY_ALIGN_32 +long_loop: + add hash, 4 + + ; *(UInt64 *)(void *)ptr = ((const UInt64 *)(const void *)ptr)[diff]; + + mov t0, [src] + add src, 8 + mov [cp_r], t0 + add cp_r, 8 + cmp src, len + cmove src, son ; if end of (son) buffer is reached, we wrap to begin + + mov DWORD PTR [d], 2 + mov [d + 4], lenLimit_x + mov [d + 8], delta1_x + add d, 12 + + inc cur + + cmp hash, hash_lim + je long_footer + cmp delta_x, [hash] + jne long_footer + movzx t0_x, BYTE PTR [diff + 1 * cur] + cmp [cur], t0_L + jne long_footer + cmp d, d_lim + jb long_loop + +long_footer: + sub cp_r, son + shr cp_r, 3 + add pos, cp_x + sub pos, cycPos_VAR + mov cycSize, cycSize_VAR + + cmp d, d_lim + jae fin + cmp hash, hash_lim + jne main_loop + jmp fin + + + +fin_error: + xor d, d + +fin: + mov RSP, Old_RSP + mov t0, [r4 + posRes_OFFS] + mov [t0], pos + mov r0, d + +MY_POP_PRESERVED_ABI_REGS +MY_ENDP + +_TEXT$LZFINDOPT ENDS + +end diff --git a/3rdparty/lzma/Asm/x86/LzmaDecOpt.asm b/3rdparty/lzma/Asm/x86/LzmaDecOpt.asm new file mode 100644 index 00000000000..f2818e77b30 --- /dev/null +++ b/3rdparty/lzma/Asm/x86/LzmaDecOpt.asm @@ -0,0 +1,1303 @@ +; LzmaDecOpt.asm -- ASM version of LzmaDec_DecodeReal_3() function +; 2021-02-23: Igor Pavlov : Public domain +; +; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() +; function for check at link time. +; That code is tightly coupled with LzmaDec_TryDummy() +; and with another functions in LzmaDec.c file. +; CLzmaDec structure, (probs) array layout, input and output of +; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). + +ifndef x64 +; x64=1 +; .err +endif + +include 7zAsm.asm + +MY_ASM_START + +_TEXT$LZMADECOPT SEGMENT ALIGN(64) 'CODE' + +MY_ALIGN macro num:req + align num +endm + +MY_ALIGN_16 macro + MY_ALIGN 16 +endm + +MY_ALIGN_32 macro + MY_ALIGN 32 +endm + +MY_ALIGN_64 macro + MY_ALIGN 64 +endm + + +; _LZMA_SIZE_OPT equ 1 + +; _LZMA_PROB32 equ 1 + +ifdef _LZMA_PROB32 + PSHIFT equ 2 + PLOAD macro dest, mem + mov dest, dword ptr [mem] + endm + PSTORE macro src, mem + mov dword ptr [mem], src + endm +else + PSHIFT equ 1 + PLOAD macro dest, mem + movzx dest, word ptr [mem] + endm + PSTORE macro src, mem + mov word ptr [mem], @CatStr(src, _W) + endm +endif + +PMULT equ (1 SHL PSHIFT) +PMULT_HALF equ (1 SHL (PSHIFT - 1)) +PMULT_2 equ (1 SHL (PSHIFT + 1)) + +kMatchSpecLen_Error_Data equ (1 SHL 9) + +; x0 range +; x1 pbPos / (prob) TREE +; x2 probBranch / prm (MATCHED) / pbPos / cnt +; x3 sym +;====== r4 === RSP +; x5 cod +; x6 t1 NORM_CALC / probs_state / dist +; x7 t0 NORM_CALC / prob2 IF_BIT_1 +; x8 state +; x9 match (MATCHED) / sym2 / dist2 / lpMask_reg +; x10 kBitModelTotal_reg +; r11 probs +; x12 offs (MATCHED) / dic / len_temp +; x13 processedPos +; x14 bit (MATCHED) / dicPos +; r15 buf + + +cod equ x5 +cod_L equ x5_L +range equ x0 +state equ x8 +state_R equ r8 +buf equ r15 +processedPos equ x13 +kBitModelTotal_reg equ x10 + +probBranch equ x2 +probBranch_R equ r2 +probBranch_W equ x2_W + +pbPos equ x1 +pbPos_R equ r1 + +cnt equ x2 +cnt_R equ r2 + +lpMask_reg equ x9 +dicPos equ r14 + +sym equ x3 +sym_R equ r3 +sym_L equ x3_L + +probs equ r11 +dic equ r12 + +t0 equ x7 +t0_W equ x7_W +t0_R equ r7 + +prob2 equ t0 +prob2_W equ t0_W + +t1 equ x6 +t1_R equ r6 + +probs_state equ t1 +probs_state_R equ t1_R + +prm equ r2 +match equ x9 +match_R equ r9 +offs equ x12 +offs_R equ r12 +bit equ x14 +bit_R equ r14 + +sym2 equ x9 +sym2_R equ r9 + +len_temp equ x12 + +dist equ sym +dist2 equ x9 + + + +kNumBitModelTotalBits equ 11 +kBitModelTotal equ (1 SHL kNumBitModelTotalBits) +kNumMoveBits equ 5 +kBitModelOffset equ ((1 SHL kNumMoveBits) - 1) +kTopValue equ (1 SHL 24) + +NORM_2 macro + ; movzx t0, BYTE PTR [buf] + shl cod, 8 + mov cod_L, BYTE PTR [buf] + shl range, 8 + ; or cod, t0 + inc buf +endm + + +NORM macro + cmp range, kTopValue + jae SHORT @F + NORM_2 +@@: +endm + + +; ---------- Branch MACROS ---------- + +UPDATE_0 macro probsArray:req, probOffset:req, probDisp:req + mov prob2, kBitModelTotal_reg + sub prob2, probBranch + shr prob2, kNumMoveBits + add probBranch, prob2 + PSTORE probBranch, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +UPDATE_1 macro probsArray:req, probOffset:req, probDisp:req + sub prob2, range + sub cod, range + mov range, prob2 + mov prob2, probBranch + shr probBranch, kNumMoveBits + sub prob2, probBranch + PSTORE prob2, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +CMP_COD macro probsArray:req, probOffset:req, probDisp:req + PLOAD probBranch, probOffset * 1 + probsArray + probDisp * PMULT + NORM + mov prob2, range + shr range, kNumBitModelTotalBits + imul range, probBranch + cmp cod, range +endm + + +IF_BIT_1_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jae toLabel +endm + + +IF_BIT_1 macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + IF_BIT_1_NOUP probsArray, probOffset, probDisp, toLabel + UPDATE_0 probsArray, probOffset, probDisp +endm + + +IF_BIT_0_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jb toLabel +endm + + +; ---------- CMOV MACROS ---------- + +NORM_CALC macro prob:req + NORM + mov t0, range + shr range, kNumBitModelTotalBits + imul range, prob + sub t0, range + mov t1, cod + sub cod, range +endm + + +PUP macro prob:req, probPtr:req + sub t0, prob + ; only sar works for both 16/32 bit prob modes + sar t0, kNumMoveBits + add t0, prob + PSTORE t0, probPtr +endm + + +PUP_SUB macro prob:req, probPtr:req, symSub:req + sbb sym, symSub + PUP prob, probPtr +endm + + +PUP_COD macro prob:req, probPtr:req, symSub:req + mov t0, kBitModelOffset + cmovb cod, t1 + mov t1, sym + cmovb t0, kBitModelTotal_reg + PUP_SUB prob, probPtr, symSub +endm + + +BIT_0 macro prob:req, probNext:req + PLOAD prob, probs + 1 * PMULT + PLOAD probNext, probs + 1 * PMULT_2 + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 1 * PMULT_2 + PMULT + cmovae probNext, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 2 + PUP_SUB prob, probs + 1 * PMULT, 0 - 1 +endm + + +BIT_1 macro prob:req, probNext:req + PLOAD probNext, probs + sym_R * PMULT_2 + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + sym_R * PMULT + PMULT + cmovae probNext, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, 0 - 1 +endm + + +BIT_2 macro prob:req, symSub:req + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, symSub +endm + + +; ---------- MATCHED LITERAL ---------- + +LITM_0 macro + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, offs + and bit, match + PLOAD x1, probs + 256 * PMULT + bit_R * 1 + 1 * PMULT + lea prm, [probs + 256 * PMULT + bit_R * 1 + 1 * PMULT] + ; lea prm, [probs + 256 * PMULT + 1 * PMULT] + ; add prm, bit_R + xor offs, bit + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 0 + PUP_SUB x1, prm, -2-1 +endm + + +LITM macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + xor offs, bit + add sym, sym + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, - 1 +endm + + +LITM_2 macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + add sym, sym + + NORM_CALC x1 + + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, 256 - 1 +endm + + +; ---------- REVERSE BITS ---------- + +REV_0 macro prob:req, probNext:req + ; PLOAD prob, probs + 1 * PMULT + ; lea sym2_R, [probs + 2 * PMULT] + ; PLOAD probNext, probs + 2 * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 3 * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [probs + 3 * PMULT] + cmovae sym2_R, t1_R + PUP prob, probs + 1 * PMULT +endm + + +REV_1 macro prob:req, probNext:req, step:req + add sym2_R, step * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, sym2_R + step * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [sym2_R + step * PMULT] + cmovae sym2_R, t1_R + PUP prob, t1_R - step * PMULT_2 +endm + + +REV_2 macro prob:req, step:req + sub sym2_R, probs + shr sym2, PSHIFT + or sym, sym2 + + NORM_CALC prob + + cmovae range, t0 + lea t0, [sym - step] + cmovb sym, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + PUP prob, probs + sym2_R * PMULT +endm + + +REV_1_VAR macro prob:req + PLOAD prob, sym_R + mov probs, sym_R + add sym_R, sym2_R + + NORM_CALC prob + + cmovae range, t0 + lea t0_R, [sym_R + 1 * sym2_R] + cmovae sym_R, t0_R + mov t0, kBitModelOffset + cmovb cod, t1 + ; mov t1, kBitModelTotal + ; cmovb t0, t1 + cmovb t0, kBitModelTotal_reg + add sym2, sym2 + PUP prob, probs +endm + + + + +LIT_PROBS macro lpMaskParam:req + ; prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); + mov t0, processedPos + shl t0, 8 + add sym, t0 + and sym, lpMaskParam + add probs_state_R, pbPos_R + mov x1, LOC lc2 + lea sym, dword ptr[sym_R + 2 * sym_R] + add probs, Literal * PMULT + shl sym, x1_L + add probs, sym_R + UPDATE_0 probs_state_R, 0, IsMatch + inc processedPos +endm + + + +kNumPosBitsMax equ 4 +kNumPosStatesMax equ (1 SHL kNumPosBitsMax) + +kLenNumLowBits equ 3 +kLenNumLowSymbols equ (1 SHL kLenNumLowBits) +kLenNumHighBits equ 8 +kLenNumHighSymbols equ (1 SHL kLenNumHighBits) +kNumLenProbs equ (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +LenLow equ 0 +LenChoice equ LenLow +LenChoice2 equ (LenLow + kLenNumLowSymbols) +LenHigh equ (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +kNumStates equ 12 +kNumStates2 equ 16 +kNumLitStates equ 7 + +kStartPosModelIndex equ 4 +kEndPosModelIndex equ 14 +kNumFullDistances equ (1 SHL (kEndPosModelIndex SHR 1)) + +kNumPosSlotBits equ 6 +kNumLenToPosStates equ 4 + +kNumAlignBits equ 4 +kAlignTableSize equ (1 SHL kNumAlignBits) + +kMatchMinLen equ 2 +kMatchSpecLenStart equ (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +kStartOffset equ 1664 +SpecPos equ (-kStartOffset) +IsRep0Long equ (SpecPos + kNumFullDistances) +RepLenCoder equ (IsRep0Long + (kNumStates2 SHL kNumPosBitsMax)) +LenCoder equ (RepLenCoder + kNumLenProbs) +IsMatch equ (LenCoder + kNumLenProbs) +kAlign equ (IsMatch + (kNumStates2 SHL kNumPosBitsMax)) +IsRep equ (kAlign + kAlignTableSize) +IsRepG0 equ (IsRep + kNumStates) +IsRepG1 equ (IsRepG0 + kNumStates) +IsRepG2 equ (IsRepG1 + kNumStates) +PosSlot equ (IsRepG2 + kNumStates) +Literal equ (PosSlot + (kNumLenToPosStates SHL kNumPosSlotBits)) +NUM_BASE_PROBS equ (Literal + kStartOffset) + +if kAlign ne 0 + .err +endif + +if NUM_BASE_PROBS ne 1984 + .err +endif + + +PTR_FIELD equ dq ? + +CLzmaDec_Asm struct + lc db ? + lp db ? + pb db ? + _pad_ db ? + dicSize dd ? + + probs_Spec PTR_FIELD + probs_1664 PTR_FIELD + dic_Spec PTR_FIELD + dicBufSize PTR_FIELD + dicPos_Spec PTR_FIELD + buf_Spec PTR_FIELD + + range_Spec dd ? + code_Spec dd ? + processedPos_Spec dd ? + checkDicSize dd ? + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? + state_Spec dd ? + remainLen dd ? +CLzmaDec_Asm ends + + +CLzmaDec_Asm_Loc struct + OLD_RSP PTR_FIELD + lzmaPtr PTR_FIELD + _pad0_ PTR_FIELD + _pad1_ PTR_FIELD + _pad2_ PTR_FIELD + dicBufSize PTR_FIELD + probs_Spec PTR_FIELD + dic_Spec PTR_FIELD + + limit PTR_FIELD + bufLimit PTR_FIELD + lc2 dd ? + lpMask dd ? + pbMask dd ? + checkDicSize dd ? + + _pad_ dd ? + remainLen dd ? + dicPos_Spec PTR_FIELD + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? +CLzmaDec_Asm_Loc ends + + +GLOB_2 equ [sym_R].CLzmaDec_Asm. +GLOB equ [r1].CLzmaDec_Asm. +LOC_0 equ [r0].CLzmaDec_Asm_Loc. +LOC equ [RSP].CLzmaDec_Asm_Loc. + + +COPY_VAR macro name + mov t0, GLOB_2 name + mov LOC_0 name, t0 +endm + + +RESTORE_VAR macro name + mov t0, LOC name + mov GLOB name, t0 +endm + + + +IsMatchBranch_Pre macro reg + ; prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + mov pbPos, LOC pbMask + and pbPos, processedPos + shl pbPos, (kLenNumLowBits + 1 + PSHIFT) + lea probs_state_R, [probs + 1 * state_R] +endm + + +IsMatchBranch macro reg + IsMatchBranch_Pre + IF_BIT_1 probs_state_R, pbPos_R, IsMatch, IsMatch_label +endm + + +CheckLimits macro reg + cmp buf, LOC bufLimit + jae fin_OK + cmp dicPos, LOC limit + jae fin_OK +endm + + + +; RSP is (16x + 8) bytes aligned in WIN64-x64 +; LocalSize equ ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) + +PARAM_lzma equ REG_ABI_PARAM_0 +PARAM_limit equ REG_ABI_PARAM_1 +PARAM_bufLimit equ REG_ABI_PARAM_2 + +; MY_ALIGN_64 +MY_PROC LzmaDec_DecodeReal_3, 3 +MY_PUSH_PRESERVED_ABI_REGS + + lea r0, [RSP - (SIZEOF CLzmaDec_Asm_Loc)] + and r0, -128 + mov r5, RSP + mov RSP, r0 + mov LOC_0 Old_RSP, r5 + mov LOC_0 lzmaPtr, PARAM_lzma + + mov LOC_0 remainLen, 0 ; remainLen must be ZERO + + mov LOC_0 bufLimit, PARAM_bufLimit + mov sym_R, PARAM_lzma ; CLzmaDec_Asm_Loc pointer for GLOB_2 + mov dic, GLOB_2 dic_Spec + add PARAM_limit, dic + mov LOC_0 limit, PARAM_limit + + COPY_VAR(rep0) + COPY_VAR(rep1) + COPY_VAR(rep2) + COPY_VAR(rep3) + + mov dicPos, GLOB_2 dicPos_Spec + add dicPos, dic + mov LOC_0 dicPos_Spec, dicPos + mov LOC_0 dic_Spec, dic + + mov x1_L, GLOB_2 pb + mov t0, 1 + shl t0, x1_L + dec t0 + mov LOC_0 pbMask, t0 + + ; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + ; unsigned lc = p->prop.lc; + ; unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); + + mov x1_L, GLOB_2 lc + mov x2, 100h + mov t0, x2 + shr x2, x1_L + ; inc x1 + add x1_L, PSHIFT + mov LOC_0 lc2, x1 + mov x1_L, GLOB_2 lp + shl t0, x1_L + sub t0, x2 + mov LOC_0 lpMask, t0 + mov lpMask_reg, t0 + + ; mov probs, GLOB_2 probs_Spec + ; add probs, kStartOffset SHL PSHIFT + mov probs, GLOB_2 probs_1664 + mov LOC_0 probs_Spec, probs + + mov t0_R, GLOB_2 dicBufSize + mov LOC_0 dicBufSize, t0_R + + mov x1, GLOB_2 checkDicSize + mov LOC_0 checkDicSize, x1 + + mov processedPos, GLOB_2 processedPos_Spec + + mov state, GLOB_2 state_Spec + shl state, PSHIFT + + mov buf, GLOB_2 buf_Spec + mov range, GLOB_2 range_Spec + mov cod, GLOB_2 code_Spec + mov kBitModelTotal_reg, kBitModelTotal + xor sym, sym + + ; if (processedPos != 0 || checkDicSize != 0) + or x1, processedPos + jz @f + + add t0_R, dic + cmp dicPos, dic + cmovnz t0_R, dicPos + movzx sym, byte ptr[t0_R - 1] + +@@: + IsMatchBranch_Pre + cmp state, 4 * PMULT + jb lit_end + cmp state, kNumLitStates * PMULT + jb lit_matched_end + jmp lz_end + + + + +; ---------- LITERAL ---------- +MY_ALIGN_64 +lit_start: + xor state, state +lit_start_2: + LIT_PROBS lpMask_reg + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +lit_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 127 + jbe lit_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + BIT_2 x2, 256 - 1 + + ; mov dic, LOC dic_Spec + mov probs, LOC probs_Spec + IsMatchBranch_Pre + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_end: + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsMatch, lit_start + + ; jmp IsMatch_label + +; ---------- MATCHES ---------- +; MY_ALIGN_32 +IsMatch_label: + UPDATE_1 probs_state_R, pbPos_R, IsMatch + IF_BIT_1 probs_state_R, 0, IsRep, IsRep_label + + add probs, LenCoder * PMULT + add state, kNumStates * PMULT + +; ---------- LEN DECODE ---------- +len_decode: + mov len_temp, 8 - 1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, (1 SHL (kLenNumLowBits + PSHIFT)) + mov len_temp, -1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, LenHigh * PMULT - (1 SHL (kLenNumLowBits + PSHIFT)) + mov sym, 1 + PLOAD x1, probs + 1 * PMULT + +MY_ALIGN_32 +len8_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 64 + jb len8_loop + + mov len_temp, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - 1 - kMatchMinLen + jmp short len_mid_2 ; we use short here for MASM that doesn't optimize that code as another assembler programs + +MY_ALIGN_32 +len_mid_0: + UPDATE_0 probs, 0, 0 + add probs, pbPos_R + BIT_0 x2, x1 +len_mid_2: + BIT_1 x1, x2 + BIT_2 x2, len_temp + mov probs, LOC probs_Spec + cmp state, kNumStates * PMULT + jb copy_match + + +; ---------- DECODE DISTANCE ---------- + ; probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + + mov t0, 3 + kMatchMinLen + cmp sym, 3 + kMatchMinLen + cmovb t0, sym + add probs, PosSlot * PMULT - (kMatchMinLen SHL (kNumPosSlotBits + PSHIFT)) + shl t0, (kNumPosSlotBits + PSHIFT) + add probs, t0_R + + ; sym = Len + ; mov LOC remainLen, sym + mov len_temp, sym + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +slot_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 32 + jb slot_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + mov x1, sym + BIT_2 x2, 64-1 + + and sym, 3 + mov probs, LOC probs_Spec + cmp x1, 32 + kEndPosModelIndex / 2 + jb short_dist + + ; unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + sub x1, (32 + 1 + kNumAlignBits) + ; distance = (2 | (distance & 1)); + or sym, 2 + PLOAD x2, probs + 1 * PMULT + shl sym, kNumAlignBits + 1 + lea sym2_R, [probs + 2 * PMULT] + + jmp direct_norm + ; lea t1, [sym_R + (1 SHL kNumAlignBits)] + ; cmp range, kTopValue + ; jb direct_norm + +; ---------- DIRECT DISTANCE ---------- +MY_ALIGN_32 +direct_loop: + shr range, 1 + mov t0, cod + sub cod, range + cmovs cod, t0 + cmovns sym, t1 + + comment ~ + sub cod, range + mov x2, cod + sar x2, 31 + lea sym, dword ptr [r2 + sym_R * 2 + 1] + and x2, range + add cod, x2 + ~ + dec x1 + je direct_end + + add sym, sym +direct_norm: + lea t1, [sym_R + (1 SHL kNumAlignBits)] + cmp range, kTopValue + jae near ptr direct_loop + ; we align for 32 here with "near ptr" command above + NORM_2 + jmp direct_loop + +MY_ALIGN_32 +direct_end: + ; prob = + kAlign; + ; distance <<= kNumAlignBits; + REV_0 x2, x1 + REV_1 x1, x2, 2 + REV_1 x2, x1, 4 + REV_2 x1, 8 + +decode_dist_end: + + ; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + + mov t1, LOC rep0 + mov x1, LOC rep1 + mov x2, LOC rep2 + + mov t0, LOC checkDicSize + test t0, t0 + cmove t0, processedPos + cmp sym, t0 + jae end_of_payload + ; jmp end_of_payload ; for debug + + ; rep3 = rep2; + ; rep2 = rep1; + ; rep1 = rep0; + ; rep0 = distance + 1; + + inc sym + mov LOC rep0, sym + ; mov sym, LOC remainLen + mov sym, len_temp + mov LOC rep1, t1 + mov LOC rep2, x1 + mov LOC rep3, x2 + + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + cmp state, (kNumStates + kNumLitStates) * PMULT + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 + + +; ---------- COPY MATCH ---------- +copy_match: + + ; len += kMatchMinLen; + ; add sym, kMatchMinLen + + ; if ((rem = limit - dicPos) == 0) + ; { + ; p->dicPos = dicPos; + ; return SZ_ERROR_DATA; + ; } + mov cnt_R, LOC limit + sub cnt_R, dicPos + jz fin_dicPos_LIMIT + + ; curLen = ((rem < len) ? (unsigned)rem : len); + cmp cnt_R, sym_R + ; cmovae cnt_R, sym_R ; 64-bit + cmovae cnt, sym ; 32-bit + + mov dic, LOC dic_Spec + mov x1, LOC rep0 + + mov t0_R, dicPos + add dicPos, cnt_R + ; processedPos += curLen; + add processedPos, cnt + ; len -= curLen; + sub sym, cnt + mov LOC remainLen, sym + + sub t0_R, dic + + ; pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + sub t0_R, r1 + jae @f + + mov r1, LOC dicBufSize + add t0_R, r1 + sub r1, t0_R + cmp cnt_R, r1 + ja copy_match_cross +@@: + ; if (curLen <= dicBufSize - pos) + +; ---------- COPY MATCH FAST ---------- + ; Byte *dest = dic + dicPos; + ; mov r1, dic + ; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + ; sub t0_R, dicPos + ; dicPos += curLen; + + ; const Byte *lim = dest + curLen; + add t0_R, dic + movzx sym, byte ptr[t0_R] + add t0_R, cnt_R + neg cnt_R + ; lea r1, [dicPos - 1] +copy_common: + dec dicPos + ; cmp LOC rep0, 1 + ; je rep0Label + + ; t0_R - src_lim + ; r1 - dest_lim - 1 + ; cnt_R - (-cnt) + + IsMatchBranch_Pre + inc cnt_R + jz copy_end +MY_ALIGN_16 +@@: + mov byte ptr[cnt_R * 1 + dicPos], sym_L + movzx sym, byte ptr[cnt_R * 1 + t0_R] + inc cnt_R + jnz @b + +copy_end: +lz_end_match: + mov byte ptr[dicPos], sym_L + inc dicPos + + ; IsMatchBranch_Pre + CheckLimits +lz_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + + + +; ---------- LITERAL MATCHED ---------- + + LIT_PROBS LOC lpMask + + ; matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov x1, LOC rep0 + ; mov dic, LOC dic_Spec + mov LOC dicPos_Spec, dicPos + + ; state -= (state < 10) ? 3 : 6; + lea t0, [state_R - 6 * PMULT] + sub state, 3 * PMULT + cmp state, 7 * PMULT + cmovae state, t0 + + sub dicPos, dic + sub dicPos, r1 + jae @f + add dicPos, LOC dicBufSize +@@: + comment ~ + xor t0, t0 + sub dicPos, r1 + cmovb t0_R, LOC dicBufSize + ~ + + movzx match, byte ptr[dic + dicPos * 1] + + ifdef _LZMA_SIZE_OPT + + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, match + mov sym, 1 +MY_ALIGN_16 +litm_loop: + LITM + cmp sym, 256 + jb litm_loop + sub sym, 256 + + else + + LITM_0 + LITM + LITM + LITM + LITM + LITM + LITM + LITM_2 + + endif + + mov probs, LOC probs_Spec + IsMatchBranch_Pre + ; mov dic, LOC dic_Spec + mov dicPos, LOC dicPos_Spec + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_matched_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + ; IsMatchBranch + mov lpMask_reg, LOC lpMask + sub state, 3 * PMULT + jmp lit_start_2 + + + +; ---------- REP 0 LITERAL ---------- +MY_ALIGN_32 +IsRep0Short_label: + UPDATE_0 probs_state_R, pbPos_R, IsRep0Long + + ; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov dic, LOC dic_Spec + mov t0_R, dicPos + mov probBranch, LOC rep0 + sub t0_R, dic + + sub probs, RepLenCoder * PMULT + + ; state = state < kNumLitStates ? 9 : 11; + or state, 1 * PMULT + + ; the caller doesn't allow (dicPos >= limit) case for REP_SHORT + ; so we don't need the following (dicPos == limit) check here: + ; cmp dicPos, LOC limit + ; jae fin_dicPos_LIMIT_REP_SHORT + + inc processedPos + + IsMatchBranch_Pre + +; xor sym, sym +; sub t0_R, probBranch_R +; cmovb sym_R, LOC dicBufSize +; add t0_R, sym_R + sub t0_R, probBranch_R + jae @f + add t0_R, LOC dicBufSize +@@: + movzx sym, byte ptr[dic + t0_R * 1] + jmp lz_end_match + + +MY_ALIGN_32 +IsRep_label: + UPDATE_1 probs_state_R, 0, IsRep + + ; The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. + ; So we don't check it here. + + ; mov t0, processedPos + ; or t0, LOC checkDicSize + ; jz fin_ERROR_2 + + ; state = state < kNumLitStates ? 8 : 11; + cmp state, kNumLitStates * PMULT + mov state, 8 * PMULT + mov probBranch, 11 * PMULT + cmovae state, probBranch + + ; prob = probs + RepLenCoder; + add probs, RepLenCoder * PMULT + + IF_BIT_1 probs_state_R, 0, IsRepG0, IsRepG0_label + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsRep0Long, IsRep0Short_label + UPDATE_1 probs_state_R, pbPos_R, IsRep0Long + jmp len_decode + +MY_ALIGN_32 +IsRepG0_label: + UPDATE_1 probs_state_R, 0, IsRepG0 + mov dist2, LOC rep0 + mov dist, LOC rep1 + mov LOC rep1, dist2 + + IF_BIT_1 probs_state_R, 0, IsRepG1, IsRepG1_label + mov LOC rep0, dist + jmp len_decode + +; MY_ALIGN_32 +IsRepG1_label: + UPDATE_1 probs_state_R, 0, IsRepG1 + mov dist2, LOC rep2 + mov LOC rep2, dist + + IF_BIT_1 probs_state_R, 0, IsRepG2, IsRepG2_label + mov LOC rep0, dist2 + jmp len_decode + +; MY_ALIGN_32 +IsRepG2_label: + UPDATE_1 probs_state_R, 0, IsRepG2 + mov dist, LOC rep3 + mov LOC rep3, dist2 + mov LOC rep0, dist + jmp len_decode + + + +; ---------- SPEC SHORT DISTANCE ---------- + +MY_ALIGN_32 +short_dist: + sub x1, 32 + 1 + jbe decode_dist_end + or sym, 2 + shl sym, x1_L + lea sym_R, [probs + sym_R * PMULT + SpecPos * PMULT + 1 * PMULT] + mov sym2, PMULT ; step +MY_ALIGN_32 +spec_loop: + REV_1_VAR x2 + dec x1 + jnz spec_loop + + mov probs, LOC probs_Spec + sub sym, sym2 + sub sym, SpecPos * PMULT + sub sym_R, probs + shr sym, PSHIFT + + jmp decode_dist_end + + +; ---------- COPY MATCH CROSS ---------- +copy_match_cross: + ; t0_R - src pos + ; r1 - len to dicBufSize + ; cnt_R - total copy len + + mov t1_R, t0_R ; srcPos + mov t0_R, dic + mov r1, LOC dicBufSize ; + neg cnt_R +@@: + movzx sym, byte ptr[t1_R * 1 + t0_R] + inc t1_R + mov byte ptr[cnt_R * 1 + dicPos], sym_L + inc cnt_R + cmp t1_R, r1 + jne @b + + movzx sym, byte ptr[t0_R] + sub t0_R, cnt_R + jmp copy_common + + + + +; fin_dicPos_LIMIT_REP_SHORT: + ; mov sym, 1 + +fin_dicPos_LIMIT: + mov LOC remainLen, sym + jmp fin_OK + ; For more strict mode we can stop decoding with error + ; mov sym, 1 + ; jmp fin + + +fin_ERROR_MATCH_DIST: + + ; rep3 = rep2; + ; rep2 = rep1; + ; rep1 = rep0; + ; rep0 = distance + 1; + + add len_temp, kMatchSpecLen_Error_Data + mov LOC remainLen, len_temp + + mov LOC rep0, sym + mov LOC rep1, t1 + mov LOC rep2, x1 + mov LOC rep3, x2 + + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + cmp state, (kNumStates + kNumLitStates) * PMULT + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 + + ; jmp fin_OK + mov sym, 1 + jmp fin + +end_of_payload: + inc sym + jnz fin_ERROR_MATCH_DIST + + mov LOC remainLen, kMatchSpecLenStart + sub state, kNumStates * PMULT + +fin_OK: + xor sym, sym + +fin: + NORM + + mov r1, LOC lzmaPtr + + sub dicPos, LOC dic_Spec + mov GLOB dicPos_Spec, dicPos + mov GLOB buf_Spec, buf + mov GLOB range_Spec, range + mov GLOB code_Spec, cod + shr state, PSHIFT + mov GLOB state_Spec, state + mov GLOB processedPos_Spec, processedPos + + RESTORE_VAR(remainLen) + RESTORE_VAR(rep0) + RESTORE_VAR(rep1) + RESTORE_VAR(rep2) + RESTORE_VAR(rep3) + + mov x0, sym + + mov RSP, LOC Old_RSP + +MY_POP_PRESERVED_ABI_REGS +MY_ENDP + +_TEXT$LZMADECOPT ENDS + +end diff --git a/3rdparty/lzma/Asm/x86/Sha256Opt.asm b/3rdparty/lzma/Asm/x86/Sha256Opt.asm new file mode 100644 index 00000000000..3e9f6eda3e7 --- /dev/null +++ b/3rdparty/lzma/Asm/x86/Sha256Opt.asm @@ -0,0 +1,275 @@ +; Sha256Opt.asm -- SHA-256 optimized code for SHA-256 x86 hardware instructions +; 2022-04-17 : Igor Pavlov : Public domain + +include 7zAsm.asm + +MY_ASM_START + +; .data +; public K + +; we can use external SHA256_K_ARRAY defined in Sha256.c +; but we must guarantee that SHA256_K_ARRAY is aligned for 16-bytes + +COMMENT @ +ifdef x64 +K_CONST equ SHA256_K_ARRAY +else +K_CONST equ _SHA256_K_ARRAY +endif +EXTRN K_CONST:xmmword +@ + +CONST SEGMENT + +align 16 +Reverse_Endian_Mask db 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 + +; COMMENT @ +align 16 +K_CONST \ +DD 0428a2f98H, 071374491H, 0b5c0fbcfH, 0e9b5dba5H +DD 03956c25bH, 059f111f1H, 0923f82a4H, 0ab1c5ed5H +DD 0d807aa98H, 012835b01H, 0243185beH, 0550c7dc3H +DD 072be5d74H, 080deb1feH, 09bdc06a7H, 0c19bf174H +DD 0e49b69c1H, 0efbe4786H, 00fc19dc6H, 0240ca1ccH +DD 02de92c6fH, 04a7484aaH, 05cb0a9dcH, 076f988daH +DD 0983e5152H, 0a831c66dH, 0b00327c8H, 0bf597fc7H +DD 0c6e00bf3H, 0d5a79147H, 006ca6351H, 014292967H +DD 027b70a85H, 02e1b2138H, 04d2c6dfcH, 053380d13H +DD 0650a7354H, 0766a0abbH, 081c2c92eH, 092722c85H +DD 0a2bfe8a1H, 0a81a664bH, 0c24b8b70H, 0c76c51a3H +DD 0d192e819H, 0d6990624H, 0f40e3585H, 0106aa070H +DD 019a4c116H, 01e376c08H, 02748774cH, 034b0bcb5H +DD 0391c0cb3H, 04ed8aa4aH, 05b9cca4fH, 0682e6ff3H +DD 0748f82eeH, 078a5636fH, 084c87814H, 08cc70208H +DD 090befffaH, 0a4506cebH, 0bef9a3f7H, 0c67178f2H +; @ + +CONST ENDS + +; _TEXT$SHA256OPT SEGMENT 'CODE' + +ifndef x64 + .686 + .xmm +endif + +; jwasm-based assemblers for linux and linker from new versions of binutils +; can generate incorrect code for load [ARRAY + offset] instructions. +; 22.00: we load K_CONST offset to (rTable) register to avoid jwasm+binutils problem + rTable equ r0 + ; rTable equ K_CONST + +ifdef x64 + rNum equ REG_ABI_PARAM_2 + if (IS_LINUX eq 0) + LOCAL_SIZE equ (16 * 2) + endif +else + rNum equ r3 + LOCAL_SIZE equ (16 * 1) +endif + +rState equ REG_ABI_PARAM_0 +rData equ REG_ABI_PARAM_1 + + + + + + +MY_SHA_INSTR macro cmd, a1, a2 + db 0fH, 038H, cmd, (0c0H + a1 * 8 + a2) +endm + +cmd_sha256rnds2 equ 0cbH +cmd_sha256msg1 equ 0ccH +cmd_sha256msg2 equ 0cdH + +MY_sha256rnds2 macro a1, a2 + MY_SHA_INSTR cmd_sha256rnds2, a1, a2 +endm + +MY_sha256msg1 macro a1, a2 + MY_SHA_INSTR cmd_sha256msg1, a1, a2 +endm + +MY_sha256msg2 macro a1, a2 + MY_SHA_INSTR cmd_sha256msg2, a1, a2 +endm + +MY_PROLOG macro + ifdef x64 + if (IS_LINUX eq 0) + movdqa [r4 + 8], xmm6 + movdqa [r4 + 8 + 16], xmm7 + sub r4, LOCAL_SIZE + 8 + movdqa [r4 ], xmm8 + movdqa [r4 + 16], xmm9 + endif + else ; x86 + push r3 + push r5 + mov r5, r4 + NUM_PUSH_REGS equ 2 + PARAM_OFFSET equ (REG_SIZE * (1 + NUM_PUSH_REGS)) + if (IS_CDECL gt 0) + mov rState, [r4 + PARAM_OFFSET] + mov rData, [r4 + PARAM_OFFSET + REG_SIZE * 1] + mov rNum, [r4 + PARAM_OFFSET + REG_SIZE * 2] + else ; fastcall + mov rNum, [r4 + PARAM_OFFSET] + endif + and r4, -16 + sub r4, LOCAL_SIZE + endif +endm + +MY_EPILOG macro + ifdef x64 + if (IS_LINUX eq 0) + movdqa xmm8, [r4] + movdqa xmm9, [r4 + 16] + add r4, LOCAL_SIZE + 8 + movdqa xmm6, [r4 + 8] + movdqa xmm7, [r4 + 8 + 16] + endif + else ; x86 + mov r4, r5 + pop r5 + pop r3 + endif + MY_ENDP +endm + + +msg equ xmm0 +tmp equ xmm0 +state0_N equ 2 +state1_N equ 3 +w_regs equ 4 + + +state1_save equ xmm1 +state0 equ @CatStr(xmm, %state0_N) +state1 equ @CatStr(xmm, %state1_N) + + +ifdef x64 + state0_save equ xmm8 + mask2 equ xmm9 +else + state0_save equ [r4] + mask2 equ xmm0 +endif + +LOAD_MASK macro + movdqa mask2, XMMWORD PTR Reverse_Endian_Mask +endm + +LOAD_W macro k:req + movdqu @CatStr(xmm, %(w_regs + k)), [rData + (16 * (k))] + pshufb @CatStr(xmm, %(w_regs + k)), mask2 +endm + + +; pre1 <= 4 && pre2 >= 1 && pre1 > pre2 && (pre1 - pre2) <= 1 +pre1 equ 3 +pre2 equ 2 + + + +RND4 macro k + movdqa msg, xmmword ptr [rTable + (k) * 16] + paddd msg, @CatStr(xmm, %(w_regs + ((k + 0) mod 4))) + MY_sha256rnds2 state0_N, state1_N + pshufd msg, msg, 0eH + + if (k GE (4 - pre1)) AND (k LT (16 - pre1)) + ; w4[0] = msg1(w4[-4], w4[-3]) + MY_sha256msg1 (w_regs + ((k + pre1) mod 4)), (w_regs + ((k + pre1 - 3) mod 4)) + endif + + MY_sha256rnds2 state1_N, state0_N + + if (k GE (4 - pre2)) AND (k LT (16 - pre2)) + movdqa tmp, @CatStr(xmm, %(w_regs + ((k + pre2 - 1) mod 4))) + palignr tmp, @CatStr(xmm, %(w_regs + ((k + pre2 - 2) mod 4))), 4 + paddd @CatStr(xmm, %(w_regs + ((k + pre2) mod 4))), tmp + ; w4[0] = msg2(w4[0], w4[-1]) + MY_sha256msg2 %(w_regs + ((k + pre2) mod 4)), %(w_regs + ((k + pre2 - 1) mod 4)) + endif +endm + + + + + +REVERSE_STATE macro + ; state0 ; dcba + ; state1 ; hgfe + pshufd tmp, state0, 01bH ; abcd + pshufd state0, state1, 01bH ; efgh + movdqa state1, state0 ; efgh + punpcklqdq state0, tmp ; cdgh + punpckhqdq state1, tmp ; abef +endm + + +MY_PROC Sha256_UpdateBlocks_HW, 3 + MY_PROLOG + + lea rTable, [K_CONST] + + cmp rNum, 0 + je end_c + + movdqu state0, [rState] ; dcba + movdqu state1, [rState + 16] ; hgfe + + REVERSE_STATE + + ifdef x64 + LOAD_MASK + endif + + align 16 + nextBlock: + movdqa state0_save, state0 + movdqa state1_save, state1 + + ifndef x64 + LOAD_MASK + endif + + LOAD_W 0 + LOAD_W 1 + LOAD_W 2 + LOAD_W 3 + + + k = 0 + rept 16 + RND4 k + k = k + 1 + endm + + paddd state0, state0_save + paddd state1, state1_save + + add rData, 64 + sub rNum, 1 + jnz nextBlock + + REVERSE_STATE + + movdqu [rState], state0 + movdqu [rState + 16], state1 + + end_c: +MY_EPILOG + +; _TEXT$SHA256OPT ENDS + +end diff --git a/3rdparty/lzma/Asm/x86/XzCrc64Opt.asm b/3rdparty/lzma/Asm/x86/XzCrc64Opt.asm index 734ca5c45f4..ad22cc2fce0 100644 --- a/3rdparty/lzma/Asm/x86/XzCrc64Opt.asm +++ b/3rdparty/lzma/Asm/x86/XzCrc64Opt.asm @@ -1,5 +1,5 @@ ; XzCrc64Opt.asm -- CRC64 calculation : optimized version -; 2011-06-28 : Igor Pavlov : Public domain +; 2021-02-06 : Igor Pavlov : Public domain include 7zAsm.asm @@ -7,16 +7,15 @@ MY_ASM_START ifdef x64 - rD equ r9 - rN equ r10 - - num_VAR equ r8 - table_VAR equ r9 - - SRCDAT equ rN + rD +rD equ r9 +rN equ r10 +rT equ r5 +num_VAR equ r8 +SRCDAT4 equ dword ptr [rD + rN * 1] + CRC_XOR macro dest:req, src:req, t:req - xor dest, QWORD PTR [r5 + src * 8 + 0800h * t] + xor dest, QWORD PTR [rT + src * 8 + 0800h * t] endm CRC1b macro @@ -30,12 +29,15 @@ CRC1b macro endm MY_PROLOG macro crc_end:req + ifdef ABI_LINUX + MY_PUSH_2_REGS + else MY_PUSH_4_REGS - - mov r0, r1 - mov rN, num_VAR - mov r5, table_VAR - mov rD, r2 + endif + mov r0, REG_ABI_PARAM_0 + mov rN, REG_ABI_PARAM_2 + mov rT, REG_ABI_PARAM_3 + mov rD, REG_ABI_PARAM_1 test rN, rN jz crc_end @@: @@ -51,14 +53,14 @@ MY_PROLOG macro crc_end:req sub rN, 4 and rN, NOT 3 sub rD, rN - mov x1, [SRCDAT] + mov x1, SRCDAT4 xor r0, r1 add rN, 4 endm MY_EPILOG macro crc_end:req sub rN, 4 - mov x1, [SRCDAT] + mov x1, SRCDAT4 xor r0, r1 mov rD, rN mov rN, num_VAR @@ -69,14 +71,18 @@ MY_EPILOG macro crc_end:req CRC1b jmp crc_end @@: + ifdef ABI_LINUX + MY_POP_2_REGS + else MY_POP_4_REGS + endif endm MY_PROC XzCrc64UpdateT4, 4 MY_PROLOG crc_end_4 align 16 main_loop_4: - mov x1, [SRCDAT] + mov x1, SRCDAT4 movzx x2, x0_L movzx x3, x0_H shr r0, 16 @@ -96,21 +102,43 @@ MY_PROC XzCrc64UpdateT4, 4 MY_ENDP else +; x86 (32-bit) + +rD equ r1 +rN equ r7 +rT equ r5 + +crc_OFFS equ (REG_SIZE * 5) + +if (IS_CDECL gt 0) or (IS_LINUX gt 0) + ; cdecl or (GNU fastcall) stack: + ; (UInt32 *) table + ; size_t size + ; void * data + ; (UInt64) crc + ; ret-ip <-(r4) + data_OFFS equ (8 + crc_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + table_OFFS equ (REG_SIZE + size_OFFS) + num_VAR equ [r4 + size_OFFS] + table_VAR equ [r4 + table_OFFS] +else + ; Windows fastcall: + ; r1 = data, r2 = size + ; stack: + ; (UInt32 *) table + ; (UInt64) crc + ; ret-ip <-(r4) + table_OFFS equ (8 + crc_OFFS) + table_VAR equ [r4 + table_OFFS] + num_VAR equ table_VAR +endif - rD equ r1 - rN equ r7 - - crc_val equ (REG_SIZE * 5) - crc_table equ (8 + crc_val) - table_VAR equ [r4 + crc_table] - num_VAR equ table_VAR - - - SRCDAT equ rN + rD +SRCDAT4 equ dword ptr [rD + rN * 1] CRC macro op0:req, op1:req, dest0:req, dest1:req, src:req, t:req - op0 dest0, DWORD PTR [r5 + src * 8 + 0800h * t] - op1 dest1, DWORD PTR [r5 + src * 8 + 0800h * t + 4] + op0 dest0, DWORD PTR [rT + src * 8 + 0800h * t] + op1 dest1, DWORD PTR [rT + src * 8 + 0800h * t + 4] endm CRC_XOR macro dest0:req, dest1:req, src:req, t:req @@ -131,12 +159,18 @@ endm MY_PROLOG macro crc_end:req MY_PUSH_4_REGS - + + if (IS_CDECL gt 0) or (IS_LINUX gt 0) + proc_numParams = proc_numParams + 2 ; for ABI_LINUX + mov rN, [r4 + size_OFFS] + mov rD, [r4 + data_OFFS] + else mov rN, r2 + endif - mov x0, [r4 + crc_val] - mov x2, [r4 + crc_val + 4] - mov r5, table_VAR + mov x0, [r4 + crc_OFFS] + mov x2, [r4 + crc_OFFS + 4] + mov rT, table_VAR test rN, rN jz crc_end @@: @@ -154,13 +188,13 @@ MY_PROLOG macro crc_end:req sub rN, 4 and rN, NOT 3 sub rD, rN - xor r0, [SRCDAT] + xor r0, SRCDAT4 add rN, 4 endm MY_EPILOG macro crc_end:req sub rN, 4 - xor r0, [SRCDAT] + xor r0, SRCDAT4 mov rD, rN mov rN, num_VAR @@ -179,7 +213,7 @@ MY_PROC XzCrc64UpdateT4, 5 movzx x6, x0_L align 16 main_loop_4: - mov r3, [SRCDAT] + mov r3, SRCDAT4 xor r3, r2 CRC xor, mov, r3, r2, r6, 3 @@ -200,6 +234,6 @@ MY_PROC XzCrc64UpdateT4, 5 MY_EPILOG crc_end_4 MY_ENDP -endif +endif ; ! x64 end diff --git a/3rdparty/lzma/C/7z.h b/3rdparty/lzma/C/7z.h index 4768151948d..304f75ffc54 100644 --- a/3rdparty/lzma/C/7z.h +++ b/3rdparty/lzma/C/7z.h @@ -1,5 +1,5 @@ /* 7z.h -- 7z interface -2015-11-18 : Igor Pavlov : Public domain */ +2018-07-02 : Igor Pavlov : Public domain */ #ifndef __7Z_H #define __7Z_H @@ -91,6 +91,8 @@ typedef struct UInt64 *CoderUnpackSizes; // for all coders in all folders Byte *CodersData; + + UInt64 RangeLimit; } CSzAr; UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); @@ -98,7 +100,7 @@ UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, ILookInStream *stream, UInt64 startPos, Byte *outBuffer, size_t outSize, - ISzAlloc *allocMain); + ISzAllocPtr allocMain); typedef struct { @@ -131,7 +133,7 @@ typedef struct #define SzArEx_GetFileSize(p, i) ((p)->UnpackPositions[(i) + 1] - (p)->UnpackPositions[i]) void SzArEx_Init(CSzArEx *p); -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc); +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc); UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder); int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize); @@ -179,8 +181,8 @@ SRes SzArEx_Extract( size_t *outBufferSize, /* buffer size for output buffer */ size_t *offset, /* offset of stream for required file in *outBuffer */ size_t *outSizeProcessed, /* size of file in *outBuffer */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp); + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp); /* @@ -195,7 +197,7 @@ SZ_ERROR_FAIL */ SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, ISzAlloc *allocTemp); + ISzAllocPtr allocMain, ISzAllocPtr allocTemp); EXTERN_C_END diff --git a/3rdparty/lzma/C/7zAlloc.c b/3rdparty/lzma/C/7zAlloc.c index 3e848c9800d..c924a529fa8 100644 --- a/3rdparty/lzma/C/7zAlloc.c +++ b/3rdparty/lzma/C/7zAlloc.c @@ -1,8 +1,10 @@ /* 7zAlloc.c -- Allocation functions -2015-11-09 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + #include "7zAlloc.h" /* #define _SZ_ALLOC_DEBUG */ @@ -20,7 +22,7 @@ int g_allocCountTemp = 0; #endif -void *SzAlloc(void *p, size_t size) +void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); if (size == 0) @@ -32,7 +34,7 @@ void *SzAlloc(void *p, size_t size) return malloc(size); } -void SzFree(void *p, void *address) +void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); #ifdef _SZ_ALLOC_DEBUG @@ -45,7 +47,7 @@ void SzFree(void *p, void *address) free(address); } -void *SzAllocTemp(void *p, size_t size) +void *SzAllocTemp(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); if (size == 0) @@ -60,7 +62,7 @@ void *SzAllocTemp(void *p, size_t size) return malloc(size); } -void SzFreeTemp(void *p, void *address) +void SzFreeTemp(ISzAllocPtr p, void *address) { UNUSED_VAR(p); #ifdef _SZ_ALLOC_DEBUG diff --git a/3rdparty/lzma/C/7zAlloc.h b/3rdparty/lzma/C/7zAlloc.h index 2fd5bdbc800..44778f9b2e1 100644 --- a/3rdparty/lzma/C/7zAlloc.h +++ b/3rdparty/lzma/C/7zAlloc.h @@ -1,23 +1,19 @@ /* 7zAlloc.h -- Allocation functions -2013-03-25 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_ALLOC_H #define __7Z_ALLOC_H -#include +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN -void *SzAlloc(void *p, size_t size); -void SzFree(void *p, void *address); +void *SzAlloc(ISzAllocPtr p, size_t size); +void SzFree(ISzAllocPtr p, void *address); -void *SzAllocTemp(void *p, size_t size); -void SzFreeTemp(void *p, void *address); +void *SzAllocTemp(ISzAllocPtr p, size_t size); +void SzFreeTemp(ISzAllocPtr p, void *address); -#ifdef __cplusplus -} -#endif +EXTERN_C_END #endif diff --git a/3rdparty/lzma/C/7zArcIn.c b/3rdparty/lzma/C/7zArcIn.c index 2beed3d5ccf..0d9dec41e50 100644 --- a/3rdparty/lzma/C/7zArcIn.c +++ b/3rdparty/lzma/C/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2016-05-16 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,7 +11,7 @@ #include "CpuArch.h" #define MY_ALLOC(T, p, size, alloc) { \ - if ((p = (T *)IAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } + if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } #define MY_ALLOC_ZE(T, p, size, alloc) { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) } @@ -19,7 +19,7 @@ { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } #define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ - { if ((size) == 0) p = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } + { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } #define k7zMajorVersion 0 @@ -60,7 +60,7 @@ const Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; #define SzBitUi32s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAlloc *alloc) +static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) { if (num == 0) { @@ -75,18 +75,18 @@ static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAlloc *alloc) return SZ_OK; } -void SzBitUi32s_Free(CSzBitUi32s *p, ISzAlloc *alloc) +static void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->Defs); p->Defs = NULL; - IAlloc_Free(alloc, p->Vals); p->Vals = NULL; + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; } #define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -void SzBitUi64s_Free(CSzBitUi64s *p, ISzAlloc *alloc) +static void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->Defs); p->Defs = NULL; - IAlloc_Free(alloc, p->Vals); p->Vals = NULL; + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; } @@ -105,20 +105,22 @@ static void SzAr_Init(CSzAr *p) p->CoderUnpackSizes = NULL; p->CodersData = NULL; + + p->RangeLimit = 0; } -static void SzAr_Free(CSzAr *p, ISzAlloc *alloc) +static void SzAr_Free(CSzAr *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->PackPositions); + ISzAlloc_Free(alloc, p->PackPositions); SzBitUi32s_Free(&p->FolderCRCs, alloc); - IAlloc_Free(alloc, p->FoCodersOffsets); - IAlloc_Free(alloc, p->FoStartPackStreamIndex); - IAlloc_Free(alloc, p->FoToCoderUnpackSizes); - IAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); - IAlloc_Free(alloc, p->CoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoCodersOffsets); + ISzAlloc_Free(alloc, p->FoStartPackStreamIndex); + ISzAlloc_Free(alloc, p->FoToCoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); + ISzAlloc_Free(alloc, p->CoderUnpackSizes); - IAlloc_Free(alloc, p->CodersData); + ISzAlloc_Free(alloc, p->CodersData); SzAr_Init(p); } @@ -147,16 +149,16 @@ void SzArEx_Init(CSzArEx *p) SzBitUi64s_Init(&p->CTime); } -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc) +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->UnpackPositions); - IAlloc_Free(alloc, p->IsDirs); + ISzAlloc_Free(alloc, p->UnpackPositions); + ISzAlloc_Free(alloc, p->IsDirs); - IAlloc_Free(alloc, p->FolderToFile); - IAlloc_Free(alloc, p->FileToFolder); + ISzAlloc_Free(alloc, p->FolderToFile); + ISzAlloc_Free(alloc, p->FileToFolder); - IAlloc_Free(alloc, p->FileNameOffsets); - IAlloc_Free(alloc, p->FileNames); + ISzAlloc_Free(alloc, p->FileNameOffsets); + ISzAlloc_Free(alloc, p->FileNames); SzBitUi32s_Free(&p->CRCs, alloc); SzBitUi32s_Free(&p->Attribs, alloc); @@ -305,7 +307,7 @@ static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) return sum; } -static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAlloc *alloc) +static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) { Byte allAreDefined; Byte *v2; @@ -328,12 +330,12 @@ static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, I { unsigned numBits = (unsigned)numItems & 7; if (numBits != 0) - v2[numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); + v2[(size_t)numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); } return SZ_OK; } -static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAlloc *alloc) +static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { UInt32 i; CSzData sd; @@ -354,7 +356,7 @@ static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *c return SZ_OK; } -static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAlloc *alloc) +static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { SzBitUi32s_Free(crcs, alloc); RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc)); @@ -380,7 +382,7 @@ static SRes SkipBitUi32s(CSzData *sd, UInt32 numItems) return SZ_OK; } -static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAlloc *alloc) +static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) { RINOK(SzReadNumber32(sd, &p->NumPackStreams)); @@ -502,7 +504,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) return SZ_ERROR_ARCHIVE; if (propsSize >= 0x80) return SZ_ERROR_UNSUPPORTED; - coder->PropsOffset = sd->Data - dataStart; + coder->PropsOffset = (size_t)(sd->Data - dataStart); coder->PropsSize = (Byte)propsSize; sd->Data += (size_t)propsSize; sd->Size -= (size_t)propsSize; @@ -635,7 +637,7 @@ static SRes ReadUnpackInfo(CSzAr *p, CSzData *sd2, UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, - ISzAlloc *alloc) + ISzAllocPtr alloc) { CSzData sd; @@ -666,7 +668,7 @@ static SRes ReadUnpackInfo(CSzAr *p, MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); startBufPtr = sd.Data; @@ -677,7 +679,7 @@ static SRes ReadUnpackInfo(CSzAr *p, { UInt32 numCoders, ci, numInStreams = 0; - p->FoCodersOffsets[fo] = sd.Data - startBufPtr; + p->FoCodersOffsets[fo] = (size_t)(sd.Data - startBufPtr); RINOK(SzReadNumber32(&sd, &numCoders)); if (numCoders == 0 || numCoders > k_Scan_NumCoders_MAX) @@ -797,7 +799,7 @@ static SRes ReadUnpackInfo(CSzAr *p, p->FoToCoderUnpackSizes[fo] = numCodersOutStreams; { - size_t dataSize = sd.Data - startBufPtr; + const size_t dataSize = (size_t)(sd.Data - startBufPtr); p->FoStartPackStreamIndex[fo] = packStreamIndex; p->FoCodersOffsets[fo] = dataSize; MY_ALLOC_ZE_AND_CPY(p->CodersData, dataSize, startBufPtr, alloc); @@ -885,7 +887,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) if (numStreams != 1 || !SzBitWithVals_Check(&p->FolderCRCs, i)) numSubDigests += numStreams; } - ssi->sdNumSubStreams.Size = sd->Data - ssi->sdNumSubStreams.Data; + ssi->sdNumSubStreams.Size = (size_t)(sd->Data - ssi->sdNumSubStreams.Data); continue; } if (type == k7zIdCRC || type == k7zIdSize || type == k7zIdEnd) @@ -907,7 +909,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) { ssi->sdSizes.Data = sd->Data; RINOK(SkipNumbers(sd, numUnpackSizesInData)); - ssi->sdSizes.Size = sd->Data - ssi->sdSizes.Data; + ssi->sdSizes.Size = (size_t)(sd->Data - ssi->sdSizes.Data); RINOK(ReadID(sd, &type)); } @@ -919,7 +921,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) { ssi->sdCRCs.Data = sd->Data; RINOK(SkipBitUi32s(sd, numSubDigests)); - ssi->sdCRCs.Size = sd->Data - ssi->sdCRCs.Data; + ssi->sdCRCs.Size = (size_t)(sd->Data - ssi->sdCRCs.Data); } else { @@ -934,7 +936,7 @@ static SRes SzReadStreamsInfo(CSzAr *p, UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, UInt64 *dataOffset, CSubStreamInfo *ssi, - ISzAlloc *alloc) + ISzAllocPtr alloc) { UInt64 type; @@ -947,7 +949,11 @@ static SRes SzReadStreamsInfo(CSzAr *p, if (type == k7zIdPackInfo) { RINOK(ReadNumber(sd, dataOffset)); + if (*dataOffset > p->RangeLimit) + return SZ_ERROR_ARCHIVE; RINOK(ReadPackInfo(p, sd, alloc)); + if (p->PackPositions[p->NumPackStreams] > p->RangeLimit - *dataOffset) + return SZ_ERROR_ARCHIVE; RINOK(ReadID(sd, &type)); } if (type == k7zIdUnpackInfo) @@ -976,7 +982,7 @@ static SRes SzReadAndDecodePackedStreams( UInt32 numFoldersMax, UInt64 baseOffset, CSzAr *p, - ISzAlloc *allocTemp) + ISzAllocPtr allocTemp) { UInt64 dataStartPos; UInt32 fo; @@ -1028,12 +1034,12 @@ static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size return SZ_ERROR_ARCHIVE; for (p = data + pos; #ifdef _WIN32 - *(const UInt16 *)p != 0 + *(const UInt16 *)(const void *)p != 0 #else p[0] != 0 || p[1] != 0 #endif ; p += 2); - pos = p - data + 2; + pos = (size_t)(p - data) + 2; *offsets++ = (pos >> 1); } while (--numFiles); @@ -1043,7 +1049,7 @@ static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, CSzData *sd2, const CBuf *tempBufs, UInt32 numTempBufs, - ISzAlloc *alloc) + ISzAllocPtr alloc) { CSzData sd; UInt32 i; @@ -1096,8 +1102,8 @@ static SRes SzReadHeader2( CSzData *sd, ILookInStream *inStream, CBuf *tempBufs, UInt32 *numTempBufs, - ISzAlloc *allocMain, - ISzAlloc *allocTemp + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp ) { CSubStreamInfo ssi; @@ -1133,6 +1139,8 @@ static SRes SzReadHeader2( SRes res; SzAr_Init(&tempAr); + tempAr.RangeLimit = p->db.RangeLimit; + res = SzReadAndDecodePackedStreams(inStream, sd, tempBufs, NUM_ADDITIONAL_STREAMS_MAX, p->startPosAfterHeader, &tempAr, allocTemp); *numTempBufs = tempAr.NumFolders; @@ -1472,8 +1480,8 @@ static SRes SzReadHeader( CSzArEx *p, CSzData *sd, ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { UInt32 i; UInt32 numTempBufs = 0; @@ -1501,8 +1509,8 @@ static SRes SzReadHeader( static SRes SzArEx_Open2( CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { Byte header[k7zStartHeaderSize]; Int64 startArcPos; @@ -1513,7 +1521,7 @@ static SRes SzArEx_Open2( SRes res; startArcPos = 0; - RINOK(inStream->Seek(inStream, &startArcPos, SZ_SEEK_CUR)); + RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)); RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)); @@ -1526,11 +1534,13 @@ static SRes SzArEx_Open2( nextHeaderSize = GetUi64(header + 20); nextHeaderCRC = GetUi32(header + 28); - p->startPosAfterHeader = startArcPos + k7zStartHeaderSize; + p->startPosAfterHeader = (UInt64)startArcPos + k7zStartHeaderSize; if (CrcCalc(header + 12, 20) != GetUi32(header + 8)) return SZ_ERROR_CRC; + p->db.RangeLimit = nextHeaderOffset; + nextHeaderSizeT = (size_t)nextHeaderSize; if (nextHeaderSizeT != nextHeaderSize) return SZ_ERROR_MEM; @@ -1542,14 +1552,14 @@ static SRes SzArEx_Open2( { Int64 pos = 0; - RINOK(inStream->Seek(inStream, &pos, SZ_SEEK_END)); - if ((UInt64)pos < startArcPos + nextHeaderOffset || - (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset || - (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) + RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); + if ((UInt64)pos < (UInt64)startArcPos + nextHeaderOffset || + (UInt64)pos < (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset || + (UInt64)pos < (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) return SZ_ERROR_INPUT_EOF; } - RINOK(LookInStream_SeekTo(inStream, startArcPos + k7zStartHeaderSize + nextHeaderOffset)); + RINOK(LookInStream_SeekTo(inStream, (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset)); if (!Buf_Create(&buf, nextHeaderSizeT, allocTemp)) return SZ_ERROR_MEM; @@ -1575,6 +1585,8 @@ static SRes SzArEx_Open2( Buf_Init(&tempBuf); SzAr_Init(&tempAr); + tempAr.RangeLimit = p->db.RangeLimit; + res = SzReadAndDecodePackedStreams(inStream, &sd, &tempBuf, 1, p->startPosAfterHeader, &tempAr, allocTemp); SzAr_Free(&tempAr, allocTemp); @@ -1623,7 +1635,7 @@ static SRes SzArEx_Open2( SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, ISzAlloc *allocTemp) + ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); if (res != SZ_OK) @@ -1641,8 +1653,8 @@ SRes SzArEx_Extract( size_t *outBufferSize, size_t *offset, size_t *outSizeProcessed, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { UInt32 folderIndex = p->FileToFolder[fileIndex]; SRes res = SZ_OK; @@ -1652,7 +1664,7 @@ SRes SzArEx_Extract( if (folderIndex == (UInt32)-1) { - IAlloc_Free(allocMain, *tempBuf); + ISzAlloc_Free(allocMain, *tempBuf); *blockIndex = folderIndex; *tempBuf = NULL; *outBufferSize = 0; @@ -1664,7 +1676,7 @@ SRes SzArEx_Extract( UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); /* UInt64 unpackSizeSpec = - p->UnpackPositions[p->FolderToFile[folderIndex + 1]] - + p->UnpackPositions[p->FolderToFile[(size_t)folderIndex + 1]] - p->UnpackPositions[p->FolderToFile[folderIndex]]; */ size_t unpackSize = (size_t)unpackSizeSpec; @@ -1672,7 +1684,7 @@ SRes SzArEx_Extract( if (unpackSize != unpackSizeSpec) return SZ_ERROR_MEM; *blockIndex = folderIndex; - IAlloc_Free(allocMain, *tempBuf); + ISzAlloc_Free(allocMain, *tempBuf); *tempBuf = NULL; if (res == SZ_OK) @@ -1680,7 +1692,7 @@ SRes SzArEx_Extract( *outBufferSize = unpackSize; if (unpackSize != 0) { - *tempBuf = (Byte *)IAlloc_Alloc(allocMain, unpackSize); + *tempBuf = (Byte *)ISzAlloc_Alloc(allocMain, unpackSize); if (*tempBuf == NULL) res = SZ_ERROR_MEM; } @@ -1697,7 +1709,7 @@ SRes SzArEx_Extract( { UInt64 unpackPos = p->UnpackPositions[fileIndex]; *offset = (size_t)(unpackPos - p->UnpackPositions[p->FolderToFile[folderIndex]]); - *outSizeProcessed = (size_t)(p->UnpackPositions[fileIndex + 1] - unpackPos); + *outSizeProcessed = (size_t)(p->UnpackPositions[(size_t)fileIndex + 1] - unpackPos); if (*offset + *outSizeProcessed > *outBufferSize) return SZ_ERROR_FAIL; if (SzBitWithVals_Check(&p->CRCs, fileIndex)) @@ -1744,7 +1756,7 @@ size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex) UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) { - Bool needSlash; + BoolInt needSlash; if (!p->FileNameOffsets) { *(--dest) = 0; diff --git a/3rdparty/lzma/C/7zBuf.c b/3rdparty/lzma/C/7zBuf.c index 089a5c4f21f..8865c32a81b 100644 --- a/3rdparty/lzma/C/7zBuf.c +++ b/3rdparty/lzma/C/7zBuf.c @@ -1,5 +1,5 @@ /* 7zBuf.c -- Byte Buffer -2013-01-21 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,7 +11,7 @@ void Buf_Init(CBuf *p) p->size = 0; } -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc) { p->size = 0; if (size == 0) @@ -19,8 +19,8 @@ int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) p->data = 0; return 1; } - p->data = (Byte *)alloc->Alloc(alloc, size); - if (p->data != 0) + p->data = (Byte *)ISzAlloc_Alloc(alloc, size); + if (p->data) { p->size = size; return 1; @@ -28,9 +28,9 @@ int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) return 0; } -void Buf_Free(CBuf *p, ISzAlloc *alloc) +void Buf_Free(CBuf *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->data); + ISzAlloc_Free(alloc, p->data); p->data = 0; p->size = 0; } diff --git a/3rdparty/lzma/C/7zBuf.h b/3rdparty/lzma/C/7zBuf.h index 65f1d7a716e..81d1b5b646c 100644 --- a/3rdparty/lzma/C/7zBuf.h +++ b/3rdparty/lzma/C/7zBuf.h @@ -1,5 +1,5 @@ /* 7zBuf.h -- Byte Buffer -2013-01-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_BUF_H #define __7Z_BUF_H @@ -15,8 +15,8 @@ typedef struct } CBuf; void Buf_Init(CBuf *p); -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc); -void Buf_Free(CBuf *p, ISzAlloc *alloc); +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc); +void Buf_Free(CBuf *p, ISzAllocPtr alloc); typedef struct { @@ -27,8 +27,8 @@ typedef struct void DynBuf_Construct(CDynBuf *p); void DynBuf_SeekToBeg(CDynBuf *p); -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc); -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc); +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc); +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc); EXTERN_C_END diff --git a/3rdparty/lzma/C/7zBuf2.c b/3rdparty/lzma/C/7zBuf2.c index 430f76bf830..20834741654 100644 --- a/3rdparty/lzma/C/7zBuf2.c +++ b/3rdparty/lzma/C/7zBuf2.c @@ -1,5 +1,5 @@ /* 7zBuf2.c -- Byte Buffer -2014-08-22 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -19,19 +19,20 @@ void DynBuf_SeekToBeg(CDynBuf *p) p->pos = 0; } -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc) +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc) { if (size > p->size - p->pos) { size_t newSize = p->pos + size; Byte *data; newSize += newSize / 4; - data = (Byte *)alloc->Alloc(alloc, newSize); - if (data == 0) + data = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!data) return 0; p->size = newSize; - memcpy(data, p->data, p->pos); - alloc->Free(alloc, p->data); + if (p->pos != 0) + memcpy(data, p->data, p->pos); + ISzAlloc_Free(alloc, p->data); p->data = data; } if (size != 0) @@ -42,9 +43,9 @@ int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc) return 1; } -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc) +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->data); + ISzAlloc_Free(alloc, p->data); p->data = 0; p->size = 0; p->pos = 0; diff --git a/3rdparty/lzma/C/7zCrc.c b/3rdparty/lzma/C/7zCrc.c index dc6d6abdc0a..f186324ddc6 100644 --- a/3rdparty/lzma/C/7zCrc.c +++ b/3rdparty/lzma/C/7zCrc.c @@ -1,5 +1,5 @@ /* 7zCrc.c -- CRC32 init -2015-03-10 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -26,8 +26,20 @@ typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table); +extern CRC_FUNC g_CrcUpdateT4; +CRC_FUNC g_CrcUpdateT4; +extern +CRC_FUNC g_CrcUpdateT8; CRC_FUNC g_CrcUpdateT8; +extern +CRC_FUNC g_CrcUpdateT0_32; +CRC_FUNC g_CrcUpdateT0_32; +extern +CRC_FUNC g_CrcUpdateT0_64; +CRC_FUNC g_CrcUpdateT0_64; +extern +CRC_FUNC g_CrcUpdate; CRC_FUNC g_CrcUpdate; UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; @@ -44,6 +56,7 @@ UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size) #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) { const Byte *p = (const Byte *)data; @@ -53,6 +66,166 @@ UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const U return v; } + +/* ---------- hardware CRC ---------- */ + +#ifdef MY_CPU_LE + +#if defined(MY_CPU_ARM_OR_ARM64) + +// #pragma message("ARM*") + + #if defined(_MSC_VER) + #if defined(MY_CPU_ARM64) + #if (_MSC_VER >= 1910) + #define USE_ARM64_CRC + #endif + #endif + #elif (defined(__clang__) && (__clang_major__ >= 3)) \ + || (defined(__GNUC__) && (__GNUC__ > 4)) + #if !defined(__ARM_FEATURE_CRC32) + #define __ARM_FEATURE_CRC32 1 + #if (!defined(__clang__) || (__clang_major__ > 3)) // fix these numbers + #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc"))) + #endif + #endif + #if defined(__ARM_FEATURE_CRC32) + #define USE_ARM64_CRC + #include + #endif + #endif + +#else + +// no hardware CRC + +// #define USE_CRC_EMU + +#ifdef USE_CRC_EMU + +#pragma message("ARM64 CRC emulation") + +MY_FORCE_INLINE +UInt32 __crc32b(UInt32 v, UInt32 data) +{ + const UInt32 *table = g_CrcTable; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); + return v; +} + +MY_FORCE_INLINE +UInt32 __crc32w(UInt32 v, UInt32 data) +{ + const UInt32 *table = g_CrcTable; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + return v; +} + +MY_FORCE_INLINE +UInt32 __crc32d(UInt32 v, UInt64 data) +{ + const UInt32 *table = g_CrcTable; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; + return v; +} + +#endif // USE_CRC_EMU + +#endif // defined(MY_CPU_ARM64) && defined(MY_CPU_LE) + + + +#if defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) + +#define T0_32_UNROLL_BYTES (4 * 4) +#define T0_64_UNROLL_BYTES (4 * 8) + +#ifndef ATTRIB_CRC +#define ATTRIB_CRC +#endif +// #pragma message("USE ARM HW CRC") + +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table); +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + UNUSED_VAR(table); + + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_32_UNROLL_BYTES - 1)) != 0; size--) + v = __crc32b(v, *p++); + + if (size >= T0_32_UNROLL_BYTES) + { + const Byte *lim = p + size; + size &= (T0_32_UNROLL_BYTES - 1); + lim -= size; + do + { + v = __crc32w(v, *(const UInt32 *)(const void *)(p)); + v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; + v = __crc32w(v, *(const UInt32 *)(const void *)(p)); + v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; + } + while (p != lim); + } + + for (; size != 0; size--) + v = __crc32b(v, *p++); + + return v; +} + +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table); +ATTRIB_CRC +UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + UNUSED_VAR(table); + + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_64_UNROLL_BYTES - 1)) != 0; size--) + v = __crc32b(v, *p++); + + if (size >= T0_64_UNROLL_BYTES) + { + const Byte *lim = p + size; + size &= (T0_64_UNROLL_BYTES - 1); + lim -= size; + do + { + v = __crc32d(v, *(const UInt64 *)(const void *)(p)); + v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; + v = __crc32d(v, *(const UInt64 *)(const void *)(p)); + v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; + } + while (p != lim); + } + + for (; size != 0; size--) + v = __crc32b(v, *p++); + + return v; +} + +#endif // defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) + +#endif // MY_CPU_LE + + + + void MY_FAST_CALL CrcGenerateTable() { UInt32 i; @@ -61,12 +234,12 @@ void MY_FAST_CALL CrcGenerateTable() UInt32 r = i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); g_CrcTable[i] = r; } - for (; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC_NUM_TABLES; i++) { - UInt32 r = g_CrcTable[i - 256]; + UInt32 r = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); } @@ -86,8 +259,8 @@ void MY_FAST_CALL CrcGenerateTable() #ifdef MY_CPU_X86_OR_AMD64 if (!CPU_Is_InOrder()) - g_CrcUpdate = CrcUpdateT8; #endif + g_CrcUpdate = CrcUpdateT8; #endif #else @@ -101,7 +274,7 @@ void MY_FAST_CALL CrcGenerateTable() g_CrcUpdate = CrcUpdateT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT8; - // g_CrcUpdate = CrcUpdateT8; + g_CrcUpdate = CrcUpdateT8; #endif } else if (p[0] != 1 || p[1] != 2) @@ -111,18 +284,39 @@ void MY_FAST_CALL CrcGenerateTable() { for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) { - UInt32 x = g_CrcTable[i - 256]; + UInt32 x = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = CRC_UINT32_SWAP(x); } g_CrcUpdateT4 = CrcUpdateT1_BeT4; g_CrcUpdate = CrcUpdateT1_BeT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT1_BeT8; - // g_CrcUpdate = CrcUpdateT1_BeT8; + g_CrcUpdate = CrcUpdateT1_BeT8; #endif } } #endif + #endif + #ifdef MY_CPU_LE + #ifdef USE_ARM64_CRC + if (CPU_IsSupported_CRC32()) + { + g_CrcUpdateT0_32 = CrcUpdateT0_32; + g_CrcUpdateT0_64 = CrcUpdateT0_64; + g_CrcUpdate = + #if defined(MY_CPU_ARM) + CrcUpdateT0_32; + #else + CrcUpdateT0_64; + #endif + } + #endif + + #ifdef USE_CRC_EMU + g_CrcUpdateT0_32 = CrcUpdateT0_32; + g_CrcUpdateT0_64 = CrcUpdateT0_64; + g_CrcUpdate = CrcUpdateT0_64; + #endif #endif } diff --git a/3rdparty/lzma/C/7zCrcOpt.c b/3rdparty/lzma/C/7zCrcOpt.c index d1e1cd7bae1..69fad9ca2b4 100644 --- a/3rdparty/lzma/C/7zCrcOpt.c +++ b/3rdparty/lzma/C/7zCrcOpt.c @@ -1,5 +1,5 @@ /* 7zCrcOpt.c -- CRC32 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,6 +9,7 @@ #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table) { const Byte *p = (const Byte *)data; @@ -16,18 +17,19 @@ UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const U v = CRC_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = - table[0x300 + ((v ) & 0xFF)] - ^ table[0x200 + ((v >> 8) & 0xFF)] - ^ table[0x100 + ((v >> 16) & 0xFF)] - ^ table[0x000 + ((v >> 24))]; + (table + 0x300)[((v ) & 0xFF)] + ^ (table + 0x200)[((v >> 8) & 0xFF)] + ^ (table + 0x100)[((v >> 16) & 0xFF)] + ^ (table + 0x000)[((v >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p); return v; } +UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table) { const Byte *p = (const Byte *)data; @@ -36,18 +38,18 @@ UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const U for (; size >= 8; size -= 8, p += 8) { UInt32 d; - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = - table[0x700 + ((v ) & 0xFF)] - ^ table[0x600 + ((v >> 8) & 0xFF)] - ^ table[0x500 + ((v >> 16) & 0xFF)] - ^ table[0x400 + ((v >> 24))]; - d = *((const UInt32 *)p + 1); + (table + 0x700)[((v ) & 0xFF)] + ^ (table + 0x600)[((v >> 8) & 0xFF)] + ^ (table + 0x500)[((v >> 16) & 0xFF)] + ^ (table + 0x400)[((v >> 24))]; + d = *((const UInt32 *)(const void *)p + 1); v ^= - table[0x300 + ((d ) & 0xFF)] - ^ table[0x200 + ((d >> 8) & 0xFF)] - ^ table[0x100 + ((d >> 16) & 0xFF)] - ^ table[0x000 + ((d >> 24))]; + (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p); @@ -72,12 +74,12 @@ UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, co v = CRC_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = - table[0x000 + ((v ) & 0xFF)] - ^ table[0x100 + ((v >> 8) & 0xFF)] - ^ table[0x200 + ((v >> 16) & 0xFF)] - ^ table[0x300 + ((v >> 24))]; + (table + 0x000)[((v ) & 0xFF)] + ^ (table + 0x100)[((v >> 8) & 0xFF)] + ^ (table + 0x200)[((v >> 16) & 0xFF)] + ^ (table + 0x300)[((v >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); @@ -94,18 +96,18 @@ UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, co for (; size >= 8; size -= 8, p += 8) { UInt32 d; - v ^= *(const UInt32 *)p; + v ^= *(const UInt32 *)(const void *)p; v = - table[0x400 + ((v ) & 0xFF)] - ^ table[0x500 + ((v >> 8) & 0xFF)] - ^ table[0x600 + ((v >> 16) & 0xFF)] - ^ table[0x700 + ((v >> 24))]; - d = *((const UInt32 *)p + 1); + (table + 0x400)[((v ) & 0xFF)] + ^ (table + 0x500)[((v >> 8) & 0xFF)] + ^ (table + 0x600)[((v >> 16) & 0xFF)] + ^ (table + 0x700)[((v >> 24))]; + d = *((const UInt32 *)(const void *)p + 1); v ^= - table[0x000 + ((d ) & 0xFF)] - ^ table[0x100 + ((d >> 8) & 0xFF)] - ^ table[0x200 + ((d >> 16) & 0xFF)] - ^ table[0x300 + ((d >> 24))]; + (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); diff --git a/3rdparty/lzma/C/7zDec.c b/3rdparty/lzma/C/7zDec.c index c45d6bf8d4b..fbfd016e1e6 100644 --- a/3rdparty/lzma/C/7zDec.c +++ b/3rdparty/lzma/C/7zDec.c @@ -1,5 +1,5 @@ /* 7zDec.c -- Decoding from 7z folder -2015-11-18 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -21,17 +21,20 @@ #endif #define k_Copy 0 -#define k_Delta 3 +#ifndef _7Z_NO_METHOD_LZMA2 #define k_LZMA2 0x21 +#endif #define k_LZMA 0x30101 -#define k_BCJ 0x3030103 #define k_BCJ2 0x303011B +#ifndef _7Z_NO_METHODS_FILTERS +#define k_Delta 3 +#define k_BCJ 0x3030103 #define k_PPC 0x3030205 #define k_IA64 0x3030401 #define k_ARM 0x3030501 #define k_ARMT 0x3030701 #define k_SPARC 0x3030805 - +#endif #ifdef _7ZIP_PPMD_SUPPPORT @@ -39,28 +42,28 @@ typedef struct { - IByteIn p; + IByteIn vt; const Byte *cur; const Byte *end; const Byte *begin; UInt64 processed; - Bool extra; + BoolInt extra; SRes res; - ILookInStream *inStream; + const ILookInStream *inStream; } CByteInToLook; -static Byte ReadByte(void *pp) +static Byte ReadByte(const IByteIn *pp) { - CByteInToLook *p = (CByteInToLook *)pp; + CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt); if (p->cur != p->end) return *p->cur++; if (p->res == SZ_OK) { - size_t size = p->cur - p->begin; + size_t size = (size_t)(p->cur - p->begin); p->processed += size; - p->res = p->inStream->Skip(p->inStream, size); + p->res = ILookInStream_Skip(p->inStream, size); size = (1 << 25); - p->res = p->inStream->Look(p->inStream, (const void **)&p->begin, &size); + p->res = ILookInStream_Look(p->inStream, (const void **)&p->begin, &size); p->cur = p->begin; p->end = p->begin + size; if (size != 0) @@ -70,14 +73,14 @@ static Byte ReadByte(void *pp) return 0; } -static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) +static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, const ILookInStream *inStream, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CPpmd7 ppmd; CByteInToLook s; SRes res = SZ_OK; - s.p.Read = ReadByte; + s.vt.Read = ReadByte; s.inStream = inStream; s.begin = s.end = s.cur = NULL; s.extra = False; @@ -101,28 +104,32 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, I Ppmd7_Init(&ppmd, order); } { - CPpmd7z_RangeDec rc; - Ppmd7z_RangeDec_CreateVTable(&rc); - rc.Stream = &s.p; - if (!Ppmd7z_RangeDec_Init(&rc)) + ppmd.rc.dec.Stream = &s.vt; + if (!Ppmd7z_RangeDec_Init(&ppmd.rc.dec)) res = SZ_ERROR_DATA; - else if (s.extra) - res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); - else + else if (!s.extra) { - SizeT i; - for (i = 0; i < outSize; i++) + Byte *buf = outBuffer; + const Byte *lim = buf + outSize; + for (; buf != lim; buf++) { - int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.p); + int sym = Ppmd7z_DecodeSymbol(&ppmd); if (s.extra || sym < 0) break; - outBuffer[i] = (Byte)sym; + *buf = (Byte)sym; } - if (i != outSize) - res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); - else if (s.processed + (s.cur - s.begin) != inSize || !Ppmd7z_RangeDec_IsFinishedOK(&rc)) + if (buf != lim) + res = SZ_ERROR_DATA; + else if (!Ppmd7z_RangeDec_IsFinishedOK(&ppmd.rc.dec)) + { + /* if (Ppmd7z_DecodeSymbol(&ppmd) != PPMD7_SYM_END || !Ppmd7z_RangeDec_IsFinishedOK(&ppmd.rc.dec)) */ res = SZ_ERROR_DATA; + } } + if (s.extra) + res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); + else if (s.processed + (size_t)(s.cur - s.begin) != inSize) + res = SZ_ERROR_DATA; } Ppmd7_Free(&ppmd, allocMain); return res; @@ -132,7 +139,7 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, I static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzmaDec state; SRes res = SZ_OK; @@ -149,14 +156,14 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I size_t lookahead = (1 << 18); if (lookahead > inSize) lookahead = (size_t)inSize; - res = inStream->Look(inStream, &inBuf, &lookahead); + res = ILookInStream_Look(inStream, &inBuf, &lookahead); if (res != SZ_OK) break; { SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -178,7 +185,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I break; } - res = inStream->Skip((void *)inStream, inProcessed); + res = ILookInStream_Skip(inStream, inProcessed); if (res != SZ_OK) break; } @@ -192,7 +199,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I #ifndef _7Z_NO_METHOD_LZMA2 static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzma2Dec state; SRes res = SZ_OK; @@ -211,14 +218,14 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, size_t lookahead = (1 << 18); if (lookahead > inSize) lookahead = (size_t)inSize; - res = inStream->Look(inStream, &inBuf, &lookahead); + res = ILookInStream_Look(inStream, &inBuf, &lookahead); if (res != SZ_OK) break; { SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; ELzmaStatus status; - res = Lzma2Dec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -237,7 +244,7 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, break; } - res = inStream->Skip((void *)inStream, inProcessed); + res = ILookInStream_Skip(inStream, inProcessed); if (res != SZ_OK) break; } @@ -258,18 +265,18 @@ static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer size_t curSize = (1 << 18); if (curSize > inSize) curSize = (size_t)inSize; - RINOK(inStream->Look(inStream, &inBuf, &curSize)); + RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)); if (curSize == 0) return SZ_ERROR_INPUT_EOF; memcpy(outBuffer, inBuf, curSize); outBuffer += curSize; inSize -= curSize; - RINOK(inStream->Skip((void *)inStream, curSize)); + RINOK(ILookInStream_Skip(inStream, curSize)); } return SZ_OK; } -static Bool IS_MAIN_METHOD(UInt32 m) +static BoolInt IS_MAIN_METHOD(UInt32 m) { switch (m) { @@ -286,7 +293,7 @@ static Bool IS_MAIN_METHOD(UInt32 m) return False; } -static Bool IS_SUPPORTED_CODER(const CSzCoderInfo *c) +static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) { return c->NumStreams == 1 @@ -365,14 +372,16 @@ static SRes CheckSupportedFolder(const CSzFolder *f) return SZ_ERROR_UNSUPPORTED; } +#ifndef _7Z_NO_METHODS_FILTERS #define CASE_BRA_CONV(isa) case k_ ## isa: isa ## _Convert(outBuffer, outSize, 0, 0); break; +#endif static SRes SzFolder_Decode2(const CSzFolder *folder, const Byte *propsData, const UInt64 *unpackSizes, const UInt64 *packPositions, ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain, Byte *tempBuf[]) { UInt32 ci; @@ -404,7 +413,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, outSizeCur = (SizeT)unpackSize; if (outSizeCur != unpackSize) return SZ_ERROR_MEM; - temp = (Byte *)IAlloc_Alloc(allocMain, outSizeCur); + temp = (Byte *)ISzAlloc_Alloc(allocMain, outSizeCur); if (!temp && outSizeCur != 0) return SZ_ERROR_MEM; outBufCur = tempBuf[1 - ci] = temp; @@ -421,7 +430,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, return SZ_ERROR_UNSUPPORTED; } offset = packPositions[si]; - inSize = packPositions[si + 1] - offset; + inSize = packPositions[(size_t)si + 1] - offset; RINOK(LookInStream_SeekTo(inStream, startPos + offset)); if (coder->MethodID == k_Copy) @@ -460,7 +469,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, tempSizes[2] = (SizeT)s3Size; if (tempSizes[2] != s3Size) return SZ_ERROR_MEM; - tempBuf[2] = (Byte *)IAlloc_Alloc(allocMain, tempSizes[2]); + tempBuf[2] = (Byte *)ISzAlloc_Alloc(allocMain, tempSizes[2]); if (!tempBuf[2] && tempSizes[2] != 0) return SZ_ERROR_MEM; @@ -549,7 +558,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, ILookInStream *inStream, UInt64 startPos, Byte *outBuffer, size_t outSize, - ISzAlloc *allocMain) + ISzAllocPtr allocMain) { SRes res; CSzFolder folder; @@ -557,7 +566,7 @@ SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, const Byte *data = p->CodersData + p->FoCodersOffsets[folderIndex]; sd.Data = data; - sd.Size = p->FoCodersOffsets[folderIndex + 1] - p->FoCodersOffsets[folderIndex]; + sd.Size = p->FoCodersOffsets[(size_t)folderIndex + 1] - p->FoCodersOffsets[folderIndex]; res = SzGetNextFolderItem(&folder, &sd); @@ -579,7 +588,7 @@ SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, outBuffer, (SizeT)outSize, allocMain, tempBuf); for (i = 0; i < 3; i++) - IAlloc_Free(allocMain, tempBuf[i]); + ISzAlloc_Free(allocMain, tempBuf[i]); if (res == SZ_OK) if (SzBitWithVals_Check(&p->FolderCRCs, folderIndex)) diff --git a/3rdparty/lzma/C/7zFile.c b/3rdparty/lzma/C/7zFile.c index 041e5b15465..13d2efa47f4 100644 --- a/3rdparty/lzma/C/7zFile.c +++ b/3rdparty/lzma/C/7zFile.c @@ -1,5 +1,5 @@ /* 7zFile.c -- File IO -2009-11-24 : Igor Pavlov : Public domain */ +2021-04-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,9 +7,19 @@ #ifndef USE_WINDOWS_FILE -#ifndef UNDER_CE -#include -#endif + #include + + #ifndef USE_FOPEN + #include + #include + #ifdef _WIN32 + #include + typedef int ssize_t; + typedef int off_t; + #else + #include + #endif + #endif #else @@ -23,30 +33,36 @@ And message can be "Network connection was lost" */ -#define kChunkSizeMax (1 << 22) - #endif +#define kChunkSizeMax (1 << 22) + void File_Construct(CSzFile *p) { #ifdef USE_WINDOWS_FILE p->handle = INVALID_HANDLE_VALUE; - #else + #elif defined(USE_FOPEN) p->file = NULL; + #else + p->fd = -1; #endif } #if !defined(UNDER_CE) || !defined(USE_WINDOWS_FILE) + static WRes File_Open(CSzFile *p, const char *name, int writeMode) { #ifdef USE_WINDOWS_FILE + p->handle = CreateFileA(name, writeMode ? GENERIC_WRITE : GENERIC_READ, FILE_SHARE_READ, NULL, writeMode ? CREATE_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); - #else + + #elif defined(USE_FOPEN) + p->file = fopen(name, writeMode ? "wb+" : "rb"); return (p->file != 0) ? 0 : #ifdef UNDER_CE @@ -54,13 +70,34 @@ static WRes File_Open(CSzFile *p, const char *name, int writeMode) #else errno; #endif + + #else + + int flags = (writeMode ? (O_CREAT | O_EXCL | O_WRONLY) : O_RDONLY); + #ifdef O_BINARY + flags |= O_BINARY; + #endif + p->fd = open(name, flags, 0666); + return (p->fd != -1) ? 0 : errno; + #endif } WRes InFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 0); } -WRes OutFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 1); } + +WRes OutFile_Open(CSzFile *p, const char *name) +{ + #if defined(USE_WINDOWS_FILE) || defined(USE_FOPEN) + return File_Open(p, name, 1); + #else + p->fd = creat(name, 0666); + return (p->fd != -1) ? 0 : errno; + #endif +} + #endif + #ifdef USE_WINDOWS_FILE static WRes File_OpenW(CSzFile *p, const WCHAR *name, int writeMode) { @@ -78,74 +115,124 @@ WRes OutFile_OpenW(CSzFile *p, const WCHAR *name) { return File_OpenW(p, name, 1 WRes File_Close(CSzFile *p) { #ifdef USE_WINDOWS_FILE + if (p->handle != INVALID_HANDLE_VALUE) { if (!CloseHandle(p->handle)) return GetLastError(); p->handle = INVALID_HANDLE_VALUE; } - #else + + #elif defined(USE_FOPEN) + if (p->file != NULL) { int res = fclose(p->file); if (res != 0) + { + if (res == EOF) + return errno; return res; + } p->file = NULL; } + + #else + + if (p->fd != -1) + { + if (close(p->fd) != 0) + return errno; + p->fd = -1; + } + #endif + return 0; } + WRes File_Read(CSzFile *p, void *data, size_t *size) { size_t originalSize = *size; + *size = 0; if (originalSize == 0) return 0; #ifdef USE_WINDOWS_FILE - *size = 0; do { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; + const DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; DWORD processed = 0; - BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); + const BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); data = (void *)((Byte *)data + processed); originalSize -= processed; *size += processed; if (!res) return GetLastError(); + // debug : we can break here for partial reading mode + if (processed == 0) + break; + } + while (originalSize > 0); + + #elif defined(USE_FOPEN) + + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const size_t processed = fread(data, 1, curSize, p->file); + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= processed; + *size += processed; + if (processed != curSize) + return ferror(p->file); + // debug : we can break here for partial reading mode if (processed == 0) break; } while (originalSize > 0); - return 0; #else - - *size = fread(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - + + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const ssize_t processed = read(p->fd, data, curSize); + if (processed == -1) + return errno; + if (processed == 0) + break; + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= (size_t)processed; + *size += (size_t)processed; + // debug : we can break here for partial reading mode + // break; + } + while (originalSize > 0); + #endif + + return 0; } + WRes File_Write(CSzFile *p, const void *data, size_t *size) { size_t originalSize = *size; + *size = 0; if (originalSize == 0) return 0; #ifdef USE_WINDOWS_FILE - *size = 0; do { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; + const DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; DWORD processed = 0; - BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); - data = (void *)((Byte *)data + processed); + const BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); + data = (const void *)((const Byte *)data + processed); originalSize -= processed; *size += processed; if (!res) @@ -154,26 +241,52 @@ WRes File_Write(CSzFile *p, const void *data, size_t *size) break; } while (originalSize > 0); - return 0; + + #elif defined(USE_FOPEN) + + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const size_t processed = fwrite(data, 1, curSize, p->file); + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= processed; + *size += processed; + if (processed != curSize) + return ferror(p->file); + if (processed == 0) + break; + } + while (originalSize > 0); #else - *size = fwrite(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - + do + { + const size_t curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : originalSize; + const ssize_t processed = write(p->fd, data, curSize); + if (processed == -1) + return errno; + if (processed == 0) + break; + data = (void *)((Byte *)data + (size_t)processed); + originalSize -= (size_t)processed; + *size += (size_t)processed; + } + while (originalSize > 0); + #endif + + return 0; } + WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) { #ifdef USE_WINDOWS_FILE - LARGE_INTEGER value; DWORD moveMethod; - value.LowPart = (DWORD)*pos; - value.HighPart = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ + UInt32 low = (UInt32)*pos; + LONG high = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ switch (origin) { case SZ_SEEK_SET: moveMethod = FILE_BEGIN; break; @@ -181,34 +294,52 @@ WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) case SZ_SEEK_END: moveMethod = FILE_END; break; default: return ERROR_INVALID_PARAMETER; } - value.LowPart = SetFilePointer(p->handle, value.LowPart, &value.HighPart, moveMethod); - if (value.LowPart == 0xFFFFFFFF) + low = SetFilePointer(p->handle, (LONG)low, &high, moveMethod); + if (low == (UInt32)0xFFFFFFFF) { WRes res = GetLastError(); if (res != NO_ERROR) return res; } - *pos = ((Int64)value.HighPart << 32) | value.LowPart; + *pos = ((Int64)high << 32) | low; return 0; #else - int moveMethod; - int res; + int moveMethod; // = origin; + switch (origin) { case SZ_SEEK_SET: moveMethod = SEEK_SET; break; case SZ_SEEK_CUR: moveMethod = SEEK_CUR; break; case SZ_SEEK_END: moveMethod = SEEK_END; break; - default: return 1; + default: return EINVAL; } - res = fseek(p->file, (long)*pos, moveMethod); - *pos = ftell(p->file); - return res; - #endif + #if defined(USE_FOPEN) + { + int res = fseek(p->file, (long)*pos, moveMethod); + if (res == -1) + return errno; + *pos = ftell(p->file); + if (*pos == -1) + return errno; + return 0; + } + #else + { + off_t res = lseek(p->fd, (off_t)*pos, moveMethod); + if (res == -1) + return errno; + *pos = res; + return 0; + } + + #endif // USE_FOPEN + #endif // USE_WINDOWS_FILE } + WRes File_GetLength(CSzFile *p, UInt64 *length) { #ifdef USE_WINDOWS_FILE @@ -224,13 +355,31 @@ WRes File_GetLength(CSzFile *p, UInt64 *length) *length = (((UInt64)sizeHigh) << 32) + sizeLow; return 0; - #else + #elif defined(USE_FOPEN) long pos = ftell(p->file); int res = fseek(p->file, 0, SEEK_END); *length = ftell(p->file); fseek(p->file, pos, SEEK_SET); return res; + + #else + + off_t pos; + *length = 0; + pos = lseek(p->fd, 0, SEEK_CUR); + if (pos != -1) + { + const off_t len2 = lseek(p->fd, 0, SEEK_END); + const off_t res2 = lseek(p->fd, pos, SEEK_SET); + if (len2 != -1) + { + *length = (UInt64)len2; + if (res2 != -1) + return 0; + } + } + return errno; #endif } @@ -238,49 +387,56 @@ WRes File_GetLength(CSzFile *p, UInt64 *length) /* ---------- FileSeqInStream ---------- */ -static SRes FileSeqInStream_Read(void *pp, void *buf, size_t *size) +static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CFileSeqInStream *p = (CFileSeqInStream *)pp; - return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; + CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); + WRes wres = File_Read(&p->file, buf, size); + p->wres = wres; + return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } void FileSeqInStream_CreateVTable(CFileSeqInStream *p) { - p->s.Read = FileSeqInStream_Read; + p->vt.Read = FileSeqInStream_Read; } /* ---------- FileInStream ---------- */ -static SRes FileInStream_Read(void *pp, void *buf, size_t *size) +static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) { - CFileInStream *p = (CFileInStream *)pp; - return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); + WRes wres = File_Read(&p->file, buf, size); + p->wres = wres; + return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } -static SRes FileInStream_Seek(void *pp, Int64 *pos, ESzSeek origin) +static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) { - CFileInStream *p = (CFileInStream *)pp; - return File_Seek(&p->file, pos, origin); + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); + WRes wres = File_Seek(&p->file, pos, origin); + p->wres = wres; + return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } void FileInStream_CreateVTable(CFileInStream *p) { - p->s.Read = FileInStream_Read; - p->s.Seek = FileInStream_Seek; + p->vt.Read = FileInStream_Read; + p->vt.Seek = FileInStream_Seek; } /* ---------- FileOutStream ---------- */ -static size_t FileOutStream_Write(void *pp, const void *data, size_t size) +static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CFileOutStream *p = (CFileOutStream *)pp; - File_Write(&p->file, data, &size); + CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); + WRes wres = File_Write(&p->file, data, &size); + p->wres = wres; return size; } void FileOutStream_CreateVTable(CFileOutStream *p) { - p->s.Write = FileOutStream_Write; + p->vt.Write = FileOutStream_Write; } diff --git a/3rdparty/lzma/C/7zFile.h b/3rdparty/lzma/C/7zFile.h index 658987ed51a..788abb6b9db 100644 --- a/3rdparty/lzma/C/7zFile.h +++ b/3rdparty/lzma/C/7zFile.h @@ -1,17 +1,20 @@ /* 7zFile.h -- File IO -2013-01-18 : Igor Pavlov : Public domain */ +2021-02-15 : Igor Pavlov : Public domain */ #ifndef __7Z_FILE_H #define __7Z_FILE_H #ifdef _WIN32 #define USE_WINDOWS_FILE +// #include #endif #ifdef USE_WINDOWS_FILE #include #else -#include +// note: USE_FOPEN mode is limited to 32-bit file size +// #define USE_FOPEN +// #include #endif #include "7zTypes.h" @@ -24,8 +27,10 @@ typedef struct { #ifdef USE_WINDOWS_FILE HANDLE handle; - #else + #elif defined(USE_FOPEN) FILE *file; + #else + int fd; #endif } CSzFile; @@ -54,8 +59,9 @@ WRes File_GetLength(CSzFile *p, UInt64 *length); typedef struct { - ISeqInStream s; + ISeqInStream vt; CSzFile file; + WRes wres; } CFileSeqInStream; void FileSeqInStream_CreateVTable(CFileSeqInStream *p); @@ -63,8 +69,9 @@ void FileSeqInStream_CreateVTable(CFileSeqInStream *p); typedef struct { - ISeekInStream s; + ISeekInStream vt; CSzFile file; + WRes wres; } CFileInStream; void FileInStream_CreateVTable(CFileInStream *p); @@ -72,8 +79,9 @@ void FileInStream_CreateVTable(CFileInStream *p); typedef struct { - ISeqOutStream s; + ISeqOutStream vt; CSzFile file; + WRes wres; } CFileOutStream; void FileOutStream_CreateVTable(CFileOutStream *p); diff --git a/3rdparty/lzma/C/7zStream.c b/3rdparty/lzma/C/7zStream.c index 88f9c42b1dd..28a14604fb1 100644 --- a/3rdparty/lzma/C/7zStream.c +++ b/3rdparty/lzma/C/7zStream.c @@ -1,5 +1,5 @@ /* 7zStream.c -- 7z Stream functions -2013-11-12 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,12 +7,12 @@ #include "7zTypes.h" -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType) +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)); if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed); @@ -21,40 +21,42 @@ SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorT return SZ_OK; } -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size) +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size) { return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf) +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) { size_t processed = 1; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)); return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; } -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset) + + +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) { - Int64 t = offset; - return stream->Seek(stream, &t, SZ_SEEK_SET); + Int64 t = (Int64)offset; + return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); } -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size) +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size) { const void *lookBuf; if (*size == 0) return SZ_OK; - RINOK(stream->Look(stream, &lookBuf, size)); + RINOK(ILookInStream_Look(stream, &lookBuf, size)); memcpy(buf, lookBuf, *size); - return stream->Skip(stream, *size); + return ILookInStream_Skip(stream, *size); } -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType) +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ILookInStream_Read(stream, buf, &processed)); if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed); @@ -63,61 +65,67 @@ SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes erro return SZ_OK; } -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size) +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size) { return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -static SRes LookToRead_Look_Lookahead(void *pp, const void **buf, size_t *size) + + +#define GET_LookToRead2 CLookToRead2 *p = CONTAINER_FROM_VTBL(pp, CLookToRead2, vt); + +static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf, size_t *size) { SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) + if (size2 == 0 && *size != 0) { p->pos = 0; - size2 = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, &size2); + p->size = 0; + size2 = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, &size2); p->size = size2; } - if (size2 < *size) + if (*size > size2) *size = size2; *buf = p->buf + p->pos; return res; } -static SRes LookToRead_Look_Exact(void *pp, const void **buf, size_t *size) +static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, size_t *size) { SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) + if (size2 == 0 && *size != 0) { p->pos = 0; - if (*size > LookToRead_BUF_SIZE) - *size = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, size); + p->size = 0; + if (*size > p->bufSize) + *size = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, size); size2 = p->size = *size; } - if (size2 < *size) + if (*size > size2) *size = size2; *buf = p->buf + p->pos; return res; } -static SRes LookToRead_Skip(void *pp, size_t offset) +static SRes LookToRead2_Skip(const ILookInStream *pp, size_t offset) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 p->pos += offset; return SZ_OK; } -static SRes LookToRead_Read(void *pp, void *buf, size_t *size) +static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t rem = p->size - p->pos; if (rem == 0) - return p->realStream->Read(p->realStream, buf, size); + return ISeekInStream_Read(p->realStream, buf, size); if (rem > *size) rem = *size; memcpy(buf, p->buf + p->pos, rem); @@ -126,46 +134,43 @@ static SRes LookToRead_Read(void *pp, void *buf, size_t *size) return SZ_OK; } -static SRes LookToRead_Seek(void *pp, Int64 *pos, ESzSeek origin) +static SRes LookToRead2_Seek(const ILookInStream *pp, Int64 *pos, ESzSeek origin) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 p->pos = p->size = 0; - return p->realStream->Seek(p->realStream, pos, origin); + return ISeekInStream_Seek(p->realStream, pos, origin); } -void LookToRead_CreateVTable(CLookToRead *p, int lookahead) +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead) { - p->s.Look = lookahead ? - LookToRead_Look_Lookahead : - LookToRead_Look_Exact; - p->s.Skip = LookToRead_Skip; - p->s.Read = LookToRead_Read; - p->s.Seek = LookToRead_Seek; + p->vt.Look = lookahead ? + LookToRead2_Look_Lookahead : + LookToRead2_Look_Exact; + p->vt.Skip = LookToRead2_Skip; + p->vt.Read = LookToRead2_Read; + p->vt.Seek = LookToRead2_Seek; } -void LookToRead_Init(CLookToRead *p) -{ - p->pos = p->size = 0; -} -static SRes SecToLook_Read(void *pp, void *buf, size_t *size) + +static SRes SecToLook_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CSecToLook *p = (CSecToLook *)pp; + CSecToLook *p = CONTAINER_FROM_VTBL(pp, CSecToLook, vt); return LookInStream_LookRead(p->realStream, buf, size); } void SecToLook_CreateVTable(CSecToLook *p) { - p->s.Read = SecToLook_Read; + p->vt.Read = SecToLook_Read; } -static SRes SecToRead_Read(void *pp, void *buf, size_t *size) +static SRes SecToRead_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CSecToRead *p = (CSecToRead *)pp; - return p->realStream->Read(p->realStream, buf, size); + CSecToRead *p = CONTAINER_FROM_VTBL(pp, CSecToRead, vt); + return ILookInStream_Read(p->realStream, buf, size); } void SecToRead_CreateVTable(CSecToRead *p) { - p->s.Read = SecToRead_Read; + p->vt.Read = SecToRead_Read; } diff --git a/3rdparty/lzma/C/7zTypes.h b/3rdparty/lzma/C/7zTypes.h index 778413ef424..f7d7071830c 100644 --- a/3rdparty/lzma/C/7zTypes.h +++ b/3rdparty/lzma/C/7zTypes.h @@ -1,11 +1,13 @@ /* 7zTypes.h -- Basic types -2013-11-12 : Igor Pavlov : Public domain */ +2022-04-01 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H #ifdef _WIN32 /* #include */ +#else +#include #endif #include @@ -42,17 +44,125 @@ EXTERN_C_BEGIN typedef int SRes; + +#ifdef _MSC_VER + #if _MSC_VER > 1200 + #define MY_ALIGN(n) __declspec(align(n)) + #else + #define MY_ALIGN(n) + #endif +#else + #define MY_ALIGN(n) __attribute__ ((aligned(n))) +#endif + + #ifdef _WIN32 + /* typedef DWORD WRes; */ typedef unsigned WRes; -#else +#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) + +// #define MY_HRES_ERROR__INTERNAL_ERROR MY_SRes_HRESULT_FROM_WRes(ERROR_INTERNAL_ERROR) + +#else // _WIN32 + +// #define ENV_HAVE_LSTAT typedef int WRes; + +// (FACILITY_ERRNO = 0x800) is 7zip's FACILITY constant to represent (errno) errors in HRESULT +#define MY__FACILITY_ERRNO 0x800 +#define MY__FACILITY_WIN32 7 +#define MY__FACILITY__WRes MY__FACILITY_ERRNO + +#define MY_HRESULT_FROM_errno_CONST_ERROR(x) ((HRESULT)( \ + ( (HRESULT)(x) & 0x0000FFFF) \ + | (MY__FACILITY__WRes << 16) \ + | (HRESULT)0x80000000 )) + +#define MY_SRes_HRESULT_FROM_WRes(x) \ + ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : MY_HRESULT_FROM_errno_CONST_ERROR(x)) + +// we call macro HRESULT_FROM_WIN32 for system errors (WRes) that are (errno) +#define HRESULT_FROM_WIN32(x) MY_SRes_HRESULT_FROM_WRes(x) + +/* +#define ERROR_FILE_NOT_FOUND 2L +#define ERROR_ACCESS_DENIED 5L +#define ERROR_NO_MORE_FILES 18L +#define ERROR_LOCK_VIOLATION 33L +#define ERROR_FILE_EXISTS 80L +#define ERROR_DISK_FULL 112L +#define ERROR_NEGATIVE_SEEK 131L +#define ERROR_ALREADY_EXISTS 183L +#define ERROR_DIRECTORY 267L +#define ERROR_TOO_MANY_POSTS 298L + +#define ERROR_INTERNAL_ERROR 1359L +#define ERROR_INVALID_REPARSE_DATA 4392L +#define ERROR_REPARSE_TAG_INVALID 4393L +#define ERROR_REPARSE_TAG_MISMATCH 4394L +*/ + +// we use errno equivalents for some WIN32 errors: + +#define ERROR_INVALID_PARAMETER EINVAL +#define ERROR_INVALID_FUNCTION EINVAL +#define ERROR_ALREADY_EXISTS EEXIST +#define ERROR_FILE_EXISTS EEXIST +#define ERROR_PATH_NOT_FOUND ENOENT +#define ERROR_FILE_NOT_FOUND ENOENT +#define ERROR_DISK_FULL ENOSPC +// #define ERROR_INVALID_HANDLE EBADF + +// we use FACILITY_WIN32 for errors that has no errno equivalent +// Too many posts were made to a semaphore. +#define ERROR_TOO_MANY_POSTS ((HRESULT)0x8007012AL) +#define ERROR_INVALID_REPARSE_DATA ((HRESULT)0x80071128L) +#define ERROR_REPARSE_TAG_INVALID ((HRESULT)0x80071129L) + +// if (MY__FACILITY__WRes != FACILITY_WIN32), +// we use FACILITY_WIN32 for COM errors: +#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) +#define E_INVALIDARG ((HRESULT)0x80070057L) +#define MY__E_ERROR_NEGATIVE_SEEK ((HRESULT)0x80070083L) + +/* +// we can use FACILITY_ERRNO for some COM errors, that have errno equivalents: +#define E_OUTOFMEMORY MY_HRESULT_FROM_errno_CONST_ERROR(ENOMEM) +#define E_INVALIDARG MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) +#define MY__E_ERROR_NEGATIVE_SEEK MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) +*/ + +#define TEXT(quote) quote + +#define FILE_ATTRIBUTE_READONLY 0x0001 +#define FILE_ATTRIBUTE_HIDDEN 0x0002 +#define FILE_ATTRIBUTE_SYSTEM 0x0004 +#define FILE_ATTRIBUTE_DIRECTORY 0x0010 +#define FILE_ATTRIBUTE_ARCHIVE 0x0020 +#define FILE_ATTRIBUTE_DEVICE 0x0040 +#define FILE_ATTRIBUTE_NORMAL 0x0080 +#define FILE_ATTRIBUTE_TEMPORARY 0x0100 +#define FILE_ATTRIBUTE_SPARSE_FILE 0x0200 +#define FILE_ATTRIBUTE_REPARSE_POINT 0x0400 +#define FILE_ATTRIBUTE_COMPRESSED 0x0800 +#define FILE_ATTRIBUTE_OFFLINE 0x1000 +#define FILE_ATTRIBUTE_NOT_CONTENT_INDEXED 0x2000 +#define FILE_ATTRIBUTE_ENCRYPTED 0x4000 + +#define FILE_ATTRIBUTE_UNIX_EXTENSION 0x8000 /* trick for Unix */ + #endif + #ifndef RINOK #define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } #endif +#ifndef RINOK_WRes +#define RINOK_WRes(x) { WRes __result__ = (x); if (__result__ != 0) return __result__; } +#endif + typedef unsigned char Byte; typedef short Int16; typedef unsigned short UInt16; @@ -65,6 +175,40 @@ typedef int Int32; typedef unsigned int UInt32; #endif + +#ifndef _WIN32 + +typedef int INT; +typedef Int32 INT32; +typedef unsigned int UINT; +typedef UInt32 UINT32; +typedef INT32 LONG; // LONG, ULONG and DWORD must be 32-bit for _WIN32 compatibility +typedef UINT32 ULONG; + +#undef DWORD +typedef UINT32 DWORD; + +#define VOID void + +#define HRESULT LONG + +typedef void *LPVOID; +// typedef void VOID; +// typedef ULONG_PTR DWORD_PTR, *PDWORD_PTR; +// gcc / clang on Unix : sizeof(long==sizeof(void*) in 32 or 64 bits) +typedef long INT_PTR; +typedef unsigned long UINT_PTR; +typedef long LONG_PTR; +typedef unsigned long DWORD_PTR; + +typedef size_t SIZE_T; + +#endif // _WIN32 + + +#define MY_HRES_ERROR__INTERNAL_ERROR ((HRESULT)0x8007054FL) + + #ifdef _SZ_NO_INT_64 /* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. @@ -93,7 +237,8 @@ typedef UInt32 SizeT; typedef size_t SizeT; #endif -typedef int Bool; +typedef int BoolInt; +/* typedef BoolInt Bool; */ #define True 1 #define False 0 @@ -112,48 +257,84 @@ typedef int Bool; #define MY_NO_INLINE #endif +#define MY_FORCE_INLINE __forceinline + #define MY_CDECL __cdecl #define MY_FAST_CALL __fastcall -#else +#else // _MSC_VER +#if (defined(__GNUC__) && (__GNUC__ >= 4)) \ + || (defined(__clang__) && (__clang_major__ >= 4)) \ + || defined(__INTEL_COMPILER) \ + || defined(__xlC__) +#define MY_NO_INLINE __attribute__((noinline)) +// #define MY_FORCE_INLINE __attribute__((always_inline)) inline +#else #define MY_NO_INLINE +#endif + +#define MY_FORCE_INLINE + + #define MY_CDECL -#define MY_FAST_CALL +#if defined(_M_IX86) \ + || defined(__i386__) +// #define MY_FAST_CALL __attribute__((fastcall)) +// #define MY_FAST_CALL __attribute__((cdecl)) +#define MY_FAST_CALL +#elif defined(MY_CPU_AMD64) +// #define MY_FAST_CALL __attribute__((ms_abi)) +#define MY_FAST_CALL +#else +#define MY_FAST_CALL #endif +#endif // _MSC_VER + /* The following interfaces use first parameter as pointer to structure */ -typedef struct +typedef struct IByteIn IByteIn; +struct IByteIn { - Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */ -} IByteIn; + Byte (*Read)(const IByteIn *p); /* reads one byte, returns 0 in case of EOF or error */ +}; +#define IByteIn_Read(p) (p)->Read(p) -typedef struct + +typedef struct IByteOut IByteOut; +struct IByteOut { - void (*Write)(void *p, Byte b); -} IByteOut; + void (*Write)(const IByteOut *p, Byte b); +}; +#define IByteOut_Write(p, b) (p)->Write(p, b) -typedef struct + +typedef struct ISeqInStream ISeqInStream; +struct ISeqInStream { - SRes (*Read)(void *p, void *buf, size_t *size); + SRes (*Read)(const ISeqInStream *p, void *buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) < input(*size)) is allowed */ -} ISeqInStream; +}; +#define ISeqInStream_Read(p, buf, size) (p)->Read(p, buf, size) /* it can return SZ_ERROR_INPUT_EOF */ -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size); +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf); -typedef struct + +typedef struct ISeqOutStream ISeqOutStream; +struct ISeqOutStream { - size_t (*Write)(void *p, const void *buf, size_t size); + size_t (*Write)(const ISeqOutStream *p, const void *buf, size_t size); /* Returns: result - the number of actually written bytes. (result < size) means error */ -} ISeqOutStream; +}; +#define ISeqOutStream_Write(p, buf, size) (p)->Write(p, buf, size) typedef enum { @@ -162,78 +343,162 @@ typedef enum SZ_SEEK_END = 2 } ESzSeek; -typedef struct + +typedef struct ISeekInStream ISeekInStream; +struct ISeekInStream { - SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ISeekInStream; + SRes (*Read)(const ISeekInStream *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(const ISeekInStream *p, Int64 *pos, ESzSeek origin); +}; +#define ISeekInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ISeekInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) -typedef struct + +typedef struct ILookInStream ILookInStream; +struct ILookInStream { - SRes (*Look)(void *p, const void **buf, size_t *size); + SRes (*Look)(const ILookInStream *p, const void **buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) > input(*size)) is not allowed (output(*size) < input(*size)) is allowed */ - SRes (*Skip)(void *p, size_t offset); + SRes (*Skip)(const ILookInStream *p, size_t offset); /* offset must be <= output(*size) of Look */ - SRes (*Read)(void *p, void *buf, size_t *size); + SRes (*Read)(const ILookInStream *p, void *buf, size_t *size); /* reads directly (without buffer). It's same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ILookInStream; + SRes (*Seek)(const ILookInStream *p, Int64 *pos, ESzSeek origin); +}; + +#define ILookInStream_Look(p, buf, size) (p)->Look(p, buf, size) +#define ILookInStream_Skip(p, offset) (p)->Skip(p, offset) +#define ILookInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ILookInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) + -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset); /* reads via ILookInStream::Read */ -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size); + -#define LookToRead_BUF_SIZE (1 << 14) typedef struct { - ILookInStream s; - ISeekInStream *realStream; + ILookInStream vt; + const ISeekInStream *realStream; + size_t pos; - size_t size; - Byte buf[LookToRead_BUF_SIZE]; -} CLookToRead; + size_t size; /* it's data size */ + + /* the following variables must be set outside */ + Byte *buf; + size_t bufSize; +} CLookToRead2; + +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead); + +#define LookToRead2_Init(p) { (p)->pos = (p)->size = 0; } -void LookToRead_CreateVTable(CLookToRead *p, int lookahead); -void LookToRead_Init(CLookToRead *p); typedef struct { - ISeqInStream s; - ILookInStream *realStream; + ISeqInStream vt; + const ILookInStream *realStream; } CSecToLook; void SecToLook_CreateVTable(CSecToLook *p); + + typedef struct { - ISeqInStream s; - ILookInStream *realStream; + ISeqInStream vt; + const ILookInStream *realStream; } CSecToRead; void SecToRead_CreateVTable(CSecToRead *p); -typedef struct + +typedef struct ICompressProgress ICompressProgress; + +struct ICompressProgress { - SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); + SRes (*Progress)(const ICompressProgress *p, UInt64 inSize, UInt64 outSize); /* Returns: result. (result != SZ_OK) means break. Value (UInt64)(Int64)-1 for size means unknown value. */ -} ICompressProgress; +}; +#define ICompressProgress_Progress(p, inSize, outSize) (p)->Progress(p, inSize, outSize) -typedef struct + + +typedef struct ISzAlloc ISzAlloc; +typedef const ISzAlloc * ISzAllocPtr; + +struct ISzAlloc { - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; + void *(*Alloc)(ISzAllocPtr p, size_t size); + void (*Free)(ISzAllocPtr p, void *address); /* address can be 0 */ +}; + +#define ISzAlloc_Alloc(p, size) (p)->Alloc(p, size) +#define ISzAlloc_Free(p, a) (p)->Free(p, a) + +/* deprecated */ +#define IAlloc_Alloc(p, size) ISzAlloc_Alloc(p, size) +#define IAlloc_Free(p, a) ISzAlloc_Free(p, a) + + + + + +#ifndef MY_offsetof + #ifdef offsetof + #define MY_offsetof(type, m) offsetof(type, m) + /* + #define MY_offsetof(type, m) FIELD_OFFSET(type, m) + */ + #else + #define MY_offsetof(type, m) ((size_t)&(((type *)0)->m)) + #endif +#endif + + -#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) -#define IAlloc_Free(p, a) (p)->Free((p), a) +#ifndef MY_container_of + +/* +#define MY_container_of(ptr, type, m) container_of(ptr, type, m) +#define MY_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m) +#define MY_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m))) +#define MY_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m)))) +*/ + +/* + GCC shows warning: "perhaps the 'offsetof' macro was used incorrectly" + GCC 3.4.4 : classes with constructor + GCC 4.8.1 : classes with non-public variable members" +*/ + +#define MY_container_of(ptr, type, m) ((type *)(void *)((char *)(void *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) + +#endif + +#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(void *)(ptr)) + +/* +#define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +*/ +#define CONTAINER_FROM_VTBL(ptr, type, m) MY_container_of(ptr, type, m) + +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +/* +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL(ptr, type, m) +*/ + + +#define MY_memset_0_ARRAY(a) memset((a), 0, sizeof(a)) #ifdef _WIN32 @@ -251,6 +516,14 @@ typedef struct #endif +#define k_PropVar_TimePrec_0 0 +#define k_PropVar_TimePrec_Unix 1 +#define k_PropVar_TimePrec_DOS 2 +#define k_PropVar_TimePrec_HighPrec 3 +#define k_PropVar_TimePrec_Base 16 +#define k_PropVar_TimePrec_100ns (k_PropVar_TimePrec_Base + 7) +#define k_PropVar_TimePrec_1ns (k_PropVar_TimePrec_Base + 9) + EXTERN_C_END #endif diff --git a/3rdparty/lzma/C/7zVersion.h b/3rdparty/lzma/C/7zVersion.h index 4f15081de78..49ea81dd3be 100644 --- a/3rdparty/lzma/C/7zVersion.h +++ b/3rdparty/lzma/C/7zVersion.h @@ -1,14 +1,21 @@ -#define MY_VER_MAJOR 16 -#define MY_VER_MINOR 04 +#define MY_VER_MAJOR 22 +#define MY_VER_MINOR 01 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "16.04" -#define MY_VERSION "16.04" -#define MY_DATE "2016-10-04" +#define MY_VERSION_NUMBERS "22.01" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2022-07-15" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2016 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2022 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR @@ -16,4 +23,5 @@ #define MY_COPYRIGHT MY_COPYRIGHT_PD #endif -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " : " MY_COPYRIGHT " : " MY_DATE +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/3rdparty/lzma/C/7zip_gcc_c.mak b/3rdparty/lzma/C/7zip_gcc_c.mak new file mode 100644 index 00000000000..24505f37748 --- /dev/null +++ b/3rdparty/lzma/C/7zip_gcc_c.mak @@ -0,0 +1,343 @@ + +MY_ARCH_2 = $(MY_ARCH) + +MY_ASM = jwasm +MY_ASM = asmc + +PROGPATH = $(O)/$(PROG) +PROGPATH_STATIC = $(O)/$(PROG)s + + +# for object file +CFLAGS_BASE_LIST = -c +# for ASM file +# CFLAGS_BASE_LIST = -S +CFLAGS_BASE = $(MY_ARCH_2) -O2 $(CFLAGS_BASE_LIST) -Wall -Werror -Wextra $(CFLAGS_WARN) \ + -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + + +ifdef SystemDrive +IS_MINGW = 1 +else +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif +endif + +ifdef IS_MINGW +LDFLAGS_STATIC_2 = -static +else +ifndef DEF_FILE +ifndef IS_NOT_STANDALONE +ifndef MY_DYNAMIC_LINK +ifneq ($(CC), clang) +LDFLAGS_STATIC_2 = +# -static +# -static-libstdc++ -static-libgcc +endif +endif +endif +endif +endif + +LDFLAGS_STATIC = -DNDEBUG $(LDFLAGS_STATIC_2) + +ifdef DEF_FILE + + +ifdef IS_MINGW +SHARED_EXT=.dll +LDFLAGS = -shared -DEF $(DEF_FILE) $(LDFLAGS_STATIC) +else +SHARED_EXT=.so +LDFLAGS = -shared -fPIC $(LDFLAGS_STATIC) +CC_SHARED=-fPIC +endif + + +else + +LDFLAGS = $(LDFLAGS_STATIC) +# -s is not required for clang, do we need it for GGC ??? +# -s + +#-static -static-libgcc -static-libstdc++ + +ifdef IS_MINGW +SHARED_EXT=.exe +else +SHARED_EXT= +endif + +endif + + +PROGPATH = $(O)/$(PROG)$(SHARED_EXT) +PROGPATH_STATIC = $(O)/$(PROG)s$(SHARED_EXT) + +ifndef O +O=_o +endif + +ifdef IS_MINGW + +ifdef MSYSTEM +RM = rm -f +MY_MKDIR=mkdir -p +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) +else +RM = del +MY_MKDIR=mkdir +DEL_OBJ_EXE = -$(RM) $(O)\*.o $(O)\$(PROG).exe $(O)\$(PROG).dll +endif + + +LIB2 = -lOle32 -loleaut32 -luuid -ladvapi32 -lUser32 + +CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE +# -Wno-delete-non-virtual-dtor + + +else + +RM = rm -f +MY_MKDIR=mkdir -p +# CFLAGS_BASE := $(CFLAGS_BASE) -D_7ZIP_ST +# CXXFLAGS_EXTRA = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + +# LOCAL_LIBS=-lpthread +# LOCAL_LIBS_DLL=$(LOCAL_LIBS) -ldl +LIB2 = -lpthread -ldl + +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) + +endif + + + +CFLAGS = $(LOCAL_FLAGS) $(CFLAGS_BASE2) $(CFLAGS_BASE) $(CC_SHARED) -o $@ + + +ifdef IS_X64 +AFLAGS_ABI = -elf64 -DABI_LINUX +else +AFLAGS_ABI = -elf -DABI_LINUX -DABI_CDECL +# -DABI_CDECL +# -DABI_LINUX +# -DABI_CDECL +endif +AFLAGS = $(AFLAGS_ABI) -Fo$(O)/ + + +CXX_WARN_FLAGS = +#-Wno-invalid-offsetof +#-Wno-reorder + +CXXFLAGS = $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(CXXFLAGS_EXTRA) $(CC_SHARED) -o $@ $(CXX_WARN_FLAGS) + +STATIC_TARGET= +ifdef COMPL_STATIC +STATIC_TARGET=$(PROGPATH_STATIC) +endif + + +all: $(O) $(PROGPATH) $(STATIC_TARGET) + +$(O): + $(MY_MKDIR) $(O) + +LFLAGS_ALL = -s $(MY_ARCH_2) $(LDFLAGS) $(LD_arch) $(OBJS) $(MY_LIBS) $(LIB2) +$(PROGPATH): $(OBJS) + $(CXX) -o $(PROGPATH) $(LFLAGS_ALL) + +$(PROGPATH_STATIC): $(OBJS) + $(CXX) -static -o $(PROGPATH_STATIC) $(LFLAGS_ALL) + + +ifndef NO_DEFAULT_RES +$O/resource.o: resource.rc + windres.exe $(RFLAGS) resource.rc $O/resource.o +endif + + + +$O/7zAlloc.o: ../../../C/7zAlloc.c + $(CC) $(CFLAGS) $< +$O/7zArcIn.o: ../../../C/7zArcIn.c + $(CC) $(CFLAGS) $< +$O/7zBuf.o: ../../../C/7zBuf.c + $(CC) $(CFLAGS) $< +$O/7zBuf2.o: ../../../C/7zBuf2.c + $(CC) $(CFLAGS) $< +$O/7zCrc.o: ../../../C/7zCrc.c + $(CC) $(CFLAGS) $< +$O/7zDec.o: ../../../C/7zDec.c + $(CC) $(CFLAGS) $< +$O/7zFile.o: ../../../C/7zFile.c + $(CC) $(CFLAGS) $< +$O/7zStream.o: ../../../C/7zStream.c + $(CC) $(CFLAGS) $< +$O/Aes.o: ../../../C/Aes.c + $(CC) $(CFLAGS) $< +$O/Alloc.o: ../../../C/Alloc.c + $(CC) $(CFLAGS) $< +$O/Bcj2.o: ../../../C/Bcj2.c + $(CC) $(CFLAGS) $< +$O/Bcj2Enc.o: ../../../C/Bcj2Enc.c + $(CC) $(CFLAGS) $< +$O/Blake2s.o: ../../../C/Blake2s.c + $(CC) $(CFLAGS) $< +$O/Bra.o: ../../../C/Bra.c + $(CC) $(CFLAGS) $< +$O/Bra86.o: ../../../C/Bra86.c + $(CC) $(CFLAGS) $< +$O/BraIA64.o: ../../../C/BraIA64.c + $(CC) $(CFLAGS) $< +$O/BwtSort.o: ../../../C/BwtSort.c + $(CC) $(CFLAGS) $< + +$O/CpuArch.o: ../../../C/CpuArch.c + $(CC) $(CFLAGS) $< +$O/Delta.o: ../../../C/Delta.c + $(CC) $(CFLAGS) $< +$O/DllSecur.o: ../../../C/DllSecur.c + $(CC) $(CFLAGS) $< +$O/HuffEnc.o: ../../../C/HuffEnc.c + $(CC) $(CFLAGS) $< +$O/LzFind.o: ../../../C/LzFind.c + $(CC) $(CFLAGS) $< + +# ifdef MT_FILES +$O/LzFindMt.o: ../../../C/LzFindMt.c + $(CC) $(CFLAGS) $< +$O/LzFindOpt.o: ../../../C/LzFindOpt.c + $(CC) $(CFLAGS) $< + +$O/Threads.o: ../../../C/Threads.c + $(CC) $(CFLAGS) $< +# endif + +$O/LzmaEnc.o: ../../../C/LzmaEnc.c + $(CC) $(CFLAGS) $< +$O/Lzma86Dec.o: ../../../C/Lzma86Dec.c + $(CC) $(CFLAGS) $< +$O/Lzma86Enc.o: ../../../C/Lzma86Enc.c + $(CC) $(CFLAGS) $< +$O/Lzma2Dec.o: ../../../C/Lzma2Dec.c + $(CC) $(CFLAGS) $< +$O/Lzma2DecMt.o: ../../../C/Lzma2DecMt.c + $(CC) $(CFLAGS) $< +$O/Lzma2Enc.o: ../../../C/Lzma2Enc.c + $(CC) $(CFLAGS) $< +$O/LzmaLib.o: ../../../C/LzmaLib.c + $(CC) $(CFLAGS) $< +$O/MtCoder.o: ../../../C/MtCoder.c + $(CC) $(CFLAGS) $< +$O/MtDec.o: ../../../C/MtDec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7.o: ../../../C/Ppmd7.c + $(CC) $(CFLAGS) $< +$O/Ppmd7aDec.o: ../../../C/Ppmd7aDec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7Dec.o: ../../../C/Ppmd7Dec.c + $(CC) $(CFLAGS) $< +$O/Ppmd7Enc.o: ../../../C/Ppmd7Enc.c + $(CC) $(CFLAGS) $< +$O/Ppmd8.o: ../../../C/Ppmd8.c + $(CC) $(CFLAGS) $< +$O/Ppmd8Dec.o: ../../../C/Ppmd8Dec.c + $(CC) $(CFLAGS) $< +$O/Ppmd8Enc.o: ../../../C/Ppmd8Enc.c + $(CC) $(CFLAGS) $< +$O/Sha1.o: ../../../C/Sha1.c + $(CC) $(CFLAGS) $< +$O/Sha256.o: ../../../C/Sha256.c + $(CC) $(CFLAGS) $< +$O/Sort.o: ../../../C/Sort.c + $(CC) $(CFLAGS) $< +$O/Xz.o: ../../../C/Xz.c + $(CC) $(CFLAGS) $< +$O/XzCrc64.o: ../../../C/XzCrc64.c + $(CC) $(CFLAGS) $< + + +ifdef USE_ASM +ifdef IS_X64 +USE_X86_ASM=1 +else +ifdef IS_X86 +USE_X86_ASM=1 +endif +endif +endif + +ifdef USE_X86_ASM +$O/7zCrcOpt.o: ../../../Asm/x86/7zCrcOpt.asm + $(MY_ASM) $(AFLAGS) $< +$O/XzCrc64Opt.o: ../../../Asm/x86/XzCrc64Opt.asm + $(MY_ASM) $(AFLAGS) $< +$O/AesOpt.o: ../../../Asm/x86/AesOpt.asm + $(MY_ASM) $(AFLAGS) $< +$O/Sha1Opt.o: ../../../Asm/x86/Sha1Opt.asm + $(MY_ASM) $(AFLAGS) $< +$O/Sha256Opt.o: ../../../Asm/x86/Sha256Opt.asm + $(MY_ASM) $(AFLAGS) $< +else +$O/7zCrcOpt.o: ../../7zCrcOpt.c + $(CC) $(CFLAGS) $< +$O/XzCrc64Opt.o: ../../XzCrc64Opt.c + $(CC) $(CFLAGS) $< +$O/Sha1Opt.o: ../../Sha1Opt.c + $(CC) $(CFLAGS) $< +$O/Sha256Opt.o: ../../Sha256Opt.c + $(CC) $(CFLAGS) $< +$O/AesOpt.o: ../../AesOpt.c + $(CC) $(CFLAGS) $< +endif + + +ifdef USE_LZMA_DEC_ASM + +ifdef IS_X64 +$O/LzmaDecOpt.o: ../../../Asm/x86/LzmaDecOpt.asm + $(MY_ASM) $(AFLAGS) $< +endif + +ifdef IS_ARM64 +$O/LzmaDecOpt.o: ../../../Asm/arm64/LzmaDecOpt.S ../../../Asm/arm64/7zAsm.S + $(CC) $(CFLAGS) $< +endif + +$O/LzmaDec.o: ../../LzmaDec.c + $(CC) $(CFLAGS) -D_LZMA_DEC_OPT $< + +else + +$O/LzmaDec.o: ../../LzmaDec.c + $(CC) $(CFLAGS) $< + +endif + + + +$O/XzDec.o: ../../../C/XzDec.c + $(CC) $(CFLAGS) $< +$O/XzEnc.o: ../../../C/XzEnc.c + $(CC) $(CFLAGS) $< +$O/XzIn.o: ../../../C/XzIn.c + $(CC) $(CFLAGS) $< + + +$O/7zMain.o: ../../../C/Util/7z/7zMain.c + $(CC) $(CFLAGS) $< +$O/LzmaUtil.o: ../../../C/Util/Lzma/LzmaUtil.c + $(CC) $(CFLAGS) $< +$O/7zipInstall.o: ../../../C/Util/7zipInstall/7zipInstall.c + $(CC) $(CFLAGS) $< +$O/7zipUninstall.o: ../../../C/Util/7zipUninstall/7zipUninstall.c + $(CC) $(CFLAGS) $< + + +clean: + -$(DEL_OBJ_EXE) diff --git a/3rdparty/lzma/C/Aes.c b/3rdparty/lzma/C/Aes.c index b10b9e58a55..27e32e62294 100644 --- a/3rdparty/lzma/C/Aes.c +++ b/3rdparty/lzma/C/Aes.c @@ -1,10 +1,17 @@ /* Aes.c -- AES encryption / decryption -2016-05-21 : Igor Pavlov : Public domain */ +2021-05-13 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "Aes.h" #include "CpuArch.h" +#include "Aes.h" + +AES_CODE_FUNC g_AesCbc_Decode; +#ifndef _SFX +AES_CODE_FUNC g_AesCbc_Encode; +AES_CODE_FUNC g_AesCtr_Code; +UInt32 g_Aes_SupportedFunctions_Flags; +#endif static UInt32 T[256 * 4]; static const Byte Sbox[256] = { @@ -25,23 +32,10 @@ static const Byte Sbox[256] = { 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); - -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); - -AES_CODE_FUNC g_AesCbc_Encode; -AES_CODE_FUNC g_AesCbc_Decode; -AES_CODE_FUNC g_AesCtr_Code; static UInt32 D[256 * 4]; static Byte InvS[256]; -static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; - #define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF) #define Ui32(a0, a1, a2, a3) ((UInt32)(a0) | ((UInt32)(a1) << 8) | ((UInt32)(a2) << 16) | ((UInt32)(a3) << 24)) @@ -49,7 +43,43 @@ static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0 #define gb0(x) ( (x) & 0xFF) #define gb1(x) (((x) >> ( 8)) & 0xFF) #define gb2(x) (((x) >> (16)) & 0xFF) -#define gb3(x) (((x) >> (24)) & 0xFF) +#define gb3(x) (((x) >> (24))) + +#define gb(n, x) gb ## n(x) + +#define TT(x) (T + (x << 8)) +#define DD(x) (D + (x << 8)) + + +// #define _SHOW_AES_STATUS + +#ifdef MY_CPU_X86_OR_AMD64 + #define USE_HW_AES +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_AES + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_AES + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_AES + #endif + #endif +#endif + +#ifdef USE_HW_AES +#ifdef _SHOW_AES_STATUS +#include +#define _PRF(x) x +#else +#define _PRF(x) +#endif +#endif + void AesGenTables(void) { @@ -63,10 +93,10 @@ void AesGenTables(void) UInt32 a1 = Sbox[i]; UInt32 a2 = xtime(a1); UInt32 a3 = a2 ^ a1; - T[ i] = Ui32(a2, a1, a1, a3); - T[0x100 + i] = Ui32(a3, a2, a1, a1); - T[0x200 + i] = Ui32(a1, a3, a2, a1); - T[0x300 + i] = Ui32(a1, a1, a3, a2); + TT(0)[i] = Ui32(a2, a1, a1, a3); + TT(1)[i] = Ui32(a3, a2, a1, a1); + TT(2)[i] = Ui32(a1, a3, a2, a1); + TT(3)[i] = Ui32(a1, a1, a3, a2); } { UInt32 a1 = InvS[i]; @@ -77,29 +107,59 @@ void AesGenTables(void) UInt32 aB = a8 ^ a2 ^ a1; UInt32 aD = a8 ^ a4 ^ a1; UInt32 aE = a8 ^ a4 ^ a2; - D[ i] = Ui32(aE, a9, aD, aB); - D[0x100 + i] = Ui32(aB, aE, a9, aD); - D[0x200 + i] = Ui32(aD, aB, aE, a9); - D[0x300 + i] = Ui32(a9, aD, aB, aE); + DD(0)[i] = Ui32(aE, a9, aD, aB); + DD(1)[i] = Ui32(aB, aE, a9, aD); + DD(2)[i] = Ui32(aD, aB, aE, a9); + DD(3)[i] = Ui32(a9, aD, aB, aE); } } - g_AesCbc_Encode = AesCbc_Encode; - g_AesCbc_Decode = AesCbc_Decode; - g_AesCtr_Code = AesCtr_Code; + { + AES_CODE_FUNC d = AesCbc_Decode; + #ifndef _SFX + AES_CODE_FUNC e = AesCbc_Encode; + AES_CODE_FUNC c = AesCtr_Code; + UInt32 flags = 0; + #endif - #ifdef MY_CPU_X86_OR_AMD64 - if (CPU_Is_Aes_Supported()) + #ifdef USE_HW_AES + if (CPU_IsSupported_AES()) { - g_AesCbc_Encode = AesCbc_Encode_Intel; - g_AesCbc_Decode = AesCbc_Decode_Intel; - g_AesCtr_Code = AesCtr_Code_Intel; + // #pragma message ("AES HW") + _PRF(printf("\n===AES HW\n")); + d = AesCbc_Decode_HW; + + #ifndef _SFX + e = AesCbc_Encode_HW; + c = AesCtr_Code_HW; + flags = k_Aes_SupportedFunctions_HW; + #endif + + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_VAES_AVX2()) + { + _PRF(printf("\n===vaes avx2\n")); + d = AesCbc_Decode_HW_256; + #ifndef _SFX + c = AesCtr_Code_HW_256; + flags |= k_Aes_SupportedFunctions_HW_256; + #endif + } + #endif } #endif + + g_AesCbc_Decode = d; + #ifndef _SFX + g_AesCbc_Encode = e; + g_AesCtr_Code = c; + g_Aes_SupportedFunctions_Flags = flags; + #endif + } } -#define HT(i, x, s) (T + (x << 8))[gb ## x(s[(i + x) & 3])] +#define HT(i, x, s) TT(x)[gb(x, s[(i + x) & 3])] #define HT4(m, i, s, p) m[i] = \ HT(i, 0, s) ^ \ @@ -113,11 +173,11 @@ void AesGenTables(void) HT4(m, 2, s, p); \ HT4(m, 3, s, p); \ -#define FT(i, x) Sbox[gb ## x(m[(i + x) & 3])] +#define FT(i, x) Sbox[gb(x, m[(i + x) & 3])] #define FT4(i) dest[i] = Ui32(FT(i, 0), FT(i, 1), FT(i, 2), FT(i, 3)) ^ w[i]; -#define HD(i, x, s) (D + (x << 8))[gb ## x(s[(i - x) & 3])] +#define HD(i, x, s) DD(x)[gb(x, s[(i - x) & 3])] #define HD4(m, i, s, p) m[i] = \ HD(i, 0, s) ^ \ @@ -131,13 +191,16 @@ void AesGenTables(void) HD4(m, 2, s, p); \ HD4(m, 3, s, p); \ -#define FD(i, x) InvS[gb ## x(m[(i - x) & 3])] +#define FD(i, x) InvS[gb(x, m[(i - x) & 3])] #define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i]; void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) { - unsigned i, wSize; - wSize = keySize + 28; + unsigned i, m; + const UInt32 *wLim; + UInt32 t; + UInt32 rcon = 1; + keySize /= 4; w[0] = ((UInt32)keySize / 2) + 3; w += 4; @@ -145,16 +208,26 @@ void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) for (i = 0; i < keySize; i++, key += 4) w[i] = GetUi32(key); - for (; i < wSize; i++) + t = w[(size_t)keySize - 1]; + wLim = w + (size_t)keySize * 3 + 28; + m = 0; + do { - UInt32 t = w[i - 1]; - unsigned rem = i % keySize; - if (rem == 0) - t = Ui32(Sbox[gb1(t)] ^ Rcon[i / keySize], Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); - else if (keySize > 6 && rem == 4) + if (m == 0) + { + t = Ui32(Sbox[gb1(t)] ^ rcon, Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); + rcon <<= 1; + if (rcon & 0x100) + rcon = 0x1b; + m = keySize; + } + else if (m == 4 && keySize > 6) t = Ui32(Sbox[gb0(t)], Sbox[gb1(t)], Sbox[gb2(t)], Sbox[gb3(t)]); - w[i] = w[i - keySize] ^ t; + m--; + t ^= w[0]; + w[keySize] = t; } + while (++w != wLim); } void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) @@ -167,10 +240,10 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) { UInt32 r = w[i]; w[i] = - D[ (unsigned)Sbox[gb0(r)]] ^ - D[0x100 + (unsigned)Sbox[gb1(r)]] ^ - D[0x200 + (unsigned)Sbox[gb2(r)]] ^ - D[0x300 + (unsigned)Sbox[gb3(r)]]; + DD(0)[Sbox[gb0(r)]] ^ + DD(1)[Sbox[gb1(r)]] ^ + DD(2)[Sbox[gb2(r)]] ^ + DD(3)[Sbox[gb3(r)]]; } } @@ -178,6 +251,7 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) src and dest are pointers to 4 UInt32 words. src and dest can point to same block */ +// MY_FORCE_INLINE static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) { UInt32 s[4]; @@ -201,6 +275,7 @@ static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) FT4(0); FT4(1); FT4(2); FT4(3); } +MY_FORCE_INLINE static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src) { UInt32 s[4]; @@ -276,20 +351,25 @@ void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) for (; numBlocks != 0; numBlocks--) { UInt32 temp[4]; - Byte buf[16]; - int i; + unsigned i; if (++p[0] == 0) p[1]++; Aes_Encode(p + 4, temp, p); - SetUi32(buf, temp[0]); - SetUi32(buf + 4, temp[1]); - SetUi32(buf + 8, temp[2]); - SetUi32(buf + 12, temp[3]); - - for (i = 0; i < 16; i++) - *data++ ^= buf[i]; + for (i = 0; i < 4; i++, data += 4) + { + UInt32 t = temp[i]; + + #ifdef MY_CPU_LE_UNALIGN + *((UInt32 *)(void *)data) ^= t; + #else + data[0] = (Byte)(data[0] ^ (t & 0xFF)); + data[1] = (Byte)(data[1] ^ ((t >> 8) & 0xFF)); + data[2] = (Byte)(data[2] ^ ((t >> 16) & 0xFF)); + data[3] = (Byte)(data[3] ^ ((t >> 24))); + #endif + } } } diff --git a/3rdparty/lzma/C/Aes.h b/3rdparty/lzma/C/Aes.h index 64979b5bcbf..2aa22564266 100644 --- a/3rdparty/lzma/C/Aes.h +++ b/3rdparty/lzma/C/Aes.h @@ -1,5 +1,5 @@ /* Aes.h -- AES encryption / decryption -2013-01-18 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #ifndef __AES_H #define __AES_H @@ -26,12 +26,34 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *aes, const Byte *key, unsigned keySize) /* ivAes - 16-byte aligned pointer to iv+keyMode+roundKeys sequence: UInt32[AES_NUM_IVMRK_WORDS] */ void AesCbc_Init(UInt32 *ivAes, const Byte *iv); /* iv size is AES_BLOCK_SIZE */ + /* data - 16-byte aligned pointer to data */ /* numBlocks - the number of 16-byte blocks in data array */ typedef void (MY_FAST_CALL *AES_CODE_FUNC)(UInt32 *ivAes, Byte *data, size_t numBlocks); -extern AES_CODE_FUNC g_AesCbc_Encode; + extern AES_CODE_FUNC g_AesCbc_Decode; +#ifndef _SFX +extern AES_CODE_FUNC g_AesCbc_Encode; extern AES_CODE_FUNC g_AesCtr_Code; +#define k_Aes_SupportedFunctions_HW (1 << 2) +#define k_Aes_SupportedFunctions_HW_256 (1 << 3) +extern UInt32 g_Aes_SupportedFunctions_Flags; +#endif + + +#define DECLARE__AES_CODE_FUNC(funcName) \ + void MY_FAST_CALL funcName(UInt32 *ivAes, Byte *data, size_t numBlocks); + +DECLARE__AES_CODE_FUNC (AesCbc_Encode) +DECLARE__AES_CODE_FUNC (AesCbc_Decode) +DECLARE__AES_CODE_FUNC (AesCtr_Code) + +DECLARE__AES_CODE_FUNC (AesCbc_Encode_HW) +DECLARE__AES_CODE_FUNC (AesCbc_Decode_HW) +DECLARE__AES_CODE_FUNC (AesCtr_Code_HW) + +DECLARE__AES_CODE_FUNC (AesCbc_Decode_HW_256) +DECLARE__AES_CODE_FUNC (AesCtr_Code_HW_256) EXTERN_C_END diff --git a/3rdparty/lzma/C/AesOpt.c b/3rdparty/lzma/C/AesOpt.c index 8234f0d1465..8be8ff69d1e 100644 --- a/3rdparty/lzma/C/AesOpt.c +++ b/3rdparty/lzma/C/AesOpt.c @@ -1,184 +1,776 @@ -/* AesOpt.c -- Intel's AES -2013-11-12 : Igor Pavlov : Public domain */ +/* AesOpt.c -- AES optimized code for x86 AES hardware instructions +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if _MSC_VER >= 1500 && !defined(__clang__) -#define USE_INTEL_AES + + #if defined(__clang__) + #if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 8) + #define USE_INTEL_AES + #define ATTRIB_AES __attribute__((__target__("aes"))) + #if (__clang_major__ >= 8) + #define USE_INTEL_VAES + #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2"))) + #endif + #endif + #elif defined(__GNUC__) + #if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) + #define USE_INTEL_AES + #ifndef __AES__ + #define ATTRIB_AES __attribute__((__target__("aes"))) + #endif + #if (__GNUC__ >= 8) + #define USE_INTEL_VAES + #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2"))) + #endif + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1110) + #define USE_INTEL_AES + #if (__INTEL_COMPILER >= 1900) + #define USE_INTEL_VAES + #endif + #endif + #elif defined(_MSC_VER) + #if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) + #define USE_INTEL_AES + #if (_MSC_VER >= 1910) + #define USE_INTEL_VAES + #endif + #endif + #endif + +#ifndef ATTRIB_AES + #define ATTRIB_AES #endif +#ifndef ATTRIB_VAES + #define ATTRIB_VAES #endif + #ifdef USE_INTEL_AES #include -void MY_FAST_CALL AesCbc_Encode_Intel(__m128i *p, __m128i *data, size_t numBlocks) +#ifndef USE_INTEL_VAES +#define AES_TYPE_keys __m128i +#define AES_TYPE_data __m128i +#endif + +#define AES_FUNC_START(name) \ + void MY_FAST_CALL name(__m128i *p, __m128i *data, size_t numBlocks) + +#define AES_FUNC_START2(name) \ +AES_FUNC_START (name); \ +ATTRIB_AES \ +AES_FUNC_START (name) + +#define MM_OP(op, dest, src) dest = op(dest, src); +#define MM_OP_m(op, src) MM_OP(op, m, src); + +#define MM_XOR( dest, src) MM_OP(_mm_xor_si128, dest, src); +#define AVX_XOR(dest, src) MM_OP(_mm256_xor_si256, dest, src); + + +AES_FUNC_START2 (AesCbc_Encode_HW) { __m128i m = *p; + const __m128i k0 = p[2]; + const __m128i k1 = p[3]; + const UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; for (; numBlocks != 0; numBlocks--, data++) { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p + 3; - m = _mm_xor_si128(m, *data); - m = _mm_xor_si128(m, p[2]); + UInt32 r = numRounds2; + const __m128i *w = p + 4; + __m128i temp = *data; + MM_XOR (temp, k0); + MM_XOR (m, temp); + MM_OP_m (_mm_aesenc_si128, k1); do { - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenc_si128(m, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenc_si128, w[1]); w += 2; } - while (--numRounds2 != 0); - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenclast_si128(m, w[1]); + while (--r); + MM_OP_m (_mm_aesenclast_si128, w[0]); *data = m; } *p = m; } -#define NUM_WAYS 3 -#define AES_OP_W(op, n) { \ - const __m128i t = w[n]; \ - m0 = op(m0, t); \ - m1 = op(m1, t); \ - m2 = op(m2, t); \ - } +#define WOP_1(op) +#define WOP_2(op) WOP_1 (op) op (m1, 1); +#define WOP_3(op) WOP_2 (op) op (m2, 2); +#define WOP_4(op) WOP_3 (op) op (m3, 3); +#ifdef MY_CPU_AMD64 +#define WOP_5(op) WOP_4 (op) op (m4, 4); +#define WOP_6(op) WOP_5 (op) op (m5, 5); +#define WOP_7(op) WOP_6 (op) op (m6, 6); +#define WOP_8(op) WOP_7 (op) op (m7, 7); +#endif +/* +#define WOP_9(op) WOP_8 (op) op (m8, 8); +#define WOP_10(op) WOP_9 (op) op (m9, 9); +#define WOP_11(op) WOP_10(op) op (m10, 10); +#define WOP_12(op) WOP_11(op) op (m11, 11); +#define WOP_13(op) WOP_12(op) op (m12, 12); +#define WOP_14(op) WOP_13(op) op (m13, 13); +*/ + +#ifdef MY_CPU_AMD64 + #define NUM_WAYS 8 + #define WOP_M1 WOP_8 +#else + #define NUM_WAYS 4 + #define WOP_M1 WOP_4 +#endif + +#define WOP(op) op (m0, 0); WOP_M1(op) + + +#define DECLARE_VAR(reg, ii) __m128i reg +#define LOAD_data( reg, ii) reg = data[ii]; +#define STORE_data( reg, ii) data[ii] = reg; +#if (NUM_WAYS > 1) +#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]); +#endif + +#define AVX__DECLARE_VAR(reg, ii) __m256i reg +#define AVX__LOAD_data( reg, ii) reg = ((const __m256i *)(const void *)data)[ii]; +#define AVX__STORE_data( reg, ii) ((__m256i *)(void *)data)[ii] = reg; +#define AVX__XOR_data_M1(reg, ii) AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii])); + +#define MM_OP_key(op, reg) MM_OP(op, reg, key); + +#define AES_DEC( reg, ii) MM_OP_key (_mm_aesdec_si128, reg) +#define AES_DEC_LAST( reg, ii) MM_OP_key (_mm_aesdeclast_si128, reg) +#define AES_ENC( reg, ii) MM_OP_key (_mm_aesenc_si128, reg) +#define AES_ENC_LAST( reg, ii) MM_OP_key (_mm_aesenclast_si128, reg) +#define AES_XOR( reg, ii) MM_OP_key (_mm_xor_si128, reg) + -#define AES_DEC(n) AES_OP_W(_mm_aesdec_si128, n) -#define AES_DEC_LAST(n) AES_OP_W(_mm_aesdeclast_si128, n) -#define AES_ENC(n) AES_OP_W(_mm_aesenc_si128, n) -#define AES_ENC_LAST(n) AES_OP_W(_mm_aesenclast_si128, n) +#define AVX__AES_DEC( reg, ii) MM_OP_key (_mm256_aesdec_epi128, reg) +#define AVX__AES_DEC_LAST( reg, ii) MM_OP_key (_mm256_aesdeclast_epi128, reg) +#define AVX__AES_ENC( reg, ii) MM_OP_key (_mm256_aesenc_epi128, reg) +#define AVX__AES_ENC_LAST( reg, ii) MM_OP_key (_mm256_aesenclast_epi128, reg) +#define AVX__AES_XOR( reg, ii) MM_OP_key (_mm256_xor_si256, reg) -void MY_FAST_CALL AesCbc_Decode_Intel(__m128i *p, __m128i *data, size_t numBlocks) +#define CTR_START(reg, ii) MM_OP (_mm_add_epi64, ctr, one); reg = ctr; +#define CTR_END( reg, ii) MM_XOR (data[ii], reg); + +#define AVX__CTR_START(reg, ii) MM_OP (_mm256_add_epi64, ctr2, two); reg = _mm256_xor_si256(ctr2, key); +#define AVX__CTR_END( reg, ii) AVX_XOR (((__m256i *)(void *)data)[ii], reg); + +#define WOP_KEY(op, n) { \ + const __m128i key = w[n]; \ + WOP(op); } + +#define AVX__WOP_KEY(op, n) { \ + const __m256i key = w[n]; \ + WOP(op); } + + +#define WIDE_LOOP_START \ + dataEnd = data + numBlocks; \ + if (numBlocks >= NUM_WAYS) \ + { dataEnd -= NUM_WAYS; do { \ + + +#define WIDE_LOOP_END \ + data += NUM_WAYS; \ + } while (data <= dataEnd); \ + dataEnd += NUM_WAYS; } \ + + +#define SINGLE_LOOP \ + for (; data < dataEnd; data++) + + +#define NUM_AES_KEYS_MAX 15 + +#define WIDE_LOOP_START_AVX(OP) \ + dataEnd = data + numBlocks; \ + if (numBlocks >= NUM_WAYS * 2) \ + { __m256i keys[NUM_AES_KEYS_MAX]; \ + UInt32 ii; \ + OP \ + for (ii = 0; ii < numRounds; ii++) \ + keys[ii] = _mm256_broadcastsi128_si256(p[ii]); \ + dataEnd -= NUM_WAYS * 2; do { \ + + +#define WIDE_LOOP_END_AVX(OP) \ + data += NUM_WAYS * 2; \ + } while (data <= dataEnd); \ + dataEnd += NUM_WAYS * 2; \ + OP \ + _mm256_zeroupper(); \ + } \ + +/* MSVC for x86: If we don't call _mm256_zeroupper(), and -arch:IA32 is not specified, + MSVC still can insert vzeroupper instruction. */ + + +AES_FUNC_START2 (AesCbc_Decode_HW) { __m128i iv = *p; - for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) + const __m128i *wStart = p + *(const UInt32 *)(p + 1) * 2 + 2 - 1; + const __m128i *dataEnd; + p += 2; + + WIDE_LOOP_START { - UInt32 numRounds2 = *(const UInt32 *)(p + 1); - const __m128i *w = p + numRounds2 * 2; - __m128i m0, m1, m2; + const __m128i *w = wStart; + + WOP (DECLARE_VAR) + WOP (LOAD_data); + WOP_KEY (AES_XOR, 1) + + do { - const __m128i t = w[2]; - m0 = _mm_xor_si128(t, data[0]); - m1 = _mm_xor_si128(t, data[1]); - m2 = _mm_xor_si128(t, data[2]); + WOP_KEY (AES_DEC, 0) + w--; } - numRounds2--; + while (w != p); + WOP_KEY (AES_DEC_LAST, 0) + + MM_XOR (m0, iv); + WOP_M1 (XOR_data_M1) + iv = data[NUM_WAYS - 1]; + WOP (STORE_data); + } + WIDE_LOOP_END + + SINGLE_LOOP + { + const __m128i *w = wStart - 1; + __m128i m = _mm_xor_si128 (w[2], *data); do { - AES_DEC(1) - AES_DEC(0) + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdec_si128, w[0]); w -= 2; } - while (--numRounds2 != 0); - AES_DEC(1) - AES_DEC_LAST(0) + while (w != p); + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdeclast_si128, w[0]); + MM_XOR (m, iv); + iv = *data; + *data = m; + } + + p[-2] = iv; +} + + +AES_FUNC_START2 (AesCtr_Code_HW) +{ + __m128i ctr = *p; + UInt32 numRoundsMinus2 = *(const UInt32 *)(p + 1) * 2 - 1; + const __m128i *dataEnd; + __m128i one = _mm_cvtsi32_si128(1); + + p += 2; + + WIDE_LOOP_START + { + const __m128i *w = p; + UInt32 r = numRoundsMinus2; + WOP (DECLARE_VAR) + WOP (CTR_START); + WOP_KEY (AES_XOR, 0) + w += 1; + do { - __m128i t; - t = _mm_xor_si128(m0, iv); iv = data[0]; data[0] = t; - t = _mm_xor_si128(m1, iv); iv = data[1]; data[1] = t; - t = _mm_xor_si128(m2, iv); iv = data[2]; data[2] = t; + WOP_KEY (AES_ENC, 0) + w += 1; } + while (--r); + WOP_KEY (AES_ENC_LAST, 0) + + WOP (CTR_END); } - for (; numBlocks != 0; numBlocks--, data++) + WIDE_LOOP_END + + SINGLE_LOOP + { + UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1; + const __m128i *w = p; + __m128i m; + MM_OP (_mm_add_epi64, ctr, one); + m = _mm_xor_si128 (ctr, p[0]); + w += 1; + do + { + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenc_si128, w[1]); + w += 2; + } + while (--numRounds2); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenclast_si128, w[1]); + MM_XOR (*data, m); + } + + p[-2] = ctr; +} + + + +#ifdef USE_INTEL_VAES + +#if defined(__clang__) && defined(_MSC_VER) +#define __SSE4_2__ +#define __AES__ +#define __AVX__ +#define __AVX2__ +#define __VAES__ +#define __AVX512F__ +#define __AVX512VL__ +#endif + +#include + +#define VAES_FUNC_START2(name) \ +AES_FUNC_START (name); \ +ATTRIB_VAES \ +AES_FUNC_START (name) + +VAES_FUNC_START2 (AesCbc_Decode_HW_256) +{ + __m128i iv = *p; + const __m128i *dataEnd; + UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1; + p += 2; + + WIDE_LOOP_START_AVX(;) { - UInt32 numRounds2 = *(const UInt32 *)(p + 1); - const __m128i *w = p + numRounds2 * 2; - __m128i m = _mm_xor_si128(w[2], *data); - numRounds2--; + const __m256i *w = keys + numRounds - 2; + + WOP (AVX__DECLARE_VAR) + WOP (AVX__LOAD_data); + AVX__WOP_KEY (AVX__AES_XOR, 1) + do { - m = _mm_aesdec_si128(m, w[1]); - m = _mm_aesdec_si128(m, w[0]); + AVX__WOP_KEY (AVX__AES_DEC, 0) + w--; + } + while (w != keys); + AVX__WOP_KEY (AVX__AES_DEC_LAST, 0) + + AVX_XOR (m0, _mm256_setr_m128i(iv, data[0])); + WOP_M1 (AVX__XOR_data_M1) + iv = data[NUM_WAYS * 2 - 1]; + WOP (AVX__STORE_data); + } + WIDE_LOOP_END_AVX(;) + + SINGLE_LOOP + { + const __m128i *w = p + *(const UInt32 *)(p + 1 - 2) * 2 + 1 - 3; + __m128i m = _mm_xor_si128 (w[2], *data); + do + { + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdec_si128, w[0]); w -= 2; } - while (--numRounds2 != 0); - m = _mm_aesdec_si128(m, w[1]); - m = _mm_aesdeclast_si128(m, w[0]); + while (w != p); + MM_OP_m (_mm_aesdec_si128, w[1]); + MM_OP_m (_mm_aesdeclast_si128, w[0]); - m = _mm_xor_si128(m, iv); + MM_XOR (m, iv); iv = *data; *data = m; } - *p = iv; + + p[-2] = iv; } -void MY_FAST_CALL AesCtr_Code_Intel(__m128i *p, __m128i *data, size_t numBlocks) + +/* +SSE2: _mm_cvtsi32_si128 : movd +AVX: _mm256_setr_m128i : vinsertf128 +AVX2: _mm256_add_epi64 : vpaddq ymm, ymm, ymm + _mm256_extracti128_si256 : vextracti128 + _mm256_broadcastsi128_si256 : vbroadcasti128 +*/ + +#define AVX__CTR_LOOP_START \ + ctr2 = _mm256_setr_m128i(_mm_sub_epi64(ctr, one), ctr); \ + two = _mm256_setr_m128i(one, one); \ + two = _mm256_add_epi64(two, two); \ + +// two = _mm256_setr_epi64x(2, 0, 2, 0); + +#define AVX__CTR_LOOP_ENC \ + ctr = _mm256_extracti128_si256 (ctr2, 1); \ + +VAES_FUNC_START2 (AesCtr_Code_HW_256) { __m128i ctr = *p; - __m128i one; - one.m128i_u64[0] = 1; - one.m128i_u64[1] = 0; - for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) + UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1; + const __m128i *dataEnd; + __m128i one = _mm_cvtsi32_si128(1); + __m256i ctr2, two; + p += 2; + + WIDE_LOOP_START_AVX (AVX__CTR_LOOP_START) { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p; - __m128i m0, m1, m2; - { - const __m128i t = w[2]; - ctr = _mm_add_epi64(ctr, one); m0 = _mm_xor_si128(ctr, t); - ctr = _mm_add_epi64(ctr, one); m1 = _mm_xor_si128(ctr, t); - ctr = _mm_add_epi64(ctr, one); m2 = _mm_xor_si128(ctr, t); - } - w += 3; + const __m256i *w = keys; + UInt32 r = numRounds - 2; + WOP (AVX__DECLARE_VAR) + AVX__WOP_KEY (AVX__CTR_START, 0); + + w += 1; do { - AES_ENC(0) - AES_ENC(1) - w += 2; + AVX__WOP_KEY (AVX__AES_ENC, 0) + w += 1; } - while (--numRounds2 != 0); - AES_ENC(0) - AES_ENC_LAST(1) - data[0] = _mm_xor_si128(data[0], m0); - data[1] = _mm_xor_si128(data[1], m1); - data[2] = _mm_xor_si128(data[2], m2); + while (--r); + AVX__WOP_KEY (AVX__AES_ENC_LAST, 0) + + WOP (AVX__CTR_END); } - for (; numBlocks != 0; numBlocks--, data++) + WIDE_LOOP_END_AVX (AVX__CTR_LOOP_ENC) + + SINGLE_LOOP { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; + UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1; const __m128i *w = p; __m128i m; - ctr = _mm_add_epi64(ctr, one); - m = _mm_xor_si128(ctr, p[2]); - w += 3; + MM_OP (_mm_add_epi64, ctr, one); + m = _mm_xor_si128 (ctr, p[0]); + w += 1; do { - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenc_si128(m, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenc_si128, w[1]); w += 2; } - while (--numRounds2 != 0); - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenclast_si128(m, w[1]); - *data = _mm_xor_si128(*data, m); + while (--numRounds2); + MM_OP_m (_mm_aesenc_si128, w[0]); + MM_OP_m (_mm_aesenclast_si128, w[1]); + MM_XOR (*data, m); } - *p = ctr; + + p[-2] = ctr; } +#endif // USE_INTEL_VAES + +#else // USE_INTEL_AES + +/* no USE_INTEL_AES */ + +#pragma message("AES HW_SW stub was used") + +#define AES_TYPE_keys UInt32 +#define AES_TYPE_data Byte + +#define AES_FUNC_START(name) \ + void MY_FAST_CALL name(UInt32 *p, Byte *data, size_t numBlocks) \ + +#define AES_COMPAT_STUB(name) \ + AES_FUNC_START(name); \ + AES_FUNC_START(name ## _HW) \ + { name(p, data, numBlocks); } + +AES_COMPAT_STUB (AesCbc_Encode) +AES_COMPAT_STUB (AesCbc_Decode) +AES_COMPAT_STUB (AesCtr_Code) + +#endif // USE_INTEL_AES + + +#ifndef USE_INTEL_VAES + +#pragma message("VAES HW_SW stub was used") + +#define VAES_COMPAT_STUB(name) \ + void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \ + void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks) \ + { name((AES_TYPE_keys *)(void *)p, (AES_TYPE_data *)(void *)data, numBlocks); } + +VAES_COMPAT_STUB (AesCbc_Decode_HW) +VAES_COMPAT_STUB (AesCtr_Code_HW) + +#endif // ! USE_INTEL_VAES + + +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_AES + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_AES + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_AES + #endif + #endif + +#ifdef USE_HW_AES + +// #pragma message("=== AES HW === ") + +#if defined(__clang__) || defined(__GNUC__) + #ifdef MY_CPU_ARM64 + #define ATTRIB_AES __attribute__((__target__("+crypto"))) + #else + #define ATTRIB_AES __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif +#else + // _MSC_VER + // for arm32 + #define _ARM_USE_NEW_NEON_INTRINSICS +#endif + +#ifndef ATTRIB_AES + #define ATTRIB_AES +#endif + +#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#include #else +#include +#endif + +typedef uint8x16_t v128; + +#define AES_FUNC_START(name) \ + void MY_FAST_CALL name(v128 *p, v128 *data, size_t numBlocks) + +#define AES_FUNC_START2(name) \ +AES_FUNC_START (name); \ +ATTRIB_AES \ +AES_FUNC_START (name) + +#define MM_OP(op, dest, src) dest = op(dest, src); +#define MM_OP_m(op, src) MM_OP(op, m, src); +#define MM_OP1_m(op) m = op(m); -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); +#define MM_XOR( dest, src) MM_OP(veorq_u8, dest, src); +#define MM_XOR_m( src) MM_XOR(m, src); -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *p, Byte *data, size_t numBlocks) +#define AES_E_m(k) MM_OP_m (vaeseq_u8, k); +#define AES_E_MC_m(k) AES_E_m (k); MM_OP1_m(vaesmcq_u8); + + +AES_FUNC_START2 (AesCbc_Encode_HW) { - AesCbc_Encode(p, data, numBlocks); + v128 m = *p; + const v128 k0 = p[2]; + const v128 k1 = p[3]; + const v128 k2 = p[4]; + const v128 k3 = p[5]; + const v128 k4 = p[6]; + const v128 k5 = p[7]; + const v128 k6 = p[8]; + const v128 k7 = p[9]; + const v128 k8 = p[10]; + const v128 k9 = p[11]; + const UInt32 numRounds2 = *(const UInt32 *)(p + 1); + const v128 *w = p + ((size_t)numRounds2 * 2); + const v128 k_z1 = w[1]; + const v128 k_z0 = w[2]; + for (; numBlocks != 0; numBlocks--, data++) + { + MM_XOR_m (*data); + AES_E_MC_m (k0) + AES_E_MC_m (k1) + AES_E_MC_m (k2) + AES_E_MC_m (k3) + AES_E_MC_m (k4) + AES_E_MC_m (k5) + AES_E_MC_m (k6) + AES_E_MC_m (k7) + AES_E_MC_m (k8) + if (numRounds2 >= 6) + { + AES_E_MC_m (k9) + AES_E_MC_m (p[12]) + if (numRounds2 != 6) + { + AES_E_MC_m (p[13]) + AES_E_MC_m (p[14]) + } + } + AES_E_m (k_z1); + MM_XOR_m (k_z0); + *data = m; + } + *p = m; } -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *p, Byte *data, size_t numBlocks) + +#define WOP_1(op) +#define WOP_2(op) WOP_1 (op) op (m1, 1); +#define WOP_3(op) WOP_2 (op) op (m2, 2); +#define WOP_4(op) WOP_3 (op) op (m3, 3); +#define WOP_5(op) WOP_4 (op) op (m4, 4); +#define WOP_6(op) WOP_5 (op) op (m5, 5); +#define WOP_7(op) WOP_6 (op) op (m6, 6); +#define WOP_8(op) WOP_7 (op) op (m7, 7); + + #define NUM_WAYS 8 + #define WOP_M1 WOP_8 + +#define WOP(op) op (m0, 0); WOP_M1(op) + +#define DECLARE_VAR(reg, ii) v128 reg +#define LOAD_data( reg, ii) reg = data[ii]; +#define STORE_data( reg, ii) data[ii] = reg; +#if (NUM_WAYS > 1) +#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]); +#endif + +#define MM_OP_key(op, reg) MM_OP (op, reg, key); + +#define AES_D_m(k) MM_OP_m (vaesdq_u8, k); +#define AES_D_IMC_m(k) AES_D_m (k); MM_OP1_m (vaesimcq_u8); + +#define AES_XOR( reg, ii) MM_OP_key (veorq_u8, reg) +#define AES_D( reg, ii) MM_OP_key (vaesdq_u8, reg) +#define AES_E( reg, ii) MM_OP_key (vaeseq_u8, reg) + +#define AES_D_IMC( reg, ii) AES_D (reg, ii); reg = vaesimcq_u8(reg) +#define AES_E_MC( reg, ii) AES_E (reg, ii); reg = vaesmcq_u8(reg) + +#define CTR_START(reg, ii) MM_OP (vaddq_u64, ctr, one); reg = vreinterpretq_u8_u64(ctr); +#define CTR_END( reg, ii) MM_XOR (data[ii], reg); + +#define WOP_KEY(op, n) { \ + const v128 key = w[n]; \ + WOP(op); } + +#define WIDE_LOOP_START \ + dataEnd = data + numBlocks; \ + if (numBlocks >= NUM_WAYS) \ + { dataEnd -= NUM_WAYS; do { \ + +#define WIDE_LOOP_END \ + data += NUM_WAYS; \ + } while (data <= dataEnd); \ + dataEnd += NUM_WAYS; } \ + +#define SINGLE_LOOP \ + for (; data < dataEnd; data++) + + +AES_FUNC_START2 (AesCbc_Decode_HW) { - AesCbc_Decode(p, data, numBlocks); + v128 iv = *p; + const v128 *wStart = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; + const v128 *dataEnd; + p += 2; + + WIDE_LOOP_START + { + const v128 *w = wStart; + WOP (DECLARE_VAR) + WOP (LOAD_data); + WOP_KEY (AES_D_IMC, 2) + do + { + WOP_KEY (AES_D_IMC, 1) + WOP_KEY (AES_D_IMC, 0) + w -= 2; + } + while (w != p); + WOP_KEY (AES_D, 1) + WOP_KEY (AES_XOR, 0) + MM_XOR (m0, iv); + WOP_M1 (XOR_data_M1) + iv = data[NUM_WAYS - 1]; + WOP (STORE_data); + } + WIDE_LOOP_END + + SINGLE_LOOP + { + const v128 *w = wStart; + v128 m = *data; + AES_D_IMC_m (w[2]) + do + { + AES_D_IMC_m (w[1]); + AES_D_IMC_m (w[0]); + w -= 2; + } + while (w != p); + AES_D_m (w[1]); + MM_XOR_m (w[0]); + MM_XOR_m (iv); + iv = *data; + *data = m; + } + + p[-2] = iv; } -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *p, Byte *data, size_t numBlocks) + +AES_FUNC_START2 (AesCtr_Code_HW) { - AesCtr_Code(p, data, numBlocks); + uint64x2_t ctr = vreinterpretq_u64_u8(*p); + const v128 *wEnd = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; + const v128 *dataEnd; + uint64x2_t one = vdupq_n_u64(0); + one = vsetq_lane_u64(1, one, 0); + p += 2; + + WIDE_LOOP_START + { + const v128 *w = p; + WOP (DECLARE_VAR) + WOP (CTR_START); + do + { + WOP_KEY (AES_E_MC, 0) + WOP_KEY (AES_E_MC, 1) + w += 2; + } + while (w != wEnd); + WOP_KEY (AES_E_MC, 0) + WOP_KEY (AES_E, 1) + WOP_KEY (AES_XOR, 2) + WOP (CTR_END); + } + WIDE_LOOP_END + + SINGLE_LOOP + { + const v128 *w = p; + v128 m; + CTR_START (m, 0); + do + { + AES_E_MC_m (w[0]); + AES_E_MC_m (w[1]); + w += 2; + } + while (w != wEnd); + AES_E_MC_m (w[0]); + AES_E_m (w[1]); + MM_XOR_m (w[2]); + CTR_END (m, 0); + } + + p[-2] = vreinterpretq_u8_u64(ctr); } -#endif +#endif // USE_HW_AES + +#endif // MY_CPU_ARM_OR_ARM64 diff --git a/3rdparty/lzma/C/Alloc.c b/3rdparty/lzma/C/Alloc.c index 3f6ac41918e..d1af76c5c30 100644 --- a/3rdparty/lzma/C/Alloc.c +++ b/3rdparty/lzma/C/Alloc.c @@ -1,10 +1,12 @@ /* Alloc.c -- Memory allocation functions -2015-02-21 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + #ifdef _WIN32 -#include +#include #endif #include @@ -14,20 +16,127 @@ /* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ #ifdef _SZ_ALLOC_DEBUG + #include int g_allocCount = 0; int g_allocCountMid = 0; int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintAddr(p) + #endif + + void *MyAlloc(size_t size) { if (size == 0) - return 0; + return NULL; + PRINT_ALLOC("Alloc ", g_allocCount, size, NULL); #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + // PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else @@ -37,10 +146,8 @@ void *MyAlloc(size_t size) void MyFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif + PRINT_FREE("Free ", g_allocCount, address); + free(address); } @@ -49,40 +156,44 @@ void MyFree(void *address) void *MidAlloc(size_t size) { if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + return NULL; + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif - if (address == 0) + PRINT_FREE("Free-Mid", g_allocCountMid, address); + + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } -#ifndef MEM_LARGE_PAGES -#undef _7ZIP_LARGE_PAGES +#ifdef _7ZIP_LARGE_PAGES + +#ifdef MEM_LARGE_PAGES + #define MY__MEM_LARGE_PAGES MEM_LARGE_PAGES +#else + #define MY__MEM_LARGE_PAGES 0x20000000 #endif -#ifdef _7ZIP_LARGE_PAGES +extern +SIZE_T g_LargePageSize; SIZE_T g_LargePageSize = 0; -typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); -#endif +typedef SIZE_T (WINAPI *GetLargePageMinimumP)(VOID); + +#endif // _7ZIP_LARGE_PAGES void SetLargePageSize() { #ifdef _7ZIP_LARGE_PAGES - SIZE_T size = 0; + SIZE_T size; GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (largePageMinimum == 0) + if (!largePageMinimum) return; size = largePageMinimum(); if (size == 0 || (size & (size - 1)) != 0) @@ -95,31 +206,36 @@ void SetLargePageSize() void *BigAlloc(size_t size) { if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); - #endif + return NULL; + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); #ifdef _7ZIP_LARGE_PAGES - if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18)) { - void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)), - MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); - if (res != 0) - return res; + SIZE_T ps = g_LargePageSize; + if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) + { + size_t size2; + ps--; + size2 = (size + ps) & ~ps; + if (size2 >= size) + { + void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MY__MEM_LARGE_PAGES, PAGE_READWRITE); + if (res) + return res; + } + } } #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void BigFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif + PRINT_FREE("Free-Big", g_allocCountBig, address); - if (address == 0) + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } @@ -127,10 +243,221 @@ void BigFree(void *address) #endif -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } -ISzAlloc g_Alloc = { SzAlloc, SzFree }; +static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } +static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +#ifdef _WIN32 +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } +static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; +#endif + +/* + uintptr_t : C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + + +#if !defined(_WIN32) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) + #define USE_posix_memalign +#endif + +#ifndef USE_posix_memalign +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} -static void *SzBigAlloc(void *p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } -static void SzBigFree(void *p, void *address) { UNUSED_VAR(p); BigFree(address); } -ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + + +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +} diff --git a/3rdparty/lzma/C/Alloc.h b/3rdparty/lzma/C/Alloc.h index bac0d87e2a0..3be2041eb75 100644 --- a/3rdparty/lzma/C/Alloc.h +++ b/3rdparty/lzma/C/Alloc.h @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2015-02-21 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -13,7 +13,7 @@ void MyFree(void *address); #ifdef _WIN32 -void SetLargePageSize(); +void SetLargePageSize(void); void *MidAlloc(size_t size); void MidFree(void *address); @@ -29,8 +29,29 @@ void BigFree(void *address); #endif -extern ISzAlloc g_Alloc; -extern ISzAlloc g_BigAlloc; +extern const ISzAlloc g_Alloc; + +#ifdef _WIN32 +extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +#else +#define g_BigAlloc g_AlignedAlloc +#define g_MidAlloc g_AlignedAlloc +#endif + +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + EXTERN_C_END diff --git a/3rdparty/lzma/C/Bcj2.c b/3rdparty/lzma/C/Bcj2.c index 3c88e44fc1f..c7b956708f6 100644 --- a/3rdparty/lzma/C/Bcj2.c +++ b/3rdparty/lzma/C/Bcj2.c @@ -1,5 +1,5 @@ /* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2015-08-01 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -61,7 +61,8 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) Byte *dest = p->dest; if (dest == p->destLim) return SZ_OK; - *dest = p->temp[p->state++ - BCJ2_DEC_STATE_ORIG_0]; + *dest = p->temp[(size_t)p->state - BCJ2_DEC_STATE_ORIG_0]; + p->state++; p->dest = dest + 1; } } @@ -122,7 +123,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) const Byte *src = p->bufs[BCJ2_STREAM_MAIN]; const Byte *srcLim; Byte *dest; - SizeT num = p->lims[BCJ2_STREAM_MAIN] - src; + SizeT num = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - src); if (num == 0) { @@ -133,7 +134,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) dest = p->dest; if (num > (SizeT)(p->destLim - dest)) { - num = p->destLim - dest; + num = (SizeT)(p->destLim - dest); if (num == 0) { p->state = BCJ2_DEC_STATE_ORIG; @@ -167,7 +168,7 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) break; } - num = src - p->bufs[BCJ2_STREAM_MAIN]; + num = (SizeT)(src - p->bufs[BCJ2_STREAM_MAIN]); if (src == srcLim) { @@ -227,14 +228,14 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) p->ip += 4; val -= p->ip; dest = p->dest; - rem = p->destLim - dest; + rem = (SizeT)(p->destLim - dest); if (rem < 4) { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; + p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; + p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; + p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; + p->temp[3] = (Byte)val; p->dest = dest + rem; p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; break; diff --git a/3rdparty/lzma/C/Bcj2Enc.c b/3rdparty/lzma/C/Bcj2Enc.c index 0ea36462cbe..682362a15d5 100644 --- a/3rdparty/lzma/C/Bcj2Enc.c +++ b/3rdparty/lzma/C/Bcj2Enc.c @@ -1,5 +1,5 @@ /* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2014-11-10 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,7 +12,6 @@ #define PRF(x) #endif -#include #include #include "Bcj2.h" @@ -53,7 +52,7 @@ void Bcj2Enc_Init(CBcj2Enc *p) p->probs[i] = kBitModelTotal >> 1; } -static Bool MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) { if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) { @@ -105,7 +104,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) const Byte *src = p->src; const Byte *srcLim; Byte *dest; - SizeT num = p->srcLim - src; + SizeT num = (SizeT)(p->srcLim - src); if (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) { @@ -119,7 +118,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) dest = p->bufs[BCJ2_STREAM_MAIN]; if (num > (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest)) { - num = p->lims[BCJ2_STREAM_MAIN] - dest; + num = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest); if (num == 0) { p->state = BCJ2_STREAM_MAIN; @@ -153,7 +152,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) break; } - num = src - p->src; + num = (SizeT)(src - p->src); if (src == srcLim) { @@ -166,7 +165,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) { Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - Bool needConvert; + BoolInt needConvert; p->bufs[BCJ2_STREAM_MAIN] = dest + 1; p->ip += (UInt32)num + 1; @@ -254,7 +253,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p) { const Byte *src = p->src; const Byte *srcLim = p->srcLim; - unsigned finishMode = p->finishMode; + EBcj2Enc_FinishMode finishMode = p->finishMode; p->src = p->temp; p->srcLim = p->temp + p->tempPos; @@ -271,7 +270,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p) unsigned i; p->tempPos = tempPos; for (i = 0; i < tempPos; i++) - p->temp[i] = p->temp[i + num]; + p->temp[i] = p->temp[(size_t)i + num]; p->src = src; p->srcLim = srcLim; diff --git a/3rdparty/lzma/C/Bra.c b/3rdparty/lzma/C/Bra.c index cdb94569202..3b854d9cad7 100644 --- a/3rdparty/lzma/C/Bra.c +++ b/3rdparty/lzma/C/Bra.c @@ -1,135 +1,230 @@ /* Bra.c -- Converters for RISC code -2010-04-16 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "CpuArch.h" #include "Bra.h" SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 8; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip += 4; + p = data; + lim = data + size; + + if (encoding) + + for (;;) { - if (data[i + 3] == 0xEB) + for (;;) { - UInt32 dest; - UInt32 src = ((UInt32)data[i + 2] << 16) | ((UInt32)data[i + 1] << 8) | (data[i + 0]); - src <<= 2; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 2; - data[i + 2] = (Byte)(dest >> 16); - data[i + 1] = (Byte)(dest >> 8); - data[i + 0] = (Byte)dest; + if (p >= lim) + return (SizeT)(p - data); + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v += ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } + + for (;;) + { + for (;;) + { + if (p >= lim) + return (SizeT)(p - data); + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v -= ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); } } - return i; } + SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 4; - for (i = 0; i <= size; i += 2) + Byte *p; + const Byte *lim; + size &= ~(size_t)1; + p = data; + lim = data + size - 4; + + if (encoding) + + for (;;) { - if ((data[i + 1] & 0xF8) == 0xF0 && - (data[i + 3] & 0xF8) == 0xF8) + UInt32 b1; + for (;;) { - UInt32 dest; - UInt32 src = - (((UInt32)data[i + 1] & 0x7) << 19) | - ((UInt32)data[i + 0] << 11) | - (((UInt32)data[i + 3] & 0x7) << 8) | - (data[i + 2]); - - src <<= 1; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 1; + UInt32 b3; + if (p > lim) + return (SizeT)(p - data); + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v += cur; + } + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return (SizeT)(p - data); + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v -= cur; + } + + /* + SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); + SetUi16(p - 2, (UInt16)(v | 0xF800)); + */ - data[i + 1] = (Byte)(0xF0 | ((dest >> 19) & 0x7)); - data[i + 0] = (Byte)(dest >> 11); - data[i + 3] = (Byte)(0xF8 | ((dest >> 8) & 0x7)); - data[i + 2] = (Byte)dest; - i += 2; + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); } } - return i; } + SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) { - if ((data[i] >> 2) == 0x12 && (data[i + 3] & 3) == 1) + for (;;) { - UInt32 src = ((UInt32)(data[i + 0] & 3) << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3] & (~3)); - - UInt32 dest; + if (p >= lim) + return (SizeT)(p - data); + p += 4; + /* if ((v & 0xFC000003) == 0x48000001) */ + if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) + break; + } + { + UInt32 v = GetBe32(p - 4); if (encoding) - dest = ip + (UInt32)i + src; + v += ip + (UInt32)(p - data); else - dest = src - (ip + (UInt32)i); - data[i + 0] = (Byte)(0x48 | ((dest >> 24) & 0x3)); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] &= 0x3; - data[i + 3] |= dest; + v -= ip + (UInt32)(p - data); + v &= 0x03FFFFFF; + v |= 0x48000000; + SetBe32(p - 4, v); } } - return i; } + SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - UInt32 i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) { - if ((data[i] == 0x40 && (data[i + 1] & 0xC0) == 0x00) || - (data[i] == 0x7F && (data[i + 1] & 0xC0) == 0xC0)) + for (;;) { - UInt32 src = - ((UInt32)data[i + 0] << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3]); - UInt32 dest; - - src <<= 2; + if (p >= lim) + return (SizeT)(p - data); + /* + v = GetBe32(p); + p += 4; + m = v + ((UInt32)5 << 29); + m ^= (UInt32)7 << 29; + m += (UInt32)1 << 22; + if ((m & ((UInt32)0x1FF << 23)) == 0) + break; + */ + p += 4; + if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || + (p[-4] == 0x7F && (p[-3] >= 0xC0))) + break; + } + { + UInt32 v = GetBe32(p - 4); + v <<= 2; if (encoding) - dest = ip + i + src; + v += ip + (UInt32)(p - data); else - dest = src - (ip + i); - dest >>= 2; + v -= ip + (UInt32)(p - data); - dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFFFFFF) | (dest & 0x3FFFFF) | 0x40000000; - - data[i + 0] = (Byte)(dest >> 24); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] = (Byte)dest; + v &= 0x01FFFFFF; + v -= (UInt32)1 << 24; + v ^= 0xFF000000; + v >>= 2; + v |= 0x40000000; + SetBe32(p - 4, v); } } - return i; } diff --git a/3rdparty/lzma/C/Bra86.c b/3rdparty/lzma/C/Bra86.c index 6db15e7ecd3..10a0fbd161c 100644 --- a/3rdparty/lzma/C/Bra86.c +++ b/3rdparty/lzma/C/Bra86.c @@ -1,5 +1,5 @@ /* Bra86.c -- Converter for x86 code (BCJ) -2013-11-12 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -25,7 +25,7 @@ SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding break; { - SizeT d = (SizeT)(p - data - pos); + SizeT d = (SizeT)(p - data) - pos; pos = (SizeT)(p - data); if (p >= limit) { @@ -37,7 +37,7 @@ SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding else { mask >>= (unsigned)d; - if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(mask >> 1) + 1]))) + if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) { mask = (mask >> 1) | 4; pos++; diff --git a/3rdparty/lzma/C/BraIA64.c b/3rdparty/lzma/C/BraIA64.c index fa60356b288..d1dbc62c55b 100644 --- a/3rdparty/lzma/C/BraIA64.c +++ b/3rdparty/lzma/C/BraIA64.c @@ -1,69 +1,53 @@ /* BraIA64.c -- Converter for IA-64 code -2013-11-12 : Igor Pavlov : Public domain */ +2017-01-26 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "CpuArch.h" #include "Bra.h" -static const Byte kBranchTable[32] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 4, 4, 6, 6, 0, 0, 7, 7, - 4, 4, 0, 0, 4, 4, 0, 0 -}; - SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { SizeT i; if (size < 16) return 0; size -= 16; - for (i = 0; i <= size; i += 16) + i = 0; + do { - UInt32 instrTemplate = data[i] & 0x1F; - UInt32 mask = kBranchTable[instrTemplate]; - UInt32 bitPos = 5; - int slot; - for (slot = 0; slot < 3; slot++, bitPos += 41) + unsigned m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3; + if (m) { - UInt32 bytePos, bitRes; - UInt64 instruction, instNorm; - int j; - if (((mask >> slot) & 1) == 0) - continue; - bytePos = (bitPos >> 3); - bitRes = bitPos & 0x7; - instruction = 0; - for (j = 0; j < 6; j++) - instruction += (UInt64)data[i + j + bytePos] << (8 * j); - - instNorm = instruction >> bitRes; - if (((instNorm >> 37) & 0xF) == 0x5 && ((instNorm >> 9) & 0x7) == 0) + m++; + do { - UInt32 src = (UInt32)((instNorm >> 13) & 0xFFFFF); - UInt32 dest; - src |= ((UInt32)(instNorm >> 36) & 1) << 20; - - src <<= 4; - - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - - dest >>= 4; - - instNorm &= ~((UInt64)(0x8FFFFF) << 13); - instNorm |= ((UInt64)(dest & 0xFFFFF) << 13); - instNorm |= ((UInt64)(dest & 0x100000) << (36 - 20)); - - instruction &= (1 << bitRes) - 1; - instruction |= (instNorm << bitRes); - for (j = 0; j < 6; j++) - data[i + j + bytePos] = (Byte)(instruction >> (8 * j)); + Byte *p = data + (i + (size_t)m * 5 - 8); + if (((p[3] >> m) & 15) == 5 + && (((p[-1] | ((UInt32)p[0] << 8)) >> m) & 0x70) == 0) + { + unsigned raw = GetUi32(p); + unsigned v = raw >> m; + v = (v & 0xFFFFF) | ((v & (1 << 23)) >> 3); + + v <<= 4; + if (encoding) + v += ip + (UInt32)i; + else + v -= ip + (UInt32)i; + v >>= 4; + + v &= 0x1FFFFF; + v += 0x700000; + v &= 0x8FFFFF; + raw &= ~((UInt32)0x8FFFFF << m); + raw |= (v << m); + SetUi32(p, raw); + } } + while (++m <= 4); } + i += 16; } + while (i <= size); return i; } diff --git a/3rdparty/lzma/C/Compiler.h b/3rdparty/lzma/C/Compiler.h index 5bba7ee5618..a9816fa5ad0 100644 --- a/3rdparty/lzma/C/Compiler.h +++ b/3rdparty/lzma/C/Compiler.h @@ -1,9 +1,13 @@ /* Compiler.h -2015-08-02 : Igor Pavlov : Public domain */ +2021-01-05 : Igor Pavlov : Public domain */ #ifndef __7Z_COMPILER_H #define __7Z_COMPILER_H + #ifdef __clang__ + #pragma clang diagnostic ignored "-Wunused-private-field" + #endif + #ifdef _MSC_VER #ifdef UNDER_CE @@ -21,9 +25,16 @@ #pragma warning(disable : 4514) // unreferenced inline function has been removed #pragma warning(disable : 4702) // unreachable code #pragma warning(disable : 4710) // not inlined + #pragma warning(disable : 4714) // function marked as __forceinline not inlined #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information #endif + #ifdef __clang__ + #pragma clang diagnostic ignored "-Wdeprecated-declarations" + #pragma clang diagnostic ignored "-Wmicrosoft-exception-spec" + // #pragma clang diagnostic ignored "-Wreserved-id-macro" + #endif + #endif #define UNUSED_VAR(x) (void)x; diff --git a/3rdparty/lzma/C/CpuArch.c b/3rdparty/lzma/C/CpuArch.c index 554ffa4fc7b..fa9afe3970b 100644 --- a/3rdparty/lzma/C/CpuArch.c +++ b/3rdparty/lzma/C/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2016-02-25: Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -55,6 +55,47 @@ static UInt32 CheckFlag(UInt32 flag) #define CHECK_CPUID_IS_SUPPORTED #endif +#ifndef USE_ASM + #ifdef _MSC_VER + #if _MSC_VER >= 1600 + #define MY__cpuidex __cpuidex + #else + +/* + __cpuid (function == 4) requires subfunction number in ECX. + MSDN: The __cpuid intrinsic clears the ECX register before calling the cpuid instruction. + __cpuid() in new MSVC clears ECX. + __cpuid() in old MSVC (14.00) doesn't clear ECX + We still can use __cpuid for low (function) values that don't require ECX, + but __cpuid() in old MSVC will be incorrect for some function values: (function == 4). + So here we use the hack for old MSVC to send (subFunction) in ECX register to cpuid instruction, + where ECX value is first parameter for FAST_CALL / NO_INLINE function, + So the caller of MY__cpuidex_HACK() sets ECX as subFunction, and + old MSVC for __cpuid() doesn't change ECX and cpuid instruction gets (subFunction) value. + + DON'T remove MY_NO_INLINE and MY_FAST_CALL for MY__cpuidex_HACK() !!! +*/ + +static +MY_NO_INLINE +void MY_FAST_CALL MY__cpuidex_HACK(UInt32 subFunction, int *CPUInfo, UInt32 function) +{ + UNUSED_VAR(subFunction); + __cpuid(CPUInfo, function); +} + + #define MY__cpuidex(info, func, func2) MY__cpuidex_HACK(func2, info, func) + #pragma message("======== MY__cpuidex_HACK WAS USED ========") + #endif + #else + #define MY__cpuidex(info, func, func2) __cpuid(info, func) + #pragma message("======== (INCORRECT ?) cpuid WAS USED ========") + #endif +#endif + + + + void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) { #ifdef USE_ASM @@ -99,23 +140,25 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) #endif "=c" (*c) , "=d" (*d) - : "0" (function)) ; + : "0" (function), "c"(0) ) ; #endif #else int CPUInfo[4]; - __cpuid(CPUInfo, function); - *a = CPUInfo[0]; - *b = CPUInfo[1]; - *c = CPUInfo[2]; - *d = CPUInfo[3]; + + MY__cpuidex(CPUInfo, (int)function, 0); + + *a = (UInt32)CPUInfo[0]; + *b = (UInt32)CPUInfo[1]; + *c = (UInt32)CPUInfo[2]; + *d = (UInt32)CPUInfo[3]; #endif } -Bool x86cpuid_CheckAndRead(Cx86cpuid *p) +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) { CHECK_CPUID_IS_SUPPORTED MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); @@ -144,7 +187,7 @@ int x86cpuid_GetFirm(const Cx86cpuid *p) return -1; } -Bool CPU_Is_InOrder() +BoolInt CPU_Is_InOrder() { Cx86cpuid p; int firm; @@ -174,8 +217,8 @@ Bool CPU_Is_InOrder() } #if !defined(MY_CPU_AMD64) && defined(_WIN32) -#include -static Bool CPU_Sys_Is_SSE_Supported() +#include +static BoolInt CPU_Sys_Is_SSE_Supported() { OSVERSIONINFO vi; vi.dwOSVersionInfoSize = sizeof(vi); @@ -188,13 +231,248 @@ static Bool CPU_Sys_Is_SSE_Supported() #define CHECK_SYS_SSE_SUPPORT #endif -Bool CPU_Is_Aes_Supported() + +static UInt32 X86_CPUID_ECX_Get_Flags() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_CheckAndRead(&p)) + return 0; + return p.c; +} + +BoolInt CPU_IsSupported_AES() +{ + return (X86_CPUID_ECX_Get_Flags() >> 25) & 1; +} + +BoolInt CPU_IsSupported_SSSE3() +{ + return (X86_CPUID_ECX_Get_Flags() >> 9) & 1; +} + +BoolInt CPU_IsSupported_SSE41() +{ + return (X86_CPUID_ECX_Get_Flags() >> 19) & 1; +} + +BoolInt CPU_IsSupported_SHA() { Cx86cpuid p; CHECK_SYS_SSE_SUPPORT if (!x86cpuid_CheckAndRead(&p)) return False; - return (p.c >> 25) & 1; + + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + return (d[1] >> 29) & 1; + } +} + +// #include + +#ifdef _WIN32 +#include +#endif + +BoolInt CPU_IsSupported_AVX2() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + + #ifdef _WIN32 + #define MY__PF_XSAVE_ENABLED 17 + if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + return False; + #endif + + if (!x86cpuid_CheckAndRead(&p)) + return False; + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (d[1] >> 5); // avx2 + } +} + +BoolInt CPU_IsSupported_VAES_AVX2() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + + #ifdef _WIN32 + #define MY__PF_XSAVE_ENABLED 17 + if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + return False; + #endif + + if (!x86cpuid_CheckAndRead(&p)) + return False; + if (p.maxFunc < 7) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (d[1] >> 5) // avx2 + // & (d[1] >> 31) // avx512vl + & (d[2] >> 9); // vaes // VEX-256/EVEX + } +} + +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } +} + + +#elif defined(MY_CPU_ARM_OR_ARM64) + +#ifdef _WIN32 + +#include + +BoolInt CPU_IsSupported_CRC32() { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRYPTO() { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_NEON() { return IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } + +#else + +#if defined(__APPLE__) + +/* +#include +#include +static void Print_sysctlbyname(const char *name) +{ + size_t bufSize = 256; + char buf[256]; + int res = sysctlbyname(name, &buf, &bufSize, NULL, 0); + { + int i; + printf("\nres = %d : %s : '%s' : bufSize = %d, numeric", res, name, buf, (unsigned)bufSize); + for (i = 0; i < 20; i++) + printf(" %2x", (unsigned)(Byte)buf[i]); + + } +} +*/ + +static BoolInt My_sysctlbyname_Get_BoolInt(const char *name) +{ + UInt32 val = 0; + if (My_sysctlbyname_Get_UInt32(name, &val) == 0 && val == 1) + return 1; + return 0; +} + + /* + Print_sysctlbyname("hw.pagesize"); + Print_sysctlbyname("machdep.cpu.brand_string"); + */ + +BoolInt CPU_IsSupported_CRC32(void) +{ + return My_sysctlbyname_Get_BoolInt("hw.optional.armv8_crc32"); +} + +BoolInt CPU_IsSupported_NEON(void) +{ + return My_sysctlbyname_Get_BoolInt("hw.optional.neon"); +} + +#ifdef MY_CPU_ARM64 +#define APPLE_CRYPTO_SUPPORT_VAL 1 +#else +#define APPLE_CRYPTO_SUPPORT_VAL 0 +#endif + +BoolInt CPU_IsSupported_SHA1(void) { return APPLE_CRYPTO_SUPPORT_VAL; } +BoolInt CPU_IsSupported_SHA2(void) { return APPLE_CRYPTO_SUPPORT_VAL; } +BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; } + + +#else // __APPLE__ + +#include + +#define USE_HWCAP + +#ifdef USE_HWCAP + +#include + + #define MY_HWCAP_CHECK_FUNC_2(name1, name2) \ + BoolInt CPU_IsSupported_ ## name1() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name2)) ? 1 : 0; } + +#ifdef MY_CPU_ARM64 + #define MY_HWCAP_CHECK_FUNC(name) \ + MY_HWCAP_CHECK_FUNC_2(name, name) + MY_HWCAP_CHECK_FUNC_2(NEON, ASIMD) +// MY_HWCAP_CHECK_FUNC (ASIMD) +#elif defined(MY_CPU_ARM) + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP2) & (HWCAP2_ ## name)) ? 1 : 0; } + MY_HWCAP_CHECK_FUNC_2(NEON, NEON) +#endif + +#else // USE_HWCAP + + #define MY_HWCAP_CHECK_FUNC(name) \ + BoolInt CPU_IsSupported_ ## name() { return 0; } + MY_HWCAP_CHECK_FUNC(NEON) + +#endif // USE_HWCAP + +MY_HWCAP_CHECK_FUNC (CRC32) +MY_HWCAP_CHECK_FUNC (SHA1) +MY_HWCAP_CHECK_FUNC (SHA2) +MY_HWCAP_CHECK_FUNC (AES) + +#endif // __APPLE__ +#endif // _WIN32 + +#endif // MY_CPU_ARM_OR_ARM64 + + + +#ifdef __APPLE__ + +#include + +int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize) +{ + return sysctlbyname(name, buf, bufSize, NULL, 0); +} + +int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val) +{ + size_t bufSize = sizeof(*val); + int res = My_sysctlbyname_Get(name, val, &bufSize); + if (res == 0 && bufSize != sizeof(*val)) + return EFAULT; + return res; } #endif diff --git a/3rdparty/lzma/C/CpuArch.h b/3rdparty/lzma/C/CpuArch.h index e42ce281424..4856fbb12a2 100644 --- a/3rdparty/lzma/C/CpuArch.h +++ b/3rdparty/lzma/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2016-06-09: Igor Pavlov : Public domain */ +2022-07-15 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -14,50 +14,156 @@ MY_CPU_BE means that CPU is BIG ENDIAN. If MY_CPU_LE and MY_CPU_BE are not defined, we don't know about ENDIANNESS of platform. MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. + +MY_CPU_64BIT means that processor can work with 64-bit registers. + MY_CPU_64BIT can be used to select fast code branch + MY_CPU_64BIT doesn't mean that (sizeof(void *) == 8) */ -#if defined(_M_X64) \ - || defined(_M_AMD64) \ - || defined(__x86_64__) \ - || defined(__AMD64__) \ - || defined(__amd64__) +#if defined(_M_X64) \ + || defined(_M_AMD64) \ + || defined(__x86_64__) \ + || defined(__AMD64__) \ + || defined(__amd64__) #define MY_CPU_AMD64 + #ifdef __ILP32__ + #define MY_CPU_NAME "x32" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "x64" + #define MY_CPU_SIZEOF_POINTER 8 + #endif + #define MY_CPU_64BIT #endif -#if defined(MY_CPU_AMD64) \ - || defined(_M_IA64) \ - || defined(__AARCH64EL__) \ - || defined(__AARCH64EB__) + +#if defined(_M_IX86) \ + || defined(__i386__) + #define MY_CPU_X86 + #define MY_CPU_NAME "x86" + /* #define MY_CPU_32BIT */ + #define MY_CPU_SIZEOF_POINTER 4 +#endif + + +#if defined(_M_ARM64) \ + || defined(__AARCH64EL__) \ + || defined(__AARCH64EB__) \ + || defined(__aarch64__) + #define MY_CPU_ARM64 + #define MY_CPU_NAME "arm64" + #define MY_CPU_64BIT +#endif + + +#if defined(_M_ARM) \ + || defined(_M_ARM_NT) \ + || defined(_M_ARMT) \ + || defined(__arm__) \ + || defined(__thumb__) \ + || defined(__ARMEL__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEL__) \ + || defined(__THUMBEB__) + #define MY_CPU_ARM + + #if defined(__thumb__) || defined(__THUMBEL__) || defined(_M_ARMT) + #define MY_CPU_NAME "armt" + #else + #define MY_CPU_NAME "arm" + #endif + /* #define MY_CPU_32BIT */ + #define MY_CPU_SIZEOF_POINTER 4 +#endif + + +#if defined(_M_IA64) \ + || defined(__ia64__) + #define MY_CPU_IA64 + #define MY_CPU_NAME "ia64" + #define MY_CPU_64BIT +#endif + + +#if defined(__mips64) \ + || defined(__mips64__) \ + || (defined(__mips) && (__mips == 64 || __mips == 4 || __mips == 3)) + #define MY_CPU_NAME "mips64" + #define MY_CPU_64BIT +#elif defined(__mips__) + #define MY_CPU_NAME "mips" + /* #define MY_CPU_32BIT */ +#endif + + +#if defined(__ppc64__) \ + || defined(__powerpc64__) \ + || defined(__ppc__) \ + || defined(__powerpc__) \ + || defined(__PPC__) \ + || defined(_POWER) + +#if defined(__ppc64__) \ + || defined(__powerpc64__) \ + || defined(_LP64) \ + || defined(__64BIT__) + #ifdef __ILP32__ + #define MY_CPU_NAME "ppc64-32" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "ppc64" + #define MY_CPU_SIZEOF_POINTER 8 + #endif #define MY_CPU_64BIT +#else + #define MY_CPU_NAME "ppc" + #define MY_CPU_SIZEOF_POINTER 4 + /* #define MY_CPU_32BIT */ #endif +#endif + -#if defined(_M_IX86) || defined(__i386__) -#define MY_CPU_X86 +#if defined(__riscv) \ + || defined(__riscv__) + #if __riscv_xlen == 32 + #define MY_CPU_NAME "riscv32" + #elif __riscv_xlen == 64 + #define MY_CPU_NAME "riscv64" + #else + #define MY_CPU_NAME "riscv" + #endif #endif + #if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) #define MY_CPU_X86_OR_AMD64 #endif -#if defined(MY_CPU_X86) \ - || defined(_M_ARM) \ - || defined(__ARMEL__) \ - || defined(__THUMBEL__) \ - || defined(__ARMEB__) \ - || defined(__THUMBEB__) - #define MY_CPU_32BIT +#if defined(MY_CPU_ARM) || defined(MY_CPU_ARM64) +#define MY_CPU_ARM_OR_ARM64 #endif -#if defined(_WIN32) && defined(_M_ARM) -#define MY_CPU_ARM_LE -#endif -#if defined(_WIN32) && defined(_M_IA64) -#define MY_CPU_IA64_LE +#ifdef _WIN32 + + #ifdef MY_CPU_ARM + #define MY_CPU_ARM_LE + #endif + + #ifdef MY_CPU_ARM64 + #define MY_CPU_ARM64_LE + #endif + + #ifdef _M_IA64 + #define MY_CPU_IA64_LE + #endif + #endif + #if defined(MY_CPU_X86_OR_AMD64) \ || defined(MY_CPU_ARM_LE) \ + || defined(MY_CPU_ARM64_LE) \ || defined(MY_CPU_IA64_LE) \ || defined(__LITTLE_ENDIAN__) \ || defined(__ARMEL__) \ @@ -86,14 +192,75 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #define MY_CPU_BE #endif + #if defined(MY_CPU_LE) && defined(MY_CPU_BE) -Stop_Compiling_Bad_Endian + #error Stop_Compiling_Bad_Endian +#endif + + +#if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) + #error Stop_Compiling_Bad_32_64_BIT +#endif + +#ifdef __SIZEOF_POINTER__ + #ifdef MY_CPU_SIZEOF_POINTER + #if MY_CPU_SIZEOF_POINTER != __SIZEOF_POINTER__ + #error Stop_Compiling_Bad_MY_CPU_PTR_SIZE + #endif + #else + #define MY_CPU_SIZEOF_POINTER __SIZEOF_POINTER__ + #endif +#endif + +#if defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 4) +#if defined (_LP64) + #error Stop_Compiling_Bad_MY_CPU_PTR_SIZE +#endif +#endif + +#ifdef _MSC_VER + #if _MSC_VER >= 1300 + #define MY_CPU_pragma_pack_push_1 __pragma(pack(push, 1)) + #define MY_CPU_pragma_pop __pragma(pack(pop)) + #else + #define MY_CPU_pragma_pack_push_1 + #define MY_CPU_pragma_pop + #endif +#else + #ifdef __xlC__ + #define MY_CPU_pragma_pack_push_1 _Pragma("pack(1)") + #define MY_CPU_pragma_pop _Pragma("pack()") + #else + #define MY_CPU_pragma_pack_push_1 _Pragma("pack(push, 1)") + #define MY_CPU_pragma_pop _Pragma("pack(pop)") + #endif #endif +#ifndef MY_CPU_NAME + #ifdef MY_CPU_LE + #define MY_CPU_NAME "LE" + #elif defined(MY_CPU_BE) + #define MY_CPU_NAME "BE" + #else + /* + #define MY_CPU_NAME "" + */ + #endif +#endif + + + + + #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ - /* || defined(__AARCH64EL__) */ + || defined(MY_CPU_ARM64) + #define MY_CPU_LE_UNALIGN + #define MY_CPU_LE_UNALIGN_64 + #elif defined(__ARM_FEATURE_UNALIGNED) + /* gcc9 for 32-bit arm can use LDRD instruction that requires 32-bit alignment. + So we can't use unaligned 64-bit operations. */ #define MY_CPU_LE_UNALIGN #endif #endif @@ -103,11 +270,15 @@ Stop_Compiling_Bad_Endian #define GetUi16(p) (*(const UInt16 *)(const void *)(p)) #define GetUi32(p) (*(const UInt32 *)(const void *)(p)) +#ifdef MY_CPU_LE_UNALIGN_64 #define GetUi64(p) (*(const UInt64 *)(const void *)(p)) +#endif -#define SetUi16(p, v) { *(UInt16 *)(p) = (v); } -#define SetUi32(p, v) { *(UInt32 *)(p) = (v); } -#define SetUi64(p, v) { *(UInt64 *)(p) = (v); } +#define SetUi16(p, v) { *(UInt16 *)(void *)(p) = (v); } +#define SetUi32(p, v) { *(UInt32 *)(void *)(p) = (v); } +#ifdef MY_CPU_LE_UNALIGN_64 +#define SetUi64(p, v) { *(UInt64 *)(void *)(p) = (v); } +#endif #else @@ -121,8 +292,6 @@ Stop_Compiling_Bad_Endian ((UInt32)((const Byte *)(p))[2] << 16) | \ ((UInt32)((const Byte *)(p))[3] << 24)) -#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) - #define SetUi16(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ _ppp_[0] = (Byte)_vvv_; \ _ppp_[1] = (Byte)(_vvv_ >> 8); } @@ -133,6 +302,13 @@ Stop_Compiling_Bad_Endian _ppp_[2] = (Byte)(_vvv_ >> 16); \ _ppp_[3] = (Byte)(_vvv_ >> 24); } +#endif + + +#ifndef MY_CPU_LE_UNALIGN_64 + +#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) + #define SetUi64(p, v) { Byte *_ppp2_ = (Byte *)(p); UInt64 _vvv2_ = (v); \ SetUi32(_ppp2_ , (UInt32)_vvv2_); \ SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); } @@ -140,23 +316,37 @@ Stop_Compiling_Bad_Endian #endif -#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300) + + +#ifdef __has_builtin + #define MY__has_builtin(x) __has_builtin(x) +#else + #define MY__has_builtin(x) 0 +#endif + +#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ defined(_MSC_VER) && (_MSC_VER >= 1300) /* Note: we use bswap instruction, that is unsupported in 386 cpu */ #include +#pragma intrinsic(_byteswap_ushort) #pragma intrinsic(_byteswap_ulong) #pragma intrinsic(_byteswap_uint64) -#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) + +/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ +#define GetBe32(p) _byteswap_ulong (*(const UInt32 *)(const void *)(p)) +#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const void *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) -#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#elif defined(MY_CPU_LE_UNALIGN) && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) -#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p)) +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const void *)(p)) */ +#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const void *)(p)) +#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const void *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = __builtin_bswap32(v) @@ -179,10 +369,14 @@ Stop_Compiling_Bad_Endian #endif +#ifndef GetBe16 + #define GetBe16(p) ( (UInt16) ( \ ((UInt16)((const Byte *)(p))[0] << 8) | \ ((const Byte *)(p))[1] )) +#endif + #ifdef MY_CPU_X86_OR_AMD64 @@ -206,16 +400,44 @@ enum void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); -Bool x86cpuid_CheckAndRead(Cx86cpuid *p); +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); int x86cpuid_GetFirm(const Cx86cpuid *p); #define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) #define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) #define x86cpuid_GetStepping(ver) (ver & 0xF) -Bool CPU_Is_InOrder(); -Bool CPU_Is_Aes_Supported(); +BoolInt CPU_Is_InOrder(void); + +BoolInt CPU_IsSupported_AES(void); +BoolInt CPU_IsSupported_AVX2(void); +BoolInt CPU_IsSupported_VAES_AVX2(void); +BoolInt CPU_IsSupported_SSSE3(void); +BoolInt CPU_IsSupported_SSE41(void); +BoolInt CPU_IsSupported_SHA(void); +BoolInt CPU_IsSupported_PageGB(void); + +#elif defined(MY_CPU_ARM_OR_ARM64) + +BoolInt CPU_IsSupported_CRC32(void); +BoolInt CPU_IsSupported_NEON(void); + +#if defined(_WIN32) +BoolInt CPU_IsSupported_CRYPTO(void); +#define CPU_IsSupported_SHA1 CPU_IsSupported_CRYPTO +#define CPU_IsSupported_SHA2 CPU_IsSupported_CRYPTO +#define CPU_IsSupported_AES CPU_IsSupported_CRYPTO +#else +BoolInt CPU_IsSupported_SHA1(void); +BoolInt CPU_IsSupported_SHA2(void); +BoolInt CPU_IsSupported_AES(void); +#endif + +#endif +#if defined(__APPLE__) +int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize); +int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val); #endif EXTERN_C_END diff --git a/3rdparty/lzma/C/Delta.c b/3rdparty/lzma/C/Delta.c index e3edd21edbb..c4a4499fe2a 100644 --- a/3rdparty/lzma/C/Delta.c +++ b/3rdparty/lzma/C/Delta.c @@ -1,5 +1,5 @@ /* Delta.c -- Delta converter -2009-05-26 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,53 +12,158 @@ void Delta_Init(Byte *state) state[i] = 0; } -static void MyMemCpy(Byte *dest, const Byte *src, unsigned size) -{ - unsigned i; - for (i = 0; i < size; i++) - dest[i] = src[i]; -} void Delta_Encode(Byte *state, unsigned delta, Byte *data, SizeT size) { - Byte buf[DELTA_STATE_SIZE]; - unsigned j = 0; - MyMemCpy(buf, state, delta); + Byte temp[DELTA_STATE_SIZE]; + + if (size == 0) + return; + + { + unsigned i = 0; + do + temp[i] = state[i]; + while (++i != delta); + } + + if (size <= delta) + { + unsigned i = 0, k; + do + { + Byte b = *data; + *data++ = (Byte)(b - temp[i]); + temp[i] = b; + } + while (++i != size); + + k = 0; + + do + { + if (i == delta) + i = 0; + state[k] = temp[i++]; + } + while (++k != delta); + + return; + } + { - SizeT i; - for (i = 0; i < size;) + Byte *p = data + size - delta; + { + unsigned i = 0; + do + state[i] = *p++; + while (++i != delta); + } { - for (j = 0; j < delta && i < size; i++, j++) + const Byte *lim = data + delta; + ptrdiff_t dif = -(ptrdiff_t)delta; + + if (((ptrdiff_t)size + dif) & 1) { - Byte b = data[i]; - data[i] = (Byte)(b - buf[j]); - buf[j] = b; + --p; *p = (Byte)(*p - p[dif]); } + + while (p != lim) + { + --p; *p = (Byte)(*p - p[dif]); + --p; *p = (Byte)(*p - p[dif]); + } + + dif = -dif; + + do + { + --p; *p = (Byte)(*p - temp[--dif]); + } + while (dif != 0); } } - if (j == delta) - j = 0; - MyMemCpy(state, buf + j, delta - j); - MyMemCpy(state + delta - j, buf, j); } + void Delta_Decode(Byte *state, unsigned delta, Byte *data, SizeT size) { - Byte buf[DELTA_STATE_SIZE]; - unsigned j = 0; - MyMemCpy(buf, state, delta); + unsigned i; + const Byte *lim; + + if (size == 0) + return; + + i = 0; + lim = data + size; + + if (size <= delta) + { + do + *data = (Byte)(*data + state[i++]); + while (++data != lim); + + for (; delta != i; state++, delta--) + *state = state[i]; + data -= i; + } + else { - SizeT i; - for (i = 0; i < size;) + /* + #define B(n) b ## n + #define I(n) Byte B(n) = state[n]; + #define U(n) { B(n) = (Byte)((B(n)) + *data++); data[-1] = (B(n)); } + #define F(n) if (data != lim) { U(n) } + + if (delta == 1) + { + I(0) + if ((lim - data) & 1) { U(0) } + while (data != lim) { U(0) U(0) } + data -= 1; + } + else if (delta == 2) { - for (j = 0; j < delta && i < size; i++, j++) + I(0) I(1) + lim -= 1; while (data < lim) { U(0) U(1) } + lim += 1; F(0) + data -= 2; + } + else if (delta == 3) + { + I(0) I(1) I(2) + lim -= 2; while (data < lim) { U(0) U(1) U(2) } + lim += 2; F(0) F(1) + data -= 3; + } + else if (delta == 4) + { + I(0) I(1) I(2) I(3) + lim -= 3; while (data < lim) { U(0) U(1) U(2) U(3) } + lim += 3; F(0) F(1) F(2) + data -= 4; + } + else + */ + { + do + { + *data = (Byte)(*data + state[i++]); + data++; + } + while (i != delta); + { - buf[j] = data[i] = (Byte)(buf[j] + data[i]); + ptrdiff_t dif = -(ptrdiff_t)delta; + do + *data = (Byte)(*data + data[dif]); + while (++data != lim); + data += dif; } } } - if (j == delta) - j = 0; - MyMemCpy(state, buf + j, delta - j); - MyMemCpy(state + delta - j, buf, j); + + do + *state++ = *data; + while (++data != lim); } diff --git a/3rdparty/lzma/C/DllSecur.c b/3rdparty/lzma/C/DllSecur.c index 90352ce10f7..dce0c96c08c 100644 --- a/3rdparty/lzma/C/DllSecur.c +++ b/3rdparty/lzma/C/DllSecur.c @@ -1,16 +1,20 @@ /* DllSecur.c -- DLL loading security -2016-10-04 : Igor Pavlov : Public domain */ +2022-07-15 : Igor Pavlov : Public domain */ #include "Precomp.h" #ifdef _WIN32 -#include +#include #include "DllSecur.h" #ifndef UNDER_CE +#if defined(__GNUC__) && (__GNUC__ >= 8) + #pragma GCC diagnostic ignored "-Wcast-function-type" +#endif + typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); #define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400 @@ -28,10 +32,33 @@ static const char * const g_Dlls = "CRYPTBASE\0" "OLEACC\0" "CLBCATQ\0" + "VERSION\0" ; #endif +// #define MY_CAST_FUNC (void(*)()) +#define MY_CAST_FUNC + +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + void LoadSecurityDlls() { #ifndef UNDER_CE @@ -45,7 +72,7 @@ void LoadSecurityDlls() if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) { Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) - GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); if (setDllDirs) if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) return; @@ -70,7 +97,7 @@ void LoadSecurityDlls() for (;;) { char c = *dll++; - buf[pos + k] = c; + buf[pos + k] = (Byte)c; k++; if (c == 0) break; diff --git a/3rdparty/lzma/C/DllSecur.h b/3rdparty/lzma/C/DllSecur.h index e01f2247dcc..64ff26cd93a 100644 --- a/3rdparty/lzma/C/DllSecur.h +++ b/3rdparty/lzma/C/DllSecur.h @@ -1,5 +1,5 @@ /* DllSecur.h -- DLL loading for security -2016-06-08 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __DLL_SECUR_H #define __DLL_SECUR_H @@ -10,7 +10,8 @@ EXTERN_C_BEGIN #ifdef _WIN32 -void LoadSecurityDlls(); +void My_SetDefaultDllDirectories(void); +void LoadSecurityDlls(void); #endif diff --git a/3rdparty/lzma/C/LzFind.c b/3rdparty/lzma/C/LzFind.c index 2d05fa39534..1b73c28484c 100644 --- a/3rdparty/lzma/C/LzFind.c +++ b/3rdparty/lzma/C/LzFind.c @@ -1,70 +1,130 @@ /* LzFind.c -- Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2021-11-29 : Igor Pavlov : Public domain */ #include "Precomp.h" #include +// #include +#include "CpuArch.h" #include "LzFind.h" #include "LzHash.h" +#define kBlockMoveAlign (1 << 7) // alignment for memmove() +#define kBlockSizeAlign (1 << 16) // alignment for block allocation +#define kBlockSizeReserveMin (1 << 24) // it's 1/256 from 4 GB dictinary + #define kEmptyHashValue 0 -#define kMaxValForNormalize ((UInt32)0xFFFFFFFF) -#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */ -#define kNormalizeMask (~(UInt32)(kNormalizeStepMin - 1)) -#define kMaxHistorySize ((UInt32)7 << 29) -#define kStartMaxLen 3 +#define kMaxValForNormalize ((UInt32)0) +// #define kMaxValForNormalize ((UInt32)(1 << 20) + 0xFFF) // for debug + +// #define kNormalizeAlign (1 << 7) // alignment for speculated accesses + +#define GET_AVAIL_BYTES(p) \ + Inline_MatchFinder_GetNumAvailableBytes(p) + -static void LzInWindow_Free(CMatchFinder *p, ISzAlloc *alloc) +// #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) +#define kFix5HashSize kFix4HashSize + +/* + HASH2_CALC: + if (hv) match, then cur[0] and cur[1] also match +*/ +#define HASH2_CALC hv = GetUi16(cur); + +// (crc[0 ... 255] & 0xFF) provides one-to-one correspondence to [0 ... 255] + +/* + HASH3_CALC: + if (cur[0]) and (h2) match, then cur[1] also match + if (cur[0]) and (hv) match, then cur[1] and cur[2] also match +*/ +#define HASH3_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } + +#define HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + hv = (temp ^ (p->crc[cur[3]] << kLzHash_CrcShift_1)) & p->hashMask; } + +#define HASH5_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + temp ^= (p->crc[cur[3]] << kLzHash_CrcShift_1); \ + /* h4 = temp & p->hash4Mask; */ /* (kHash4Size - 1); */ \ + hv = (temp ^ (p->crc[cur[4]] << kLzHash_CrcShift_2)) & p->hashMask; } + +#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; + + +static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) { if (!p->directInput) { - alloc->Free(alloc, p->bufferBase); + ISzAlloc_Free(alloc, p->bufferBase); p->bufferBase = NULL; } } -/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAlloc *alloc) +static int LzInWindow_Create2(CMatchFinder *p, UInt32 blockSize, ISzAllocPtr alloc) { - UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; - if (p->directInput) - { - p->blockSize = blockSize; - return 1; - } + if (blockSize == 0) + return 0; if (!p->bufferBase || p->blockSize != blockSize) { + // size_t blockSizeT; LzInWindow_Free(p, alloc); p->blockSize = blockSize; - p->bufferBase = (Byte *)alloc->Alloc(alloc, (size_t)blockSize); + // blockSizeT = blockSize; + + // printf("\nblockSize = 0x%x\n", blockSize); + /* + #if defined _WIN64 + // we can allocate 4GiB, but still use UInt32 for (p->blockSize) + // we use UInt32 type for (p->blockSize), because + // we don't want to wrap over 4 GiB, + // when we use (p->streamPos - p->pos) that is UInt32. + if (blockSize >= (UInt32)0 - (UInt32)kBlockSizeAlign) + { + blockSizeT = ((size_t)1 << 32); + printf("\nchanged to blockSizeT = 4GiB\n"); + } + #endif + */ + + p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, blockSize); + // printf("\nbufferBase = %p\n", p->bufferBase); + // return 0; // for debug } return (p->bufferBase != NULL); } -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } +static const Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } -UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } +static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return GET_AVAIL_BYTES(p); } -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) -{ - p->posLimit -= subValue; - p->pos -= subValue; - p->streamPos -= subValue; -} +MY_NO_INLINE static void MatchFinder_ReadBlock(CMatchFinder *p) { if (p->streamEndWasReached || p->result != SZ_OK) return; - /* We use (p->streamPos - p->pos) value. (p->streamPos < p->pos) is allowed. */ + /* We use (p->streamPos - p->pos) value. + (p->streamPos < p->pos) is allowed. */ if (p->directInput) { - UInt32 curSize = 0xFFFFFFFF - (p->streamPos - p->pos); + UInt32 curSize = 0xFFFFFFFF - GET_AVAIL_BYTES(p); if (curSize > p->directInputRem) curSize = (UInt32)p->directInputRem; p->directInputRem -= curSize; @@ -76,12 +136,24 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) for (;;) { - Byte *dest = p->buffer + (p->streamPos - p->pos); - size_t size = (p->bufferBase + p->blockSize - dest); + Byte *dest = p->buffer + GET_AVAIL_BYTES(p); + size_t size = (size_t)(p->bufferBase + p->blockSize - dest); if (size == 0) + { + /* we call ReadBlock() after NeedMove() and MoveBlock(). + NeedMove() and MoveBlock() povide more than (keepSizeAfter) + to the end of (blockSize). + So we don't execute this branch in normal code flow. + We can go here, if we will call ReadBlock() before NeedMove(), MoveBlock(). + */ + // p->result = SZ_ERROR_FAIL; // we can show error here return; + } + + // #define kRead 3 + // if (size > kRead) size = kRead; // for debug - p->result = p->stream->Read(p->stream, dest, &size); + p->result = ISeqInStream_Read(p->stream, dest, &size); if (p->result != SZ_OK) return; if (size == 0) @@ -90,41 +162,52 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) return; } p->streamPos += (UInt32)size; - if (p->streamPos - p->pos > p->keepSizeAfter) + if (GET_AVAIL_BYTES(p) > p->keepSizeAfter) return; + /* here and in another (p->keepSizeAfter) checks we keep on 1 byte more than was requested by Create() function + (GET_AVAIL_BYTES(p) >= p->keepSizeAfter) - minimal required size */ } + + // on exit: (p->result != SZ_OK || p->streamEndWasReached || GET_AVAIL_BYTES(p) > p->keepSizeAfter) } + + +MY_NO_INLINE void MatchFinder_MoveBlock(CMatchFinder *p) { + const size_t offset = (size_t)(p->buffer - p->bufferBase) - p->keepSizeBefore; + const size_t keepBefore = (offset & (kBlockMoveAlign - 1)) + p->keepSizeBefore; + p->buffer = p->bufferBase + keepBefore; memmove(p->bufferBase, - p->buffer - p->keepSizeBefore, - (size_t)(p->streamPos - p->pos) + p->keepSizeBefore); - p->buffer = p->bufferBase + p->keepSizeBefore; + p->bufferBase + (offset & ~((size_t)kBlockMoveAlign - 1)), + keepBefore + (size_t)GET_AVAIL_BYTES(p)); } +/* We call MoveBlock() before ReadBlock(). + So MoveBlock() can be wasteful operation, if the whole input data + can fit in current block even without calling MoveBlock(). + in important case where (dataSize <= historySize) + condition (p->blockSize > dataSize + p->keepSizeAfter) is met + So there is no MoveBlock() in that case case. +*/ + int MatchFinder_NeedMove(CMatchFinder *p) { if (p->directInput) return 0; - /* if (p->streamEndWasReached) return 0; */ + if (p->streamEndWasReached || p->result != SZ_OK) + return 0; return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); } void MatchFinder_ReadIfRequired(CMatchFinder *p) { - if (p->streamEndWasReached) - return; - if (p->keepSizeAfter >= p->streamPos - p->pos) + if (p->keepSizeAfter >= GET_AVAIL_BYTES(p)) MatchFinder_ReadBlock(p); } -static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p) -{ - if (MatchFinder_NeedMove(p)) - MatchFinder_MoveBlock(p); - MatchFinder_ReadBlock(p); -} + static void MatchFinder_SetDefaultSettings(CMatchFinder *p) { @@ -138,84 +221,124 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p) void MatchFinder_Construct(CMatchFinder *p) { - UInt32 i; + unsigned i; p->bufferBase = NULL; p->directInput = 0; p->hash = NULL; + p->expectedDataSize = (UInt64)(Int64)-1; MatchFinder_SetDefaultSettings(p); for (i = 0; i < 256; i++) { - UInt32 r = i; + UInt32 r = (UInt32)i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); p->crc[i] = r; } } -static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAlloc *alloc) +static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->hash); + ISzAlloc_Free(alloc, p->hash); p->hash = NULL; } -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc) +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) { MatchFinder_FreeThisClassMemory(p, alloc); LzInWindow_Free(p, alloc); } -static CLzRef* AllocRefs(size_t num, ISzAlloc *alloc) +static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) { size_t sizeInBytes = (size_t)num * sizeof(CLzRef); if (sizeInBytes / sizeof(CLzRef) != num) return NULL; - return (CLzRef *)alloc->Alloc(alloc, sizeInBytes); + return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); } -int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc) +#if (kBlockSizeReserveMin < kBlockSizeAlign * 2) + #error Stop_Compiling_Bad_Reserve +#endif + + + +static UInt32 GetBlockSize(CMatchFinder *p, UInt32 historySize) { - UInt32 sizeReserv; - + UInt32 blockSize = (p->keepSizeBefore + p->keepSizeAfter); + /* if (historySize > kMaxHistorySize) - { - MatchFinder_Free(p, alloc); return 0; - } + */ + // printf("\nhistorySize == 0x%x\n", historySize); - sizeReserv = historySize >> 1; - if (historySize >= ((UInt32)3 << 30)) sizeReserv = historySize >> 3; - else if (historySize >= ((UInt32)2 << 30)) sizeReserv = historySize >> 2; + if (p->keepSizeBefore < historySize || blockSize < p->keepSizeBefore) // if 32-bit overflow + return 0; - sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19); + { + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)kBlockSizeAlign; + const UInt32 rem = kBlockSizeMax - blockSize; + const UInt32 reserve = (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)) + + (1 << 12) + kBlockMoveAlign + kBlockSizeAlign; // do not overflow 32-bit here + if (blockSize >= kBlockSizeMax + || rem < kBlockSizeReserveMin) // we reject settings that will be slow + return 0; + if (reserve >= rem) + blockSize = kBlockSizeMax; + else + { + blockSize += reserve; + blockSize &= ~(UInt32)(kBlockSizeAlign - 1); + } + } + // printf("\n LzFind_blockSize = %x\n", blockSize); + // printf("\n LzFind_blockSize = %d\n", blockSize >> 20); + return blockSize; +} + +int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, + UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, + ISzAllocPtr alloc) +{ + /* we need one additional byte in (p->keepSizeBefore), + since we use MoveBlock() after (p->pos++) and before dictionary using */ + // keepAddBufferBefore = (UInt32)0xFFFFFFFF - (1 << 22); // for debug p->keepSizeBefore = historySize + keepAddBufferBefore + 1; - p->keepSizeAfter = matchMaxLen + keepAddBufferAfter; - - /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */ - - if (LzInWindow_Create(p, sizeReserv, alloc)) + + keepAddBufferAfter += matchMaxLen; + /* we need (p->keepSizeAfter >= p->numHashBytes) */ + if (keepAddBufferAfter < p->numHashBytes) + keepAddBufferAfter = p->numHashBytes; + // keepAddBufferAfter -= 2; // for debug + p->keepSizeAfter = keepAddBufferAfter; + + if (p->directInput) + p->blockSize = 0; + if (p->directInput || LzInWindow_Create2(p, GetBlockSize(p, historySize), alloc)) { - UInt32 newCyclicBufferSize = historySize + 1; + const UInt32 newCyclicBufferSize = historySize + 1; // do not change it UInt32 hs; p->matchMaxLen = matchMaxLen; { + // UInt32 hs4; p->fixedHashSize = 0; - if (p->numHashBytes == 2) - hs = (1 << 16) - 1; - else + hs = (1 << 16) - 1; + if (p->numHashBytes != 2) { - hs = historySize - 1; + hs = historySize; + if (hs > p->expectedDataSize) + hs = (UInt32)p->expectedDataSize; + if (hs != 0) + hs--; hs |= (hs >> 1); hs |= (hs >> 2); hs |= (hs >> 4); hs |= (hs >> 8); + // we propagated 16 bits in (hs). Low 16 bits must be set later hs >>= 1; - hs |= 0xFFFF; /* don't change it! It's required for Deflate */ - if (hs > (1 << 24)) + if (hs >= (1 << 24)) { if (p->numHashBytes == 3) hs = (1 << 24) - 1; @@ -223,12 +346,30 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, hs >>= 1; /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ } + + // hs = ((UInt32)1 << 25) - 1; // for test + + // (hash_size >= (1 << 16)) : Required for (numHashBytes > 2) + hs |= (1 << 16) - 1; /* don't change it! */ + + // bt5: we adjust the size with recommended minimum size + if (p->numHashBytes >= 5) + hs |= (256 << kLzHash_CrcShift_2) - 1; } p->hashMask = hs; hs++; + + /* + hs4 = (1 << 20); + if (hs4 > hs) + hs4 = hs; + // hs4 = (1 << 16); // for test + p->hash4Mask = hs4 - 1; + */ + if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; - if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size; + // if (p->numHashBytes > 4) p->fixedHashSize += hs4; // kHash4Size; hs += p->fixedHashSize; } @@ -237,13 +378,17 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, size_t numSons; p->historySize = historySize; p->hashSizeSum = hs; - p->cyclicBufferSize = newCyclicBufferSize; + p->cyclicBufferSize = newCyclicBufferSize; // it must be = (historySize + 1) numSons = newCyclicBufferSize; if (p->btMode) numSons <<= 1; newSize = hs + numSons; + // aligned size is not required here, but it can be better for some loops + #define NUM_REFS_ALIGN_MASK 0xF + newSize = (newSize + NUM_REFS_ALIGN_MASK) & ~(size_t)NUM_REFS_ALIGN_MASK; + if (p->hash && p->numRefs == newSize) return 1; @@ -263,107 +408,395 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, return 0; } + static void MatchFinder_SetLimits(CMatchFinder *p) { - UInt32 limit = kMaxValForNormalize - p->pos; - UInt32 limit2 = p->cyclicBufferSize - p->cyclicBufferPos; - - if (limit2 < limit) - limit = limit2; - limit2 = p->streamPos - p->pos; + UInt32 k; + UInt32 n = kMaxValForNormalize - p->pos; + if (n == 0) + n = (UInt32)(Int32)-1; // we allow (pos == 0) at start even with (kMaxValForNormalize == 0) - if (limit2 <= p->keepSizeAfter) + k = p->cyclicBufferSize - p->cyclicBufferPos; + if (k < n) + n = k; + + k = GET_AVAIL_BYTES(p); { - if (limit2 > 0) - limit2 = 1; + const UInt32 ksa = p->keepSizeAfter; + UInt32 mm = p->matchMaxLen; + if (k > ksa) + k -= ksa; // we must limit exactly to keepSizeAfter for ReadBlock + else if (k >= mm) + { + // the limitation for (p->lenLimit) update + k -= mm; // optimization : to reduce the number of checks + k++; + // k = 1; // non-optimized version : for debug + } + else + { + mm = k; + if (k != 0) + k = 1; + } + p->lenLimit = mm; } - else - limit2 -= p->keepSizeAfter; - - if (limit2 < limit) - limit = limit2; + if (k < n) + n = k; - { - UInt32 lenLimit = p->streamPos - p->pos; - if (lenLimit > p->matchMaxLen) - lenLimit = p->matchMaxLen; - p->lenLimit = lenLimit; - } - p->posLimit = p->pos + limit; + p->posLimit = p->pos + n; } -void MatchFinder_Init_2(CMatchFinder *p, int readData) + +void MatchFinder_Init_LowHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash; + const size_t numItems = p->fixedHashSize; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_HighHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash + p->fixedHashSize; + const size_t numItems = (size_t)p->hashMask + 1; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_4(CMatchFinder *p) { - UInt32 i; - UInt32 *hash = p->hash; - UInt32 num = p->hashSizeSum; - for (i = 0; i < num; i++) - hash[i] = kEmptyHashValue; - - p->cyclicBufferPos = 0; p->buffer = p->bufferBase; - p->pos = p->streamPos = p->cyclicBufferSize; + { + /* kEmptyHashValue = 0 (Zero) is used in hash tables as NO-VALUE marker. + the code in CMatchFinderMt expects (pos = 1) */ + p->pos = + p->streamPos = + 1; // it's smallest optimal value. do not change it + // 0; // for debug + } p->result = SZ_OK; p->streamEndWasReached = 0; - - if (readData) - MatchFinder_ReadBlock(p); - - MatchFinder_SetLimits(p); } + +// (CYC_TO_POS_OFFSET == 0) is expected by some optimized code +#define CYC_TO_POS_OFFSET 0 +// #define CYC_TO_POS_OFFSET 1 // for debug + void MatchFinder_Init(CMatchFinder *p) { - MatchFinder_Init_2(p, True); + MatchFinder_Init_HighHash(p); + MatchFinder_Init_LowHash(p); + MatchFinder_Init_4(p); + // if (readData) + MatchFinder_ReadBlock(p); + + /* if we init (cyclicBufferPos = pos), then we can use one variable + instead of both (cyclicBufferPos) and (pos) : only before (cyclicBufferPos) wrapping */ + p->cyclicBufferPos = (p->pos - CYC_TO_POS_OFFSET); // init with relation to (pos) + // p->cyclicBufferPos = 0; // smallest value + // p->son[0] = p->son[1] = 0; // unused: we can init skipped record for speculated accesses. + MatchFinder_SetLimits(p); } - -static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) + + + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900) + #define USE_SATUR_SUB_128 + #define USE_AVX2 + #define ATTRIB_SSE41 __attribute__((__target__("sse4.1"))) + #define ATTRIB_AVX2 __attribute__((__target__("avx2"))) + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1600) + #define USE_SATUR_SUB_128 + #if (_MSC_VER >= 1900) + #define USE_AVX2 + #include // avx + #endif + #endif + #endif + +// #elif defined(MY_CPU_ARM_OR_ARM64) +#elif defined(MY_CPU_ARM64) + + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #define USE_SATUR_SUB_128 + #ifdef MY_CPU_ARM64 + // #define ATTRIB_SSE41 __attribute__((__target__(""))) + #else + // #define ATTRIB_SSE41 __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif + + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1910) + #define USE_SATUR_SUB_128 + #endif + #endif + + #if defined(_MSC_VER) && defined(MY_CPU_ARM64) + #include + #else + #include + #endif + +#endif + +/* +#ifndef ATTRIB_SSE41 + #define ATTRIB_SSE41 +#endif +#ifndef ATTRIB_AVX2 + #define ATTRIB_AVX2 +#endif +*/ + +#ifdef USE_SATUR_SUB_128 + +// #define _SHOW_HW_STATUS + +#ifdef _SHOW_HW_STATUS +#include +#define _PRF(x) x +_PRF(;) +#else +#define _PRF(x) +#endif + +#ifdef MY_CPU_ARM_OR_ARM64 + +#ifdef MY_CPU_ARM64 +// #define FORCE_SATUR_SUB_128 +#endif + +typedef uint32x4_t v128; +#define SASUB_128(i) \ + *(v128 *)(void *)(items + (i) * 4) = \ + vsubq_u32(vmaxq_u32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); + +#else + +#include // sse4.1 + +typedef __m128i v128; +#define SASUB_128(i) \ + *(v128 *)(void *)(items + (i) * 4) = \ + _mm_sub_epi32(_mm_max_epu32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); // SSE 4.1 + +#endif + + + +MY_NO_INLINE +static +#ifdef ATTRIB_SSE41 +ATTRIB_SSE41 +#endif +void +MY_FAST_CALL +LzFind_SaturSub_128(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - return (p->pos - p->historySize - 1) & kNormalizeMask; + v128 sub2 = + #ifdef MY_CPU_ARM_OR_ARM64 + vdupq_n_u32(subValue); + #else + _mm_set_epi32((Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + #endif + do + { + SASUB_128(0) + SASUB_128(1) + SASUB_128(2) + SASUB_128(3) + items += 4 * 4; + } + while (items != lim); } -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) + + +#ifdef USE_AVX2 + +#include // avx + +#define SASUB_256(i) *(__m256i *)(void *)(items + (i) * 8) = _mm256_sub_epi32(_mm256_max_epu32(*(const __m256i *)(const void *)(items + (i) * 8), sub2), sub2); // AVX2 + +MY_NO_INLINE +static +#ifdef ATTRIB_AVX2 +ATTRIB_AVX2 +#endif +void +MY_FAST_CALL +LzFind_SaturSub_256(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - size_t i; - for (i = 0; i < numItems; i++) + __m256i sub2 = _mm256_set_epi32( + (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, + (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + do { - UInt32 value = items[i]; - if (value <= subValue) - value = kEmptyHashValue; - else - value -= subValue; - items[i] = value; + SASUB_256(0) + SASUB_256(1) + items += 2 * 8; } + while (items != lim); } +#endif // USE_AVX2 -static void MatchFinder_Normalize(CMatchFinder *p) +#ifndef FORCE_SATUR_SUB_128 +typedef void (MY_FAST_CALL *LZFIND_SATUR_SUB_CODE_FUNC)( + UInt32 subValue, CLzRef *items, const CLzRef *lim); +static LZFIND_SATUR_SUB_CODE_FUNC g_LzFind_SaturSub; +#endif // FORCE_SATUR_SUB_128 + +#endif // USE_SATUR_SUB_128 + + +// kEmptyHashValue must be zero +// #define SASUB_32(i) v = items[i]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; items[i] = m; +#define SASUB_32(i) v = items[i]; if (v < subValue) v = subValue; items[i] = v - subValue; + +#ifdef FORCE_SATUR_SUB_128 + +#define DEFAULT_SaturSub LzFind_SaturSub_128 + +#else + +#define DEFAULT_SaturSub LzFind_SaturSub_32 + +MY_NO_INLINE +static +void +MY_FAST_CALL +LzFind_SaturSub_32(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - UInt32 subValue = MatchFinder_GetSubValue(p); - MatchFinder_Normalize3(subValue, p->hash, p->numRefs); - MatchFinder_ReduceOffsets(p, subValue); + do + { + UInt32 v; + SASUB_32(0) + SASUB_32(1) + SASUB_32(2) + SASUB_32(3) + SASUB_32(4) + SASUB_32(5) + SASUB_32(6) + SASUB_32(7) + items += 8; + } + while (items != lim); } +#endif + + +MY_NO_INLINE +void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) +{ + #define K_NORM_ALIGN_BLOCK_SIZE (1 << 6) + + CLzRef *lim; + + for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (K_NORM_ALIGN_BLOCK_SIZE - 1)) != 0; numItems--) + { + UInt32 v; + SASUB_32(0); + items++; + } + + { + #define K_NORM_ALIGN_MASK (K_NORM_ALIGN_BLOCK_SIZE / 4 - 1) + lim = items + (numItems & ~(size_t)K_NORM_ALIGN_MASK); + numItems &= K_NORM_ALIGN_MASK; + if (items != lim) + { + #if defined(USE_SATUR_SUB_128) && !defined(FORCE_SATUR_SUB_128) + if (g_LzFind_SaturSub) + g_LzFind_SaturSub(subValue, items, lim); + else + #endif + DEFAULT_SaturSub(subValue, items, lim); + } + items = lim; + } + + + for (; numItems != 0; numItems--) + { + UInt32 v; + SASUB_32(0); + items++; + } +} + + + +// call MatchFinder_CheckLimits() only after (p->pos++) update + +MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { + if (// !p->streamEndWasReached && p->result == SZ_OK && + p->keepSizeAfter == GET_AVAIL_BYTES(p)) + { + // we try to read only in exact state (p->keepSizeAfter == GET_AVAIL_BYTES(p)) + if (MatchFinder_NeedMove(p)) + MatchFinder_MoveBlock(p); + MatchFinder_ReadBlock(p); + } + if (p->pos == kMaxValForNormalize) - MatchFinder_Normalize(p); - if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos) - MatchFinder_CheckAndMoveAndRead(p); + if (GET_AVAIL_BYTES(p) >= p->numHashBytes) // optional optimization for last bytes of data. + /* + if we disable normalization for last bytes of data, and + if (data_size == 4 GiB), we don't call wastfull normalization, + but (pos) will be wrapped over Zero (0) in that case. + And we cannot resume later to normal operation + */ + { + // MatchFinder_Normalize(p); + /* after normalization we need (p->pos >= p->historySize + 1); */ + /* we can reduce subValue to aligned value, if want to keep alignment + of (p->pos) and (p->buffer) for speculated accesses. */ + const UInt32 subValue = (p->pos - p->historySize - 1) /* & ~(UInt32)(kNormalizeAlign - 1) */; + // const UInt32 subValue = (1 << 15); // for debug + // printf("\nMatchFinder_Normalize() subValue == 0x%x\n", subValue); + size_t numSonRefs = p->cyclicBufferSize; + if (p->btMode) + numSonRefs <<= 1; + Inline_MatchFinder_ReduceOffsets(p, subValue); + MatchFinder_Normalize3(subValue, p->hash, (size_t)p->hashSizeSum + numSonRefs); + } + if (p->cyclicBufferPos == p->cyclicBufferSize) p->cyclicBufferPos = 0; + MatchFinder_SetLimits(p); } -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *d, unsigned maxLen) { + /* son[_cyclicBufferPos] = curMatch; for (;;) { UInt32 delta = pos - curMatch; if (cutValue-- == 0 || delta >= _cyclicBufferSize) - return distances; + return d; { const Byte *pb = cur - delta; curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; @@ -375,35 +808,91 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, break; if (maxLen < len) { - *distances++ = maxLen = len; - *distances++ = delta - 1; + maxLen = len; + *d++ = len; + *d++ = delta - 1; if (len == lenLimit) - return distances; + return d; + } + } + } + } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + + do + { + UInt32 delta; + + if (curMatch == 0) + break; + // if (curMatch2 >= curMatch) return NULL; + delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + if (cur[maxLen] == cur[(ptrdiff_t)maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + d[0] = (UInt32)(lim - cur); + d[1] = delta - 1; + return d + 2; + } + } + { + const unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + d[0] = (UInt32)len; + d[1] = delta - 1; + d += 2; + } } } } } + while (--cutValue); + + return d; } + +MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *d, UInt32 maxLen) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - for (;;) + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; + + UInt32 cmCheck; + + // if (curMatch >= pos) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; } + + cmCheck = (UInt32)(pos - _cyclicBufferSize); + if ((UInt32)pos <= _cyclicBufferSize) + cmCheck = 0; + + if (cmCheck < curMatch) + do { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + const UInt32 delta = pos - curMatch; { - *ptr0 = *ptr1 = kEmptyHashValue; - return distances; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + const UInt32 pair0 = pair[0]; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -412,52 +901,65 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt break; if (maxLen < len) { - *distances++ = maxLen = len; - *distances++ = delta - 1; + maxLen = (UInt32)len; + *d++ = (UInt32)len; + *d++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; + *ptr1 = pair0; *ptr0 = pair[1]; - return distances; + return d; } } } if (pb[len] < cur[len]) { *ptr1 = curMatch; + // const UInt32 curMatch2 = pair[1]; + // if (curMatch2 >= curMatch) { *ptr0 = *ptr1 = kEmptyHashValue; return NULL; } + // curMatch = curMatch2; + curMatch = pair[1]; ptr1 = pair + 1; - curMatch = *ptr1; len1 = len; } else { *ptr0 = curMatch; + curMatch = pair[0]; ptr0 = pair; - curMatch = *ptr0; len0 = len; } } } + while(--cutValue && cmCheck < curMatch); + + *ptr0 = *ptr1 = kEmptyHashValue; + return d; } + static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - for (;;) + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; + + UInt32 cmCheck; + + cmCheck = (UInt32)(pos - _cyclicBufferSize); + if ((UInt32)pos <= _cyclicBufferSize) + cmCheck = 0; + + if (// curMatch >= pos || // failure + cmCheck < curMatch) + do { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + const UInt32 delta = pos - curMatch; { - *ptr0 = *ptr1 = kEmptyHashValue; - return; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); if (pb[len] == cur[len]) { while (++len != lenLimit) @@ -475,79 +977,108 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const if (pb[len] < cur[len]) { *ptr1 = curMatch; + curMatch = pair[1]; ptr1 = pair + 1; - curMatch = *ptr1; len1 = len; } else { *ptr0 = curMatch; + curMatch = pair[0]; ptr0 = pair; - curMatch = *ptr0; len0 = len; } } } + while(--cutValue && cmCheck < curMatch); + + *ptr0 = *ptr1 = kEmptyHashValue; + return; } + #define MOVE_POS \ ++p->cyclicBufferPos; \ p->buffer++; \ - if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); + { const UInt32 pos1 = p->pos + 1; p->pos = pos1; if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } -#define MOVE_POS_RET MOVE_POS return offset; +#define MOVE_POS_RET MOVE_POS return distances; -static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } +MY_NO_INLINE +static void MatchFinder_MovePos(CMatchFinder *p) +{ + /* we go here at the end of stream data, when (avail < num_hash_bytes) + We don't update sons[cyclicBufferPos << btMode]. + So (sons) record will contain junk. And we cannot resume match searching + to normal operation, even if we will provide more input data in buffer. + p->sons[p->cyclicBufferPos << p->btMode] = 0; // kEmptyHashValue + if (p->btMode) + p->sons[(p->cyclicBufferPos << p->btMode) + 1] = 0; // kEmptyHashValue + */ + MOVE_POS; +} #define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + unsigned lenLimit; UInt32 hv; Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; -#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) -#define SKIP_HEADER(minLen) GET_MATCHES_HEADER2(minLen, continue) +#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return distances) +#define SKIP_HEADER(minLen) do { GET_MATCHES_HEADER2(minLen, continue) + +#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue + +#define SKIP_FOOTER SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS; } while (--num); -#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue +#define GET_MATCHES_FOOTER_BASE(_maxLen_, func) \ + distances = func(MF_PARAMS(p), \ + distances, (UInt32)_maxLen_); MOVE_POS_RET; + +#define GET_MATCHES_FOOTER_BT(_maxLen_) \ + GET_MATCHES_FOOTER_BASE(_maxLen_, GetMatchesSpec1) + +#define GET_MATCHES_FOOTER_HC(_maxLen_) \ + GET_MATCHES_FOOTER_BASE(_maxLen_, Hc_GetMatchesSpec) -#define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; -#define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; #define UPDATE_maxLen { \ - ptrdiff_t diff = (ptrdiff_t)0 - d2; \ + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ - maxLen = (UInt32)(c - cur); } + maxLen = (unsigned)(c - cur); } -static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; GET_MATCHES_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 1) + GET_MATCHES_FOOTER_BT(1) } -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 2) + GET_MATCHES_FOOTER_BT(2) } -static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +#define SET_mmm \ + mmm = p->cyclicBufferSize; \ + if (pos < mmm) \ + mmm = pos; + + +static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, d2, maxLen, offset, pos; + UInt32 mmm; + UInt32 h2, d2, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(3) @@ -558,33 +1089,37 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) d2 = pos - hash[h2]; - curMatch = hash[kFix3HashSize + hv]; + curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = pos; - hash[kFix3HashSize + hv] = pos; + (hash + kFix3HashSize)[hv] = pos; + + SET_mmm maxLen = 2; - offset = 0; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { UPDATE_maxLen - distances[0] = maxLen; + distances[0] = (UInt32)maxLen; distances[1] = d2 - 1; - offset = 2; + distances += 2; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS_RET; } } - GET_MATCHES_FOOTER(offset, maxLen) + GET_MATCHES_FOOTER_BT(maxLen) } -static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 mmm; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -593,54 +1128,65 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; - curMatch = hash[kFix4HashSize + hv]; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + hv] = pos; + SET_mmm - maxLen = 0; - offset = 0; - - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - } + maxLen = 3; - if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + for (;;) { - maxLen = 3; - distances[offset + 1] = d3 - 1; - offset += 2; - d2 = d3; - } + if (d2 < mmm && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + // distances[-2] = 3; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; - if (offset != 0) - { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; + SkipMatchesSpec(MF_PARAMS(p)); + MOVE_POS_RET } + break; } - if (maxLen < 3) - maxLen = 3; - - GET_MATCHES_FOOTER(offset, maxLen) + GET_MATCHES_FOOTER_BT(maxLen) } -/* -static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos; + UInt32 mmm; + UInt32 h2, h3, d2, d3, maxLen, pos; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -649,74 +1195,71 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - d4 = pos - hash[kFix4HashSize + h4]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + // d4 = pos - (hash + kFix4HashSize)[h4]; + + curMatch = (hash + kFix5HashSize)[hv]; - curMatch = hash[kFix5HashSize + hv]; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + // (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + h4] = pos; - hash[kFix5HashSize + hv] = pos; + SET_mmm - maxLen = 0; - offset = 0; + maxLen = 4; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + for (;;) { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - if (*(cur - d2 + 2) == cur[2]) - distances[0] = maxLen = 3; - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) { - distances[2] = maxLen = 3; - distances[3] = d3 - 1; - offset = 4; + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + distances[1] = d3 - 1; + distances += 2; + d2 = d3; + } + else + break; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + distances[1] = d3 - 1; + distances += 2; d2 = d3; } - } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[0] = maxLen = 3; - distances[1] = d3 - 1; - offset = 2; - d2 = d3; - } - - if (d2 != d4 && d4 < p->cyclicBufferSize - && *(cur - d4) == *cur - && *(cur - d4 + 3) == *(cur + 3)) - { - maxLen = 4; - distances[offset + 1] = d4 - 1; - offset += 2; - d2 = d4; - } - - if (offset != 0) - { + else + break; + + distances[-2] = 3; + if (*(cur - d2 + 3) != cur[3]) + break; UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS_RET; } + break; } - - if (maxLen < 4) - maxLen = 4; - GET_MATCHES_FOOTER(offset, maxLen) + GET_MATCHES_FOOTER_BT(maxLen) } -*/ -static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 mmm; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -725,56 +1268,65 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - - curMatch = hash[kFix4HashSize + hv]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; - maxLen = 0; - offset = 0; + SET_mmm - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - } - - if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - maxLen = 3; - distances[offset + 1] = d3 - 1; - offset += 2; - d2 = d3; - } - - if (offset != 0) + maxLen = 3; + + for (;;) { + if (d2 < mmm && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + // distances[-2] = 3; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + d2 = d3; + distances[1] = d3 - 1; + distances += 2; + } + else + break; + UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; MOVE_POS_RET; } + break; } - if (maxLen < 3) - maxLen = 3; - - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET + GET_MATCHES_FOOTER_HC(maxLen); } -/* -static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos + UInt32 mmm; + UInt32 h2, h3, d2, d3, maxLen, pos; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -782,242 +1334,237 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - d4 = pos - hash[kFix4HashSize + h4]; - curMatch = hash[kFix5HashSize + hv]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + // d4 = pos - (hash + kFix4HashSize)[h4]; + + curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + h4] = pos; - hash[kFix5HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + // (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; - maxLen = 0; - offset = 0; + SET_mmm + + maxLen = 4; - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + for (;;) { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - if (*(cur - d2 + 2) == cur[2]) - distances[0] = maxLen = 3; - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + if (d2 < mmm && *(cur - d2) == *cur) + { + distances[0] = 2; + distances[1] = d2 - 1; + distances += 2; + if (*(cur - d2 + 2) == cur[2]) + { + } + else if (d3 < mmm && *(cur - d3) == *cur) + { + distances[1] = d3 - 1; + distances += 2; + d2 = d3; + } + else + break; + } + else if (d3 < mmm && *(cur - d3) == *cur) { - distances[2] = maxLen = 3; - distances[3] = d3 - 1; - offset = 4; + distances[1] = d3 - 1; + distances += 2; d2 = d3; } - } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[0] = maxLen = 3; - distances[1] = d3 - 1; - offset = 2; - d2 = d3; - } - - if (d2 != d4 && d4 < p->cyclicBufferSize - && *(cur - d4) == *cur - && *(cur - d4 + 3) == *(cur + 3)) - { - maxLen = 4; - distances[offset + 1] = d4 - 1; - offset += 2; - d2 = d4; - } - - if (offset != 0) - { + else + break; + + distances[-2] = 3; + if (*(cur - d2 + 3) != cur[3]) + break; UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[-2] = maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; MOVE_POS_RET; } + break; } - if (maxLen < 4) - maxLen = 4; - - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET + GET_MATCHES_FOOTER_HC(maxLen); } -*/ -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) + +UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances, 2) - (distances)); - MOVE_POS_RET + GET_MATCHES_FOOTER_HC(2) } + static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(2) { - SKIP_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(3) { - SKIP_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - SKIP_FOOTER } - while (--num != 0); + SKIP_FOOTER } static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(3) { UInt32 h2; UInt32 *hash; - SKIP_HEADER(3) HASH3_CALC; hash = p->hash; - curMatch = hash[kFix3HashSize + hv]; + curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = - hash[kFix3HashSize + hv] = p->pos; - SKIP_FOOTER + (hash + kFix3HashSize)[hv] = p->pos; } - while (--num != 0); + SKIP_FOOTER } static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(4) { UInt32 h2, h3; UInt32 *hash; - SKIP_HEADER(4) HASH4_CALC; hash = p->hash; - curMatch = hash[kFix4HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + hv] = p->pos; - SKIP_FOOTER + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; } - while (--num != 0); + SKIP_FOOTER } -/* static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do + SKIP_HEADER(5) { - UInt32 h2, h3, h4; + UInt32 h2, h3; UInt32 *hash; - SKIP_HEADER(5) HASH5_CALC; hash = p->hash; - curMatch = hash[kFix5HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + h4] = - hash[kFix5HashSize + hv] = p->pos; - SKIP_FOOTER + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + // (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; } - while (--num != 0); + SKIP_FOOTER } -*/ + + +#define HC_SKIP_HEADER(minLen) \ + do { if (p->lenLimit < minLen) { MatchFinder_MovePos(p); num--; continue; } { \ + Byte *cur; \ + UInt32 *hash; \ + UInt32 *son; \ + UInt32 pos = p->pos; \ + UInt32 num2 = num; \ + /* (p->pos == p->posLimit) is not allowed here !!! */ \ + { const UInt32 rem = p->posLimit - pos; if (num2 > rem) num2 = rem; } \ + num -= num2; \ + { const UInt32 cycPos = p->cyclicBufferPos; \ + son = p->son + cycPos; \ + p->cyclicBufferPos = cycPos + num2; } \ + cur = p->buffer; \ + hash = p->hash; \ + do { \ + UInt32 curMatch; \ + UInt32 hv; + + +#define HC_SKIP_FOOTER \ + cur++; pos++; *son++ = curMatch; \ + } while (--num2); \ + p->buffer = cur; \ + p->pos = pos; \ + if (pos == p->posLimit) MatchFinder_CheckLimits(p); \ + }} while(num); \ + static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { + HC_SKIP_HEADER(4) + UInt32 h2, h3; - UInt32 *hash; - SKIP_HEADER(4) HASH4_CALC; - hash = p->hash; - curMatch = hash[kFix4HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = pos; + + HC_SKIP_FOOTER } -/* + static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - UInt32 h2, h3, h4; - UInt32 *hash; - SKIP_HEADER(5) - HASH5_CALC; - hash = p->hash; - curMatch = p->hash[kFix5HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + h4] = - hash[kFix5HashSize + hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); + HC_SKIP_HEADER(5) + + UInt32 h2, h3; + HASH5_CALC + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + // (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = pos; + + HC_SKIP_FOOTER } -*/ + void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { - do - { - SKIP_HEADER(3) + HC_SKIP_HEADER(3) + HASH_ZIP_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); + curMatch = hash[hv]; + hash[hv] = pos; + + HC_SKIP_FOOTER } -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) + +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable) { vTable->Init = (Mf_Init_Func)MatchFinder_Init; vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; if (!p->btMode) { - /* if (p->numHashBytes <= 4) */ + if (p->numHashBytes <= 4) { vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; } - /* else { vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; } - */ } else if (p->numHashBytes == 2) { @@ -1029,16 +1576,53 @@ void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; } - else /* if (p->numHashBytes == 4) */ + else if (p->numHashBytes == 4) { vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; } - /* else { vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; } - */ +} + + + +void LzFindPrepare() +{ + #ifndef FORCE_SATUR_SUB_128 + #ifdef USE_SATUR_SUB_128 + LZFIND_SATUR_SUB_CODE_FUNC f = NULL; + #ifdef MY_CPU_ARM_OR_ARM64 + { + if (CPU_IsSupported_NEON()) + { + // #pragma message ("=== LzFind NEON") + _PRF(printf("\n=== LzFind NEON\n")); + f = LzFind_SaturSub_128; + } + // f = 0; // for debug + } + #else // MY_CPU_ARM_OR_ARM64 + if (CPU_IsSupported_SSE41()) + { + // #pragma message ("=== LzFind SSE41") + _PRF(printf("\n=== LzFind SSE41\n")); + f = LzFind_SaturSub_128; + + #ifdef USE_AVX2 + if (CPU_IsSupported_AVX2()) + { + // #pragma message ("=== LzFind AVX2") + _PRF(printf("\n=== LzFind AVX2\n")); + f = LzFind_SaturSub_256; + } + #endif + } + #endif // MY_CPU_ARM_OR_ARM64 + g_LzFind_SaturSub = f; + #endif // USE_SATUR_SUB_128 + #endif // FORCE_SATUR_SUB_128 } diff --git a/3rdparty/lzma/C/LzFind.h b/3rdparty/lzma/C/LzFind.h index d119944f445..eea873ff61e 100644 --- a/3rdparty/lzma/C/LzFind.h +++ b/3rdparty/lzma/C/LzFind.h @@ -1,5 +1,5 @@ /* LzFind.h -- Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2021-07-13 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H @@ -15,7 +15,7 @@ typedef struct _CMatchFinder Byte *buffer; UInt32 pos; UInt32 posLimit; - UInt32 streamPos; + UInt32 streamPos; /* wrap over Zero is allowed (streamPos < pos). Use (UInt32)(streamPos - pos) */ UInt32 lenLimit; UInt32 cyclicBufferPos; @@ -47,19 +47,23 @@ typedef struct _CMatchFinder SRes result; UInt32 crc[256]; size_t numRefs; + + UInt64 expectedDataSize; } CMatchFinder; -#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) +#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((const Byte *)(p)->buffer) -#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos) +#define Inline_MatchFinder_GetNumAvailableBytes(p) ((UInt32)((p)->streamPos - (p)->pos)) +/* #define Inline_MatchFinder_IsFinishedOK(p) \ ((p)->streamEndWasReached \ && (p)->streamPos == (p)->pos \ && (!(p)->directInput || (p)->directInputRem == 0)) +*/ int MatchFinder_NeedMove(CMatchFinder *p); -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); +/* Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); */ void MatchFinder_MoveBlock(CMatchFinder *p); void MatchFinder_ReadIfRequired(CMatchFinder *p); @@ -71,13 +75,24 @@ void MatchFinder_Construct(CMatchFinder *p); */ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc); -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc); + ISzAllocPtr alloc); +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); +// void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); + +/* +#define Inline_MatchFinder_InitPos(p, val) \ + (p)->pos = (val); \ + (p)->streamPos = (val); +*/ + +#define Inline_MatchFinder_ReduceOffsets(p, subValue) \ + (p)->pos -= (subValue); \ + (p)->streamPos -= (subValue); + UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, UInt32 *distances, UInt32 maxLen); /* @@ -89,7 +104,7 @@ Conditions: typedef void (*Mf_Init_Func)(void *object); typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object); typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); -typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances); +typedef UInt32 * (*Mf_GetMatches_Func)(void *object, UInt32 *distances); typedef void (*Mf_Skip_Func)(void *object, UInt32); typedef struct _IMatchFinder @@ -99,19 +114,23 @@ typedef struct _IMatchFinder Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos; Mf_GetMatches_Func GetMatches; Mf_Skip_Func Skip; -} IMatchFinder; +} IMatchFinder2; -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable); -void MatchFinder_Init_2(CMatchFinder *p, int readData); +void MatchFinder_Init_LowHash(CMatchFinder *p); +void MatchFinder_Init_HighHash(CMatchFinder *p); +void MatchFinder_Init_4(CMatchFinder *p); void MatchFinder_Init(CMatchFinder *p); -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); +UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); +UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); +void LzFindPrepare(void); + EXTERN_C_END #endif diff --git a/3rdparty/lzma/C/LzFindMt.c b/3rdparty/lzma/C/LzFindMt.c index a4ffa5ef9c8..4e67fc3f2f1 100644 --- a/3rdparty/lzma/C/LzFindMt.c +++ b/3rdparty/lzma/C/LzFindMt.c @@ -1,95 +1,215 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2021-12-21 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "LzHash.h" +// #include + +#include "CpuArch.h" +#include "LzHash.h" #include "LzFindMt.h" +// #define LOG_ITERS + +// #define LOG_THREAD + +#ifdef LOG_THREAD +#include +#define PRF(x) x +#else +#define PRF(x) +#endif + +#ifdef LOG_ITERS +#include +extern UInt64 g_NumIters_Tree; +extern UInt64 g_NumIters_Loop; +extern UInt64 g_NumIters_Bytes; +#define LOG_ITER(x) x +#else +#define LOG_ITER(x) +#endif + +#define kMtHashBlockSize ((UInt32)1 << 17) +#define kMtHashNumBlocks (1 << 1) + +#define GET_HASH_BLOCK_OFFSET(i) (((i) & (kMtHashNumBlocks - 1)) * kMtHashBlockSize) + +#define kMtBtBlockSize ((UInt32)1 << 16) +#define kMtBtNumBlocks (1 << 4) + +#define GET_BT_BLOCK_OFFSET(i) (((i) & (kMtBtNumBlocks - 1)) * (size_t)kMtBtBlockSize) + +/* + HASH functions: + We use raw 8/16 bits from a[1] and a[2], + xored with crc(a[0]) and crc(a[3]). + We check a[0], a[3] only. We don't need to compare a[1] and a[2] in matches. + our crc() function provides one-to-one correspondence for low 8-bit values: + (crc[0...0xFF] & 0xFF) <-> [0...0xFF] +*/ + +#define MF(mt) ((mt)->MatchFinder) +#define MF_CRC (p->crc) + +// #define MF(mt) (&(mt)->MatchFinder) +// #define MF_CRC (p->MatchFinder.crc) + +#define MT_HASH2_CALC \ + h2 = (MF_CRC[cur[0]] ^ cur[1]) & (kHash2Size - 1); + +#define MT_HASH3_CALC { \ + UInt32 temp = MF_CRC[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } + +/* +#define MT_HASH3_CALC__NO_2 { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } + +#define __MT_HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + h4 = (temp ^ (p->crc[cur[3]] << kLzHash_CrcShift_1)) & p->hash4Mask; } + // (kHash4Size - 1); +*/ + + +MY_NO_INLINE static void MtSync_Construct(CMtSync *p) { + p->affinity = 0; p->wasCreated = False; p->csWasInitialized = False; p->csWasEntered = False; Thread_Construct(&p->thread); Event_Construct(&p->canStart); - Event_Construct(&p->wasStarted); Event_Construct(&p->wasStopped); Semaphore_Construct(&p->freeSemaphore); Semaphore_Construct(&p->filledSemaphore); } -static void MtSync_GetNextBlock(CMtSync *p) + +#define DEBUG_BUFFER_LOCK // define it to debug lock state + +#ifdef DEBUG_BUFFER_LOCK +#include +#define BUFFER_MUST_BE_LOCKED(p) if (!(p)->csWasEntered) exit(1); +#define BUFFER_MUST_BE_UNLOCKED(p) if ( (p)->csWasEntered) exit(1); +#else +#define BUFFER_MUST_BE_LOCKED(p) +#define BUFFER_MUST_BE_UNLOCKED(p) +#endif + +#define LOCK_BUFFER(p) { \ + BUFFER_MUST_BE_UNLOCKED(p); \ + CriticalSection_Enter(&(p)->cs); \ + (p)->csWasEntered = True; } + +#define UNLOCK_BUFFER(p) { \ + BUFFER_MUST_BE_LOCKED(p); \ + CriticalSection_Leave(&(p)->cs); \ + (p)->csWasEntered = False; } + + +MY_NO_INLINE +static UInt32 MtSync_GetNextBlock(CMtSync *p) { + UInt32 numBlocks = 0; if (p->needStart) { + BUFFER_MUST_BE_UNLOCKED(p) p->numProcessedBlocks = 1; p->needStart = False; p->stopWriting = False; p->exit = False; - Event_Reset(&p->wasStarted); Event_Reset(&p->wasStopped); - Event_Set(&p->canStart); - Event_Wait(&p->wasStarted); } else { - CriticalSection_Leave(&p->cs); - p->csWasEntered = False; - p->numProcessedBlocks++; + UNLOCK_BUFFER(p) + // we free current block + numBlocks = p->numProcessedBlocks++; Semaphore_Release1(&p->freeSemaphore); } + + // buffer is UNLOCKED here Semaphore_Wait(&p->filledSemaphore); - CriticalSection_Enter(&p->cs); - p->csWasEntered = True; + LOCK_BUFFER(p); + return numBlocks; } -/* MtSync_StopWriting must be called if Writing was started */ +/* if Writing (Processing) thread was started, we must call MtSync_StopWriting() */ + +MY_NO_INLINE static void MtSync_StopWriting(CMtSync *p) { - UInt32 myNumBlocks = p->numProcessedBlocks; if (!Thread_WasCreated(&p->thread) || p->needStart) return; - p->stopWriting = True; + + PRF(printf("\nMtSync_StopWriting %p\n", p)); + if (p->csWasEntered) { - CriticalSection_Leave(&p->cs); - p->csWasEntered = False; + /* we don't use buffer in this thread after StopWriting(). + So we UNLOCK buffer. + And we restore default UNLOCKED state for stopped thread */ + UNLOCK_BUFFER(p) } - Semaphore_Release1(&p->freeSemaphore); - + + /* We send (p->stopWriting) message and release freeSemaphore + to free current block. + So the thread will see (p->stopWriting) at some + iteration after Wait(freeSemaphore). + The thread doesn't need to fill all avail free blocks, + so we can get fast thread stop. + */ + + p->stopWriting = True; + Semaphore_Release1(&p->freeSemaphore); // check semaphore count !!! + + PRF(printf("\nMtSync_StopWriting %p : Event_Wait(&p->wasStopped)\n", p)); Event_Wait(&p->wasStopped); + PRF(printf("\nMtSync_StopWriting %p : Event_Wait() finsihed\n", p)); + + /* 21.03 : we don't restore samaphore counters here. + We will recreate and reinit samaphores in next start */ - while (myNumBlocks++ != p->numProcessedBlocks) - { - Semaphore_Wait(&p->filledSemaphore); - Semaphore_Release1(&p->freeSemaphore); - } p->needStart = True; } + +MY_NO_INLINE static void MtSync_Destruct(CMtSync *p) { + PRF(printf("\nMtSync_Destruct %p\n", p)); + if (Thread_WasCreated(&p->thread)) { + /* we want thread to be in Stopped state before sending EXIT command. + note: stop(btSync) will stop (htSync) also */ MtSync_StopWriting(p); + /* thread in Stopped state here : (p->needStart == true) */ p->exit = True; - if (p->needStart) - Event_Set(&p->canStart); - Thread_Wait(&p->thread); - Thread_Close(&p->thread); + // if (p->needStart) // it's (true) + Event_Set(&p->canStart); // we send EXIT command to thread + Thread_Wait_Close(&p->thread); // we wait thread finishing } + if (p->csWasInitialized) { CriticalSection_Delete(&p->cs); p->csWasInitialized = False; } + p->csWasEntered = False; Event_Close(&p->canStart); - Event_Close(&p->wasStarted); Event_Close(&p->wasStopped); Semaphore_Close(&p->freeSemaphore); Semaphore_Close(&p->filledSemaphore); @@ -97,77 +217,251 @@ static void MtSync_Destruct(CMtSync *p) p->wasCreated = False; } -#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static SRes MtSync_Create2(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks) +// #define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } +// we want to get real system error codes here instead of SZ_ERROR_THREAD +#define RINOK_THREAD(x) RINOK(x) + + +// call it before each new file (when new starting is required): +MY_NO_INLINE +static SRes MtSync_Init(CMtSync *p, UInt32 numBlocks) +{ + WRes wres; + // BUFFER_MUST_BE_UNLOCKED(p) + if (!p->needStart || p->csWasEntered) + return SZ_ERROR_FAIL; + wres = Semaphore_OptCreateInit(&p->freeSemaphore, numBlocks, numBlocks); + if (wres == 0) + wres = Semaphore_OptCreateInit(&p->filledSemaphore, 0, numBlocks); + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +static WRes MtSync_Create_WRes(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj) { + WRes wres; + if (p->wasCreated) return SZ_OK; RINOK_THREAD(CriticalSection_Init(&p->cs)); p->csWasInitialized = True; + p->csWasEntered = False; RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStarted)); RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped)); - - RINOK_THREAD(Semaphore_Create(&p->freeSemaphore, numBlocks, numBlocks)); - RINOK_THREAD(Semaphore_Create(&p->filledSemaphore, 0, numBlocks)); p->needStart = True; - - RINOK_THREAD(Thread_Create(&p->thread, startAddress, obj)); + p->exit = True; /* p->exit is unused before (canStart) Event. + But in case of some unexpected code failure we will get fast exit from thread */ + + // return ERROR_TOO_MANY_POSTS; // for debug + // return EINVAL; // for debug + + if (p->affinity != 0) + wres = Thread_Create_With_Affinity(&p->thread, startAddress, obj, (CAffinityMask)p->affinity); + else + wres = Thread_Create(&p->thread, startAddress, obj); + + RINOK_THREAD(wres); p->wasCreated = True; return SZ_OK; } -static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks) + +MY_NO_INLINE +static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj) { - SRes res = MtSync_Create2(p, startAddress, obj, numBlocks); - if (res != SZ_OK) - MtSync_Destruct(p); - return res; + const WRes wres = MtSync_Create_WRes(p, startAddress, obj); + if (wres == 0) + return 0; + MtSync_Destruct(p); + return MY_SRes_HRESULT_FROM_WRes(wres); } -void MtSync_Init(CMtSync *p) { p->needStart = True; } + +// ---------- HASH THREAD ---------- #define kMtMaxValForNormalize 0xFFFFFFFF +// #define kMtMaxValForNormalize ((1 << 21)) // for debug +// #define kNormalizeAlign (1 << 7) // alignment for speculated accesses -#define DEF_GetHeads2(name, v, action) \ - static void GetHeads ## name(const Byte *p, UInt32 pos, \ - UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc) \ - { action; for (; numHeads != 0; numHeads--) { \ - const UInt32 value = (v); p++; *heads++ = pos - hash[value]; hash[value] = pos++; } } +#ifdef MY_CPU_LE_UNALIGN + #define GetUi24hi_from32(p) ((UInt32)GetUi32(p) >> 8) +#else + #define GetUi24hi_from32(p) ((p)[1] ^ ((UInt32)(p)[2] << 8) ^ ((UInt32)(p)[3] << 16)) +#endif + +#define GetHeads_DECL(name) \ + static void GetHeads ## name(const Byte *p, UInt32 pos, \ + UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc) + +#define GetHeads_LOOP(v) \ + for (; numHeads != 0; numHeads--) { \ + const UInt32 value = (v); \ + p++; \ + *heads++ = pos - hash[value]; \ + hash[value] = pos++; } +#define DEF_GetHeads2(name, v, action) \ + GetHeads_DECL(name) { action \ + GetHeads_LOOP(v) } + #define DEF_GetHeads(name, v) DEF_GetHeads2(name, v, ;) -DEF_GetHeads2(2, (p[0] | ((UInt32)p[1] << 8)), UNUSED_VAR(hashMask); UNUSED_VAR(crc); ) -DEF_GetHeads(3, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8)) & hashMask) -DEF_GetHeads(4, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5)) & hashMask) -DEF_GetHeads(4b, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ ((UInt32)p[3] << 16)) & hashMask) -/* DEF_GetHeads(5, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5) ^ (crc[p[4]] << 3)) & hashMask) */ +DEF_GetHeads2(2, GetUi16(p), UNUSED_VAR(hashMask); UNUSED_VAR(crc); ) +DEF_GetHeads(3, (crc[p[0]] ^ GetUi16(p + 1)) & hashMask) +DEF_GetHeads2(3b, GetUi16(p) ^ ((UInt32)(p)[2] << 16), UNUSED_VAR(hashMask); UNUSED_VAR(crc); ) +// BT3 is not good for crc collisions for big hashMask values. + +/* +GetHeads_DECL(3b) +{ + UNUSED_VAR(hashMask); + UNUSED_VAR(crc); + { + const Byte *pLim = p + numHeads; + if (numHeads == 0) + return; + pLim--; + while (p < pLim) + { + UInt32 v1 = GetUi32(p); + UInt32 v0 = v1 & 0xFFFFFF; + UInt32 h0, h1; + p += 2; + v1 >>= 8; + h0 = hash[v0]; hash[v0] = pos; heads[0] = pos - h0; pos++; + h1 = hash[v1]; hash[v1] = pos; heads[1] = pos - h1; pos++; + heads += 2; + } + if (p == pLim) + { + UInt32 v0 = GetUi16(p) ^ ((UInt32)(p)[2] << 16); + *heads = pos - hash[v0]; + hash[v0] = pos; + } + } +} +*/ + +/* +GetHeads_DECL(4) +{ + unsigned sh = 0; + UNUSED_VAR(crc) + while ((hashMask & 0x80000000) == 0) + { + hashMask <<= 1; + sh++; + } + GetHeads_LOOP((GetUi32(p) * 0xa54a1) >> sh) +} +#define GetHeads4b GetHeads4 +*/ + +#define USE_GetHeads_LOCAL_CRC + +#ifdef USE_GetHeads_LOCAL_CRC + +GetHeads_DECL(4) +{ + UInt32 crc0[256]; + UInt32 crc1[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + { + UInt32 v = crc[i]; + crc0[i] = v & hashMask; + crc1[i] = (v << kLzHash_CrcShift_1) & hashMask; + // crc1[i] = rotlFixed(v, 8) & hashMask; + } + } + GetHeads_LOOP(crc0[p[0]] ^ crc1[p[3]] ^ (UInt32)GetUi16(p+1)) +} + +GetHeads_DECL(4b) +{ + UInt32 crc0[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + crc0[i] = crc[i] & hashMask; + } + GetHeads_LOOP(crc0[p[0]] ^ GetUi24hi_from32(p)) +} + +GetHeads_DECL(5) +{ + UInt32 crc0[256]; + UInt32 crc1[256]; + UInt32 crc2[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + { + UInt32 v = crc[i]; + crc0[i] = v & hashMask; + crc1[i] = (v << kLzHash_CrcShift_1) & hashMask; + crc2[i] = (v << kLzHash_CrcShift_2) & hashMask; + } + } + GetHeads_LOOP(crc0[p[0]] ^ crc1[p[3]] ^ crc2[p[4]] ^ (UInt32)GetUi16(p+1)) +} + +GetHeads_DECL(5b) +{ + UInt32 crc0[256]; + UInt32 crc1[256]; + { + unsigned i; + for (i = 0; i < 256; i++) + { + UInt32 v = crc[i]; + crc0[i] = v & hashMask; + crc1[i] = (v << kLzHash_CrcShift_1) & hashMask; + } + } + GetHeads_LOOP(crc0[p[0]] ^ crc1[p[4]] ^ GetUi24hi_from32(p)) +} + +#else + +DEF_GetHeads(4, (crc[p[0]] ^ (crc[p[3]] << kLzHash_CrcShift_1) ^ (UInt32)GetUi16(p+1)) & hashMask) +DEF_GetHeads(4b, (crc[p[0]] ^ GetUi24hi_from32(p)) & hashMask) +DEF_GetHeads(5, (crc[p[0]] ^ (crc[p[3]] << kLzHash_CrcShift_1) ^ (crc[p[4]] << kLzHash_CrcShift_2) ^ (UInt32)GetUi16(p + 1)) & hashMask) +DEF_GetHeads(5b, (crc[p[0]] ^ (crc[p[4]] << kLzHash_CrcShift_1) ^ GetUi24hi_from32(p)) & hashMask) + +#endif + static void HashThreadFunc(CMatchFinderMt *mt) { CMtSync *p = &mt->hashSync; + PRF(printf("\nHashThreadFunc\n")); + for (;;) { - UInt32 numProcessedBlocks = 0; + UInt32 blockIndex = 0; + PRF(printf("\nHashThreadFunc : Event_Wait(&p->canStart)\n")); Event_Wait(&p->canStart); - Event_Set(&p->wasStarted); + PRF(printf("\nHashThreadFunc : Event_Wait(&p->canStart) : after \n")); + if (p->exit) + { + PRF(printf("\nHashThreadFunc : exit \n")); + return; + } + + MatchFinder_Init_HighHash(MF(mt)); + for (;;) { - if (p->exit) - return; - if (p->stopWriting) - { - p->numProcessedBlocks = numProcessedBlocks; - Event_Set(&p->wasStopped); - break; - } + PRF(printf("Hash thread block = %d pos = %d\n", (unsigned)blockIndex, mt->MatchFinder->pos)); { - CMatchFinder *mf = mt->MatchFinder; + CMatchFinder *mf = MF(mt); if (MatchFinder_NeedMove(mf)) { CriticalSection_Enter(&mt->btSync.cs); @@ -180,163 +474,178 @@ static void HashThreadFunc(CMatchFinderMt *mt) mt->pointerToCurPos -= offset; mt->buffer -= offset; } - CriticalSection_Leave(&mt->btSync.cs); CriticalSection_Leave(&mt->hashSync.cs); + CriticalSection_Leave(&mt->btSync.cs); continue; } Semaphore_Wait(&p->freeSemaphore); + if (p->exit) // exit is unexpected here. But we check it here for some failure case + return; + + // for faster stop : we check (p->stopWriting) after Wait(freeSemaphore) + if (p->stopWriting) + break; + MatchFinder_ReadIfRequired(mf); - if (mf->pos > (kMtMaxValForNormalize - kMtHashBlockSize)) - { - UInt32 subValue = (mf->pos - mf->historySize - 1); - MatchFinder_ReduceOffsets(mf, subValue); - MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, (size_t)mf->hashMask + 1); - } { - UInt32 *heads = mt->hashBuf + ((numProcessedBlocks++) & kMtHashNumBlocksMask) * kMtHashBlockSize; - UInt32 num = mf->streamPos - mf->pos; + UInt32 *heads = mt->hashBuf + GET_HASH_BLOCK_OFFSET(blockIndex++); + UInt32 num = Inline_MatchFinder_GetNumAvailableBytes(mf); heads[0] = 2; heads[1] = num; + + /* heads[1] contains the number of avail bytes: + if (avail < mf->numHashBytes) : + { + it means that stream was finished + HASH_THREAD and BT_TREAD must move position for heads[1] (avail) bytes. + HASH_THREAD doesn't stop, + HASH_THREAD fills only the header (2 numbers) for all next blocks: + {2, NumHashBytes - 1}, {2,0}, {2,0}, ... , {2,0} + } + else + { + HASH_THREAD and BT_TREAD must move position for (heads[0] - 2) bytes; + } + */ + if (num >= mf->numHashBytes) { num = num - mf->numHashBytes + 1; if (num > kMtHashBlockSize - 2) num = kMtHashBlockSize - 2; + + if (mf->pos > (UInt32)kMtMaxValForNormalize - num) + { + const UInt32 subValue = (mf->pos - mf->historySize - 1); // & ~(UInt32)(kNormalizeAlign - 1); + Inline_MatchFinder_ReduceOffsets(mf, subValue); + MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, (size_t)mf->hashMask + 1); + } + + heads[0] = 2 + num; mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); - heads[0] += num; } - mf->pos += num; + + mf->pos += num; // wrap over zero is allowed at the end of stream mf->buffer += num; } } Semaphore_Release1(&p->filledSemaphore); - } - } -} + } // for() processing end -static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) -{ - MtSync_GetNextBlock(&p->hashSync); - p->hashBufPosLimit = p->hashBufPos = ((p->hashSync.numProcessedBlocks - 1) & kMtHashNumBlocksMask) * kMtHashBlockSize; - p->hashBufPosLimit += p->hashBuf[p->hashBufPos++]; - p->hashNumAvail = p->hashBuf[p->hashBufPos++]; + // p->numBlocks_Sent = blockIndex; + Event_Set(&p->wasStopped); + } // for() thread end } -#define kEmptyHashValue 0 -/* #define MFMT_GM_INLINE */ + + +// ---------- BT THREAD ---------- + +/* we use one variable instead of two (cyclicBufferPos == pos) before CyclicBuf wrap. + here we define fixed offset of (p->pos) from (p->cyclicBufferPos) */ +#define CYC_TO_POS_OFFSET 0 +// #define CYC_TO_POS_OFFSET 1 // for debug + +#define MFMT_GM_INLINE #ifdef MFMT_GM_INLINE -#define NO_INLINE MY_FAST_CALL +/* + we use size_t for (pos) instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ -static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes) -{ - do - { - UInt32 *distances = _distances + 1; - UInt32 curMatch = pos - *hash++; - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - UInt32 cutValue = _cutValue; - UInt32 maxLen = _maxLen; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - break; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - if (++len != lenLimit && pb[len] == cur[len]) - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - *distances++ = maxLen = len; - *distances++ = delta - 1; - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - break; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - } - pos++; - _cyclicBufferPos++; - cur++; - { - UInt32 num = (UInt32)(distances - _distances); - *_distances = num - 1; - _distances += num; - limit -= num; - } - } - while (limit > 0 && --size != 0); - *posRes = pos; - return limit; -} +UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes); #endif -static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) + +static void BtGetMatches(CMatchFinderMt *p, UInt32 *d) { UInt32 numProcessed = 0; UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); - distances[1] = p->hashNumAvail; + /* GetMatchesSpec() functions don't create (len = 1) + in [len, dist] match pairs, if (p->numHashBytes >= 2) + Also we suppose here that (matchMaxLen >= 2). + So the following code for (reserve) is not required + UInt32 reserve = (p->matchMaxLen * 2); + const UInt32 kNumHashBytes_Max = 5; // BT_HASH_BYTES_MAX + if (reserve < kNumHashBytes_Max - 1) + reserve = kNumHashBytes_Max - 1; + const UInt32 limit = kMtBtBlockSize - (reserve); + */ + + const UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); + + d[1] = p->hashNumAvail; + + if (p->failure_BT) + { + // printf("\n == 1 BtGetMatches() p->failure_BT\n"); + d[0] = 0; + // d[1] = 0; + return; + } while (curPos < limit) { if (p->hashBufPos == p->hashBufPosLimit) { - MatchFinderMt_GetNextBlock_Hash(p); - distances[1] = numProcessed + p->hashNumAvail; - if (p->hashNumAvail >= p->numHashBytes) + // MatchFinderMt_GetNextBlock_Hash(p); + UInt32 avail; + { + const UInt32 bi = MtSync_GetNextBlock(&p->hashSync); + const UInt32 k = GET_HASH_BLOCK_OFFSET(bi); + const UInt32 *h = p->hashBuf + k; + avail = h[1]; + p->hashBufPosLimit = k + h[0]; + p->hashNumAvail = avail; + p->hashBufPos = k + 2; + } + + { + /* we must prevent UInt32 overflow for avail total value, + if avail was increased with new hash block */ + UInt32 availSum = numProcessed + avail; + if (availSum < numProcessed) + availSum = (UInt32)(Int32)-1; + d[1] = availSum; + } + + if (avail >= p->numHashBytes) continue; - distances[0] = curPos + p->hashNumAvail; - distances += curPos; - for (; p->hashNumAvail != 0; p->hashNumAvail--) - *distances++ = 0; + + // if (p->hashBufPos != p->hashBufPosLimit) exit(1); + + /* (avail < p->numHashBytes) + It means that stream was finished. + And (avail) - is a number of remaining bytes, + we fill (d) for (avail) bytes for LZ_THREAD (receiver). + but we don't update (p->pos) and (p->cyclicBufferPos) here in BT_THREAD */ + + /* here we suppose that we have space enough: + (kMtBtBlockSize - curPos >= p->hashNumAvail) */ + p->hashNumAvail = 0; + d[0] = curPos + avail; + d += curPos; + for (; avail != 0; avail--) + *d++ = 0; return; } { UInt32 size = p->hashBufPosLimit - p->hashBufPos; - UInt32 lenLimit = p->matchMaxLen; UInt32 pos = p->pos; UInt32 cyclicBufferPos = p->cyclicBufferPos; + UInt32 lenLimit = p->matchMaxLen; if (lenLimit >= p->hashNumAvail) lenLimit = p->hashNumAvail; { @@ -348,10 +657,18 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) size = size2; } + if (pos > (UInt32)kMtMaxValForNormalize - size) + { + const UInt32 subValue = (pos - p->cyclicBufferSize); // & ~(UInt32)(kNormalizeAlign - 1); + pos -= subValue; + p->pos = pos; + MatchFinder_Normalize3(subValue, p->son, (size_t)p->cyclicBufferSize * 2); + } + #ifndef MFMT_GM_INLINE while (curPos < limit && size-- != 0) { - UInt32 *startDistances = distances + curPos; + UInt32 *startDistances = d + curPos; UInt32 num = (UInt32)(GetMatchesSpec1(lenLimit, pos - p->hashBuf[p->hashBufPos++], pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, startDistances + 1, p->numHashBytes - 1) - startDistances); @@ -363,79 +680,112 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) } #else { - UInt32 posRes; - curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes); - p->hashBufPos += posRes - pos; - cyclicBufferPos += posRes - pos; - p->buffer += posRes - pos; - pos = posRes; + UInt32 posRes = pos; + const UInt32 *d_end; + { + d_end = GetMatchesSpecN_2( + p->buffer + lenLimit - 1, + pos, p->buffer, p->son, p->cutValue, d + curPos, + p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + d + limit, p->hashBuf + p->hashBufPos + size, + cyclicBufferPos, p->cyclicBufferSize, + &posRes); + } + { + if (!d_end) + { + // printf("\n == 2 BtGetMatches() p->failure_BT\n"); + // internal data failure + p->failure_BT = True; + d[0] = 0; + // d[1] = 0; + return; + } + } + curPos = (UInt32)(d_end - d); + { + const UInt32 processed = posRes - pos; + pos = posRes; + p->hashBufPos += processed; + cyclicBufferPos += processed; + p->buffer += processed; + } } #endif - numProcessed += pos - p->pos; - p->hashNumAvail -= pos - p->pos; - p->pos = pos; + { + const UInt32 processed = pos - p->pos; + numProcessed += processed; + p->hashNumAvail -= processed; + p->pos = pos; + } if (cyclicBufferPos == p->cyclicBufferSize) cyclicBufferPos = 0; p->cyclicBufferPos = cyclicBufferPos; } } - distances[0] = curPos; + d[0] = curPos; } + static void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) { CMtSync *sync = &p->hashSync; + + BUFFER_MUST_BE_UNLOCKED(sync) + if (!sync->needStart) { - CriticalSection_Enter(&sync->cs); - sync->csWasEntered = True; + LOCK_BUFFER(sync) } - BtGetMatches(p, p->btBuf + (globalBlockIndex & kMtBtNumBlocksMask) * kMtBtBlockSize); - - if (p->pos > kMtMaxValForNormalize - kMtBtBlockSize) - { - UInt32 subValue = p->pos - p->cyclicBufferSize; - MatchFinder_Normalize3(subValue, p->son, (size_t)p->cyclicBufferSize * 2); - p->pos -= subValue; - } + BtGetMatches(p, p->btBuf + GET_BT_BLOCK_OFFSET(globalBlockIndex)); + + /* We suppose that we have called GetNextBlock() from start. + So buffer is LOCKED */ - if (!sync->needStart) - { - CriticalSection_Leave(&sync->cs); - sync->csWasEntered = False; - } + UNLOCK_BUFFER(sync) } -void BtThreadFunc(CMatchFinderMt *mt) + +MY_NO_INLINE +static void BtThreadFunc(CMatchFinderMt *mt) { CMtSync *p = &mt->btSync; for (;;) { UInt32 blockIndex = 0; Event_Wait(&p->canStart); - Event_Set(&p->wasStarted); + for (;;) { + PRF(printf(" BT thread block = %d pos = %d\n", (unsigned)blockIndex, mt->pos)); + /* (p->exit == true) is possible after (p->canStart) at first loop iteration + and is unexpected after more Wait(freeSemaphore) iterations */ if (p->exit) return; + + Semaphore_Wait(&p->freeSemaphore); + + // for faster stop : we check (p->stopWriting) after Wait(freeSemaphore) if (p->stopWriting) - { - p->numProcessedBlocks = blockIndex; - MtSync_StopWriting(&mt->hashSync); - Event_Set(&p->wasStopped); break; - } - Semaphore_Wait(&p->freeSemaphore); + BtFillBlock(mt, blockIndex++); + Semaphore_Release1(&p->filledSemaphore); } + + // we stop HASH_THREAD here + MtSync_StopWriting(&mt->hashSync); + + // p->numBlocks_Sent = blockIndex; + Event_Set(&p->wasStopped); } } + void MatchFinderMt_Construct(CMatchFinderMt *p) { p->hashBuf = NULL; @@ -443,24 +793,47 @@ void MatchFinderMt_Construct(CMatchFinderMt *p) MtSync_Construct(&p->btSync); } -static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAlloc *alloc) +static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->hashBuf); + ISzAlloc_Free(alloc, p->hashBuf); p->hashBuf = NULL; } -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc) +void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc) { - MtSync_Destruct(&p->hashSync); + /* + HASH_THREAD can use CriticalSection(s) btSync.cs and hashSync.cs. + So we must be sure that HASH_THREAD will not use CriticalSection(s) + after deleting CriticalSection here. + + we call ReleaseStream(p) + that calls StopWriting(btSync) + that calls StopWriting(hashSync), if it's required to stop HASH_THREAD. + after StopWriting() it's safe to destruct MtSync(s) in any order */ + + MatchFinderMt_ReleaseStream(p); + MtSync_Destruct(&p->btSync); + MtSync_Destruct(&p->hashSync); + + LOG_ITER( + printf("\nTree %9d * %7d iter = %9d = sum : bytes = %9d\n", + (UInt32)(g_NumIters_Tree / 1000), + (UInt32)(((UInt64)g_NumIters_Loop * 1000) / (g_NumIters_Tree + 1)), + (UInt32)(g_NumIters_Loop / 1000), + (UInt32)(g_NumIters_Bytes / 1000) + )); + MatchFinderMt_FreeMem(p, alloc); } + #define kHashBufferSize (kMtHashBlockSize * kMtHashNumBlocks) #define kBtBufferSize (kMtBtBlockSize * kMtBtNumBlocks) -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE HashThreadFunc2(void *p) { HashThreadFunc((CMatchFinderMt *)p); return 0; } -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE BtThreadFunc2(void *p) + +static THREAD_FUNC_DECL HashThreadFunc2(void *p) { HashThreadFunc((CMatchFinderMt *)p); return 0; } +static THREAD_FUNC_DECL BtThreadFunc2(void *p) { Byte allocaDummy[0x180]; unsigned i = 0; @@ -471,16 +844,17 @@ static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE BtThreadFunc2(void *p) return 0; } + SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc) + UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc) { - CMatchFinder *mf = p->MatchFinder; + CMatchFinder *mf = MF(p); p->historySize = historySize; if (kMtBtBlockSize <= matchMaxLen * 4) return SZ_ERROR_PARAM; if (!p->hashBuf) { - p->hashBuf = (UInt32 *)alloc->Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); + p->hashBuf = (UInt32 *)ISzAlloc_Alloc(alloc, ((size_t)kHashBufferSize + (size_t)kBtBufferSize) * sizeof(UInt32)); if (!p->hashBuf) return SZ_ERROR_MEM; p->btBuf = p->hashBuf + kHashBufferSize; @@ -490,241 +864,457 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB if (!MatchFinder_Create(mf, historySize, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter, alloc)) return SZ_ERROR_MEM; - RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p, kMtHashNumBlocks)); - RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p, kMtBtNumBlocks)); + RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p)); + RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p)); return SZ_OK; } -/* Call it after ReleaseStream / SetStream */ -void MatchFinderMt_Init(CMatchFinderMt *p) + +SRes MatchFinderMt_InitMt(CMatchFinderMt *p) +{ + RINOK(MtSync_Init(&p->hashSync, kMtHashNumBlocks)); + return MtSync_Init(&p->btSync, kMtBtNumBlocks); +} + + +static void MatchFinderMt_Init(CMatchFinderMt *p) { - CMatchFinder *mf = p->MatchFinder; - p->btBufPos = p->btBufPosLimit = 0; - p->hashBufPos = p->hashBufPosLimit = 0; + CMatchFinder *mf = MF(p); + + p->btBufPos = + p->btBufPosLimit = NULL; + p->hashBufPos = + p->hashBufPosLimit = 0; + p->hashNumAvail = 0; // 21.03 + + p->failure_BT = False; /* Init without data reading. We don't want to read data in this thread */ - MatchFinder_Init_2(mf, False); + MatchFinder_Init_4(mf); + + MatchFinder_Init_LowHash(mf); p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf); p->btNumAvailBytes = 0; - p->lzPos = p->historySize + 1; + p->failure_LZ_BT = False; + // p->failure_LZ_LZ = False; + + p->lzPos = + 1; // optimal smallest value + // 0; // for debug: ignores match to start + // kNormalizeAlign; // for debug p->hash = mf->hash; p->fixedHashSize = mf->fixedHashSize; + // p->hash4Mask = mf->hash4Mask; p->crc = mf->crc; + // memcpy(p->crc, mf->crc, sizeof(mf->crc)); p->son = mf->son; p->matchMaxLen = mf->matchMaxLen; p->numHashBytes = mf->numHashBytes; - p->pos = mf->pos; - p->buffer = mf->buffer; - p->cyclicBufferPos = mf->cyclicBufferPos; + + /* (mf->pos) and (mf->streamPos) were already initialized to 1 in MatchFinder_Init_4() */ + // mf->streamPos = mf->pos = 1; // optimal smallest value + // 0; // for debug: ignores match to start + // kNormalizeAlign; // for debug + + /* we must init (p->pos = mf->pos) for BT, because + BT code needs (p->pos == delta_value_for_empty_hash_record == mf->pos) */ + p->pos = mf->pos; // do not change it + + p->cyclicBufferPos = (p->pos - CYC_TO_POS_OFFSET); p->cyclicBufferSize = mf->cyclicBufferSize; + p->buffer = mf->buffer; p->cutValue = mf->cutValue; + // p->son[0] = p->son[1] = 0; // unused: to init skipped record for speculated accesses. } + /* ReleaseStream is required to finish multithreading */ void MatchFinderMt_ReleaseStream(CMatchFinderMt *p) { + // Sleep(1); // for debug MtSync_StopWriting(&p->btSync); + // Sleep(200); // for debug /* p->MatchFinder->ReleaseStream(); */ } -static void MatchFinderMt_Normalize(CMatchFinderMt *p) -{ - MatchFinder_Normalize3(p->lzPos - p->historySize - 1, p->hash, p->fixedHashSize); - p->lzPos = p->historySize + 1; -} -static void MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) +MY_NO_INLINE +static UInt32 MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) { - UInt32 blockIndex; - MtSync_GetNextBlock(&p->btSync); - blockIndex = ((p->btSync.numProcessedBlocks - 1) & kMtBtNumBlocksMask); - p->btBufPosLimit = p->btBufPos = blockIndex * kMtBtBlockSize; - p->btBufPosLimit += p->btBuf[p->btBufPos++]; - p->btNumAvailBytes = p->btBuf[p->btBufPos++]; - if (p->lzPos >= kMtMaxValForNormalize - kMtBtBlockSize) - MatchFinderMt_Normalize(p); + if (p->failure_LZ_BT) + p->btBufPos = p->failureBuf; + else + { + const UInt32 bi = MtSync_GetNextBlock(&p->btSync); + const UInt32 *bt = p->btBuf + GET_BT_BLOCK_OFFSET(bi); + { + const UInt32 numItems = bt[0]; + p->btBufPosLimit = bt + numItems; + p->btNumAvailBytes = bt[1]; + p->btBufPos = bt + 2; + if (numItems < 2 || numItems > kMtBtBlockSize) + { + p->failureBuf[0] = 0; + p->btBufPos = p->failureBuf; + p->btBufPosLimit = p->failureBuf + 1; + p->failure_LZ_BT = True; + // p->btNumAvailBytes = 0; + /* we don't want to decrease AvailBytes, that was load before. + that can be unxepected for the code that have loaded anopther value before */ + } + } + + if (p->lzPos >= (UInt32)kMtMaxValForNormalize - (UInt32)kMtBtBlockSize) + { + /* we don't check (lzPos) over exact avail bytes in (btBuf). + (fixedHashSize) is small, so normalization is fast */ + const UInt32 subValue = (p->lzPos - p->historySize - 1); // & ~(UInt32)(kNormalizeAlign - 1); + p->lzPos -= subValue; + MatchFinder_Normalize3(subValue, p->hash, p->fixedHashSize); + } + } + return p->btNumAvailBytes; } + + static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) { return p->pointerToCurPos; } + #define GET_NEXT_BLOCK_IF_REQUIRED if (p->btBufPos == p->btBufPosLimit) MatchFinderMt_GetNextBlock_Bt(p); + static UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p) { - GET_NEXT_BLOCK_IF_REQUIRED; - return p->btNumAvailBytes; + if (p->btBufPos != p->btBufPosLimit) + return p->btNumAvailBytes; + return MatchFinderMt_GetNextBlock_Bt(p); } -static UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) + +// #define CHECK_FAILURE_LZ(_match_, _pos_) if (_match_ >= _pos_) { p->failure_LZ_LZ = True; return d; } +#define CHECK_FAILURE_LZ(_match_, _pos_) + +static UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) { - UInt32 h2, curMatch2; + UInt32 h2, c2; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; + const UInt32 m = p->lzPos; MT_HASH2_CALC - curMatch2 = hash[h2]; - hash[h2] = lzPos; + c2 = hash[h2]; + hash[h2] = m; - if (curMatch2 >= matchMinPos) - if (cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) + if (c2 >= matchMinPos) + { + CHECK_FAILURE_LZ(c2, m) + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - *distances++ = 2; - *distances++ = lzPos - curMatch2 - 1; + *d++ = 2; + *d++ = m - c2 - 1; } + } - return distances; + return d; } -static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) +static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) { - UInt32 h2, h3, curMatch2, curMatch3; + UInt32 h2, h3, c2, c3; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; + const UInt32 m = p->lzPos; MT_HASH3_CALC - curMatch2 = hash[ h2]; - curMatch3 = hash[kFix3HashSize + h3]; + c2 = hash[h2]; + c3 = (hash + kFix3HashSize)[h3]; - hash[ h2] = lzPos; - hash[kFix3HashSize + h3] = lzPos; + hash[h2] = m; + (hash + kFix3HashSize)[h3] = m; - if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) + if (c2 >= matchMinPos) { - distances[1] = lzPos - curMatch2 - 1; - if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2]) + CHECK_FAILURE_LZ(c2, m) + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - distances[0] = 3; - return distances + 2; + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) + { + d[0] = 3; + return d + 2; + } + d[0] = 2; + d += 2; } - distances[0] = 2; - distances += 2; } - if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0]) + if (c3 >= matchMinPos) { - *distances++ = 3; - *distances++ = lzPos - curMatch3 - 1; + CHECK_FAILURE_LZ(c3, m) + if (cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) + { + *d++ = 3; + *d++ = m - c3 - 1; + } } - return distances; + return d; } + +#define INCREASE_LZ_POS p->lzPos++; p->pointerToCurPos++; + /* -static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) +static +UInt32* MatchFinderMt_GetMatches_Bt4(CMatchFinderMt *p, UInt32 *d) { - UInt32 h2, h3, h4, curMatch2, curMatch3, curMatch4; + const UInt32 *bt = p->btBufPos; + const UInt32 len = *bt++; + const UInt32 *btLim = bt + len; + UInt32 matchMinPos; + UInt32 avail = p->btNumAvailBytes - 1; + p->btBufPos = btLim; + + { + p->btNumAvailBytes = avail; + + #define BT_HASH_BYTES_MAX 5 + + matchMinPos = p->lzPos; + + if (len != 0) + matchMinPos -= bt[1]; + else if (avail < (BT_HASH_BYTES_MAX - 1) - 1) + { + INCREASE_LZ_POS + return d; + } + else + { + const UInt32 hs = p->historySize; + if (matchMinPos > hs) + matchMinPos -= hs; + else + matchMinPos = 1; + } + } + + for (;;) + { + + UInt32 h2, h3, c2, c3; UInt32 *hash = p->hash; const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; - MT_HASH4_CALC - - curMatch2 = hash[ h2]; - curMatch3 = hash[kFix3HashSize + h3]; - curMatch4 = hash[kFix4HashSize + h4]; + UInt32 m = p->lzPos; + MT_HASH3_CALC + + c2 = hash[h2]; + c3 = (hash + kFix3HashSize)[h3]; + + hash[h2] = m; + (hash + kFix3HashSize)[h3] = m; + + if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) + { + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) + { + d[0] = 3; + d += 2; + break; + } + // else + { + d[0] = 2; + d += 2; + } + } + if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) + { + *d++ = 3; + *d++ = m - c3 - 1; + } + break; + } + + if (len != 0) + { + do + { + const UInt32 v0 = bt[0]; + const UInt32 v1 = bt[1]; + bt += 2; + d[0] = v0; + d[1] = v1; + d += 2; + } + while (bt != btLim); + } + INCREASE_LZ_POS + return d; +} +*/ + + +static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) +{ + UInt32 h2, h3, /* h4, */ c2, c3 /* , c4 */; + UInt32 *hash = p->hash; + const Byte *cur = p->pointerToCurPos; + const UInt32 m = p->lzPos; + MT_HASH3_CALC + // MT_HASH4_CALC + c2 = hash[h2]; + c3 = (hash + kFix3HashSize)[h3]; + // c4 = (hash + kFix4HashSize)[h4]; - hash[ h2] = lzPos; - hash[kFix3HashSize + h3] = lzPos; - hash[kFix4HashSize + h4] = lzPos; + hash[h2] = m; + (hash + kFix3HashSize)[h3] = m; + // (hash + kFix4HashSize)[h4] = m; + + #define _USE_H2 - if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) + #ifdef _USE_H2 + if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { - distances[1] = lzPos - curMatch2 - 1; - if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2]) + d[1] = m - c2 - 1; + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 2] == cur[2]) { - distances[0] = (cur[(ptrdiff_t)curMatch2 - lzPos + 3] == cur[3]) ? 4 : 3; - return distances + 2; + // d[0] = (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 3] == cur[3]) ? 4 : 3; + // return d + 2; + + if (cur[(ptrdiff_t)c2 - (ptrdiff_t)m + 3] == cur[3]) + { + d[0] = 4; + return d + 2; + } + d[0] = 3; + d += 2; + + #ifdef _USE_H4 + if (c4 >= matchMinPos) + if ( + cur[(ptrdiff_t)c4 - (ptrdiff_t)m] == cur[0] && + cur[(ptrdiff_t)c4 - (ptrdiff_t)m + 3] == cur[3] + ) + { + *d++ = 4; + *d++ = m - c4 - 1; + } + #endif + return d; } - distances[0] = 2; - distances += 2; + d[0] = 2; + d += 2; } + #endif - if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0]) + if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) { - distances[1] = lzPos - curMatch3 - 1; - if (cur[(ptrdiff_t)curMatch3 - lzPos + 3] == cur[3]) + d[1] = m - c3 - 1; + if (cur[(ptrdiff_t)c3 - (ptrdiff_t)m + 3] == cur[3]) { - distances[0] = 4; - return distances + 2; + d[0] = 4; + return d + 2; } - distances[0] = 3; - distances += 2; + d[0] = 3; + d += 2; } - if (curMatch4 >= matchMinPos) + #ifdef _USE_H4 + if (c4 >= matchMinPos) if ( - cur[(ptrdiff_t)curMatch4 - lzPos] == cur[0] && - cur[(ptrdiff_t)curMatch4 - lzPos + 3] == cur[3] + cur[(ptrdiff_t)c4 - (ptrdiff_t)m] == cur[0] && + cur[(ptrdiff_t)c4 - (ptrdiff_t)m + 3] == cur[3] ) { - *distances++ = 4; - *distances++ = lzPos - curMatch4 - 1; + *d++ = 4; + *d++ = m - c4 - 1; } + #endif - return distances; + return d; } -*/ -#define INCREASE_LZ_POS p->lzPos++; p->pointerToCurPos++; -static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances) +static UInt32* MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) { - const UInt32 *btBuf = p->btBuf + p->btBufPos; - UInt32 len = *btBuf++; - p->btBufPos += 1 + len; + const UInt32 *bt = p->btBufPos; + const UInt32 len = *bt++; + const UInt32 *btLim = bt + len; + p->btBufPos = btLim; p->btNumAvailBytes--; + INCREASE_LZ_POS { - UInt32 i; - for (i = 0; i < len; i += 2) + while (bt != btLim) { - *distances++ = *btBuf++; - *distances++ = *btBuf++; + const UInt32 v0 = bt[0]; + const UInt32 v1 = bt[1]; + bt += 2; + d[0] = v0; + d[1] = v1; + d += 2; } } - INCREASE_LZ_POS - return len; + return d; } -static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances) -{ - const UInt32 *btBuf = p->btBuf + p->btBufPos; - UInt32 len = *btBuf++; - p->btBufPos += 1 + len; + +static UInt32* MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) +{ + const UInt32 *bt = p->btBufPos; + UInt32 len = *bt++; + const UInt32 avail = p->btNumAvailBytes - 1; + p->btNumAvailBytes = avail; + p->btBufPos = bt + len; if (len == 0) { - /* change for bt5 ! */ - if (p->btNumAvailBytes-- >= 4) - len = (UInt32)(p->MixMatchesFunc(p, p->lzPos - p->historySize, distances) - (distances)); + #define BT_HASH_BYTES_MAX 5 + if (avail >= (BT_HASH_BYTES_MAX - 1) - 1) + { + UInt32 m = p->lzPos; + if (m > p->historySize) + m -= p->historySize; + else + m = 1; + d = p->MixMatchesFunc(p, m, d); + } } else { - /* Condition: there are matches in btBuf with length < p->numHashBytes */ - UInt32 *distances2; - p->btNumAvailBytes--; - distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances); + /* + first match pair from BinTree: (match_len, match_dist), + (match_len >= numHashBytes). + MixMatchesFunc() inserts only hash matches that are nearer than (match_dist) + */ + d = p->MixMatchesFunc(p, p->lzPos - bt[1], d); + // if (d) // check for failure do { - *distances2++ = *btBuf++; - *distances2++ = *btBuf++; + const UInt32 v0 = bt[0]; + const UInt32 v1 = bt[1]; + bt += 2; + d[0] = v0; + d[1] = v1; + d += 2; } - while ((len -= 2) != 0); - len = (UInt32)(distances2 - (distances)); + while (len -= 2); } INCREASE_LZ_POS - return len; + return d; } #define SKIP_HEADER2_MT do { GET_NEXT_BLOCK_IF_REQUIRED #define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; -#define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0); +#define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += (size_t)*p->btBufPos + 1; } while (--num != 0); static void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) { @@ -746,58 +1336,65 @@ static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) SKIP_HEADER_MT(3) UInt32 h2, h3; MT_HASH3_CALC - hash[kFix3HashSize + h3] = + (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT } /* +// MatchFinderMt4_Skip() is similar to MatchFinderMt3_Skip(). +// The difference is that MatchFinderMt3_Skip() updates hash for last 3 bytes of stream. + static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) { SKIP_HEADER_MT(4) - UInt32 h2, h3, h4; - MT_HASH4_CALC - hash[kFix4HashSize + h4] = - hash[kFix3HashSize + h3] = + UInt32 h2, h3; // h4 + MT_HASH3_CALC + // MT_HASH4_CALC + // (hash + kFix4HashSize)[h4] = + (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT } */ -void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) +void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder2 *vTable) { vTable->Init = (Mf_Init_Func)MatchFinderMt_Init; vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinderMt_GetNumAvailableBytes; vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinderMt_GetPointerToCurrentPos; vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches; - switch (p->MatchFinder->numHashBytes) + switch (MF(p)->numHashBytes) { case 2: p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)0; + p->MixMatchesFunc = (Mf_Mix_Matches)NULL; vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; break; case 3: - p->GetHeadsFunc = GetHeads3; + p->GetHeadsFunc = MF(p)->bigHash ? GetHeads3b : GetHeads3; p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2; vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip; break; - default: - /* case 4: */ - p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads4b : GetHeads4; + case 4: + p->GetHeadsFunc = MF(p)->bigHash ? GetHeads4b : GetHeads4; + + // it's fast inline version of GetMatches() + // vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches_Bt4; + p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches3; vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip; break; - /* default: - p->GetHeadsFunc = GetHeads5; + p->GetHeadsFunc = MF(p)->bigHash ? GetHeads5b : GetHeads5; p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt4_Skip; + vTable->Skip = + (Mf_Skip_Func)MatchFinderMt3_Skip; + // (Mf_Skip_Func)MatchFinderMt4_Skip; break; - */ } } diff --git a/3rdparty/lzma/C/LzFindMt.h b/3rdparty/lzma/C/LzFindMt.h index 89b91fefd6e..660b7244d19 100644 --- a/3rdparty/lzma/C/LzFindMt.h +++ b/3rdparty/lzma/C/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2015-05-03 : Igor Pavlov : Public domain */ +2021-07-12 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H @@ -9,31 +9,26 @@ EXTERN_C_BEGIN -#define kMtHashBlockSize (1 << 13) -#define kMtHashNumBlocks (1 << 3) -#define kMtHashNumBlocksMask (kMtHashNumBlocks - 1) - -#define kMtBtBlockSize (1 << 14) -#define kMtBtNumBlocks (1 << 6) -#define kMtBtNumBlocksMask (kMtBtNumBlocks - 1) - typedef struct _CMtSync { - Bool wasCreated; - Bool needStart; - Bool exit; - Bool stopWriting; - + UInt32 numProcessedBlocks; CThread thread; + UInt64 affinity; + + BoolInt wasCreated; + BoolInt needStart; + BoolInt csWasInitialized; + BoolInt csWasEntered; + + BoolInt exit; + BoolInt stopWriting; + CAutoResetEvent canStart; - CAutoResetEvent wasStarted; CAutoResetEvent wasStopped; CSemaphore freeSemaphore; CSemaphore filledSemaphore; - Bool csWasInitialized; - Bool csWasEntered; CCriticalSection cs; - UInt32 numProcessedBlocks; + // UInt32 numBlocks_Sent; } CMtSync; typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances); @@ -49,18 +44,23 @@ typedef struct _CMatchFinderMt /* LZ */ const Byte *pointerToCurPos; UInt32 *btBuf; - UInt32 btBufPos; - UInt32 btBufPosLimit; + const UInt32 *btBufPos; + const UInt32 *btBufPosLimit; UInt32 lzPos; UInt32 btNumAvailBytes; UInt32 *hash; UInt32 fixedHashSize; + // UInt32 hash4Mask; UInt32 historySize; const UInt32 *crc; Mf_Mix_Matches MixMatchesFunc; - + UInt32 failure_LZ_BT; // failure in BT transfered to LZ + // UInt32 failure_LZ_LZ; // failure in LZ tables + UInt32 failureBuf[1]; + // UInt32 crc[256]; + /* LZ + BT */ CMtSync btSync; Byte btDummy[kMtCacheLineDummy]; @@ -70,6 +70,8 @@ typedef struct _CMatchFinderMt UInt32 hashBufPos; UInt32 hashBufPosLimit; UInt32 hashNumAvail; + UInt32 failure_BT; + CLzRef *son; UInt32 matchMaxLen; @@ -77,7 +79,7 @@ typedef struct _CMatchFinderMt UInt32 pos; const Byte *buffer; UInt32 cyclicBufferPos; - UInt32 cyclicBufferSize; /* it must be historySize + 1 */ + UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */ UInt32 cutValue; /* BT + Hash */ @@ -87,13 +89,19 @@ typedef struct _CMatchFinderMt /* Hash */ Mf_GetHeads GetHeadsFunc; CMatchFinder *MatchFinder; + // CMatchFinder MatchFinder; } CMatchFinderMt; +// only for Mt part void MatchFinderMt_Construct(CMatchFinderMt *p); -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc); +void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc); + SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc); -void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable); + UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc); +void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder2 *vTable); + +/* call MatchFinderMt_InitMt() before IMatchFinder::Init() */ +SRes MatchFinderMt_InitMt(CMatchFinderMt *p); void MatchFinderMt_ReleaseStream(CMatchFinderMt *p); EXTERN_C_END diff --git a/3rdparty/lzma/C/LzFindOpt.c b/3rdparty/lzma/C/LzFindOpt.c new file mode 100644 index 00000000000..8ff006e0743 --- /dev/null +++ b/3rdparty/lzma/C/LzFindOpt.c @@ -0,0 +1,578 @@ +/* LzFindOpt.c -- multithreaded Match finder for LZ algorithms +2021-07-13 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "LzFind.h" + +// #include "LzFindMt.h" + +// #define LOG_ITERS + +// #define LOG_THREAD + +#ifdef LOG_THREAD +#include +#define PRF(x) x +#else +// #define PRF(x) +#endif + +#ifdef LOG_ITERS +#include +UInt64 g_NumIters_Tree; +UInt64 g_NumIters_Loop; +UInt64 g_NumIters_Bytes; +#define LOG_ITER(x) x +#else +#define LOG_ITER(x) +#endif + +// ---------- BT THREAD ---------- + +#define USE_SON_PREFETCH +#define USE_LONG_MATCH_OPT + +#define kEmptyHashValue 0 + +// #define CYC_TO_POS_OFFSET 0 + +// #define CYC_TO_POS_OFFSET 1 // for debug + +/* +MY_NO_INLINE +UInt32 * MY_FAST_CALL GetMatchesSpecN_1(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, UInt32 *posRes) +{ + do + { + UInt32 delta; + if (hash == size) + break; + delta = *hash++; + + if (delta == 0 || delta > (UInt32)pos) + return NULL; + + lenLimit++; + + if (delta == (UInt32)pos) + { + CLzRef *ptr1 = son + ((size_t)pos << 1) - CYC_TO_POS_OFFSET * 2; + *d++ = 0; + ptr1[0] = kEmptyHashValue; + ptr1[1] = kEmptyHashValue; + } +else +{ + UInt32 *_distances = ++d; + + CLzRef *ptr0 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2 + 1; + CLzRef *ptr1 = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + + const Byte *len0 = cur, *len1 = cur; + UInt32 cutValue = _cutValue; + const Byte *maxLen = cur + _maxLen; + + for (LOG_ITER(g_NumIters_Tree++);;) + { + LOG_ITER(g_NumIters_Loop++); + { + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + CLzRef *pair = son + ((size_t)(((ptrdiff_t)pos - CYC_TO_POS_OFFSET) + diff) << 1); + const Byte *len = (len0 < len1 ? len0 : len1); + + #ifdef USE_SON_PREFETCH + const UInt32 pair0 = *pair; + #endif + + if (len[diff] == len[0]) + { + if (++len != lenLimit && len[diff] == len[0]) + while (++len != lenLimit) + { + LOG_ITER(g_NumIters_Bytes++); + if (len[diff] != len[0]) + break; + } + if (maxLen < len) + { + maxLen = len; + *d++ = (UInt32)(len - cur); + *d++ = delta - 1; + + if (len == lenLimit) + { + const UInt32 pair1 = pair[1]; + *ptr1 = + #ifdef USE_SON_PREFETCH + pair0; + #else + pair[0]; + #endif + *ptr0 = pair1; + + _distances[-1] = (UInt32)(d - _distances); + + #ifdef USE_LONG_MATCH_OPT + + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + + { + for (;;) + { + hash++; + pos++; + cur++; + lenLimit++; + { + CLzRef *ptr = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + #if 0 + *(UInt64 *)(void *)ptr = ((const UInt64 *)(const void *)ptr)[diff]; + #else + const UInt32 p0 = ptr[0 + (diff * 2)]; + const UInt32 p1 = ptr[1 + (diff * 2)]; + ptr[0] = p0; + ptr[1] = p1; + // ptr[0] = ptr[0 + (diff * 2)]; + // ptr[1] = ptr[1 + (diff * 2)]; + #endif + } + // PrintSon(son + 2, pos - 1); + // printf("\npos = %x delta = %x\n", pos, delta); + len++; + *d++ = 2; + *d++ = (UInt32)(len - cur); + *d++ = delta - 1; + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + } + } + #endif + + break; + } + } + } + + { + const UInt32 curMatch = (UInt32)pos - delta; // (UInt32)(pos + diff); + if (len[diff] < len[0]) + { + delta = pair[1]; + if (delta >= curMatch) + return NULL; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = *pair; + if (delta >= curMatch) + return NULL; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } + + delta = (UInt32)pos - delta; + + if (--cutValue == 0 || delta >= pos) + { + *ptr0 = *ptr1 = kEmptyHashValue; + _distances[-1] = (UInt32)(d - _distances); + break; + } + } + } + } // for (tree iterations) +} + pos++; + cur++; + } + while (d < limit); + *posRes = (UInt32)pos; + return d; +} +*/ + +/* define cbs if you use 2 functions. + GetMatchesSpecN_1() : (pos < _cyclicBufferSize) + GetMatchesSpecN_2() : (pos >= _cyclicBufferSize) + + do not define cbs if you use 1 function: + GetMatchesSpecN_2() +*/ + +// #define cbs _cyclicBufferSize + +/* + we use size_t for (pos) and (_cyclicBufferPos_ instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ + +UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes); + +MY_NO_INLINE +UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes) +{ + do // while (hash != size) + { + UInt32 delta; + + #ifndef cbs + UInt32 cbs; + #endif + + if (hash == size) + break; + + delta = *hash++; + + if (delta == 0) + return NULL; + + lenLimit++; + + #ifndef cbs + cbs = _cyclicBufferSize; + if ((UInt32)pos < cbs) + { + if (delta > (UInt32)pos) + return NULL; + cbs = (UInt32)pos; + } + #endif + + if (delta >= cbs) + { + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + *d++ = 0; + ptr1[0] = kEmptyHashValue; + ptr1[1] = kEmptyHashValue; + } +else +{ + UInt32 *_distances = ++d; + + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + + UInt32 cutValue = _cutValue; + const Byte *len0 = cur, *len1 = cur; + const Byte *maxLen = cur + _maxLen; + + // if (cutValue == 0) { *ptr0 = *ptr1 = kEmptyHashValue; } else + for (LOG_ITER(g_NumIters_Tree++);;) + { + LOG_ITER(g_NumIters_Loop++); + { + // SPEC code + CLzRef *pair = son + ((size_t)((ptrdiff_t)_cyclicBufferPos - (ptrdiff_t)delta + + (ptrdiff_t)(UInt32)(_cyclicBufferPos < delta ? cbs : 0) + ) << 1); + + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + const Byte *len = (len0 < len1 ? len0 : len1); + + #ifdef USE_SON_PREFETCH + const UInt32 pair0 = *pair; + #endif + + if (len[diff] == len[0]) + { + if (++len != lenLimit && len[diff] == len[0]) + while (++len != lenLimit) + { + LOG_ITER(g_NumIters_Bytes++); + if (len[diff] != len[0]) + break; + } + if (maxLen < len) + { + maxLen = len; + *d++ = (UInt32)(len - cur); + *d++ = delta - 1; + + if (len == lenLimit) + { + const UInt32 pair1 = pair[1]; + *ptr1 = + #ifdef USE_SON_PREFETCH + pair0; + #else + pair[0]; + #endif + *ptr0 = pair1; + + _distances[-1] = (UInt32)(d - _distances); + + #ifdef USE_LONG_MATCH_OPT + + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + + { + for (;;) + { + *d++ = 2; + *d++ = (UInt32)(lenLimit - cur); + *d++ = delta - 1; + cur++; + lenLimit++; + // SPEC + _cyclicBufferPos++; + { + // SPEC code + CLzRef *dest = son + ((size_t)(_cyclicBufferPos) << 1); + const CLzRef *src = dest + ((diff + + (ptrdiff_t)(UInt32)((_cyclicBufferPos < delta) ? cbs : 0)) << 1); + // CLzRef *ptr = son + ((size_t)(pos) << 1) - CYC_TO_POS_OFFSET * 2; + #if 0 + *(UInt64 *)(void *)dest = *((const UInt64 *)(const void *)src); + #else + const UInt32 p0 = src[0]; + const UInt32 p1 = src[1]; + dest[0] = p0; + dest[1] = p1; + #endif + } + pos++; + hash++; + if (hash == size || *hash != delta || lenLimit[diff] != lenLimit[0] || d >= limit) + break; + } // for() end for long matches + } + #endif + + break; // break from TREE iterations + } + } + } + { + const UInt32 curMatch = (UInt32)pos - delta; // (UInt32)(pos + diff); + if (len[diff] < len[0]) + { + delta = pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + if (delta >= curMatch) + return NULL; + } + else + { + delta = *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + if (delta >= curMatch) + return NULL; + } + delta = (UInt32)pos - delta; + + if (--cutValue == 0 || delta >= cbs) + { + *ptr0 = *ptr1 = kEmptyHashValue; + _distances[-1] = (UInt32)(d - _distances); + break; + } + } + } + } // for (tree iterations) +} + pos++; + _cyclicBufferPos++; + cur++; + } + while (d < limit); + *posRes = (UInt32)pos; + return d; +} + + + +/* +typedef UInt32 uint32plus; // size_t + +UInt32 * MY_FAST_CALL GetMatchesSpecN_3(uint32plus lenLimit, size_t pos, const Byte *cur, CLzRef *son, + UInt32 _cutValue, UInt32 *d, uint32plus _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, + UInt32 *posRes) +{ + do // while (hash != size) + { + UInt32 delta; + + #ifndef cbs + UInt32 cbs; + #endif + + if (hash == size) + break; + + delta = *hash++; + + if (delta == 0) + return NULL; + + #ifndef cbs + cbs = _cyclicBufferSize; + if ((UInt32)pos < cbs) + { + if (delta > (UInt32)pos) + return NULL; + cbs = (UInt32)pos; + } + #endif + + if (delta >= cbs) + { + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + *d++ = 0; + ptr1[0] = kEmptyHashValue; + ptr1[1] = kEmptyHashValue; + } +else +{ + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + UInt32 *_distances = ++d; + uint32plus len0 = 0, len1 = 0; + UInt32 cutValue = _cutValue; + uint32plus maxLen = _maxLen; + // lenLimit++; // const Byte *lenLimit = cur + _lenLimit; + + for (LOG_ITER(g_NumIters_Tree++);;) + { + LOG_ITER(g_NumIters_Loop++); + { + // const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + CLzRef *pair = son + ((size_t)((ptrdiff_t)_cyclicBufferPos - delta + + (ptrdiff_t)(UInt32)(_cyclicBufferPos < delta ? cbs : 0) + ) << 1); + const Byte *pb = cur - delta; + uint32plus len = (len0 < len1 ? len0 : len1); + + #ifdef USE_SON_PREFETCH + const UInt32 pair0 = *pair; + #endif + + if (pb[len] == cur[len]) + { + if (++len != lenLimit && pb[len] == cur[len]) + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) + { + maxLen = len; + *d++ = (UInt32)len; + *d++ = delta - 1; + if (len == lenLimit) + { + { + const UInt32 pair1 = pair[1]; + *ptr0 = pair1; + *ptr1 = + #ifdef USE_SON_PREFETCH + pair0; + #else + pair[0]; + #endif + } + + _distances[-1] = (UInt32)(d - _distances); + + #ifdef USE_LONG_MATCH_OPT + + if (hash == size || *hash != delta || pb[lenLimit] != cur[lenLimit] || d >= limit) + break; + + { + const ptrdiff_t diff = (ptrdiff_t)0 - (ptrdiff_t)delta; + for (;;) + { + *d++ = 2; + *d++ = (UInt32)lenLimit; + *d++ = delta - 1; + _cyclicBufferPos++; + { + CLzRef *dest = son + ((size_t)_cyclicBufferPos << 1); + const CLzRef *src = dest + ((diff + + (ptrdiff_t)(UInt32)(_cyclicBufferPos < delta ? cbs : 0)) << 1); + #if 0 + *(UInt64 *)(void *)dest = *((const UInt64 *)(const void *)src); + #else + const UInt32 p0 = src[0]; + const UInt32 p1 = src[1]; + dest[0] = p0; + dest[1] = p1; + #endif + } + hash++; + pos++; + cur++; + pb++; + if (hash == size || *hash != delta || pb[lenLimit] != cur[lenLimit] || d >= limit) + break; + } + } + #endif + + break; + } + } + } + { + const UInt32 curMatch = (UInt32)pos - delta; + if (pb[len] < cur[len]) + { + delta = pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } + + { + if (delta >= curMatch) + return NULL; + delta = (UInt32)pos - delta; + if (delta >= cbs + // delta >= _cyclicBufferSize || delta >= pos + || --cutValue == 0) + { + *ptr0 = *ptr1 = kEmptyHashValue; + _distances[-1] = (UInt32)(d - _distances); + break; + } + } + } + } + } // for (tree iterations) +} + pos++; + _cyclicBufferPos++; + cur++; + } + while (d < limit); + *posRes = (UInt32)pos; + return d; +} +*/ diff --git a/3rdparty/lzma/C/LzHash.h b/3rdparty/lzma/C/LzHash.h index e7c942303dd..77b898cfab5 100644 --- a/3rdparty/lzma/C/LzHash.h +++ b/3rdparty/lzma/C/LzHash.h @@ -1,57 +1,34 @@ /* LzHash.h -- HASH functions for LZ algorithms -2015-04-12 : Igor Pavlov : Public domain */ +2019-10-30 : Igor Pavlov : Public domain */ #ifndef __LZ_HASH_H #define __LZ_HASH_H +/* + (kHash2Size >= (1 << 8)) : Required + (kHash3Size >= (1 << 16)) : Required +*/ + #define kHash2Size (1 << 10) #define kHash3Size (1 << 16) -#define kHash4Size (1 << 20) +// #define kHash4Size (1 << 20) #define kFix3HashSize (kHash2Size) #define kFix4HashSize (kHash2Size + kHash3Size) -#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) - -#define HASH2_CALC hv = cur[0] | ((UInt32)cur[1] << 8); - -#define HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } - -#define HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - hv = (temp ^ (p->crc[cur[3]] << 5)) & p->hashMask; } - -#define HASH5_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - temp ^= (p->crc[cur[3]] << 5); \ - h4 = temp & (kHash4Size - 1); \ - hv = (temp ^ (p->crc[cur[4]] << 3)) & p->hashMask; } - -/* #define HASH_ZIP_CALC hv = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */ -#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; - - -#define MT_HASH2_CALC \ - h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); - -#define MT_HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } - -#define MT_HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - h4 = (temp ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); } +// #define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) + +/* + We use up to 3 crc values for hash: + crc0 + crc1 << Shift_1 + crc2 << Shift_2 + (Shift_1 = 5) and (Shift_2 = 10) is good tradeoff. + Small values for Shift are not good for collision rate. + Big value for Shift_2 increases the minimum size + of hash table, that will be slow for small files. +*/ + +#define kLzHash_CrcShift_1 5 +#define kLzHash_CrcShift_2 10 #endif diff --git a/3rdparty/lzma/C/Lzma2Dec.c b/3rdparty/lzma/C/Lzma2Dec.c index b84f88a4813..ac970a843d9 100644 --- a/3rdparty/lzma/C/Lzma2Dec.c +++ b/3rdparty/lzma/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2015-11-09 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -14,28 +14,22 @@ #include "Lzma2Dec.h" /* -00000000 - EOS -00000001 U U - Uncompressed Reset Dic -00000010 U U - Uncompressed No Reset -100uuuuu U U P P - LZMA no reset -101uuuuu U U P P - LZMA reset state -110uuuuu U U P P S - LZMA reset state + new prop -111uuuuu U U P P S - LZMA reset state + new prop + reset dic +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic u, U - Unpack Size P - Pack Size S - Props */ -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 #define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0) - -#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3) -#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2) +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) #define LZMA2_LCLP_MAX 4 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) @@ -74,14 +68,14 @@ static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props) return SZ_OK; } -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; RINOK(Lzma2Dec_GetOldProps(prop, props)); return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc); } -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; RINOK(Lzma2Dec_GetOldProps(prop, props)); @@ -91,30 +85,40 @@ SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) void Lzma2Dec_Init(CLzma2Dec *p) { p->state = LZMA2_STATE_CONTROL; - p->needInitDic = True; - p->needInitState = True; - p->needInitProp = True; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init LzmaDec_Init(&p->decoder); } -static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) +// ELzma2State +static unsigned Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) { switch (p->state) { case LZMA2_STATE_CONTROL: + p->isExtraMode = False; p->control = b; - PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos)); - PRF(printf(" %2X", (unsigned)b)); - if (p->control == 0) + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); + if (b == 0) return LZMA2_STATE_FINISHED; if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if ((p->control & 0x7F) > 2) + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) return LZMA2_STATE_ERROR; - p->unpackSize = 0; } else - p->unpackSize = (UInt32)(p->control & 0x1F) << 16; + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; + p->unpackSize = (UInt32)(b & 0x1F) << 16; + } return LZMA2_STATE_UNPACK0; case LZMA2_STATE_UNPACK0: @@ -124,8 +128,8 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_UNPACK1: p->unpackSize |= (UInt32)b; p->unpackSize++; - PRF(printf(" %8u", (unsigned)p->unpackSize)); - return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; case LZMA2_STATE_PACK0: p->packSize = (UInt32)b << 8; @@ -134,9 +138,9 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_PACK1: p->packSize |= (UInt32)b; p->packSize++; - PRF(printf(" %8u", (unsigned)p->packSize)); - return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP: - (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA); + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; case LZMA2_STATE_PROP: { @@ -145,13 +149,12 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return LZMA2_STATE_ERROR; lc = b % 9; b /= 9; - p->decoder.prop.pb = b / 5; + p->decoder.prop.pb = (Byte)(b / 5); lp = b % 5; if (lc + lp > LZMA2_LCLP_MAX) return LZMA2_STATE_ERROR; - p->decoder.prop.lc = lc; - p->decoder.prop.lp = lp; - p->needInitProp = False; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } } @@ -167,7 +170,8 @@ static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT s p->processedPos += (UInt32)size; } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); + SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) @@ -176,12 +180,17 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, *srcLen = 0; *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->state != LZMA2_STATE_FINISHED) + while (p->state != LZMA2_STATE_ERROR) { - SizeT dicPos = p->decoder.dicPos; + SizeT dicPos; + + if (p->state == LZMA2_STATE_FINISHED) + { + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } - if (p->state == LZMA2_STATE_ERROR) - return SZ_ERROR_DATA; + dicPos = p->decoder.dicPos; if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY) { @@ -198,29 +207,25 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, } (*srcLen)++; p->state = Lzma2Dec_UpdateState(p, *src++); - if (dicPos == dicLimit && p->state != LZMA2_STATE_FINISHED) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + break; continue; } { - SizeT destSizeCur = dicLimit - dicPos; - SizeT srcSizeCur = inSize - *srcLen; + SizeT inCur = inSize - *srcLen; + SizeT outCur = dicLimit - dicPos; ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY; - if (p->unpackSize <= destSizeCur) + if (outCur >= p->unpackSize) { - destSizeCur = (SizeT)p->unpackSize; + outCur = (SizeT)p->unpackSize; curFinishMode = LZMA_FINISH_END; } if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if (*srcLen == inSize) + if (inCur == 0) { *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; @@ -228,135 +233,241 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - if (initDic) - p->needInitProp = p->needInitState = True; - else if (p->needInitDic) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } - p->needInitDic = False; + BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); LzmaDec_InitDicAndState(&p->decoder, initDic, False); } - if (srcSizeCur > destSizeCur) - srcSizeCur = destSizeCur; - - if (srcSizeCur == 0) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + if (inCur > outCur) + inCur = outCur; + if (inCur == 0) + break; - LzmaDec_UpdateWithUncompressed(&p->decoder, src, srcSizeCur); + LzmaDec_UpdateWithUncompressed(&p->decoder, src, inCur); - src += srcSizeCur; - *srcLen += srcSizeCur; - p->unpackSize -= (UInt32)srcSizeCur; + src += inCur; + *srcLen += inCur; + p->unpackSize -= (UInt32)inCur; p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; } else { - SizeT outSizeProcessed; SRes res; if (p->state == LZMA2_STATE_DATA) { - unsigned mode = LZMA2_GET_LZMA_MODE(p); - Bool initDic = (mode == 3); - Bool initState = (mode != 0); - if ((!initDic && p->needInitDic) || (!initState && p->needInitState)) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } - + BoolInt initDic = (p->control >= 0xE0); + BoolInt initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->needInitDic = False; - p->needInitState = False; p->state = LZMA2_STATE_DATA_CONT; } - if (srcSizeCur > p->packSize) - srcSizeCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + destSizeCur, src, &srcSizeCur, curFinishMode, status); + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; - src += srcSizeCur; - *srcLen += srcSizeCur; - p->packSize -= (UInt32)srcSizeCur; + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); - outSizeProcessed = p->decoder.dicPos - dicPos; - p->unpackSize -= (UInt32)outSizeProcessed; + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + outCur = p->decoder.dicPos - dicPos; + p->unpackSize -= (UInt32)outCur; - RINOK(res); + if (res != 0) + break; + if (*status == LZMA_STATUS_NEEDS_MORE_INPUT) - return res; + { + if (p->packSize == 0) + break; + return SZ_OK; + } - if (srcSizeCur == 0 && outSizeProcessed == 0) + if (inCur == 0 && outCur == 0) { if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK || p->unpackSize != 0 || p->packSize != 0) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + break; p->state = LZMA2_STATE_CONTROL; } - if (*status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) - *status = LZMA_STATUS_NOT_FINISHED; + *status = LZMA_STATUS_NOT_SPECIFIED; } } } - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + *status = LZMA_STATUS_NOT_SPECIFIED; + p->state = LZMA2_STATE_ERROR; + return SZ_ERROR_DATA; } + + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) + { + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; + } + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } + } + } + } + + p->state = LZMA2_STATE_ERROR; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; +} + + + + SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen, inSize = *srcLen; *srcLen = *destLen = 0; + for (;;) { - SizeT srcSizeCur = inSize, outSizeCur, dicPos; + SizeT inCur = inSize, outCur, dicPos; ELzmaFinishMode curFinishMode; SRes res; + if (p->decoder.dicPos == p->decoder.dicBufSize) p->decoder.dicPos = 0; dicPos = p->decoder.dicPos; - if (outSize > p->decoder.dicBufSize - dicPos) - { - outSizeCur = p->decoder.dicBufSize; - curFinishMode = LZMA_FINISH_ANY; - } - else + curFinishMode = LZMA_FINISH_ANY; + outCur = p->decoder.dicBufSize - dicPos; + + if (outCur >= outSize) { - outSizeCur = dicPos + outSize; + outCur = outSize; curFinishMode = finishMode; } - res = Lzma2Dec_DecodeToDic(p, outSizeCur, src, &srcSizeCur, curFinishMode, status); - src += srcSizeCur; - inSize -= srcSizeCur; - *srcLen += srcSizeCur; - outSizeCur = p->decoder.dicPos - dicPos; - memcpy(dest, p->decoder.dic + dicPos, outSizeCur); - dest += outSizeCur; - outSize -= outSizeCur; - *destLen += outSizeCur; + res = Lzma2Dec_DecodeToDic(p, dicPos + outCur, src, &inCur, curFinishMode, status); + + src += inCur; + inSize -= inCur; + *srcLen += inCur; + outCur = p->decoder.dicPos - dicPos; + memcpy(dest, p->decoder.dic + dicPos, outCur); + dest += outCur; + outSize -= outCur; + *destLen += outCur; if (res != 0) return res; - if (outSizeCur == 0 || outSize == 0) + if (outCur == 0 || outSize == 0) return SZ_OK; } } + SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc) + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc) { CLzma2Dec p; SRes res; diff --git a/3rdparty/lzma/C/Lzma2Dec.h b/3rdparty/lzma/C/Lzma2Dec.h index e6a0f6ed6db..b8ddeac890b 100644 --- a/3rdparty/lzma/C/Lzma2Dec.h +++ b/3rdparty/lzma/C/Lzma2Dec.h @@ -1,5 +1,5 @@ /* Lzma2Dec.h -- LZMA2 Decoder -2015-05-13 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __LZMA2_DEC_H #define __LZMA2_DEC_H @@ -12,25 +12,24 @@ EXTERN_C_BEGIN typedef struct { - CLzmaDec decoder; - UInt32 packSize; - UInt32 unpackSize; unsigned state; Byte control; - Bool needInitDic; - Bool needInitState; - Bool needInitProp; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; + UInt32 packSize; + UInt32 unpackSize; + CLzmaDec decoder; } CLzma2Dec; #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc); -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc); +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc); -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc); +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); void Lzma2Dec_Init(CLzma2Dec *p); - /* finishMode: It has meaning only if the decoding reaches output limit (*destLen or dicLimit). @@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + /* ---------- One Call Interface ---------- */ /* @@ -73,7 +113,7 @@ Returns: */ SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc); + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc); EXTERN_C_END diff --git a/3rdparty/lzma/C/Lzma2DecMt.c b/3rdparty/lzma/C/Lzma2DecMt.c new file mode 100644 index 00000000000..9f1dc52baa1 --- /dev/null +++ b/3rdparty/lzma/C/Lzma2DecMt.c @@ -0,0 +1,1090 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2021-04-01 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #define _7ZIP_ST + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifndef _7ZIP_ST +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) +#endif + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) +#endif + + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + BoolInt readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + BoolInt mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * (((unsigned)1 << 11) + (1 << 8) + (1 << 6))) & (((unsigned)1 << kNumAlignBits) - ((unsigned)1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + BoolInt overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + BoolInt blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, BoolInt isCross, + BoolInt *needContinue, BoolInt *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + BoolInt needContinue2 = True; + + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + UNUSED_VAR(isCross) + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + BoolInt outFinished; + BoolInt needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)(p->inBuf), &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + p->readRes = SZ_OK; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback2 vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + BoolInt needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_ERROR_INPUT_EOF) + { + if (p->readRes != SZ_OK) + res = p->readRes; + } + else if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/3rdparty/lzma/C/Lzma2DecMt.h b/3rdparty/lzma/C/Lzma2DecMt.h new file mode 100644 index 00000000000..7791c310bcb --- /dev/null +++ b/3rdparty/lzma/C/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/3rdparty/lzma/C/Lzma2Enc.c b/3rdparty/lzma/C/Lzma2Enc.c index 1b77a2bbc1c..e61a5dfe0b3 100644 --- a/3rdparty/lzma/C/Lzma2Enc.c +++ b/3rdparty/lzma/C/Lzma2Enc.c @@ -1,9 +1,8 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2015-10-04 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" -/* #include */ #include /* #define _7ZIP_ST */ @@ -13,7 +12,7 @@ #ifndef _7ZIP_ST #include "MtCoder.h" #else -#define NUM_MT_CODER_THREADS_MAX 1 +#define MTCODER__THREADS_MAX 1 #endif #define LZMA2_CONTROL_LZMA (1 << 7) @@ -35,41 +34,97 @@ #define PRF(x) /* x */ + +/* ---------- CLimitedSeqInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *realStream; + UInt64 limit; + UInt64 processed; + int finished; +} CLimitedSeqInStream; + +static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) +{ + p->limit = (UInt64)(Int64)-1; + p->processed = 0; + p->finished = 0; +} + +static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->finished = (size2 == 0 ? 1 : 0); + p->processed += size2; + } + *size = size2; + return res; +} + + /* ---------- CLzma2EncInt ---------- */ typedef struct { CLzmaEncHandle enc; + Byte propsAreSet; + Byte propsByte; + Byte needInitState; + Byte needInitProp; UInt64 srcPos; - Byte props; - Bool needInitState; - Bool needInitProp; } CLzma2EncInt; -static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) + +static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) +{ + if (!p->propsAreSet) + { + SizeT propsSize = LZMA_PROPS_SIZE; + Byte propsEncoded[LZMA_PROPS_SIZE]; + RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); + RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); + p->propsByte = propsEncoded[0]; + p->propsAreSet = True; + } + return SZ_OK; +} + +static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) { - Byte propsEncoded[LZMA_PROPS_SIZE]; - SizeT propsSize = LZMA_PROPS_SIZE; - RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); - RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); p->srcPos = 0; - p->props = propsEncoded[0]; p->needInitState = True; p->needInitProp = True; - return SZ_OK; } + SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig); + ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); void LzmaEnc_Finish(CLzmaEncHandle pp); void LzmaEnc_SaveState(CLzmaEncHandle pp); void LzmaEnc_RestoreState(CLzmaEncHandle pp); +/* +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); +*/ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t *packSizeRes, ISeqOutStream *outStream) @@ -78,7 +133,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t packSize = packSizeLimit; UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); - Bool useCopyBlock; + BoolInt useCopyBlock; SRes res; *packSizeRes = 0; @@ -126,7 +181,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, if (outStream) { *packSizeRes += destPos; - if (outStream->Write(outStream, outBuf, destPos) != destPos) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) return SZ_ERROR_WRITE; destPos = 0; } @@ -154,7 +209,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, outBuf[destPos++] = (Byte)pm; if (p->needInitProp) - outBuf[destPos++] = p->props; + outBuf[destPos++] = p->propsByte; p->needInitProp = False; p->needInitState = False; @@ -162,7 +217,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, p->srcPos += unpackSize; if (outStream) - if (outStream->Write(outStream, outBuf, destPos) != destPos) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) return SZ_ERROR_WRITE; *packSizeRes = destPos; @@ -176,14 +231,16 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, void Lzma2EncProps_Init(CLzma2EncProps *p) { LzmaEncProps_Init(&p->lzmaProps); + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; p->numTotalThreads = -1; - p->numBlockThreads = -1; - p->blockSize = 0; } void Lzma2EncProps_Normalize(CLzma2EncProps *p) { - int t1, t1n, t2, t3; + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; { CLzmaEncProps lzmaProps = p->lzmaProps; LzmaEncProps_Normalize(&lzmaProps); @@ -191,11 +248,11 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) } t1 = p->lzmaProps.numThreads; - t2 = p->numBlockThreads; + t2 = p->numBlockThreads_Max; t3 = p->numTotalThreads; - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; if (t3 <= 0) { @@ -211,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) t1 = 1; t2 = t3; } - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; } else if (t1 <= 0) { @@ -225,46 +282,71 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) p->lzmaProps.numThreads = t1; + t2r = t2; + + fileSize = p->lzmaProps.reduceSize; + + if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO + && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzmaProps.reduceSize = p->blockSize; + LzmaEncProps_Normalize(&p->lzmaProps); + p->lzmaProps.reduceSize = fileSize; + t1 = p->lzmaProps.numThreads; - if (p->blockSize == 0) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) { - UInt32 dictSize = p->lzmaProps.dictSize; - UInt64 blockSize = (UInt64)dictSize << 2; - const UInt32 kMinSize = (UInt32)1 << 20; - const UInt32 kMaxSize = (UInt32)1 << 28; - if (blockSize < kMinSize) blockSize = kMinSize; - if (blockSize > kMaxSize) blockSize = kMaxSize; - if (blockSize < dictSize) blockSize = dictSize; - p->blockSize = (size_t)blockSize; + t2r = t2 = 1; + t3 = t1; } - - if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1) + else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) + { + /* if there is no block multi-threading, we use SOLID block */ + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + } + else { - UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1; - if (temp > p->lzmaProps.reduceSize) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) { - UInt64 numBlocks = temp / p->blockSize; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + const UInt32 dictSize = p->lzmaProps.dictSize; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + p->blockSize = blockSize; + } + + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; if (numBlocks < (unsigned)t2) { - t2 = (unsigned)numBlocks; - if (t2 == 0) - t2 = 1; - t3 = t1 * t2; + t2r = (int)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; } } } - p->numBlockThreads = t2; + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; p->numTotalThreads = t3; } static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) { - return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; + return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; } @@ -274,180 +356,112 @@ typedef struct { Byte propEncoded; CLzma2EncProps props; + UInt64 expectedDataSize; - Byte *outBuf; + Byte *tempBufLzma; - ISzAlloc *alloc; - ISzAlloc *allocBig; + ISzAllocPtr alloc; + ISzAllocPtr allocBig; - CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; + CLzma2EncInt coders[MTCODER__THREADS_MAX]; #ifndef _7ZIP_ST + + ISeqOutStream *outStream; + Byte *outBuf; + size_t outBuf_Rem; /* remainder in outBuf */ + + size_t outBufSize; /* size of allocated outBufs[i] */ + size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; + Byte *outBufs[MTCODER__BLOCKS_MAX]; + #endif } CLzma2Enc; -/* ---------- Lzma2EncThread ---------- */ -static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt64 packTotal = 0; - SRes res = SZ_OK; - - if (!mainEncoder->outBuf) + CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); + if (!p) + return NULL; + Lzma2EncProps_Init(&p->props); + Lzma2EncProps_Normalize(&p->props); + p->expectedDataSize = (UInt64)(Int64)-1; + p->tempBufLzma = NULL; + p->alloc = alloc; + p->allocBig = allocBig; { - mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); - if (!mainEncoder->outBuf) - return SZ_ERROR_MEM; + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].enc = NULL; } - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - for (;;) + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; { - size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; - res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); - if (res != SZ_OK) - break; - packTotal += packSize; - res = Progress(progress, p->srcPos, packTotal); - if (res != SZ_OK) - break; - if (packSize == 0) - break; + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; } - - LzmaEnc_Finish(p->enc); + #endif - if (res == SZ_OK) - { - Byte b = 0; - if (outStream->Write(outStream, &b, 1) != 1) - return SZ_ERROR_WRITE; - } - - return res; + return p; } #ifndef _7ZIP_ST -typedef struct +static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) { - IMtCoderCallback funcTable; - CLzma2Enc *lzma2Enc; -} CMtCallbackImp; - -static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize, - const Byte *src, size_t srcSize, int finished) -{ - CMtCallbackImp *imp = (CMtCallbackImp *)pp; - CLzma2Enc *mainEncoder = imp->lzma2Enc; - CLzma2EncInt *p = &mainEncoder->coders[index]; - - SRes res = SZ_OK; - { - size_t destLim = *destSize; - *destSize = 0; - - if (srcSize != 0) - { - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - - RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - while (p->srcPos < srcSize) - { - size_t packSize = destLim - *destSize; - res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); - if (res != SZ_OK) - break; - *destSize += packSize; - - if (packSize == 0) - { - res = SZ_ERROR_FAIL; - break; - } - - if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } - - LzmaEnc_Finish(p->enc); - if (res != SZ_OK) - return res; - } - - if (finished) + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) { - if (*destSize == destLim) - return SZ_ERROR_OUTPUT_EOF; - dest[(*destSize)++] = 0; + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; } - } - return res; + p->outBufSize = 0; } #endif -/* ---------- Lzma2Enc ---------- */ - -CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc)); - if (!p) - return NULL; - Lzma2EncProps_Init(&p->props); - Lzma2EncProps_Normalize(&p->props); - p->outBuf = 0; - p->alloc = alloc; - p->allocBig = allocBig; - { - unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->coders[i].enc = 0; - } - - #ifndef _7ZIP_ST - MtCoder_Construct(&p->mtCoder); - #endif - - return p; -} - void Lzma2Enc_Destroy(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + for (i = 0; i < MTCODER__THREADS_MAX; i++) { CLzma2EncInt *t = &p->coders[i]; if (t->enc) { LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); - t->enc = 0; + t->enc = NULL; } } + #ifndef _7ZIP_ST - MtCoder_Destruct(&p->mtCoder); + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + Lzma2Enc_FreeOutBufs(p); #endif - IAlloc_Free(p->alloc, p->outBuf); - IAlloc_Free(p->alloc, pp); + ISzAlloc_Free(p->alloc, p->tempBufLzma); + p->tempBufLzma = NULL; + + ISzAlloc_Free(p->alloc, pp); } + SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) { CLzma2Enc *p = (CLzma2Enc *)pp; @@ -460,6 +474,14 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) return SZ_OK; } + +void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; @@ -471,50 +493,311 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) return (Byte)i; } -SRes Lzma2Enc_Encode(CLzma2EncHandle pp, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) + +static SRes Lzma2Enc_EncodeMt1( + CLzma2Enc *me, + CLzma2EncInt *p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + int finished, + ICompressProgress *progress) { - CLzma2Enc *p = (CLzma2Enc *)pp; - int i; + UInt64 unpackTotal = 0; + UInt64 packTotal = 0; + size_t outLim = 0; + CLimitedSeqInStream limitedInStream; + + if (outBuf) + { + outLim = *outBufSize; + *outBufSize = 0; + } - for (i = 0; i < p->props.numBlockThreads; i++) + if (!p->enc) { - CLzma2EncInt *t = &p->coders[(unsigned)i]; - if (!t->enc) + p->propsAreSet = False; + p->enc = LzmaEnc_Create(me->alloc); + if (!p->enc) + return SZ_ERROR_MEM; + } + + limitedInStream.realStream = inStream; + if (inStream) + { + limitedInStream.vt.Read = LimitedSeqInStream_Read; + } + + if (!outBuf) + { + // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma + if (!me->tempBufLzma) { - t->enc = LzmaEnc_Create(p->alloc); - if (!t->enc) + me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); + if (!me->tempBufLzma) return SZ_ERROR_MEM; } } - #ifndef _7ZIP_ST - if (p->props.numBlockThreads > 1) + RINOK(Lzma2EncInt_InitStream(p, &me->props)); + + for (;;) { - CMtCallbackImp mtCallback; + SRes res = SZ_OK; + size_t inSizeCur = 0; + + Lzma2EncInt_InitBlock(p); + + LimitedSeqInStream_Init(&limitedInStream); + limitedInStream.limit = me->props.blockSize; + + if (inStream) + { + UInt64 expected = (UInt64)(Int64)-1; + // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize + if (me->expectedDataSize != (UInt64)(Int64)-1 + && me->expectedDataSize >= unpackTotal) + expected = me->expectedDataSize - unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && expected > me->props.blockSize) + expected = (size_t)me->props.blockSize; + + LzmaEnc_SetDataSize(p->enc, expected); + + RINOK(LzmaEnc_PrepareForLzma2(p->enc, + &limitedInStream.vt, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + else + { + inSizeCur = inDataSize - (size_t)unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && inSizeCur > me->props.blockSize) + inSizeCur = (size_t)me->props.blockSize; + + // LzmaEnc_SetDataSize(p->enc, inSizeCur); + + RINOK(LzmaEnc_MemPrepare(p->enc, + inData + (size_t)unpackTotal, inSizeCur, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + + for (;;) + { + size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; + if (outBuf) + packSize = outLim - (size_t)packTotal; + + res = Lzma2EncInt_EncodeSubblock(p, + outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize, + outBuf ? NULL : outStream); + + if (res != SZ_OK) + break; - mtCallback.funcTable.Code = MtCallbackImp_Code; - mtCallback.lzma2Enc = p; + packTotal += packSize; + if (outBuf) + *outBufSize = (size_t)packTotal; + + res = Progress(progress, unpackTotal + p->srcPos, packTotal); + if (res != SZ_OK) + break; + + /* + if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0) + break; + */ + + if (packSize == 0) + break; + } + LzmaEnc_Finish(p->enc); + + unpackTotal += p->srcPos; + + RINOK(res); + + if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) + return SZ_ERROR_FAIL; + + if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize)) + { + if (finished) + { + if (outBuf) + { + const size_t destPos = *outBufSize; + if (destPos >= outLim) + return SZ_ERROR_OUTPUT_EOF; + outBuf[destPos] = LZMA2_CONTROL_EOF; // 0 + *outBufSize = destPos + 1; + } + else + { + const Byte b = LZMA2_CONTROL_EOF; // 0; + if (ISeqOutStream_Write(outStream, &b, 1) != 1) + return SZ_ERROR_WRITE; + } + } + return SZ_OK; + } + } +} + + + +#ifndef _7ZIP_ST + +static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t destSize = me->outBufSize; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + me->outBufsDataSizes[outBufIndex] = 0; + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + progressThunk.inSize = 0; + progressThunk.outSize = 0; + + res = Lzma2Enc_EncodeMt1(me, + &me->coders[coderIndex], + NULL, dest, &destSize, + NULL, src, srcSize, + finished, + &progressThunk.vt); + + me->outBufsDataSizes[outBufIndex] = destSize; + + return res; +} + + +static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t size = me->outBufsDataSizes[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + if (me->outStream) + return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; + + if (size > me->outBuf_Rem) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBuf_Rem -= size; + me->outBuf += size; + return SZ_OK; +} + +#endif + + + +SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + + if (inStream && inData) + return SZ_ERROR_PARAM; + + if (outStream && outBuf) + return SZ_ERROR_PARAM; + + { + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].propsAreSet = False; + } + + #ifndef _7ZIP_ST + + if (p->props.numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = Lzma2Enc_MtCallback_Code; + vt.Write = Lzma2Enc_MtCallback_Write; + + p->outStream = outStream; + p->outBuf = NULL; + p->outBuf_Rem = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBuf_Rem = *outBufSize; + *outBufSize = 0; + } + + p->mtCoder.allocBig = p->allocBig; p->mtCoder.progress = progress; p->mtCoder.inStream = inStream; - p->mtCoder.outStream = outStream; - p->mtCoder.alloc = p->alloc; - p->mtCoder.mtCallback = &mtCallback.funcTable; + p->mtCoder.inData = inData; + p->mtCoder.inDataSize = inDataSize; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + p->mtCoder.blockSize = (size_t)p->props.blockSize; + if (p->mtCoder.blockSize != p->props.blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ - p->mtCoder.blockSize = p->props.blockSize; - p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; - if (p->mtCoder.destBlockSize < p->props.blockSize) { - p->mtCoder.destBlockSize = (size_t)0 - 1; - if (p->mtCoder.destBlockSize < p->props.blockSize) - return SZ_ERROR_FAIL; + size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + Lzma2Enc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - p->mtCoder.numThreads = p->props.numBlockThreads; + + p->mtCoder.numThreadsMax = (unsigned)p->props.numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - return MtCoder_Code(&p->mtCoder); + { + SRes res = MtCoder_Code(&p->mtCoder); + if (!outStream) + *outBufSize = (size_t)(p->outBuf - outBuf); + return res; + } } + #endif - return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); + + return Lzma2Enc_EncodeMt1(p, + &p->coders[0], + outStream, outBuf, outBufSize, + inStream, inData, inDataSize, + True, /* finished */ + progress); } diff --git a/3rdparty/lzma/C/Lzma2Enc.h b/3rdparty/lzma/C/Lzma2Enc.h index f409f184cc3..6a6110ff7e7 100644 --- a/3rdparty/lzma/C/Lzma2Enc.h +++ b/3rdparty/lzma/C/Lzma2Enc.h @@ -1,5 +1,5 @@ /* Lzma2Enc.h -- LZMA2 Encoder -2013-01-18 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA2_ENC_H #define __LZMA2_ENC_H @@ -8,11 +8,15 @@ EXTERN_C_BEGIN +#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 +#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) + typedef struct { CLzmaEncProps lzmaProps; - size_t blockSize; - int numBlockThreads; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; int numTotalThreads; } CLzma2EncProps; @@ -22,40 +26,29 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p); /* ---------- CLzmaEnc2Handle Interface ---------- */ /* Lzma2Enc_* functions can return the following exit codes: -Returns: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzma2EncHandle; -CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig); +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); void Lzma2Enc_Destroy(CLzma2EncHandle p); SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); +void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); -SRes Lzma2Enc_Encode(CLzma2EncHandle p, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); - -/* ---------- One Call Interface ---------- */ - -/* Lzma2Encode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -/* -SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); -*/ +SRes Lzma2Enc_Encode2(CLzma2EncHandle p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress); EXTERN_C_END diff --git a/3rdparty/lzma/C/Lzma86Enc.c b/3rdparty/lzma/C/Lzma86Enc.c index a15b58d6cf6..14fcd65c671 100644 --- a/3rdparty/lzma/C/Lzma86Enc.c +++ b/3rdparty/lzma/C/Lzma86Enc.c @@ -1,5 +1,5 @@ /* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2016-05-16 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,14 +11,12 @@ #include "Bra.h" #include "LzmaEnc.h" -#define SZE_OUT_OVERFLOW SZE_DATA_ERROR - int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, int level, UInt32 dictSize, int filterMode) { size_t outSize2 = *destLen; Byte *filteredStream; - Bool useFilter; + BoolInt useFilter; int mainResult = SZ_ERROR_OUTPUT_EOF; CLzmaEncProps props; LzmaEncProps_Init(&props); @@ -56,7 +54,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, { size_t minSize = 0; - Bool bestIsFiltered = False; + BoolInt bestIsFiltered = False; /* passes for SZ_FILTER_AUTO: 0 - BCJ + LZMA @@ -71,7 +69,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; size_t outPropsSize = 5; SRes curRes; - Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); + BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); if (curModeIsFiltered && !bestIsFiltered) break; if (useFilter && i == 0) diff --git a/3rdparty/lzma/C/LzmaDec.c b/3rdparty/lzma/C/LzmaDec.c index 12dce11fbd5..d6742e5af8c 100644 --- a/3rdparty/lzma/C/LzmaDec.c +++ b/3rdparty/lzma/C/LzmaDec.c @@ -1,32 +1,42 @@ /* LzmaDec.c -- LZMA Decoder -2016-05-16 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "LzmaDec.h" - #include +/* #include "CpuArch.h" */ +#include "LzmaDec.h" + #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) #define kNumBitModelTotalBits 11 #define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 #define RC_INIT_SIZE 5 +#ifndef _LZMA_DEC_OPT + +#define kNumMoveBits 5 #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ { UPDATE_0(p); i = (i + i); A0; } else \ { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } @@ -46,16 +56,20 @@ i -= 0x40; } #endif -#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) #define MATCHED_LITER_DEC \ - matchByte <<= 1; \ - bit = (matchByte & offs); \ - probLit = prob + offs + bit + symbol; \ - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + +#endif // _LZMA_DEC_OPT + -#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } +#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_INPUT_EOF; range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ @@ -66,25 +80,28 @@ { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4 @@ -98,54 +115,130 @@ #define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +#define kMatchSpecLen_Error_Data (1 << 9) +#define kMatchSpecLen_Error_Fail (kMatchSpecLen_Error_Data - 1) -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 0x300 +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign #endif -#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : the number of bytes to be copied with (p->rep0) offset + = kMatchSpecLenStart : the LZMA stream was finished with end mark + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state + = kMatchSpecLen_Error_Fail : Internal Code Failure + = kMatchSpecLen_Error_Data + [0 ... 273] : LZMA Data Error +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then the function doesn't write any byte to dictionary, + the function returns SZ_OK, and the caller can use (p->remainLen) and (p->reps[0]) later. + } + +Processing: + The first LZMA symbol will be decoded in any case. + All main checks for limits are at the end of main loop, + It decodes additional LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + But if (p->buf < bufLimit), the caller provided at least (LZMA_REQUIRED_INPUT_MAX + 1) bytes for + next iteration before limit (bufLimit + LZMA_REQUIRED_INPUT_MAX), + that is enough for worst case LZMA symbol with one additional RangeCoder normalization for one bit. + So that function never reads bufLimit [LZMA_REQUIRED_INPUT_MAX] byte. + Out: + RangeCoder is normalized Result: SZ_OK - OK - SZ_ERROR_DATA - Error - p->remainLen: - < kMatchSpecLenStart : normal remain - = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker (unused now) - = kMatchSpecLenStart + 2 : State Init Marker (unused now) + p->remainLen: + < kMatchSpecLenStart : the number of bytes to be copied with (p->reps[0]) offset + = kMatchSpecLenStart : the LZMA stream was finished with end mark + + SZ_ERROR_DATA - error, when the MATCH-Symbol refers out of dictionary + p->remainLen : undefined + p->reps[*] : undefined */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = p->probs; - unsigned state = p->state; +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize; @@ -164,17 +257,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = processedPos & pbMask; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { unsigned symbol; UPDATE_0(prob); prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) - prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); processedPos++; if (state < kNumLitStates) @@ -240,16 +332,21 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte else { UPDATE_1(prob); - if (checkDicSize == 0 && processedPos == 0) - return SZ_ERROR_DATA; prob = probs + IsRepG0 + state; IF_BIT_0(prob) { UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { UPDATE_0(prob); + + // that case was checked before with kBadRepCode + // if (checkDicSize == 0 && processedPos == 0) { len = kMatchSpecLen_Error_Data + 1; break; } + // The caller doesn't allow (dicPos == limit) case here + // so we don't need the following check: + // if (dicPos == limit) { state = state < kNumLitStates ? 9 : 11; len = 1; break; } + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; dicPos++; processedPos++; @@ -299,7 +396,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; lim = (1 << kLenNumLowBits); } @@ -310,15 +407,15 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - lim = (1 << kLenNumMidBits); + lim = (1 << kLenNumLowBits); } else { UPDATE_1(probLen); probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; lim = (1 << kLenNumHighBits); } } @@ -331,7 +428,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -345,7 +442,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -356,7 +453,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte UPDATE_1(probLen); probLen = prob + LenHigh; TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols + kLenNumMidSymbols; + len += kLenNumLowSymbols * 2; } } } @@ -376,16 +473,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if (posSlot < kEndPosModelIndex) { distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; + prob = probs + SpecPos; { - UInt32 mask = 1; - unsigned i = 1; + UInt32 m = 1; + distance++; do { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; + REV_BIT_VAR(prob, distance, m); } - while (--numDirectBits != 0); + while (--numDirectBits); + distance -= m; } } else @@ -412,19 +509,20 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte } */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; + len = kMatchSpecLenStart; state -= kNumStates; break; } @@ -435,20 +533,14 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte rep2 = rep1; rep1 = rep0; rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - else if (distance >= checkDicSize) + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { - p->dicPos = dicPos; - return SZ_ERROR_DATA; + len += kMatchSpecLen_Error_Data + kMatchMinLen; + // len = kMatchSpecLen_Error_Data; + // len += kMatchMinLen; + break; } - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; } len += kMatchMinLen; @@ -460,14 +552,19 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if ((rem = limit - dicPos) == 0) { - p->dicPos = dicPos; - return SZ_ERROR_DATA; + /* + We stop decoding and return SZ_OK, and we can resume decoding later. + Any error conditions can be tested later in caller code. + For more strict mode we can stop decoding with error + // len += kMatchSpecLen_Error_Data; + */ + break; } curLen = ((rem < len) ? (unsigned)rem : len); pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); - processedPos += curLen; + processedPos += (UInt32)curLen; len -= curLen; if (curLen <= dicBufSize - pos) @@ -475,7 +572,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte Byte *dest = dic + dicPos; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; const Byte *lim = dest + curLen; - dicPos += curLen; + dicPos += (SizeT)curLen; do *(dest) = (Byte)*(dest + src); while (++dest != lim); @@ -500,108 +597,147 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte p->buf = buf; p->range = range; p->code = code; - p->remainLen = len; + p->remainLen = (UInt32)len; // & (kMatchSpecLen_Error_Data - 1); // we can write real length for error matches too. p->dicPos = dicPos; p->processedPos = processedPos; p->reps[0] = rep0; p->reps[1] = rep1; p->reps[2] = rep2; p->reps[3] = rep3; - p->state = state; - + p->state = (UInt32)state; + if (len >= kMatchSpecLen_Error_Data) + return SZ_ERROR_DATA; return SZ_OK; } +#endif + + static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { - if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) + unsigned len = (unsigned)p->remainLen; + if (len == 0 /* || len >= kMatchSpecLenStart */) + return; { - Byte *dic = p->dic; SizeT dicPos = p->dicPos; - SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; - SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ - SizeT rem = limit - dicPos; - if (rem < len) - len = (unsigned)(rem); + Byte *dic; + SizeT dicBufSize; + SizeT rep0; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ + { + SizeT rem = limit - dicPos; + if (rem < len) + { + len = (unsigned)(rem); + if (len == 0) + return; + } + } if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) p->checkDicSize = p->prop.dicSize; - p->processedPos += len; - p->remainLen -= len; - while (len != 0) + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; + dic = p->dic; + rep0 = p->reps[0]; + dicBufSize = p->dicBufSize; + do { - len--; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; dicPos++; } + while (--len); p->dicPos = dicPos; } } + +/* +At staring of new stream we have one of the following symbols: + - Literal - is allowed + - Non-Rep-Match - is allowed only if it's end marker symbol + - Rep-Match - is not allowed +We use early check of (RangeCoder:Code) over kBadRepCode to simplify main decoding code +*/ + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + + +/* +LzmaDec_DecodeReal2(): + It calls LZMA_DECODE_REAL() and it adjusts limit according (p->checkDicSize). + +We correct (p->checkDicSize) after LZMA_DECODE_REAL() and in LzmaDec_WriteRem(), +and we support the following state of (p->checkDicSize): + if (total_processed < p->prop.dicSize) then + { + (total_processed == p->processedPos) + (p->checkDicSize == 0) + } + else + (p->checkDicSize == p->prop.dicSize) +*/ + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { - do + if (p->checkDicSize == 0) { - SizeT limit2 = limit; - if (p->checkDicSize == 0) - { - UInt32 rem = p->prop.dicSize - p->processedPos; - if (limit - p->dicPos > rem) - limit2 = p->dicPos + rem; - } - - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); - + UInt32 rem = p->prop.dicSize - p->processedPos; + if (limit - p->dicPos > rem) + limit = p->dicPos + rem; + } + { + int res = LZMA_DECODE_REAL(p, limit, bufLimit); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize; - - LzmaDec_WriteRem(p, limit); + return res; } - while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); +} - if (p->remainLen > kMatchSpecLenStart) - p->remainLen = kMatchSpecLenStart; - return 0; -} typedef enum { - DUMMY_ERROR, /* unexpected end of input stream */ + DUMMY_INPUT_EOF, /* need more input data */ DUMMY_LIT, DUMMY_MATCH, DUMMY_REP } ELzmaDummy; -static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) + +#define IS_DUMMY_END_MARKER_POSSIBLE(dummyRes) ((dummyRes) == DUMMY_MATCH) + +static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byte **bufOut) { UInt32 range = p->range; UInt32 code = p->code; - const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = p->probs; - unsigned state = p->state; + const Byte *bufLimit = *bufOut; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; ELzmaDummy res; + for (;;) { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, ((unsigned)1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK - /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ - prob = probs + Literal; if (p->checkDicSize != 0 || p->processedPos != 0) prob += ((UInt32)LZMA_LIT_SIZE * - ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + - (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); + ((((p->processedPos) & (((unsigned)1 << (p->prop.lp)) - 1)) << p->prop.lc) + + ((unsigned)p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); if (state < kNumLitStates) { @@ -618,10 +754,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned bit; const CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) } while (symbol < 0x100); } @@ -648,12 +785,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - NORMALIZE_CHECK; - return DUMMY_REP; + break; } else { @@ -691,7 +827,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; } @@ -702,15 +838,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; + limit = 1 << kLenNumLowBits; } else { UPDATE_1_CHECK; probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } } @@ -722,18 +858,16 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned posSlot; prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); if (posSlot >= kStartPosModelIndex) { unsigned numDirectBits = ((posSlot >> 1) - 1); - /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ - if (posSlot < kEndPosModelIndex) { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); } else { @@ -745,41 +879,44 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS code -= range & (((code - range) >> 31) - 1); /* if (code >= range) code -= range; */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; } { unsigned i = 1; + unsigned m = 1; do { - GET_BIT_CHECK(prob + i, i); + REV_BIT_CHECK(prob, i, m); } - while (--numDirectBits != 0); + while (--numDirectBits); } } } } + break; } NORMALIZE_CHECK; + + *bufOut = buf; return res; } - -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { - p->needFlush = 1; - p->remainLen = 0; + p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; if (initDic) { p->processedPos = 0; p->checkDicSize = 0; - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } if (initState) - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p) @@ -788,53 +925,96 @@ void LzmaDec_Init(CLzmaDec *p) LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} + +/* +LZMA supports optional end_marker. +So the decoder can lookahead for one additional LZMA-Symbol to check end_marker. +That additional LZMA-Symbol can require up to LZMA_REQUIRED_INPUT_MAX bytes in input stream. +When the decoder reaches dicLimit, it looks (finishMode) parameter: + if (finishMode == LZMA_FINISH_ANY), the decoder doesn't lookahead + if (finishMode != LZMA_FINISH_ANY), the decoder lookahead, if end_marker is possible for current position + +When the decoder lookahead, and the lookahead symbol is not end_marker, we have two ways: + 1) Strict mode (default) : the decoder returns SZ_ERROR_DATA. + 2) The relaxed mode (alternative mode) : we could return SZ_OK, and the caller + must check (status) value. The caller can show the error, + if the end of stream is expected, and the (status) is noit + LZMA_STATUS_FINISHED_WITH_MARK or LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK. +*/ + + +#define RETURN__NOT_FINISHED__FOR_FINISH \ + *status = LZMA_STATUS_NOT_FINISHED; \ + return SZ_ERROR_DATA; // for strict mode + // return SZ_OK; // for relaxed mode + SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); - *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->remainLen != kMatchSpecLenStart) + if (p->remainLen > kMatchSpecLenStart) { - int checkEndMarkNow; + if (p->remainLen > kMatchSpecLenStart + 2) + return p->remainLen == kMatchSpecLen_Error_Fail ? SZ_ERROR_FAIL : SZ_ERROR_DATA; + + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); - if (p->needFlush) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; - p->tempBufSize = 0; - } + if (p->checkDicSize == 0 + && p->processedPos == 0 + && p->code >= kBadRepCode) + return SZ_ERROR_DATA; + + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } + + p->remainLen = 0; + } + + for (;;) + { + if (p->remainLen == kMatchSpecLenStart) + { + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } + + LzmaDec_WriteRem(p, dicLimit); + + { + // (p->remainLen == 0 || p->dicPos == dicLimit) + + int checkEndMarkNow = 0; - checkEndMarkNow = 0; if (p->dicPos >= dicLimit) { if (p->remainLen == 0 && p->code == 0) @@ -849,92 +1029,174 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr } if (p->remainLen != 0) { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; + RETURN__NOT_FINISHED__FOR_FINISH; } checkEndMarkNow = 1; } - if (p->needInitState) - LzmaDec_InitStateReal(p); - + // (p->remainLen == 0) + if (p->tempBufSize == 0) { - SizeT processed; const Byte *bufLimit; + int dummyProcessed = -1; + if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, src, inSize); - if (dummyRes == DUMMY_ERROR) + const Byte *bufOut = src + inSize; + + ELzmaDummy dummyRes = LzmaDec_TryDummy(p, src, &bufOut); + + if (dummyRes == DUMMY_INPUT_EOF) { - memcpy(p->tempBuf, src, inSize); - p->tempBufSize = (unsigned)inSize; + size_t i; + if (inSize >= LZMA_REQUIRED_INPUT_MAX) + break; (*srcLen) += inSize; + p->tempBufSize = (unsigned)inSize; + for (i = 0; i < inSize; i++) + p->tempBuf[i] = src[i]; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + + dummyProcessed = (int)(bufOut - src); + if ((unsigned)dummyProcessed > LZMA_REQUIRED_INPUT_MAX) + break; + + if (checkEndMarkNow && !IS_DUMMY_END_MARKER_POSSIBLE(dummyRes)) { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; + unsigned i; + (*srcLen) += (unsigned)dummyProcessed; + p->tempBufSize = (unsigned)dummyProcessed; + for (i = 0; i < (unsigned)dummyProcessed; i++) + p->tempBuf[i] = src[i]; + // p->remainLen = kMatchSpecLen_Error_Data; + RETURN__NOT_FINISHED__FOR_FINISH; } + bufLimit = src; + // we will decode only one iteration } else bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; + p->buf = src; - if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) - return SZ_ERROR_DATA; - processed = (SizeT)(p->buf - src); - (*srcLen) += processed; - src += processed; - inSize -= processed; + + { + int res = LzmaDec_DecodeReal2(p, dicLimit, bufLimit); + + SizeT processed = (SizeT)(p->buf - src); + + if (dummyProcessed < 0) + { + if (processed > inSize) + break; + } + else if ((unsigned)dummyProcessed != processed) + break; + + src += processed; + inSize -= processed; + (*srcLen) += processed; + + if (res != SZ_OK) + { + p->remainLen = kMatchSpecLen_Error_Data; + return SZ_ERROR_DATA; + } + } + continue; } - else + { - unsigned rem = p->tempBufSize, lookAhead = 0; - while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) - p->tempBuf[rem++] = src[lookAhead++]; - p->tempBufSize = rem; + // we have some data in (p->tempBuf) + // in strict mode: tempBufSize is not enough for one Symbol decoding. + // in relaxed mode: tempBufSize not larger than required for one Symbol decoding. + + unsigned rem = p->tempBufSize; + unsigned ahead = 0; + int dummyProcessed = -1; + + while (rem < LZMA_REQUIRED_INPUT_MAX && ahead < inSize) + p->tempBuf[rem++] = src[ahead++]; + + // ahead - the size of new data copied from (src) to (p->tempBuf) + // rem - the size of temp buffer including new data from (src) + if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); - if (dummyRes == DUMMY_ERROR) + const Byte *bufOut = p->tempBuf + rem; + + ELzmaDummy dummyRes = LzmaDec_TryDummy(p, p->tempBuf, &bufOut); + + if (dummyRes == DUMMY_INPUT_EOF) { - (*srcLen) += lookAhead; + if (rem >= LZMA_REQUIRED_INPUT_MAX) + break; + p->tempBufSize = rem; + (*srcLen) += (SizeT)ahead; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + + dummyProcessed = (int)(bufOut - p->tempBuf); + + if ((unsigned)dummyProcessed < p->tempBufSize) + break; + + if (checkEndMarkNow && !IS_DUMMY_END_MARKER_POSSIBLE(dummyRes)) { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; + (*srcLen) += (unsigned)dummyProcessed - p->tempBufSize; + p->tempBufSize = (unsigned)dummyProcessed; + // p->remainLen = kMatchSpecLen_Error_Data; + RETURN__NOT_FINISHED__FOR_FINISH; } } + p->buf = p->tempBuf; - if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) - return SZ_ERROR_DATA; { - unsigned kkk = (unsigned)(p->buf - p->tempBuf); - if (rem < kkk) - return SZ_ERROR_FAIL; /* some internal error */ - rem -= kkk; - if (lookAhead < rem) - return SZ_ERROR_FAIL; /* some internal error */ - lookAhead -= rem; + // we decode one symbol from (p->tempBuf) here, so the (bufLimit) is equal to (p->buf) + int res = LzmaDec_DecodeReal2(p, dicLimit, p->buf); + + SizeT processed = (SizeT)(p->buf - p->tempBuf); + rem = p->tempBufSize; + + if (dummyProcessed < 0) + { + if (processed > LZMA_REQUIRED_INPUT_MAX) + break; + if (processed < rem) + break; + } + else if ((unsigned)dummyProcessed != processed) + break; + + processed -= rem; + + src += processed; + inSize -= processed; + (*srcLen) += processed; + p->tempBufSize = 0; + + if (res != SZ_OK) + { + p->remainLen = kMatchSpecLen_Error_Data; + return SZ_ERROR_DATA; + } } - (*srcLen) += lookAhead; - src += lookAhead; - inSize -= lookAhead; - p->tempBufSize = 0; } + } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + + /* Some unexpected error: internal error of code, memory corruption or hardware failure */ + p->remainLen = kMatchSpecLen_Error_Fail; + return SZ_ERROR_FAIL; } + + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen; @@ -975,19 +1237,19 @@ SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *sr } } -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->probs); + ISzAlloc_Free(alloc, p->probs); p->probs = NULL; } -static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->dic); + ISzAlloc_Free(alloc, p->dic); p->dic = NULL; } -void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc) { LzmaDec_FreeProbs(p, alloc); LzmaDec_FreeDict(p, alloc); @@ -1011,29 +1273,30 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) if (d >= (9 * 5 * 5)) return SZ_ERROR_UNSUPPORTED; - p->lc = d % 9; + p->lc = (Byte)(d % 9); d /= 9; - p->pb = d / 5; - p->lp = d % 5; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); return SZ_OK; } -static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAllocPtr alloc) { UInt32 numProbs = LzmaProps_GetNumProbs(propNew); if (!p->probs || numProbs != p->numProbs) { LzmaDec_FreeProbs(p, alloc); - p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; + p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); if (!p->probs) return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; } return SZ_OK; } -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; RINOK(LzmaProps_Decode(&propNew, props, propsSize)); @@ -1042,7 +1305,7 @@ SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, I return SZ_OK; } -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; SizeT dicBufSize; @@ -1062,7 +1325,7 @@ SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAll if (!p->dic || dicBufSize != p->dicBufSize) { LzmaDec_FreeDict(p, alloc); - p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); + p->dic = (Byte *)ISzAlloc_Alloc(alloc, dicBufSize); if (!p->dic) { LzmaDec_FreeProbs(p, alloc); @@ -1076,7 +1339,7 @@ SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAll SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc) + ELzmaStatus *status, ISzAllocPtr alloc) { CLzmaDec p; SRes res; diff --git a/3rdparty/lzma/C/LzmaDec.h b/3rdparty/lzma/C/LzmaDec.h index cc44daef214..6f1296250cf 100644 --- a/3rdparty/lzma/C/LzmaDec.h +++ b/3rdparty/lzma/C/LzmaDec.h @@ -1,5 +1,5 @@ /* LzmaDec.h -- LZMA Decoder -2013-01-18 : Igor Pavlov : Public domain */ +2020-03-19 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H @@ -12,11 +12,13 @@ EXTERN_C_BEGIN /* _LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; /* ---------- LZMA Properties ---------- */ @@ -25,7 +27,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps; @@ -47,32 +52,34 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); /* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ typedef enum { @@ -129,11 +136,11 @@ LzmaDec_Allocate* can return: SZ_ERROR_UNSUPPORTED - Unsupported properties */ -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */ @@ -142,7 +149,7 @@ void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); You must work with CLzmaDec variables directly in this interface. STEPS: - LzmaDec_Constr() + LzmaDec_Construct() LzmaDec_Allocate() for (each new stream) { @@ -174,6 +181,7 @@ Returns: LZMA_STATUS_NEEDS_MORE_INPUT LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK SZ_ERROR_DATA - Data error + SZ_ERROR_FAIL - Some unexpected error: internal error of code, memory corruption or hardware failure */ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, @@ -216,11 +224,12 @@ Returns: SZ_ERROR_MEM - Memory allocation error SZ_ERROR_UNSUPPORTED - Unsupported properties SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). + SZ_ERROR_FAIL - Some unexpected error: internal error of code, memory corruption or hardware failure */ SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc); + ELzmaStatus *status, ISzAllocPtr alloc); EXTERN_C_END diff --git a/3rdparty/lzma/C/LzmaEnc.c b/3rdparty/lzma/C/LzmaEnc.c index 70df4569b7d..c8b31a19ecb 100644 --- a/3rdparty/lzma/C/LzmaEnc.c +++ b/3rdparty/lzma/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2016-05-16 : Igor Pavlov : Public domain */ +2022-07-15: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,6 +12,7 @@ #include #endif +#include "CpuArch.h" #include "LzmaEnc.h" #include "LzFind.h" @@ -19,21 +20,25 @@ #include "LzFindMt.h" #endif +/* the following LzmaEnc_* declarations is internal LZMA interface for LZMA2 encoder */ + +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, + Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); +void LzmaEnc_Finish(CLzmaEncHandle pp); +void LzmaEnc_SaveState(CLzmaEncHandle pp); +void LzmaEnc_RestoreState(CLzmaEncHandle pp); + #ifdef SHOW_STAT static unsigned g_STAT_OFFSET = 0; #endif -#define kMaxHistorySize ((UInt32)3 << 29) -/* #define kMaxHistorySize ((UInt32)7 << 29) */ - -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) +/* for good normalization speed we still reserve 256 MB before 4 GB range */ +#define kLzmaMaxHistorySize ((UInt32)15 << 28) #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -45,7 +50,9 @@ static unsigned g_STAT_OFFSET = 0; #define kNumMoveReducingBits 4 #define kNumBitPriceShiftBits 4 -#define kBitPrice (1 << kNumBitPriceShiftBits) +// #define kBitPrice (1 << kNumBitPriceShiftBits) + +#define REP_LEN_COUNT 64 void LzmaEncProps_Init(CLzmaEncProps *p) { @@ -54,6 +61,7 @@ void LzmaEncProps_Init(CLzmaEncProps *p) p->reduceSize = (UInt64)(Int64)-1; p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; p->writeEndMark = 0; + p->affinity = 0; } void LzmaEncProps_Normalize(CLzmaEncProps *p) @@ -62,15 +70,21 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (level < 0) level = 5; p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); + if (p->dictSize == 0) + p->dictSize = + ( level <= 3 ? ((UInt32)1 << (level * 2 + 16)) : + ( level <= 6 ? ((UInt32)1 << (level + 19)) : + ( level <= 7 ? ((UInt32)1 << 25) : ((UInt32)1 << 26) + ))); + if (p->dictSize > p->reduceSize) { - unsigned i; - for (i = 11; i <= 30; i++) - { - if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } - if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } - } + UInt32 v = (UInt32)p->reduceSize; + const UInt32 kReduceMin = ((UInt32)1 << 12); + if (v < kReduceMin) + v = kReduceMin; + if (p->dictSize > v) + p->dictSize = v; } if (p->lc < 0) p->lc = 3; @@ -80,8 +94,8 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); - if (p->numHashBytes < 0) p->numHashBytes = 4; - if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); + if (p->numHashBytes < 0) p->numHashBytes = (p->btMode ? 4 : 5); + if (p->mc == 0) p->mc = (16 + ((unsigned)p->fb >> 1)) >> (p->btMode ? 0 : 1); if (p->numThreads < 0) p->numThreads = @@ -99,30 +113,97 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) return props.dictSize; } -#if (_MSC_VER >= 1400) -/* BSR code is fast for some new CPUs */ -/* #define LZMA_LOG_BSR */ + +/* +x86/x64: + +BSR: + IF (SRC == 0) ZF = 1, DEST is undefined; + AMD : DEST is unchanged; + IF (SRC != 0) ZF = 0; DEST is index of top non-zero bit + BSR is slow in some processors + +LZCNT: + IF (SRC == 0) CF = 1, DEST is size_in_bits_of_register(src) (32 or 64) + IF (SRC != 0) CF = 0, DEST = num_lead_zero_bits + IF (DEST == 0) ZF = 1; + +LZCNT works only in new processors starting from Haswell. +if LZCNT is not supported by processor, then it's executed as BSR. +LZCNT can be faster than BSR, if supported. +*/ + +// #define LZMA_LOG_BSR + +#if defined(MY_CPU_ARM_OR_ARM64) /* || defined(MY_CPU_X86_OR_AMD64) */ + + #if (defined(__clang__) && (__clang_major__ >= 6)) \ + || (defined(__GNUC__) && (__GNUC__ >= 6)) + #define LZMA_LOG_BSR + #elif defined(_MSC_VER) && (_MSC_VER >= 1300) + // #if defined(MY_CPU_ARM_OR_ARM64) + #define LZMA_LOG_BSR + // #endif + #endif #endif +// #include + #ifdef LZMA_LOG_BSR -#define kDicLogSizeMaxCompress 32 +#if defined(__clang__) \ + || defined(__GNUC__) + +/* + C code: : (30 - __builtin_clz(x)) + gcc9/gcc10 for x64 /x86 : 30 - (bsr(x) xor 31) + clang10 for x64 : 31 + (bsr(x) xor -32) +*/ + + #define MY_clz(x) ((unsigned)__builtin_clz(x)) + // __lzcnt32 + // __builtin_ia32_lzcnt_u32 + +#else // #if defined(_MSC_VER) + + #ifdef MY_CPU_ARM_OR_ARM64 + + #define MY_clz _CountLeadingZeros + + #else // if defined(MY_CPU_X86_OR_AMD64) + + // #define MY_clz __lzcnt // we can use lzcnt (unsupported by old CPU) + // _BitScanReverse code is not optimal for some MSVC compilers + #define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); zz--; \ + res = (zz + zz) + (pos >> zz); } + + #endif // MY_CPU_X86_OR_AMD64 + +#endif // _MSC_VER + + +#ifndef BSR2_RET -#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } + #define BSR2_RET(pos, res) { unsigned zz = 30 - MY_clz(pos); \ + res = (zz + zz) + (pos >> zz); } -static UInt32 GetPosSlot1(UInt32 pos) +#endif + + +unsigned GetPosSlot1(UInt32 pos); +unsigned GetPosSlot1(UInt32 pos) { - UInt32 res; + unsigned res; BSR2_RET(pos, res); return res; } #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } #define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } -#else -#define kNumLogBits (9 + sizeof(size_t) / 2) -/* #define kNumLogBits (11 + sizeof(size_t) / 8 * 3) */ +#else // ! LZMA_LOG_BSR + +#define kNumLogBits (11 + sizeof(size_t) / 8 * 3) #define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) @@ -145,18 +226,18 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) /* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ /* -#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ res = p->g_FastPos[pos >> zz] + (zz * 2); } */ /* -#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ res = p->g_FastPos[pos >> zz] + (zz * 2); } */ -#define BSR2_RET(pos, res) { UInt32 zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ res = p->g_FastPos[pos >> zz] + (zz * 2); } /* @@ -167,55 +248,57 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) #define GetPosSlot1(pos) p->g_FastPos[pos] #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } -#endif +#endif // LZMA_LOG_BSR #define LZMA_NUM_REPS 4 -typedef unsigned CState; +typedef UInt16 CState; +typedef UInt16 CExtra; typedef struct { UInt32 price; - CState state; - int prev1IsChar; - int prev2; - - UInt32 posPrev2; - UInt32 backPrev2; - - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; } COptimal; -#define kNumOpts (1 << 12) + +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 -#define kDicLogSizeMin 0 +// #define kDicLogSizeMin 0 #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) - #define kNumAlignBits 4 #define kAlignTableSize (1 << kNumAlignBits) #define kAlignMask (kAlignTableSize - 1) #define kStartPosModelIndex 4 #define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; #define LZMA_PB_MAX 4 #define LZMA_LC_MAX 8 @@ -223,15 +306,11 @@ typedef struct #define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) #define LZMA_MATCH_LEN_MIN 2 #define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) @@ -241,27 +320,31 @@ typedef struct typedef struct { - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; CLzmaProb high[kLenNumHighSymbols]; } CLenEnc; typedef struct { - CLenEnc p; - UInt32 tableSize; + unsigned tableSize; UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; } CLenPriceEnc; +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ typedef struct { UInt32 range; - Byte cache; + unsigned cache; UInt64 low; UInt64 cacheSize; Byte *buf; @@ -277,164 +360,193 @@ typedef struct { CLzmaProb *litProbs; - UInt32 state; + unsigned state; UInt32 reps[LZMA_NUM_REPS]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLenEnc lenProbs; + CLenEnc repLenProbs; + } CSaveState; +typedef UInt32 CProbPrice; + + typedef struct { void *matchFinderObj; - IMatchFinder matchFinder; + IMatchFinder2 matchFinder; - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; + unsigned optCur; + unsigned optEnd; - UInt32 longestMatchLength; - UInt32 numPairs; + unsigned longestMatchLen; + unsigned numPairs; UInt32 numAvail; - UInt32 numFastBytes; - UInt32 additionalOffset; + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; + + UInt32 backRes; unsigned lc, lp, pb; - unsigned lpMask, pbMask; unsigned lclp; - CLzmaProb *litProbs; - - Bool fastMode; - Bool writeEndMark; - Bool finished; - Bool multiThread; - Bool needInit; + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; UInt64 nowPos64; - UInt32 matchPriceCount; - UInt32 alignPriceCount; + unsigned matchPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; - UInt32 distTableSize; + unsigned distTableSize; UInt32 dictSize; SRes result; - CRangeEnc rc; - #ifndef _7ZIP_ST - Bool mtMode; + BoolInt mtMode; + // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads + // #else + // CMatchFinder matchFinderBase; #endif - CMatchFinder matchFinderBase; + + // we suppose that we have 8-bytes alignment after CMatchFinder + #ifndef _7ZIP_ST Byte pad[128]; #endif - COptimal opt[kNumOpts]; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + // we want {len , dist} pairs to be 8-bytes aligned in matches array + UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2]; + // we want 8-bytes alignment here + UInt32 alignPrices[kAlignTableSize]; UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; + CLenEnc lenProbs; + CLenEnc repLenProbs; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; + COptimal opt[kNumOpts]; + CSaveState saveState; + // BoolInt mf_Failure; #ifndef _7ZIP_ST Byte pad2[128]; #endif } CLzmaEnc; +#define MFB (p->matchFinderBase) +/* +#ifndef _7ZIP_ST +#define MFB (p->matchFinderMt.MatchFinder) +#endif +*/ + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + void LzmaEnc_SaveState(CLzmaEncHandle pp) { CLzmaEnc *p = (CLzmaEnc *)pp; CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); } + void LzmaEnc_RestoreState(CLzmaEncHandle pp) { CLzmaEnc *dest = (CLzmaEnc *)pp; const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); } + + SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) { CLzmaEnc *p = (CLzmaEnc *)pp; @@ -443,40 +555,51 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX - || props.pb > LZMA_PB_MAX - || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) - || props.dictSize > kMaxHistorySize) + || props.pb > LZMA_PB_MAX) return SZ_ERROR_PARAM; + + if (props.dictSize > kLzmaMaxHistorySize) + props.dictSize = kLzmaMaxHistorySize; + + #ifndef LZMA_LOG_BSR + { + const UInt64 dict64 = props.dictSize; + if (dict64 > ((UInt64)1 << kDicLogSizeMaxCompress)) + return SZ_ERROR_PARAM; + } + #endif + p->dictSize = props.dictSize; { - unsigned fb = props.fb; + unsigned fb = (unsigned)props.fb; if (fb < 5) fb = 5; if (fb > LZMA_MATCH_LEN_MAX) fb = LZMA_MATCH_LEN_MAX; p->numFastBytes = fb; } - p->lc = props.lc; - p->lp = props.lp; - p->pb = props.pb; + p->lc = (unsigned)props.lc; + p->lp = (unsigned)props.lp; + p->pb = (unsigned)props.pb; p->fastMode = (props.algo == 0); - p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); + // p->_maxMode = True; + MFB.btMode = (Byte)(props.btMode ? 1 : 0); { - UInt32 numHashBytes = 4; + unsigned numHashBytes = 4; if (props.btMode) { - if (props.numHashBytes < 2) - numHashBytes = 2; - else if (props.numHashBytes < 4) - numHashBytes = props.numHashBytes; + if (props.numHashBytes < 2) numHashBytes = 2; + else if (props.numHashBytes < 4) numHashBytes = (unsigned)props.numHashBytes; } - p->matchFinderBase.numHashBytes = numHashBytes; + if (props.numHashBytes >= 5) numHashBytes = 5; + + MFB.numHashBytes = numHashBytes; } - p->matchFinderBase.cutValue = props.mc; + MFB.cutValue = props.mc; - p->writeEndMark = props.writeEndMark; + p->writeEndMark = (BoolInt)props.writeEndMark; #ifndef _7ZIP_ST /* @@ -487,18 +610,34 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) } */ p->multiThread = (props.numThreads > 1); + p->matchFinderMt.btSync.affinity = + p->matchFinderMt.hashSync.affinity = props.affinity; #endif return SZ_OK; } -static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; -#define IsCharState(s) ((s) < 7) +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + MFB.expectedDataSize = expectedDataSiize; +} + + +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) #define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) #define kInfinityPrice (1 << 30) @@ -509,14 +648,16 @@ static void RangeEnc_Construct(CRangeEnc *p) p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed(p) ( (p)->processed + (size_t)((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + (size_t)((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) -static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) + +static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) { if (!p->bufBase) { - p->bufBase = (Byte *)alloc->Alloc(alloc, RC_BUF_SIZE); + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); if (!p->bufBase) return 0; p->bufLim = p->bufBase + RC_BUF_SIZE; @@ -524,19 +665,18 @@ static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) return 1; } -static void RangeEnc_Free(CRangeEnc *p, ISzAlloc *alloc) +static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->bufBase); - p->bufBase = 0; + ISzAlloc_Free(alloc, p->bufBase); + p->bufBase = NULL; } static void RangeEnc_Init(CRangeEnc *p) { - /* Stream.Init(); */ - p->low = 0; p->range = 0xFFFFFFFF; - p->cacheSize = 1; p->cache = 0; + p->low = 0; + p->cacheSize = 0; p->buf = p->bufBase; @@ -544,37 +684,48 @@ static void RangeEnc_Init(CRangeEnc *p) p->res = SZ_OK; } -static void RangeEnc_FlushStream(CRangeEnc *p) +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { - size_t num; - if (p->res != SZ_OK) - return; - num = p->buf - p->bufBase; - if (num != p->outStream->Write(p->outStream, p->bufBase, num)) - p->res = SZ_ERROR_WRITE; + const size_t num = (size_t)(p->buf - p->bufBase); + if (p->res == SZ_OK) + { + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) + p->res = SZ_ERROR_WRITE; + } p->processed += num; p->buf = p->bufBase; } -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (unsigned)(p->low >> 32) != 0) + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) { - Byte temp = p->cache; - do { Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (p->cacheSize == 0) + return; + } + high += 0xFF; + for (;;) + { + Byte *buf = p->buf; + *buf++ = (Byte)(high); p->buf = buf; if (buf == p->bufLim) RangeEnc_FlushStream(p); - temp = 0xFF; + if (--p->cacheSize == 0) + return; } - while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); } p->cacheSize++; - p->low = (UInt32)p->low << 8; } static void RangeEnc_FlushData(CRangeEnc *p) @@ -584,78 +735,121 @@ static void RangeEnc_FlushData(CRangeEnc *p) RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, unsigned numBits) -{ - do - { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } - } - while (numBits != 0); -} +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) -{ - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) - { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, bit) { \ + RC_BIT_PRE(p, prob) \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); + +#else + +#define RC_BIT(p, prob, bit) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)bit; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)bit - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (UInt32)((Int32)(mask - ttt) >> kNumMoveBits); \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) + +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) +{ + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; } -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) { - symbol |= 0x100; + UInt32 range = p->range; + sym |= 0x100; do { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - symbol <<= 1; + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) { + UInt32 range = p->range; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) { UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; for (j = 0; j < kCyclesBits; j++) { w = w * w; @@ -666,567 +860,663 @@ static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) bitCount++; } } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + ProbPrices[i] = (CProbPrice)(((unsigned)kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); } } -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const UInt32 *ProbPrices) + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) { UInt32 price = 0; - symbol |= 0x100; + sym |= 0x100; do { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); } - while (symbol < 0x10000); + while (sym >= 2); return price; } -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const UInt32 *ProbPrices) + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) { UInt32 price = 0; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); } - while (symbol < 0x10000); + while (sym < 0x10000); return price; } -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) { - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) - { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - } -} - -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) + UInt32 range = rc->range; + unsigned m = 1; + do { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); + UInt32 ttt, newBound; + unsigned bit = sym & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + sym >>= 1; + RC_BIT(rc, probs + m, bit); m = (m << 1) | bit; - symbol >>= 1; - } -} - -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; } - return price; + while (--numBits); + rc->range = range; } -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} static void LenEnc_Init(CLenEnc *p) { unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; for (i = 0; i < kLenNumHighSymbols; i++) p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) { - if (symbol < kLenNumLowSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); - } - else + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols) { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols * 2) { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); + return; } + sym -= kLenNumLowSymbols; } -} -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, const UInt32 *ProbPrices) -{ - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); - } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); } -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, const UInt32 *ProbPrices) +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) { - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; + unsigned i; + for (i = 0; i < 8; i += 2) + { + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + } } -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, const UInt32 *ProbPrices) -{ - UInt32 posState; - for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, const UInt32 *ProbPrices) +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) { - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} + UInt32 b; + + { + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (posState = 0; posState < numPosStates; posState++) + { + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); + } + } + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; + a = GET_PRICEa_1(enc->low[0]); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + */ + + // p->counter = numSymbols; + // p->counter = 64; + { + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) + { + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); -static void MovePos(CLzmaEnc *p, UInt32 num) -{ + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } + } + } +} + +/* #ifdef SHOW_STAT g_STAT_OFFSET += num; printf("\n MovePos %u", num); #endif +*/ - if (num != 0) - { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); - } -} +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } + -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { - UInt32 lenRes = 0, numPairs; + unsigned numPairs; + + p->additionalOffset++; p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); - numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + { + const UInt32 *d = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + // if (!d) { p->mf_Failure = True; *numPairsRes = 0; return 0; } + numPairs = (unsigned)(d - p->matches); + } + *numPairsRes = numPairs; #ifdef SHOW_STAT printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); g_STAT_OFFSET++; { - UInt32 i; + unsigned i; for (i = 0; i < numPairs; i += 2) printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); } #endif - if (numPairs > 0) + if (numPairs == 0) + return 0; { - lenRes = p->matches[numPairs - 2]; - if (lenRes == p->numFastBytes) + const unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; { UInt32 numAvail = p->numAvail; if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; { - const Byte *pbyCur = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - const Byte *pby = pbyCur + lenRes; - ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[numPairs - 1]; - const Byte *pbyLim = pbyCur + numAvail; - for (; pby != pbyLim && *pby == pby[dif]; pby++); - lenRes = (UInt32)(pby - pbyCur); + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; + const ptrdiff_t dif = (ptrdiff_t)-1 - (ptrdiff_t)p->matches[(size_t)numPairs - 1]; + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} + return (unsigned)(p2 - p1); } } } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; } +#define MARK_LIT ((UInt32)(Int32)-1) -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) + +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + +MY_FORCE_INLINE +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; + UInt32 prob = p->isRepG0[state]; if (repIndex == 0) { - price = GET_PRICE_0(p->isRepG0[state]); + price = GET_PRICE_0(prob); price += GET_PRICE_1(p->isRep0Long[state][posState]); } else { - price = GET_PRICE_1(p->isRepG0[state]); + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); + price += GET_PRICE_0(prob); else { - price += GET_PRICE_1(p->isRepG1[state]); + price += GET_PRICE_1(prob); price += GET_PRICE(p->isRepG2[state], repIndex - 2); } } return price; } -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) -{ - return p->repLenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) +static unsigned Backward(CLzmaEnc *p, unsigned cur) { - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do + unsigned wr = cur + 1; + p->optEnd = wr; + + for (;;) { - if (p->opt[cur].prev1IsChar) + UInt32 dist = p->opt[cur].dist; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; + cur -= len; + + if (extra) { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) + wr--; + p->opt[wr].len = (UInt32)len; + cur -= extra; + len = extra; + if (extra == 1) { - p->opt[posMem - 1].prev1IsChar = False; - p->opt[posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[posMem - 1].backPrev = p->opt[cur].backPrev2; + p->opt[wr].dist = dist; + dist = MARK_LIT; + } + else + { + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; } } + + if (cur == 0) { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; + p->backRes = dist; + p->optCur = wr; + return len; } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = (UInt32)len; } - while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; } -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * (UInt32)0x300) -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) -{ - UInt32 lenEnd, cur; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; - UInt32 *matches; - { - - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, len; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - const Byte *data; - Byte curByte, matchByte; +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - numAvail = p->numAvail; - if (numAvail < 2) - { - *backRes = (UInt32)(-1); - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) +{ + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; + UInt32 *matches; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - reps[i] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + p->optCur = p->optEnd = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else { - repLens[i] = 0; - continue; + mainLen = p->longestMatchLen; + numPairs = p->numPairs; } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; - } - if (repLens[repMaxIndex] >= p->numFastBytes) - { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) - { - *backRes = (UInt32)-1; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAsChar(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) + + numAvail = p->numAvail; + if (numAvail < 2) { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); + p->backRes = MARK_LIT; + return 1; } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; - return 1; - } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; + repLens[i] = 0; + continue; } + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; + if (len == LZMA_MATCH_LEN_MAX) // 21.03 : optimization + break; } - while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (repLens[repMaxIndex] >= p->numFastBytes) + { + unsigned len; + p->backRes = (UInt32)repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; + } + + matches = p->matches; + #define MATCHES matches + // #define MATCHES p->matches + + if (mainLen >= p->numFastBytes) + { + p->backRes = MATCHES[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + curByte = *data; + matchByte = *(data - reps[0]); - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (; ; len++) + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; + + if (last < 2 && curByte != matchByte) { - COptimal *opt; - UInt32 distance = matches[offs + 1]; + p->backRes = MARK_LIT; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; - else + MakeAs_Lit(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + // 18.06 + if (matchByte == curByte && repLens[0] == 0) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) + if (last < 2) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; + p->backRes = p->opt[1].dist; + return 1; } - if (len == matches[offs]) + } + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + // ---------- REP ---------- + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do { - offs += 2; - if (offs == numPairs) - break; + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; + opt->extra = 0; + } } + while (--repLen >= 2); } - } + + + // ---------- MATCH ---------- + { + unsigned len = repLens[0] + 1; + if (len <= mainLen) + { + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - cur = 0; + if (len < 2) + len = 2; + else + while (len > MATCHES[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = MATCHES[(size_t)offs + 1]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == MATCHES[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } + } + } + + + cur = 0; #ifdef SHOW_STAT2 /* if (position >= 0) */ { unsigned i; printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) + for (i = cur; i <= last; i++) printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); } #endif - } + + + // ---------- Optimal Parsing ---------- + for (;;) { - UInt32 numAvail; - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; + unsigned numAvail; + UInt32 numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; Byte curByte, matchByte; const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; + COptimal *curOpt, *nextOpt; - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); + if (++cur == last) + break; + + // 18.06 + if (cur >= kNumOpts - 64) + { + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) + { + price = price2; + best = j; + } + } + { + unsigned delta = best - cur; + if (delta != 0) + { + MOVE_POS(p, delta); + } + } + cur = best; + break; + } newLen = ReadMatchDistances(p, &numPairs); + if (newLen >= p->numFastBytes) { p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); + p->longestMatchLen = newLen; + break; } - position++; + curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) - { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; - } - else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) + + position++; + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + + prev = cur - curOpt->len; + + if (curOpt->len == 1) { + state = (unsigned)p->opt[prev].state; if (IsShortRep(curOpt)) state = kShortRepNextStates[state]; else @@ -1234,357 +1524,499 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) } else { - UInt32 pos; const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; + prev -= (unsigned)curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); } else { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) + state = (unsigned)p->opt[prev].state; + if (dist < LZMA_NUM_REPS) state = kRepNextStates[state]; else state = kMatchNextStates[state]; } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; + if (dist == 0) + { + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } } else { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i - 1]; + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; } } + curOpt->state = (CState)state; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; - - curPrice = curOpt->price; - nextIsChar = False; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; curByte = *data; - matchByte = *(data - (reps[0] + 1)); + matchByte = *(data - reps[0]); posState = (position & p->pbMask); - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); } - nextOpt = &p->opt[cur + 1]; - - if (curAnd1Price < nextOpt->price) + nextOpt = &p->opt[(size_t)cur + 1]; + nextIsLit = False; + + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + + if (litPrice < nextOpt->price) + { + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; + } } - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) + numAvailFull = p->numAvail; + { + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; + } + + // 18.06 + // ---------- SHORT_REP ---------- + if (IsLitState(state)) // 18.new + if (matchByte == curByte) + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt + || (nextOpt->dist != 0 + // && nextOpt->extra <= 1 // 17.old + ) + ) { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new { nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; } } - numAvailFull = p->numAvail; - { - UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) - numAvailFull = temp; - } - + if (numAvailFull < 2) continue; numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - if (!nextIsChar && matchByte != curByte) /* speed optimization */ + // numAvail <= p->numFastBytes + + // ---------- LIT : REP_0 ---------- + + if (!nextIsLit + && litPrice != 0 // 18.new + && matchByte != curByte + && numAvailFull > 2) { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - reps[0] - 1; - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; - - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} + { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; + unsigned offset = cur + len; + + if (last < offset) + last = offset; + + // do + { + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len; + opt->dist = 0; + opt->extra = 1; + } + } + // while (len >= 3); } } } } startLen = 2; /* speed optimization */ + { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) - { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - reps[repIndex] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + + // if (len < startLen) continue; // 18.new: speed optimization + { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; + unsigned offset = cur + len; + if (last < offset) + last = offset; + } + { + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; + opt->extra = 0; + } + } + while (--len2 >= 2); } - } - while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; - /* if (_maxMode) */ + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); - /* for (; lenTest2 >= 2; lenTest2--) */ + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = (UInt32)repIndex; + } } + // while (len2 >= 3); } } + } } + } } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ if (newLen > numAvail) { newLen = numAvail; - for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; + for (numPairs = 0; newLen > MATCHES[numPairs]; numPairs += 2); + MATCHES[numPairs] = (UInt32)newLen; numPairs += 2; } + + // startLen = 2; /* speed optimization */ + if (newLen >= startLen) { UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; + UInt32 dist; + unsigned offs, posSlot, len; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } offs = 0; - while (startLen > matches[offs]) + while (startLen > MATCHES[offs]) offs += 2; - curBack = matches[offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen; ; lenTest++) + dist = MATCHES[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][lenTest - LZMA_MATCH_LEN_MIN]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); { - UInt32 lenToPosState = GetLenToPosState(lenTest); - COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } + COptimal *opt; + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } } - if (/*_maxMode && */lenTest == matches[offs]) + if (len == MATCHES[offs]) { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - curBack - 1; - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // if (p->_maxMode) { + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + + // if (len2 >= 3) + { + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice2; + UInt32 price2; COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice2 = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); opt = &p->opt[offset]; - if (curAndLenPrice2 < opt->price) + // offset--; + if (price2 < opt->price) { - opt->price = curAndLenPrice2; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; } } + // while (len2 >= 3); + } + } + offs += 2; if (offs == numPairs) break; - curBack = matches[offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); + dist = MATCHES[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); } } } } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); } + + #define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) + + +static unsigned GetOptimumFast(CLzmaEnc *p) { - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; const Byte *data; - const UInt32 *matches; if (p->additionalOffset == 0) mainLen = ReadMatchDistances(p, &numPairs); else { - mainLen = p->longestMatchLength; + mainLen = p->longestMatchLen; numPairs = p->numPairs; } numAvail = p->numAvail; - *backRes = (UInt32)-1; + p->backRes = MARK_LIT; if (numAvail < 2) return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repLen = repIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} if (len >= p->numFastBytes) { - *backRes = i; - MovePos(p, len - 1); + p->backRes = (UInt32)i; + MOVE_POS(p, len - 1) return len; } if (len > repLen) @@ -1594,98 +2026,182 @@ static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) } } - matches = p->matches; if (mainLen >= p->numFastBytes) { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) return mainLen; } mainDist = 0; /* for GCC */ + if (mainLen >= 2) { - mainDist = matches[numPairs - 1]; - while (numPairs > 2 && mainLen == matches[numPairs - 4] + 1) + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) { - if (!ChangePair(matches[numPairs - 3], mainDist)) + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) break; numPairs -= 2; - mainLen = matches[numPairs - 2]; - mainDist = matches[numPairs - 1]; + mainLen--; + mainDist = dist2; } if (mainLen == 2 && mainDist >= 0x80) mainLen = 1; } - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) { - *backRes = repIndex; - MovePos(p, repLen - 1); + p->backRes = (UInt32)repIndex; + MOVE_POS(p, repLen - 1) return repLen; } if (mainLen < 2 || numAvail <= 2) return 1; - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) { - UInt32 newDistance = matches[p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } } data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len, limit; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } + } + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); return mainLen; } -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; } + static SRes CheckErrors(CLzmaEnc *p) { if (p->result != SZ_OK) return p->result; if (p->rc.res != SZ_OK) p->result = SZ_ERROR_WRITE; - if (p->matchFinderBase.result != SZ_OK) + + #ifndef _7ZIP_ST + if ( + // p->mf_Failure || + (p->mtMode && + ( // p->matchFinderMt.failure_LZ_LZ || + p->matchFinderMt.failure_LZ_BT)) + ) + { + p->result = MY_HRES_ERROR__INTERNAL_ERROR; + // printf("\nCheckErrors p->matchFinderMt.failureLZ\n"); + } + #endif + + if (MFB.result != SZ_OK) p->result = SZ_ERROR_READ; + if (p->result != SZ_OK) p->finished = True; return p->result; } -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { /* ReleaseMFStream(); */ p->finished = True; @@ -1696,55 +2212,134 @@ static SRes Flush(CLzmaEnc *p, UInt32 nowPos) return CheckErrors(p); } -static void FillAlignPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - p->alignPriceCount = 0; + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; + // p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned sym = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } } -static void FillDistancesPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) { + // int y; for (y = 0; y < 100; y++) { + UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) - { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); + unsigned i, lps; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) + { + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = (posSlot >> 1) - 1; + unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + UInt32 price = 0; + unsigned m = 1; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); + + { + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); + } } - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) + for (lps = 0; lps < kNumLenToPosStates; lps++) { - UInt32 posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) + { + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; + unsigned bit; + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } + } { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; + } } } - p->matchPriceCount = 0; + // } } -void LzmaEnc_Construct(CLzmaEnc *p) + + +static void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); - MatchFinder_Construct(&p->matchFinderBase); + MatchFinder_Construct(&MFB); #ifndef _7ZIP_ST + p->matchFinderMt.MatchFinder = &MFB; MatchFinderMt_Construct(&p->matchFinderMt); - p->matchFinderMt.MatchFinder = &p->matchFinderBase; #endif { @@ -1762,45 +2357,53 @@ void LzmaEnc_Construct(CLzmaEnc *p) p->saveState.litProbs = NULL; } -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc) +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) { void *p; - p = alloc->Alloc(alloc, sizeof(CLzmaEnc)); + p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); if (p) LzmaEnc_Construct((CLzmaEnc *)p); return p; } -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc) +static void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->litProbs); - alloc->Free(alloc, p->saveState.litProbs); + ISzAlloc_Free(alloc, p->litProbs); + ISzAlloc_Free(alloc, p->saveState.litProbs); p->litProbs = NULL; p->saveState.litProbs = NULL; } -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) +static void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { #ifndef _7ZIP_ST MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); #endif - MatchFinder_Free(&p->matchFinderBase, allocBig); + MatchFinder_Free(&MFB, allocBig); LzmaEnc_FreeLits(p, alloc); RangeEnc_Free(&p->rc, alloc); } -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig) +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); - alloc->Free(alloc, p); + ISzAlloc_Free(alloc, p); } -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) + +MY_NO_INLINE +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; if (p->needInit) { + #ifndef _7ZIP_ST + if (p->mtMode) + { + RINOK(MatchFinderMt_InitMt(&p->matchFinderMt)); + } + #endif p->matchFinder.Init(p->matchFinderObj); p->needInit = 0; } @@ -1814,13 +2417,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize if (p->nowPos64 == 0) { - UInt32 numPairs; + unsigned numPairs; Byte curByte; if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) return Flush(p, nowPos32); ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); LitEnc_Encode(&p->rc, p->litProbs, curByte); p->additionalOffset--; @@ -1828,123 +2431,253 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + for (;;) { - UInt32 pos, len, posState; - + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + if (p->fastMode) - len = GetOptimumFast(p, &pos); + len = GetOptimumFast(p); else - len = GetOptimum(p, nowPos32, &pos); + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; #ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %u pos = %u", nowPos32, len, pos); + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); #endif - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) + if (dist == MARK_LIT) { Byte curByte; - CLzmaProb *probs; const Byte *data; + unsigned state; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); + RC_BIT_0(&p->rc, probs); + p->rc.range = range; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) LitEnc_Encode(&p->rc, probs, curByte); else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); } else { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } } else { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } else { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; + } + else + { + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; p->reps[3] = p->reps[2]; + } p->reps[2] = p->reps[1]; } p->reps[1] = p->reps[0]; - p->reps[0] = distance; + p->reps[0] = dist; } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + --p->repLenEncCounter; p->state = kRepNextStates[p->state]; } } else { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); + + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + // --p->lenEnc.counter; + + dist -= LZMA_NUM_REPS; + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + p->reps[1] = p->reps[0]; + p->reps[0] = dist + 1; + + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); + { + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (sym < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; + } - if (posSlot >= kStartPosModelIndex) + if (dist >= kStartPosModelIndex) { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; + unsigned footerBits = ((posSlot >> 1) - 1); - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); + } else { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + // p->alignPriceCount++; + } } } - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - p->reps[1] = p->reps[0]; - p->reps[0] = pos; - p->matchPriceCount++; } } + + nowPos32 += (UInt32)len; p->additionalOffset -= len; - nowPos32 += len; + if (p->additionalOffset == 0) { UInt32 processed; + if (!p->fastMode) { - if (p->matchPriceCount >= (1 << 7)) + /* + if (p->alignPriceCount >= 16) // kAlignTableSize + FillAlignPrices(p); + if (p->matchPriceCount >= 128) FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, (unsigned)1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, (unsigned)1 << p->pb, &p->repLenProbs, p->ProbPrices); + } } + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) break; processed = nowPos32 - startPos32; - if (useLimits) + + if (maxPackSize) { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) break; } else if (processed >= (1 << 17)) @@ -1954,20 +2687,25 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } } } + p->nowPos64 += nowPos32 - startPos32; return Flush(p, nowPos32); } + + #define kBigHashDicLimit ((UInt32)1 << 24) -static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { UInt32 beforeSize = kNumOpts; + UInt32 dictSize; + if (!RangeEnc_Alloc(&p->rc, alloc)) return SZ_ERROR_MEM; #ifndef _7ZIP_ST - p->mtMode = (p->multiThread && !p->fastMode && (p->matchFinderBase.btMode != 0)); + p->mtMode = (p->multiThread && !p->fastMode && (MFB.btMode != 0)); #endif { @@ -1975,8 +2713,8 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) { LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)alloc->Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)alloc->Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); if (!p->litProbs || !p->saveState.litProbs) { LzmaEnc_FreeLits(p, alloc); @@ -1986,43 +2724,72 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, I } } - p->matchFinderBase.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + MFB.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + + + dictSize = p->dictSize; + if (dictSize == ((UInt32)2 << 30) || + dictSize == ((UInt32)3 << 30)) + { + /* 21.03 : here we reduce the dictionary for 2 reasons: + 1) we don't want 32-bit back_distance matches in decoder for 2 GB dictionary. + 2) we want to elimate useless last MatchFinder_Normalize3() for corner cases, + where data size is aligned for 1 GB: 5/6/8 GB. + That reducing must be >= 1 for such corner cases. */ + dictSize -= 1; + } + + if (beforeSize + dictSize < keepWindowSize) + beforeSize = keepWindowSize - dictSize; - if (beforeSize + p->dictSize < keepWindowSize) - beforeSize = keepWindowSize - p->dictSize; + /* in worst case we can look ahead for + max(LZMA_MATCH_LEN_MAX, numFastBytes + 1 + numFastBytes) bytes. + we send larger value for (keepAfter) to MantchFinder_Create(): + (numFastBytes + LZMA_MATCH_LEN_MAX + 1) + */ #ifndef _7ZIP_ST if (p->mtMode) { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); + RINOK(MatchFinderMt_Create(&p->matchFinderMt, dictSize, beforeSize, + p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 18.04 */ + , allocBig)); p->matchFinderObj = &p->matchFinderMt; + MFB.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && MFB.hashMask >= 0xFFFFFF) ? 1 : 0); MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); } else #endif { - if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) + if (!MatchFinder_Create(&MFB, dictSize, beforeSize, + p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 21.03 */ + , allocBig)) return SZ_ERROR_MEM; - p->matchFinderObj = &p->matchFinderBase; - MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); + p->matchFinderObj = &MFB; + MatchFinder_CreateVTable(&MFB, &p->matchFinder); } return SZ_OK; } -void LzmaEnc_Init(CLzmaEnc *p) +static void LzmaEnc_Init(CLzmaEnc *p) { - UInt32 i; + unsigned i; p->state = 0; - for (i = 0 ; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; RangeEnc_Init(&p->rc); + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; for (i = 0; i < kNumStates; i++) { - UInt32 j; + unsigned j; for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) { p->isMatch[i][j] = kProbInitValue; @@ -2034,42 +2801,50 @@ void LzmaEnc_Init(CLzmaEnc *p) p->isRepG2[i] = kProbInitValue; } - { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - CLzmaProb *probs = p->litProbs; - for (i = 0; i < num; i++) - probs[i] = kProbInitValue; - } - { for (i = 0; i < kNumLenToPosStates; i++) { CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; + unsigned j; for (j = 0; j < (1 << kNumPosSlotBits); j++) probs[j] = kProbInitValue; } } { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + for (i = 0; i < kNumFullDistances; i++) p->posEncoders[i] = kProbInitValue; } - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); + + p->optEnd = 0; + p->optCur = 0; + + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } + p->additionalOffset = 0; - p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; + p->pbMask = ((unsigned)1 << p->pb) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); + + // p->mf_Failure = False; } -void LzmaEnc_InitPrices(CLzmaEnc *p) + +static void LzmaEnc_InitPrices(CLzmaEnc *p) { if (!p->fastMode) { @@ -2080,14 +2855,17 @@ void LzmaEnc_InitPrices(CLzmaEnc *p) p->lenEnc.tableSize = p->repLenEnc.tableSize = p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); + + p->repLenEncCounter = REP_LEN_COUNT; + + LenPriceEnc_UpdateTables(&p->lenEnc, (unsigned)1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, (unsigned)1 << p->pb, &p->repLenProbs, p->ProbPrices); } -static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) if (p->dictSize <= ((UInt32)1 << i)) break; p->distTableSize = i * 2; @@ -2102,10 +2880,10 @@ static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *a } static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; + MFB.stream = inStream; p->needInit = 1; p->rc.outStream = outStream; return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); @@ -2113,28 +2891,29 @@ static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInS SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; + MFB.stream = inStream; p->needInit = 1; return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) { - p->matchFinderBase.directInput = 1; - p->matchFinderBase.bufferBase = (Byte *)src; - p->matchFinderBase.directInputRem = srcLen; + MFB.directInput = 1; + MFB.bufferBase = (Byte *)src; + MFB.directInputRem = srcLen; } SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; LzmaEnc_SetInputBuf(p, src, srcLen); p->needInit = 1; + LzmaEnc_SetDataSize(pp, srcLen); return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } @@ -2152,33 +2931,37 @@ void LzmaEnc_Finish(CLzmaEncHandle pp) typedef struct { - ISeqOutStream funcTable; + ISeqOutStream vt; Byte *data; SizeT rem; - Bool overflow; -} CSeqOutStreamBuf; + BoolInt overflow; +} CLzmaEnc_SeqOutStreamBuf; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqOutStreamBuf *p = (CSeqOutStreamBuf *)pp; + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); if (p->rem < size) { size = p->rem; p->overflow = True; } - memcpy(p->data, data, size); - p->rem -= size; - p->data += size; + if (size != 0) + { + memcpy(p->data, data, size); + p->rem -= size; + p->data += size; + } return size; } +/* UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) { const CLzmaEnc *p = (CLzmaEnc *)pp; return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); } - +*/ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) { @@ -2187,15 +2970,16 @@ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) } -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +// (desiredPackSize == 0) is not allowed +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { CLzmaEnc *p = (CLzmaEnc *)pp; UInt64 nowPos64; SRes res; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; @@ -2207,11 +2991,11 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, if (reInit) LzmaEnc_Init(p); LzmaEnc_InitPrices(p); - nowPos64 = p->nowPos64; RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.funcTable; - - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); + p->rc.outStream = &outStream.vt; + nowPos64 = p->nowPos64; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); *unpackSize = (UInt32)(p->nowPos64 - nowPos64); *destLen -= outStream.rem; @@ -2222,6 +3006,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, } +MY_NO_INLINE static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) { SRes res = SZ_OK; @@ -2234,12 +3019,12 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) for (;;) { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); + res = LzmaEnc_CodeOneBlock(p, 0, 0); if (res != SZ_OK || p->finished) break; if (progress) { - res = progress->Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); + res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); if (res != SZ_OK) { res = SZ_ERROR_PROGRESS; @@ -2251,7 +3036,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) LzmaEnc_Finish(p); /* - if (res == S_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&MFB)) res = SZ_ERROR_FAIL; } */ @@ -2261,7 +3046,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); @@ -2270,47 +3055,61 @@ SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *i SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) { - CLzmaEnc *p = (CLzmaEnc *)pp; - unsigned i; - UInt32 dictSize = p->dictSize; if (*size < LZMA_PROPS_SIZE) return SZ_ERROR_PARAM; *size = LZMA_PROPS_SIZE; - props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); - - if (dictSize >= ((UInt32)1 << 22)) - { - UInt32 kDictMask = ((UInt32)1 << 20) - 1; - if (dictSize < (UInt32)0xFFFFFFFF - kDictMask) - dictSize = (dictSize + kDictMask) & ~kDictMask; - } - else for (i = 11; i <= 30; i++) { - if (dictSize <= ((UInt32)2 << i)) { dictSize = (2 << i); break; } - if (dictSize <= ((UInt32)3 << i)) { dictSize = (3 << i); break; } + const CLzmaEnc *p = (const CLzmaEnc *)pp; + const UInt32 dictSize = p->dictSize; + UInt32 v; + props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); + + // we write aligned dictionary value to properties for lzma decoder + if (dictSize >= ((UInt32)1 << 21)) + { + const UInt32 kDictMask = ((UInt32)1 << 20) - 1; + v = (dictSize + kDictMask) & ~kDictMask; + if (v < dictSize) + v = dictSize; + } + else + { + unsigned i = 11 * 2; + do + { + v = (UInt32)(2 + (i & 1)) << (i >> 1); + i++; + } + while (v < dictSize); + } + + SetUi32(props + 1, v); + return SZ_OK; } +} - for (i = 0; i < 4; i++) - props[1 + i] = (Byte)(dictSize >> (8 * i)); - return SZ_OK; + +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +{ + return (unsigned)((CLzmaEnc *)pp)->writeEndMark; } SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { SRes res; CLzmaEnc *p = (CLzmaEnc *)pp; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; p->writeEndMark = writeEndMark; - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); @@ -2330,7 +3129,7 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); SRes res; @@ -2349,3 +3148,15 @@ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, LzmaEnc_Destroy(p, alloc, allocBig); return res; } + + +/* +#ifndef _7ZIP_ST +void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + lz_threads[0] = p->matchFinderMt.hashSync.thread; + lz_threads[1] = p->matchFinderMt.btSync.thread; +} +#endif +*/ diff --git a/3rdparty/lzma/C/LzmaEnc.h b/3rdparty/lzma/C/LzmaEnc.h index cffe220bbf6..bc2ed5042b9 100644 --- a/3rdparty/lzma/C/LzmaEnc.h +++ b/3rdparty/lzma/C/LzmaEnc.h @@ -1,5 +1,5 @@ /* LzmaEnc.h -- LZMA Encoder -2013-01-18 : Igor Pavlov : Public domain */ +2019-10-30 : Igor Pavlov : Public domain */ #ifndef __LZMA_ENC_H #define __LZMA_ENC_H @@ -12,12 +12,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaEncProps { - int level; /* 0 <= level <= 9 */ + int level; /* 0 <= level <= 9 */ UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (1 << 30) for 64-bit version - default = (1 << 24) */ - UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF. - Encoder uses this value to reduce dictionary size */ + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ int lc; /* 0 <= lc <= 8, default = 3 */ int lp; /* 0 <= lp <= 4, default = 0 */ int pb; /* 0 <= pb <= 4, default = 2 */ @@ -25,9 +23,14 @@ typedef struct _CLzmaEncProps int fb; /* 5 <= fb <= 273, default = 32 */ int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ + + UInt64 affinity; } CLzmaEncProps; void LzmaEncProps_Init(CLzmaEncProps *p); @@ -37,41 +40,38 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); /* ---------- CLzmaEncHandle Interface ---------- */ -/* LzmaEnc_* functions can return the following exit codes: -Returns: +/* LzmaEnc* functions can return the following exit codes: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error. + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzmaEncHandle; -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc); -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig); +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); -/* ---------- One Call Interface ---------- */ -/* LzmaEncode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ +/* ---------- One Call Interface ---------- */ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); EXTERN_C_END diff --git a/3rdparty/lzma/C/LzmaLib.h b/3rdparty/lzma/C/LzmaLib.h index 88fa87d350c..c343a8596dd 100644 --- a/3rdparty/lzma/C/LzmaLib.h +++ b/3rdparty/lzma/C/LzmaLib.h @@ -1,5 +1,5 @@ /* LzmaLib.h -- LZMA library interface -2013-01-18 : Igor Pavlov : Public domain */ +2021-04-03 : Igor Pavlov : Public domain */ #ifndef __LZMA_LIB_H #define __LZMA_LIB_H @@ -40,14 +40,16 @@ outPropsSize - level - compression level: 0 <= level <= 9; level dictSize algo fb - 0: 16 KB 0 32 - 1: 64 KB 0 32 - 2: 256 KB 0 32 - 3: 1 MB 0 32 - 4: 4 MB 0 32 + 0: 64 KB 0 32 + 1: 256 KB 0 32 + 2: 1 MB 0 32 + 3: 4 MB 0 32 + 4: 16 MB 0 32 5: 16 MB 1 32 6: 32 MB 1 32 - 7+: 64 MB 1 64 + 7: 32 MB 1 64 + 8: 64 MB 1 64 + 9: 64 MB 1 64 The default value for "level" is 5. @@ -83,6 +85,11 @@ fb - Word size (the number of fast bytes). numThreads - The number of thereads. 1 or 2. The default value is 2. Fast mode (algo = 0) can use only 1 thread. +In: + dest - output data buffer + destLen - output data buffer size + src - input data + srcLen - input data size Out: destLen - processed output size Returns: @@ -108,8 +115,8 @@ MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char LzmaUncompress -------------- In: - dest - output data - destLen - output data size + dest - output data buffer + destLen - output data buffer size src - input data srcLen - input data size Out: diff --git a/3rdparty/lzma/C/MtCoder.c b/3rdparty/lzma/C/MtCoder.c index 8d9731d0a28..99dc9090ad5 100644 --- a/3rdparty/lzma/C/MtCoder.c +++ b/3rdparty/lzma/C/MtCoder.c @@ -1,168 +1,88 @@ /* MtCoder.c -- Multi-thread Coder -2015-10-13 : Igor Pavlov : Public domain */ +2021-12-21 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -void LoopThread_Construct(CLoopThread *p) -{ - Thread_Construct(&p->thread); - Event_Construct(&p->startEvent); - Event_Construct(&p->finishedEvent); -} - -void LoopThread_Close(CLoopThread *p) -{ - Thread_Close(&p->thread); - Event_Close(&p->startEvent); - Event_Close(&p->finishedEvent); -} +#ifndef _7ZIP_ST -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp) +static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) { - CLoopThread *p = (CLoopThread *)pp; - for (;;) + CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + UInt64 inSize2 = 0; + UInt64 outSize2 = 0; + if (inSize != (UInt64)(Int64)-1) { - if (Event_Wait(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - if (p->stop) - return 0; - p->res = p->func(p->param); - if (Event_Set(&p->finishedEvent) != 0) - return SZ_ERROR_THREAD; + inSize2 = inSize - thunk->inSize; + thunk->inSize = inSize; + } + if (outSize != (UInt64)(Int64)-1) + { + outSize2 = outSize - thunk->outSize; + thunk->outSize = outSize; } + return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); } -WRes LoopThread_Create(CLoopThread *p) -{ - p->stop = 0; - RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent)); - RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent)); - return Thread_Create(&p->thread, LoopThreadFunc, p); -} -WRes LoopThread_StopAndWait(CLoopThread *p) +void MtProgressThunk_CreateVTable(CMtProgressThunk *p) { - p->stop = 1; - if (Event_Set(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - return Thread_Wait(&p->thread); + p->vt.Progress = MtProgressThunk_Progress; } -WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); } -WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); } -static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) -{ - return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; -} - -static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) -{ - unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->inSizes[i] = p->outSizes[i] = 0; - p->totalInSize = p->totalOutSize = 0; - p->progress = progress; - p->res = SZ_OK; -} -static void MtProgress_Reinit(CMtProgress *p, unsigned index) -{ - p->inSizes[index] = 0; - p->outSizes[index] = 0; -} +#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -#define UPDATE_PROGRESS(size, prev, total) \ - if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) { - SRes res; - CriticalSection_Enter(&p->cs); - UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize) - UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize) - if (p->res == SZ_OK) - p->res = Progress(p->progress, p->totalInSize, p->totalOutSize); - res = p->res; - CriticalSection_Leave(&p->cs); - return res; + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); } -static void MtProgress_SetError(CMtProgress *p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} -static void MtCoder_SetError(CMtCoder* p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} +static THREAD_FUNC_DECL ThreadFunc(void *pp); -/* ---------- MtThread ---------- */ -void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder) +static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t) { - p->mtCoder = mtCoder; - p->outBuf = 0; - p->inBuf = 0; - Event_Construct(&p->canRead); - Event_Construct(&p->canWrite); - LoopThread_Construct(&p->thread); + WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent); + if (wres == 0) + { + t->stop = False; + if (!Thread_WasCreated(&t->thread)) + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + wres = Event_Set(&t->startEvent); + } + if (wres == 0) + return SZ_OK; + return MY_SRes_HRESULT_FROM_WRes(wres); } -#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } - -static void CMtThread_CloseEvents(CMtThread *p) -{ - Event_Close(&p->canRead); - Event_Close(&p->canWrite); -} -static void CMtThread_Destruct(CMtThread *p) +static void MtCoderThread_Destruct(CMtCoderThread *t) { - CMtThread_CloseEvents(p); - - if (Thread_WasCreated(&p->thread.thread)) + if (Thread_WasCreated(&t->thread)) { - LoopThread_StopAndWait(&p->thread); - LoopThread_Close(&p->thread); + t->stop = 1; + Event_Set(&t->startEvent); + Thread_Wait_Close(&t->thread); } - if (p->mtCoder->alloc) - IAlloc_Free(p->mtCoder->alloc, p->outBuf); - p->outBuf = 0; + Event_Close(&t->startEvent); - if (p->mtCoder->alloc) - IAlloc_Free(p->mtCoder->alloc, p->inBuf); - p->inBuf = 0; + if (t->inBuf) + { + ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf); + t->inBuf = NULL; + } } -#define MY_BUF_ALLOC(buf, size, newSize) \ - if (buf == 0 || size != newSize) \ - { IAlloc_Free(p->mtCoder->alloc, buf); \ - size = newSize; buf = (Byte *)IAlloc_Alloc(p->mtCoder->alloc, size); \ - if (buf == 0) return SZ_ERROR_MEM; } - -static SRes CMtThread_Prepare(CMtThread *p) -{ - MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize) - MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize) - - p->stopReading = False; - p->stopWriting = False; - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite)); - return SZ_OK; -} static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) { @@ -170,158 +90,506 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) *processedSize = 0; while (size != 0) { - size_t curSize = size; - SRes res = stream->Read(stream, data, &curSize); - *processedSize += curSize; - data += curSize; - size -= curSize; + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; RINOK(res); - if (curSize == 0) + if (cur == 0) return SZ_OK; } return SZ_OK; } -#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1] -static SRes MtThread_Process(CMtThread *p, Bool *stop) +/* + ThreadFunc2() returns: + SZ_OK - in all normal cases (even for stream error or memory allocation error) + SZ_ERROR_THREAD - in case of failure in system synch function +*/ + +static SRes ThreadFunc2(CMtCoderThread *t) { - CMtThread *next; - *stop = True; - if (Event_Wait(&p->canRead) != 0) - return SZ_ERROR_THREAD; - - next = GET_NEXT_THREAD(p); - - if (p->stopReading) - { - next->stopReading = True; - return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD; - } + CMtCoder *mtc = t->mtCoder; + for (;;) { - size_t size = p->mtCoder->blockSize; - size_t destSize = p->outBufSize; - - RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size)); - next->stopReading = *stop = (size != p->mtCoder->blockSize); - if (Event_Set(&next->canRead) != 0) - return SZ_ERROR_THREAD; - - RINOK(p->mtCoder->mtCallback->Code(p->mtCoder->mtCallback, p->index, - p->outBuf, &destSize, p->inBuf, size, *stop)); - - MtProgress_Reinit(&p->mtCoder->mtProgress, p->index); - - if (Event_Wait(&p->canWrite) != 0) - return SZ_ERROR_THREAD; - if (p->stopWriting) - return SZ_ERROR_FAIL; - if (p->mtCoder->outStream->Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize) - return SZ_ERROR_WRITE; - return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD; + unsigned bi; + SRes res; + SRes res2; + BoolInt finished; + unsigned bufIndex; + size_t size; + const Byte *inData; + UInt64 readProcessed = 0; + + RINOK_THREAD(Event_Wait(&mtc->readEvent)) + + /* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */ + + if (mtc->stopReading) + { + return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD; + } + + res = MtProgress_GetError(&mtc->mtProgress); + + size = 0; + inData = NULL; + finished = True; + + if (res == SZ_OK) + { + size = mtc->blockSize; + if (mtc->inStream) + { + if (!t->inBuf) + { + t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize); + if (!t->inBuf) + res = SZ_ERROR_MEM; + } + if (res == SZ_OK) + { + res = FullRead(mtc->inStream, t->inBuf, &size); + readProcessed = mtc->readProcessed + size; + mtc->readProcessed = readProcessed; + } + if (res != SZ_OK) + { + mtc->readRes = res; + /* after reading error - we can stop encoding of previous blocks */ + MtProgress_SetError(&mtc->mtProgress, res); + } + else + finished = (size != mtc->blockSize); + } + else + { + size_t rem; + readProcessed = mtc->readProcessed; + rem = mtc->inDataSize - (size_t)readProcessed; + if (size > rem) + size = rem; + inData = mtc->inData + (size_t)readProcessed; + readProcessed += size; + mtc->readProcessed = readProcessed; + finished = (mtc->inDataSize == (size_t)readProcessed); + } + } + + /* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */ + + res2 = SZ_OK; + + if (Semaphore_Wait(&mtc->blocksSemaphore) != 0) + { + res2 = SZ_ERROR_THREAD; + if (res == SZ_OK) + { + res = res2; + // MtProgress_SetError(&mtc->mtProgress, res); + } + } + + bi = mtc->blockIndex; + + if (++mtc->blockIndex >= mtc->numBlocksMax) + mtc->blockIndex = 0; + + bufIndex = (unsigned)(int)-1; + + if (res == SZ_OK) + res = MtProgress_GetError(&mtc->mtProgress); + + if (res != SZ_OK) + finished = True; + + if (!finished) + { + if (mtc->numStartedThreads < mtc->numStartedThreadsLimit + && mtc->expectedDataSize != readProcessed) + { + res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]); + if (res == SZ_OK) + mtc->numStartedThreads++; + else + { + MtProgress_SetError(&mtc->mtProgress, res); + finished = True; + } + } + } + + if (finished) + mtc->stopReading = True; + + RINOK_THREAD(Event_Set(&mtc->readEvent)) + + if (res2 != SZ_OK) + return res2; + + if (res == SZ_OK) + { + CriticalSection_Enter(&mtc->cs); + bufIndex = mtc->freeBlockHead; + mtc->freeBlockHead = mtc->freeBlockList[bufIndex]; + CriticalSection_Leave(&mtc->cs); + + res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, + mtc->inStream ? t->inBuf : inData, size, finished); + + // MtProgress_Reinit(&mtc->mtProgress, t->index); + + if (res != SZ_OK) + MtProgress_SetError(&mtc->mtProgress, res); + } + + { + CMtCoderBlock *block = &mtc->blocks[bi]; + block->res = res; + block->bufIndex = bufIndex; + block->finished = finished; + } + + #ifdef MTCODER__USE_WRITE_THREAD + RINOK_THREAD(Event_Set(&mtc->writeEvents[bi])) + #else + { + unsigned wi; + { + CriticalSection_Enter(&mtc->cs); + wi = mtc->writeIndex; + if (wi == bi) + mtc->writeIndex = (unsigned)(int)-1; + else + mtc->ReadyBlocks[bi] = True; + CriticalSection_Leave(&mtc->cs); + } + + if (wi != bi) + { + if (res != SZ_OK || finished) + return 0; + continue; + } + + if (mtc->writeRes != SZ_OK) + res = mtc->writeRes; + + for (;;) + { + if (res == SZ_OK && bufIndex != (unsigned)(int)-1) + { + res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex); + if (res != SZ_OK) + { + mtc->writeRes = res; + MtProgress_SetError(&mtc->mtProgress, res); + } + } + + if (++wi >= mtc->numBlocksMax) + wi = 0; + { + BoolInt isReady; + + CriticalSection_Enter(&mtc->cs); + + if (bufIndex != (unsigned)(int)-1) + { + mtc->freeBlockList[bufIndex] = mtc->freeBlockHead; + mtc->freeBlockHead = bufIndex; + } + + isReady = mtc->ReadyBlocks[wi]; + + if (isReady) + mtc->ReadyBlocks[wi] = False; + else + mtc->writeIndex = wi; + + CriticalSection_Leave(&mtc->cs); + + RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore)) + + if (!isReady) + break; + } + + { + CMtCoderBlock *block = &mtc->blocks[wi]; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + bufIndex = block->bufIndex; + finished = block->finished; + } + } + } + #endif + + if (finished || res != SZ_OK) + return 0; } } -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) + +static THREAD_FUNC_DECL ThreadFunc(void *pp) { - CMtThread *p = (CMtThread *)pp; + CMtCoderThread *t = (CMtCoderThread *)pp; for (;;) { - Bool stop; - CMtThread *next = GET_NEXT_THREAD(p); - SRes res = MtThread_Process(p, &stop); - if (res != SZ_OK) + if (Event_Wait(&t->startEvent) != 0) + return (THREAD_FUNC_RET_TYPE)SZ_ERROR_THREAD; + if (t->stop) + return 0; { - MtCoder_SetError(p->mtCoder, res); - MtProgress_SetError(&p->mtCoder->mtProgress, res); - next->stopReading = True; - next->stopWriting = True; - Event_Set(&next->canRead); - Event_Set(&next->canWrite); - return res; + SRes res = ThreadFunc2(t); + CMtCoder *mtc = t->mtCoder; + if (res != SZ_OK) + { + MtProgress_SetError(&mtc->mtProgress, res); + } + + #ifndef MTCODER__USE_WRITE_THREAD + { + unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads); + if (numFinished == mtc->numStartedThreads) + if (Event_Set(&mtc->finishedEvent) != 0) + return (THREAD_FUNC_RET_TYPE)SZ_ERROR_THREAD; + } + #endif } - if (stop) - return 0; } } -void MtCoder_Construct(CMtCoder* p) + + +void MtCoder_Construct(CMtCoder *p) { unsigned i; - p->alloc = 0; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + + p->blockSize = 0; + p->numThreadsMax = 0; + p->expectedDataSize = (UInt64)(Int64)-1; + + p->inStream = NULL; + p->inData = NULL; + p->inDataSize = 0; + + p->progress = NULL; + p->allocBig = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + Event_Construct(&p->readEvent); + Semaphore_Construct(&p->blocksSemaphore); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) { - CMtThread *t = &p->threads[i]; + CMtCoderThread *t = &p->threads[i]; + t->mtCoder = p; t->index = i; - CMtThread_Construct(t, p); + t->inBuf = NULL; + t->stop = False; + Event_Construct(&t->startEvent); + Thread_Construct(&t->thread); } + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Construct(&p->writeEvents[i]); + #else + Event_Construct(&p->finishedEvent); + #endif + CriticalSection_Init(&p->cs); CriticalSection_Init(&p->mtProgress.cs); } -void MtCoder_Destruct(CMtCoder* p) + + + +static void MtCoder_Free(CMtCoder *p) { unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - CMtThread_Destruct(&p->threads[i]); + + /* + p->stopReading = True; + if (Event_IsCreated(&p->readEvent)) + Event_Set(&p->readEvent); + */ + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + MtCoderThread_Destruct(&p->threads[i]); + + Event_Close(&p->readEvent); + Semaphore_Close(&p->blocksSemaphore); + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Close(&p->writeEvents[i]); + #else + Event_Close(&p->finishedEvent); + #endif +} + + +void MtCoder_Destruct(CMtCoder *p) +{ + MtCoder_Free(p); + CriticalSection_Delete(&p->cs); CriticalSection_Delete(&p->mtProgress.cs); } + SRes MtCoder_Code(CMtCoder *p) { - unsigned i, numThreads = p->numThreads; + unsigned numThreads = p->numThreadsMax; + unsigned numBlocksMax; + unsigned i; SRes res = SZ_OK; - p->res = SZ_OK; + + if (numThreads > MTCODER__THREADS_MAX) + numThreads = MTCODER__THREADS_MAX; + numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads); + + if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++; + + if (numBlocksMax > MTCODER__BLOCKS_MAX) + numBlocksMax = MTCODER__BLOCKS_MAX; + + if (p->blockSize != p->allocatedBufsSize) + { + for (i = 0; i < MTCODER__THREADS_MAX; i++) + { + CMtCoderThread *t = &p->threads[i]; + if (t->inBuf) + { + ISzAlloc_Free(p->allocBig, t->inBuf); + t->inBuf = NULL; + } + } + p->allocatedBufsSize = p->blockSize; + } + + p->readRes = SZ_OK; MtProgress_Init(&p->mtProgress, p->progress); - for (i = 0; i < numThreads; i++) + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < numBlocksMax; i++) + { + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i])); + } + #else + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + #endif + + { + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent)); + RINOK_THREAD(Semaphore_OptCreateInit(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); + } + + for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++) + p->freeBlockList[i] = i + 1; + p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1; + p->freeBlockHead = 0; + + p->readProcessed = 0; + p->blockIndex = 0; + p->numBlocksMax = numBlocksMax; + p->stopReading = False; + + #ifndef MTCODER__USE_WRITE_THREAD + p->writeIndex = 0; + p->writeRes = SZ_OK; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->ReadyBlocks[i] = False; + p->numFinishedThreads = 0; + #endif + + p->numStartedThreadsLimit = numThreads; + p->numStartedThreads = 0; + + // for (i = 0; i < numThreads; i++) { - RINOK(CMtThread_Prepare(&p->threads[i])); + CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++]; + RINOK(MtCoderThread_CreateAndStart(nextThread)); } - for (i = 0; i < numThreads; i++) + RINOK_THREAD(Event_Set(&p->readEvent)) + + #ifdef MTCODER__USE_WRITE_THREAD { - CMtThread *t = &p->threads[i]; - CLoopThread *lt = &t->thread; + unsigned bi = 0; - if (!Thread_WasCreated(<->thread)) + for (;; bi++) { - lt->func = ThreadFunc; - lt->param = t; + if (bi >= numBlocksMax) + bi = 0; + + RINOK_THREAD(Event_Wait(&p->writeEvents[bi])) - if (LoopThread_Create(lt) != SZ_OK) { - res = SZ_ERROR_THREAD; - break; + const CMtCoderBlock *block = &p->blocks[bi]; + unsigned bufIndex = block->bufIndex; + BoolInt finished = block->finished; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + + if (bufIndex != (unsigned)(int)-1) + { + if (res == SZ_OK) + { + res = p->mtCallback->Write(p->mtCallbackObject, bufIndex); + if (res != SZ_OK) + MtProgress_SetError(&p->mtProgress, res); + } + + CriticalSection_Enter(&p->cs); + { + p->freeBlockList[bufIndex] = p->freeBlockHead; + p->freeBlockHead = bufIndex; + } + CriticalSection_Leave(&p->cs); + } + + RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore)) + + if (finished) + break; } } } + #else + { + WRes wres = Event_Wait(&p->finishedEvent); + res = MY_SRes_HRESULT_FROM_WRes(wres); + } + #endif if (res == SZ_OK) - { - unsigned j; - for (i = 0; i < numThreads; i++) - { - CMtThread *t = &p->threads[i]; - if (LoopThread_StartSubThread(&t->thread) != SZ_OK) - { - res = SZ_ERROR_THREAD; - p->threads[0].stopReading = True; - break; - } - } + res = p->readRes; - Event_Set(&p->threads[0].canWrite); - Event_Set(&p->threads[0].canRead); + if (res == SZ_OK) + res = p->mtProgress.res; - for (j = 0; j < i; j++) - LoopThread_WaitSubThread(&p->threads[j].thread); - } + #ifndef MTCODER__USE_WRITE_THREAD + if (res == SZ_OK) + res = p->writeRes; + #endif - for (i = 0; i < numThreads; i++) - CMtThread_CloseEvents(&p->threads[i]); - return (res == SZ_OK) ? p->res : res; + if (res != SZ_OK) + MtCoder_Free(p); + return res; } + +#endif diff --git a/3rdparty/lzma/C/MtCoder.h b/3rdparty/lzma/C/MtCoder.h index f0f06da281f..5a5f4d11b9c 100644 --- a/3rdparty/lzma/C/MtCoder.h +++ b/3rdparty/lzma/C/MtCoder.h @@ -1,98 +1,141 @@ /* MtCoder.h -- Multi-thread Coder -2009-11-19 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H -#include "Threads.h" +#include "MtDec.h" EXTERN_C_BEGIN -typedef struct -{ - CThread thread; - CAutoResetEvent startEvent; - CAutoResetEvent finishedEvent; - int stop; - - THREAD_FUNC_TYPE func; - LPVOID param; - THREAD_FUNC_RET_TYPE res; -} CLoopThread; - -void LoopThread_Construct(CLoopThread *p); -void LoopThread_Close(CLoopThread *p); -WRes LoopThread_Create(CLoopThread *p); -WRes LoopThread_StopAndWait(CLoopThread *p); -WRes LoopThread_StartSubThread(CLoopThread *p); -WRes LoopThread_WaitSubThread(CLoopThread *p); +/* + if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream + if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream +*/ +/* #define MTCODER__USE_WRITE_THREAD */ #ifndef _7ZIP_ST -#define NUM_MT_CODER_THREADS_MAX 32 + #define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1) + #define MTCODER__THREADS_MAX 64 + #define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3) #else -#define NUM_MT_CODER_THREADS_MAX 1 + #define MTCODER__THREADS_MAX 1 + #define MTCODER__BLOCKS_MAX 1 #endif + +#ifndef _7ZIP_ST + + typedef struct { - UInt64 totalInSize; - UInt64 totalOutSize; - ICompressProgress *progress; - SRes res; - CCriticalSection cs; - UInt64 inSizes[NUM_MT_CODER_THREADS_MAX]; - UInt64 outSizes[NUM_MT_CODER_THREADS_MAX]; -} CMtProgress; + ICompressProgress vt; + CMtProgress *mtProgress; + UInt64 inSize; + UInt64 outSize; +} CMtProgressThunk; + +void MtProgressThunk_CreateVTable(CMtProgressThunk *p); + +#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize); struct _CMtCoder; + typedef struct { struct _CMtCoder *mtCoder; - Byte *outBuf; - size_t outBufSize; - Byte *inBuf; - size_t inBufSize; unsigned index; - CLoopThread thread; + int stop; + Byte *inBuf; + + CAutoResetEvent startEvent; + CThread thread; +} CMtCoderThread; - Bool stopReading; - Bool stopWriting; - CAutoResetEvent canRead; - CAutoResetEvent canWrite; -} CMtThread; typedef struct { - SRes (*Code)(void *p, unsigned index, Byte *dest, size_t *destSize, + SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex, const Byte *src, size_t srcSize, int finished); -} IMtCoderCallback; + SRes (*Write)(void *p, unsigned outBufIndex); +} IMtCoderCallback2; + + +typedef struct +{ + SRes res; + unsigned bufIndex; + BoolInt finished; +} CMtCoderBlock; + typedef struct _CMtCoder { - size_t blockSize; - size_t destBlockSize; - unsigned numThreads; + /* input variables */ + size_t blockSize; /* size of input block */ + unsigned numThreadsMax; + UInt64 expectedDataSize; + ISeqInStream *inStream; - ISeqOutStream *outStream; + const Byte *inData; + size_t inDataSize; + ICompressProgress *progress; - ISzAlloc *alloc; + ISzAllocPtr allocBig; + + IMtCoderCallback2 *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + CAutoResetEvent readEvent; + CSemaphore blocksSemaphore; + + BoolInt stopReading; + SRes readRes; + + #ifdef MTCODER__USE_WRITE_THREAD + CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX]; + #else + CAutoResetEvent finishedEvent; + SRes writeRes; + unsigned writeIndex; + Byte ReadyBlocks[MTCODER__BLOCKS_MAX]; + LONG numFinishedThreads; + #endif + + unsigned numStartedThreadsLimit; + unsigned numStartedThreads; + + unsigned numBlocksMax; + unsigned blockIndex; + UInt64 readProcessed; - IMtCoderCallback *mtCallback; CCriticalSection cs; - SRes res; + + unsigned freeBlockHead; + unsigned freeBlockList[MTCODER__BLOCKS_MAX]; CMtProgress mtProgress; - CMtThread threads[NUM_MT_CODER_THREADS_MAX]; + CMtCoderBlock blocks[MTCODER__BLOCKS_MAX]; + CMtCoderThread threads[MTCODER__THREADS_MAX]; } CMtCoder; -void MtCoder_Construct(CMtCoder* p); -void MtCoder_Destruct(CMtCoder* p); + +void MtCoder_Construct(CMtCoder *p); +void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); + +#endif + + EXTERN_C_END #endif diff --git a/3rdparty/lzma/C/MtDec.c b/3rdparty/lzma/C/MtDec.c new file mode 100644 index 00000000000..45a6713966e --- /dev/null +++ b/3rdparty/lzma/C/MtDec.c @@ -0,0 +1,1139 @@ +/* MtDec.c -- Multi-thread Decoder +2021-12-21 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #include +#include + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#include "MtDec.h" + +#ifndef _7ZIP_ST + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +{ + p->progress = progress; + p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; +} + + +SRes MtProgress_Progress_ST(CMtProgress *p) +{ + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + return p->res; +} + + +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize) +{ + SRes res; + CriticalSection_Enter(&p->cs); + + p->totalInSize += inSize; + p->totalOutSize += outSize; + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + res = p->res; + + CriticalSection_Leave(&p->cs); + return res; +} + + +SRes MtProgress_GetError(CMtProgress *p) +{ + SRes res; + CriticalSection_Enter(&p->cs); + res = p->res; + CriticalSection_Leave(&p->cs); + return res; +} + + +void MtProgress_SetError(CMtProgress *p, SRes res) +{ + CriticalSection_Enter(&p->cs); + if (p->res == SZ_OK) + p->res = res; + CriticalSection_Leave(&p->cs); +} + + +#define RINOK_THREAD(x) RINOK_WRes(x) + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + + +struct __CMtDecBufLink +{ + struct __CMtDecBufLink *next; + void *pad[3]; +}; + +typedef struct __CMtDecBufLink CMtDecBufLink; + +#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) +#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) + + + +static THREAD_FUNC_DECL ThreadFunc(void *pp); + + +static WRes MtDecThread_CreateEvents(CMtDecThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + if (wres == 0) + { + wres = ArEvent_OptCreate_And_Reset(&t->canRead); + if (wres == 0) + return SZ_OK; + } + return wres; +} + + +static SRes MtDecThread_CreateAndStart(CMtDecThread *t) +{ + WRes wres = MtDecThread_CreateEvents(t); + // wres = 17; // for test + if (wres == 0) + { + if (Thread_WasCreated(&t->thread)) + return SZ_OK; + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + return SZ_OK; + } + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +void MtDecThread_FreeInBufs(CMtDecThread *t) +{ + if (t->inBuf) + { + void *link = t->inBuf; + t->inBuf = NULL; + do + { + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(t->mtDec->alloc, link); + link = next; + } + while (link); + } +} + + +static void MtDecThread_CloseThread(CMtDecThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ + Event_Set(&t->canRead); + Thread_Wait_Close(&t->thread); + } + + Event_Close(&t->canRead); + Event_Close(&t->canWrite); +} + +static void MtDec_CloseThreads(CMtDec *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_CloseThread(&p->threads[i]); +} + +static void MtDecThread_Destruct(CMtDecThread *t) +{ + MtDecThread_CloseThread(t); + MtDecThread_FreeInBufs(t); +} + + + +static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; + RINOK(res); + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + + +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + CriticalSection_Leave(&p->mtProgress.cs); + return res; +} + +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + + p->mtProgress.totalInSize += inSize; + p->mtProgress.totalOutSize += outSize; + if (p->mtProgress.res == SZ_OK && p->mtProgress.progress) + if (ICompressProgress_Progress(p->mtProgress.progress, p->mtProgress.totalInSize, p->mtProgress.totalOutSize) != SZ_OK) + p->mtProgress.res = SZ_ERROR_PROGRESS; + + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + + CriticalSection_Leave(&p->mtProgress.cs); + + return res; +} + +static void MtDec_Interrupt(CMtDec *p, UInt64 interruptIndex) +{ + CriticalSection_Enter(&p->mtProgress.cs); + if (!p->needInterrupt || interruptIndex < p->interruptIndex) + { + p->interruptIndex = interruptIndex; + p->needInterrupt = True; + } + CriticalSection_Leave(&p->mtProgress.cs); +} + +Byte *MtDec_GetCrossBuff(CMtDec *p) +{ + Byte *cr = p->crossBlock; + if (!cr) + { + cr = (Byte *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!cr) + return NULL; + p->crossBlock = cr; + } + return MTDEC__DATA_PTR_FROM_LINK(cr); +} + + +/* + ThreadFunc2() returns: + 0 - in all normal cases (even for stream error or memory allocation error) + (!= 0) - WRes error return by system threading function +*/ + +// #define MTDEC_ProgessStep (1 << 22) +#define MTDEC_ProgessStep (1 << 0) + +static WRes ThreadFunc2(CMtDecThread *t) +{ + CMtDec *p = t->mtDec; + + PRF_STR_INT("ThreadFunc2", t->index); + + // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); + + for (;;) + { + SRes res, codeRes; + BoolInt wasInterrupted, isAllocError, overflow, finish; + SRes threadingErrorSRes; + BoolInt needCode, needWrite, needContinue; + + size_t inDataSize_Start; + UInt64 inDataSize; + // UInt64 inDataSize_Full; + + UInt64 blockIndex; + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 inCodePos; + UInt64 outCodePos; + + Byte *afterEndData = NULL; + size_t afterEndData_Size = 0; + BoolInt afterEndData_IsCross = False; + + BoolInt canCreateNewThread = False; + // CMtDecCallbackInfo parse; + CMtDecThread *nextThread; + + PRF_STR_INT("=============== Event_Wait(&t->canRead)", t->index); + + RINOK_THREAD(Event_Wait(&t->canRead)); + if (p->exitThread) + return 0; + + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + + // if (t->index == 3) return 19; // for test + + blockIndex = p->blockIndex++; + + // PRF(printf("\ncanRead\n")) + + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + + finish = p->readWasFinished; + needCode = False; + needWrite = False; + isAllocError = False; + overflow = False; + + inDataSize_Start = 0; + inDataSize = 0; + // inDataSize_Full = 0; + + if (res == SZ_OK && !wasInterrupted) + { + // if (p->inStream) + { + CMtDecBufLink *prev = NULL; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + size_t crossSize = p->crossEnd - p->crossStart; + + PRF(printf("\ncrossSize = %d\n", crossSize)); + + for (;;) + { + if (!link) + { + link = (CMtDecBufLink *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!link) + { + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + link->next = NULL; + if (prev) + { + // static unsigned g_num = 0; + // printf("\n%6d : %x", ++g_num, (unsigned)(size_t)((Byte *)link - (Byte *)prev)); + prev->next = link; + } + else + t->inBuf = (void *)link; + } + + { + Byte *data = MTDEC__DATA_PTR_FROM_LINK(link); + Byte *parseData = data; + size_t size; + + if (crossSize != 0) + { + inDataSize = crossSize; + // inDataSize_Full = inDataSize; + inDataSize_Start = crossSize; + size = crossSize; + parseData = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + PRF(printf("\ncross : crossStart = %7d crossEnd = %7d finish = %1d", + (int)p->crossStart, (int)p->crossEnd, (int)finish)); + } + else + { + size = p->inBufSize; + + res = FullRead(p->inStream, data, &size); + + // size = 10; // test + + inDataSize += size; + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = size; + + p->readProcessed += size; + finish = (size != p->inBufSize); + if (finish) + p->readWasFinished = True; + + // res = E_INVALIDARG; // test + + if (res != SZ_OK) + { + // PRF(printf("\nRead error = %d\n", res)) + // we want to decode all data before error + p->readRes = res; + // p->readError_BlockIndex = blockIndex; + p->readWasFinished = True; + finish = True; + res = SZ_OK; + // break; + } + + if (inDataSize - inPrev >= MTDEC_ProgessStep) + { + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inDataSize; + } + } + + { + CMtDecCallbackInfo parse; + + parse.startCall = (prev == NULL); + parse.src = parseData; + parse.srcSize = size; + parse.srcFinished = finish; + parse.canCreateNewThread = True; + + PRF(printf("\nParse size = %d\n", (unsigned)size)); + + p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + + PRF(printf(" Parse processed = %d, state = %d \n", (unsigned)parse.srcSize, (unsigned)parse.state)); + + needWrite = True; + canCreateNewThread = parse.canCreateNewThread; + + // printf("\n\n%12I64u %12I64u", (UInt64)p->mtProgress.totalInSize, (UInt64)p->mtProgress.totalOutSize); + + if ( + // parseRes != SZ_OK || + // inDataSize - (size - parse.srcSize) > p->inBlockMax + // || + parse.state == MTDEC_PARSE_OVERFLOW + // || wasInterrupted + ) + { + // Overflow or Parse error - switch from MT decoding to ST decoding + finish = True; + overflow = True; + + { + PRF(printf("\n Overflow")); + // PRF(printf("\nisBlockFinished = %d", (unsigned)parse.blockWasFinished)); + PRF(printf("\n inDataSize = %d", (unsigned)inDataSize)); + } + + if (crossSize != 0) + memcpy(data, parseData, size); + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (crossSize != 0) + { + memcpy(data, parseData, parse.srcSize); + p->crossStart += parse.srcSize; + } + + if (parse.state != MTDEC_PARSE_CONTINUE || finish) + { + // we don't need to parse in current thread anymore + + if (parse.state == MTDEC_PARSE_END) + finish = True; + + needCode = True; + // p->crossFinished = finish; + + if (parse.srcSize == size) + { + // full parsed - no cross transfer + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (parse.state == MTDEC_PARSE_END) + { + afterEndData = parseData + parse.srcSize; + afterEndData_Size = size - parse.srcSize; + if (crossSize != 0) + afterEndData_IsCross = True; + // we reduce data size to required bytes (parsed only) + inDataSize -= afterEndData_Size; + if (!prev) + inDataSize_Start = parse.srcSize; + break; + } + + { + // partial parsed - need cross transfer + if (crossSize != 0) + inDataSize = parse.srcSize; // it's only parsed now + else + { + // partial parsed - is not in initial cross block - we need to copy new data to cross block + Byte *cr = MtDec_GetCrossBuff(p); + if (!cr) + { + { + PRF(printf("\ncross alloc error error\n")); + // res = SZ_ERROR_MEM; + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + } + + { + size_t crSize = size - parse.srcSize; + inDataSize -= crSize; + p->crossEnd = crSize; + p->crossStart = 0; + memcpy(cr, parseData + parse.srcSize, crSize); + } + } + + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = parse.srcSize; // it's partial size (parsed only) + + finish = False; + break; + } + } + + if (parse.srcSize != size) + { + res = SZ_ERROR_FAIL; + PRF(printf("\nfinished error SZ_ERROR_FAIL = %d\n", res)); + break; + } + } + } + + prev = link; + link = link->next; + + if (crossSize != 0) + { + crossSize = 0; + p->crossStart = 0; + p->crossEnd = 0; + } + } + } + + if (res == SZ_OK) + res = MtDec_GetError_Spec(p, blockIndex, &wasInterrupted); + } + + codeRes = SZ_OK; + + if (res == SZ_OK && needCode && !wasInterrupted) + { + codeRes = p->mtCallback->PreCode(p->mtCallbackObject, t->index); + if (codeRes != SZ_OK) + { + needCode = False; + finish = True; + // SZ_ERROR_MEM is expected error here. + // if (codeRes == SZ_ERROR_MEM) - we will try single-thread decoding later. + // if (codeRes != SZ_ERROR_MEM) - we can stop decoding or try single-thread decoding. + } + } + + if (res != SZ_OK || wasInterrupted) + finish = True; + + nextThread = NULL; + threadingErrorSRes = SZ_OK; + + if (!finish) + { + if (p->numStartedThreads < p->numStartedThreads_Limit && canCreateNewThread) + { + SRes res2 = MtDecThread_CreateAndStart(&p->threads[p->numStartedThreads]); + if (res2 == SZ_OK) + { + // if (p->numStartedThreads % 1000 == 0) PRF(printf("\n numStartedThreads=%d\n", p->numStartedThreads)); + p->numStartedThreads++; + } + else + { + PRF(printf("\nERROR: numStartedThreads=%d\n", p->numStartedThreads)); + if (p->numStartedThreads == 1) + { + // if only one thread is possible, we leave muti-threading code + finish = True; + needCode = False; + threadingErrorSRes = res2; + } + else + p->numStartedThreads_Limit = p->numStartedThreads; + } + } + + if (!finish) + { + unsigned nextIndex = t->index + 1; + nextThread = &p->threads[nextIndex >= p->numStartedThreads ? 0 : nextIndex]; + RINOK_THREAD(Event_Set(&nextThread->canRead)) + // We have started executing for new iteration (with next thread) + // And that next thread now is responsible for possible exit from decoding (threading_code) + } + } + + // each call of Event_Set(&nextThread->canRead) must be followed by call of Event_Set(&nextThread->canWrite) + // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case + // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): + // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration + // - otherwise we stop decoding and exit from ThreadFunc2() + + // Don't change (finish) variable in the further code + + + // ---------- CODE ---------- + + inPrev = 0; + outPrev = 0; + inCodePos = 0; + outCodePos = 0; + + if (res == SZ_OK && needCode && codeRes == SZ_OK) + { + BoolInt isStartBlock = True; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + + for (;;) + { + size_t inSize; + int stop; + + if (isStartBlock) + inSize = inDataSize_Start; + else + { + UInt64 rem = inDataSize - inCodePos; + inSize = p->inBufSize; + if (inSize > rem) + inSize = (size_t)rem; + } + + inCodePos += inSize; + stop = True; + + codeRes = p->mtCallback->Code(p->mtCallbackObject, t->index, + (const Byte *)MTDEC__DATA_PTR_FROM_LINK(link), inSize, + (inCodePos == inDataSize), // srcFinished + &inCodePos, &outCodePos, &stop); + + if (codeRes != SZ_OK) + { + PRF(printf("\nCode Interrupt error = %x\n", codeRes)); + // we interrupt only later blocks + MtDec_Interrupt(p, blockIndex); + break; + } + + if (stop || inCodePos == inDataSize) + break; + + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + if (inDelta >= MTDEC_ProgessStep || outDelta >= MTDEC_ProgessStep) + { + // Sleep(1); + res = MtDec_Progress_GetError_Spec(p, inDelta, outDelta, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inCodePos; + outPrev = outCodePos; + } + } + + link = link->next; + isStartBlock = False; + } + } + + + // ---------- WRITE ---------- + + RINOK_THREAD(Event_Wait(&t->canWrite)); + + { + BoolInt isErrorMode = False; + BoolInt canRecode = True; + BoolInt needWriteToStream = needWrite; + + if (p->exitThread) return 0; // it's never executed in normal cases + + if (p->wasInterrupted) + wasInterrupted = True; + else + { + if (codeRes != SZ_OK) // || !needCode // check it !!! + { + p->wasInterrupted = True; + p->codeRes = codeRes; + if (codeRes == SZ_ERROR_MEM) + isAllocError = True; + } + + if (threadingErrorSRes) + { + p->wasInterrupted = True; + p->threadingErrorSRes = threadingErrorSRes; + needWriteToStream = False; + } + if (isAllocError) + { + p->wasInterrupted = True; + p->isAllocError = True; + needWriteToStream = False; + } + if (overflow) + { + p->wasInterrupted = True; + p->overflow = True; + needWriteToStream = False; + } + } + + if (needCode) + { + if (wasInterrupted) + { + inCodePos = 0; + outCodePos = 0; + } + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + // if (inDelta != 0 || outDelta != 0) + res = MtProgress_ProgressAdd(&p->mtProgress, inDelta, outDelta); + } + } + + needContinue = (!finish); + + // if (res == SZ_OK && needWrite && !wasInterrupted) + if (needWrite) + { + // p->inProcessed += inCodePos; + + PRF(printf("\n--Write afterSize = %d\n", (unsigned)afterEndData_Size)); + + res = p->mtCallback->Write(p->mtCallbackObject, t->index, + res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite + afterEndData, afterEndData_Size, afterEndData_IsCross, + &needContinue, + &canRecode); + + // res = SZ_ERROR_FAIL; // for test + + PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); + PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); + + if (res != SZ_OK) + { + PRF(printf("\nWrite error = %d\n", res)); + isErrorMode = True; + p->wasInterrupted = True; + } + if (res != SZ_OK + || (!needContinue && !finish)) + { + PRF(printf("\nWrite Interrupt error = %x\n", res)); + MtDec_Interrupt(p, blockIndex); + } + } + + if (canRecode) + if (!needCode + || res != SZ_OK + || p->wasInterrupted + || codeRes != SZ_OK + || wasInterrupted + || p->numFilledThreads != 0 + || isErrorMode) + { + if (p->numFilledThreads == 0) + p->filledThreadStart = t->index; + if (inDataSize != 0 || !finish) + { + t->inDataSize_Start = inDataSize_Start; + t->inDataSize = inDataSize; + p->numFilledThreads++; + } + PRF(printf("\np->numFilledThreads = %d\n", p->numFilledThreads)); + PRF(printf("p->filledThreadStart = %d\n", p->filledThreadStart)); + } + + if (!finish) + { + RINOK_THREAD(Event_Set(&nextThread->canWrite)); + } + else + { + if (needContinue) + { + // we restore decoding with new iteration + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + } + else + { + // we exit from decoding + if (t->index == 0) + return SZ_OK; + p->exitThread = True; + } + RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + } + } + } +} + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include +#else +#include +#endif +#endif + + +static THREAD_FUNC_DECL ThreadFunc1(void *pp) +{ + WRes res; + + CMtDecThread *t = (CMtDecThread *)pp; + CMtDec *p; + + // fprintf(stdout, "\n%d = %p\n", t->index, &t); + + res = ThreadFunc2(t); + p = t->mtDec; + if (res == 0) + return (THREAD_FUNC_RET_TYPE)(UINT_PTR)p->exitThreadWRes; + { + // it's unexpected situation for some threading function error + if (p->exitThreadWRes == 0) + p->exitThreadWRes = res; + PRF(printf("\nthread exit error = %d\n", res)); + p->exitThread = True; + Event_Set(&p->threads[0].canRead); + Event_Set(&p->threads[0].canWrite); + MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); + } + return (THREAD_FUNC_RET_TYPE)(UINT_PTR)res; +} + +static MY_NO_INLINE THREAD_FUNC_DECL ThreadFunc(void *pp) +{ + #ifdef USE_ALLOCA + CMtDecThread *t = (CMtDecThread *)pp; + // fprintf(stderr, "\n%d = %p - before", t->index, &t); + t->allocaPtr = alloca(t->index * 128); + #endif + return ThreadFunc1(pp); +} + + +int MtDec_PrepareRead(CMtDec *p) +{ + if (p->crossBlock && p->crossStart == p->crossEnd) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + if (i > p->numStartedThreads + || p->numFilledThreads <= + (i >= p->filledThreadStart ? + i - p->filledThreadStart : + i + p->numStartedThreads - p->filledThreadStart)) + MtDecThread_FreeInBufs(&p->threads[i]); + } + + return (p->numFilledThreads != 0) || (p->crossStart != p->crossEnd); +} + + +const Byte *MtDec_Read(CMtDec *p, size_t *inLim) +{ + while (p->numFilledThreads != 0) + { + CMtDecThread *t = &p->threads[p->filledThreadStart]; + + if (*inLim != 0) + { + { + void *link = t->inBuf; + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(p->alloc, link); + t->inBuf = next; + } + + if (t->inDataSize == 0) + { + MtDecThread_FreeInBufs(t); + if (--p->numFilledThreads == 0) + break; + if (++p->filledThreadStart == p->numStartedThreads) + p->filledThreadStart = 0; + t = &p->threads[p->filledThreadStart]; + } + } + + { + size_t lim = t->inDataSize_Start; + if (lim != 0) + t->inDataSize_Start = 0; + else + { + UInt64 rem = t->inDataSize; + lim = p->inBufSize; + if (lim > rem) + lim = (size_t)rem; + } + t->inDataSize -= lim; + *inLim = lim; + return (const Byte *)MTDEC__DATA_PTR_FROM_LINK(t->inBuf); + } + } + + { + size_t crossSize = p->crossEnd - p->crossStart; + if (crossSize != 0) + { + const Byte *data = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + *inLim = crossSize; + p->crossStart = 0; + p->crossEnd = 0; + return data; + } + *inLim = 0; + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + return NULL; + } +} + + +void MtDec_Construct(CMtDec *p) +{ + unsigned i; + + p->inBufSize = (size_t)1 << 18; + + p->numThreadsMax = 0; + + p->inStream = NULL; + + // p->inData = NULL; + // p->inDataSize = 0; + + p->crossBlock = NULL; + p->crossStart = 0; + p->crossEnd = 0; + + p->numFilledThreads = 0; + + p->progress = NULL; + p->alloc = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + t->mtDec = p; + t->index = i; + t->inBuf = NULL; + Event_Construct(&t->canRead); + Event_Construct(&t->canWrite); + Thread_Construct(&t->thread); + } + + // Event_Construct(&p->finishedEvent); + + CriticalSection_Init(&p->mtProgress.cs); +} + + +static void MtDec_Free(CMtDec *p) +{ + unsigned i; + + p->exitThread = True; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_Destruct(&p->threads[i]); + + // Event_Close(&p->finishedEvent); + + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } +} + + +void MtDec_Destruct(CMtDec *p) +{ + MtDec_Free(p); + + CriticalSection_Delete(&p->mtProgress.cs); +} + + +SRes MtDec_Code(CMtDec *p) +{ + unsigned i; + + p->inProcessed = 0; + + p->blockIndex = 1; // it must be larger than not_defined index (0) + p->isAllocError = False; + p->overflow = False; + p->threadingErrorSRes = SZ_OK; + + p->needContinue = True; + + p->readWasFinished = False; + p->needInterrupt = False; + p->interruptIndex = (UInt64)(Int64)-1; + + p->readProcessed = 0; + p->readRes = SZ_OK; + p->codeRes = SZ_OK; + p->wasInterrupted = False; + + p->crossStart = 0; + p->crossEnd = 0; + + p->filledThreadStart = 0; + p->numFilledThreads = 0; + + { + unsigned numThreads = p->numThreadsMax; + if (numThreads > MTDEC__THREADS_MAX) + numThreads = MTDEC__THREADS_MAX; + p->numStartedThreads_Limit = numThreads; + p->numStartedThreads = 0; + } + + if (p->inBufSize != p->allocatedBufsSize) + { + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + if (t->inBuf) + MtDecThread_FreeInBufs(t); + } + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + p->allocatedBufsSize = p->inBufSize; + } + + MtProgress_Init(&p->mtProgress, p->progress); + + // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + p->exitThread = False; + p->exitThreadWRes = 0; + + { + WRes wres; + SRes sres; + CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; + // wres = MtDecThread_CreateAndStart(nextThread); + wres = MtDecThread_CreateEvents(nextThread); + if (wres == 0) { wres = Event_Set(&nextThread->canWrite); + if (wres == 0) { wres = Event_Set(&nextThread->canRead); + if (wres == 0) { THREAD_FUNC_RET_TYPE res = ThreadFunc(nextThread); + wres = (WRes)(UINT_PTR)res; + if (wres != 0) + { + p->needContinue = False; + MtDec_CloseThreads(p); + }}}} + + // wres = 17; // for test + // wres = Event_Wait(&p->finishedEvent); + + sres = MY_SRes_HRESULT_FROM_WRes(wres); + + if (sres != 0) + p->threadingErrorSRes = sres; + + if ( + // wres == 0 + // wres != 0 + // || p->mtc.codeRes == SZ_ERROR_MEM + p->isAllocError + || p->threadingErrorSRes != SZ_OK + || p->overflow) + { + // p->needContinue = True; + } + else + p->needContinue = False; + + if (p->needContinue) + return SZ_OK; + + // if (sres != SZ_OK) + return sres; + // return SZ_ERROR_FAIL; + } +} + +#endif diff --git a/3rdparty/lzma/C/MtDec.h b/3rdparty/lzma/C/MtDec.h new file mode 100644 index 00000000000..c2da46ae291 --- /dev/null +++ b/3rdparty/lzma/C/MtDec.h @@ -0,0 +1,202 @@ +/* MtDec.h -- Multi-thread Decoder +2020-03-05 : Igor Pavlov : Public domain */ + +#ifndef __MT_DEC_H +#define __MT_DEC_H + +#include "7zTypes.h" + +#ifndef _7ZIP_ST +#include "Threads.h" +#endif + +EXTERN_C_BEGIN + +#ifndef _7ZIP_ST + +#ifndef _7ZIP_ST + #define MTDEC__THREADS_MAX 32 +#else + #define MTDEC__THREADS_MAX 1 +#endif + + +typedef struct +{ + ICompressProgress *progress; + SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; + CCriticalSection cs; +} CMtProgress; + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +SRes MtProgress_Progress_ST(CMtProgress *p); +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); +SRes MtProgress_GetError(CMtProgress *p); +void MtProgress_SetError(CMtProgress *p, SRes res); + +struct _CMtDec; + +typedef struct +{ + struct _CMtDec *mtDec; + unsigned index; + void *inBuf; + + size_t inDataSize_Start; // size of input data in start block + UInt64 inDataSize; // total size of input data in all blocks + + CThread thread; + CAutoResetEvent canRead; + CAutoResetEvent canWrite; + void *allocaPtr; +} CMtDecThread; + +void MtDecThread_FreeInBufs(CMtDecThread *t); + + +typedef enum +{ + MTDEC_PARSE_CONTINUE, // continue this block with more input data + MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread + MTDEC_PARSE_NEW, // new block + MTDEC_PARSE_END // end of block threading. But we still can return to threading after Write(&needContinue) +} EMtDecParseState; + +typedef struct +{ + // in + int startCall; + const Byte *src; + size_t srcSize; + // in : (srcSize == 0) is allowed + // out : it's allowed to return less that actually was used ? + int srcFinished; + + // out + EMtDecParseState state; + BoolInt canCreateNewThread; + UInt64 outPos; // check it (size_t) +} CMtDecCallbackInfo; + + +typedef struct +{ + void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci); + + // PreCode() and Code(): + // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks + SRes (*PreCode)(void *p, unsigned coderIndex); + SRes (*Code)(void *p, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop); + // stop - means stop another Code calls + + + /* Write() must be called, if Parse() was called + set (needWrite) if + { + && (was not interrupted by progress) + && (was not interrupted in previous block) + } + + out: + if (*needContinue), decoder still need to continue decoding with new iteration, + even after MTDEC_PARSE_END + if (*canRecode), we didn't flush current block data, so we still can decode current block later. + */ + SRes (*Write)(void *p, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, BoolInt isCross, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode); + +} IMtDecCallback2; + + + +typedef struct _CMtDec +{ + /* input variables */ + + size_t inBufSize; /* size of input block */ + unsigned numThreadsMax; + // size_t inBlockMax; + unsigned numThreadsMax_2; + + ISeqInStream *inStream; + // const Byte *inData; + // size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr alloc; + + IMtDecCallback2 *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + BoolInt exitThread; + WRes exitThreadWRes; + + UInt64 blockIndex; + BoolInt isAllocError; + BoolInt overflow; + SRes threadingErrorSRes; + + BoolInt needContinue; + + // CAutoResetEvent finishedEvent; + + SRes readRes; + SRes codeRes; + + BoolInt wasInterrupted; + + unsigned numStartedThreads_Limit; + unsigned numStartedThreads; + + Byte *crossBlock; + size_t crossStart; + size_t crossEnd; + UInt64 readProcessed; + BoolInt readWasFinished; + UInt64 inProcessed; + + unsigned filledThreadStart; + unsigned numFilledThreads; + + #ifndef _7ZIP_ST + BoolInt needInterrupt; + UInt64 interruptIndex; + CMtProgress mtProgress; + CMtDecThread threads[MTDEC__THREADS_MAX]; + #endif +} CMtDec; + + +void MtDec_Construct(CMtDec *p); +void MtDec_Destruct(CMtDec *p); + +/* +MtDec_Code() returns: + SZ_OK - in most cases + MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function +*/ + +SRes MtDec_Code(CMtDec *p); +Byte *MtDec_GetCrossBuff(CMtDec *p); + +int MtDec_PrepareRead(CMtDec *p); +const Byte *MtDec_Read(CMtDec *p, size_t *inLim); + +#endif + +EXTERN_C_END + +#endif diff --git a/3rdparty/lzma/C/Ppmd.h b/3rdparty/lzma/C/Ppmd.h index 5655b26d73c..b198792087e 100644 --- a/3rdparty/lzma/C/Ppmd.h +++ b/3rdparty/lzma/C/Ppmd.h @@ -1,5 +1,5 @@ /* Ppmd.h -- PPMD codec common code -2016-05-16 : Igor Pavlov : Public domain +2021-04-13 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #ifndef __PPMD_H @@ -9,7 +9,16 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ EXTERN_C_BEGIN -#ifdef MY_CPU_32BIT +#if defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 4) +/* + PPMD code always uses 32-bit internal fields in PPMD structures to store internal references in main block. + if (PPMD_32BIT is defined), the PPMD code stores internal pointers to 32-bit reference fields. + if (PPMD_32BIT is NOT defined), the PPMD code stores internal UInt32 offsets to reference fields. + if (pointer size is 64-bit), then (PPMD_32BIT) mode is not allowed, + if (pointer size is 32-bit), then (PPMD_32BIT) mode is optional, + and it's allowed to disable PPMD_32BIT mode even if pointer is 32-bit. + PPMD code works slightly faster in (PPMD_32BIT) mode. +*/ #define PPMD_32BIT #endif @@ -28,7 +37,7 @@ EXTERN_C_BEGIN #define PPMD_N4 ((128 + 3 - 1 * PPMD_N1 - 2 * PPMD_N2 - 3 * PPMD_N3) / 4) #define PPMD_NUM_INDEXES (PPMD_N1 + PPMD_N2 + PPMD_N3 + PPMD_N4) -#pragma pack(push, 1) +MY_CPU_pragma_pack_push_1 /* Most compilers works OK here even without #pragma pack(push, 1), but some GCC compilers need it. */ /* SEE-contexts for PPM-contexts with masked symbols */ @@ -40,44 +49,117 @@ typedef struct } CPpmd_See; #define Ppmd_See_Update(p) if ((p)->Shift < PPMD_PERIOD_BITS && --(p)->Count == 0) \ - { (p)->Summ <<= 1; (p)->Count = (Byte)(3 << (p)->Shift++); } + { (p)->Summ = (UInt16)((p)->Summ << 1); (p)->Count = (Byte)(3 << (p)->Shift++); } + typedef struct { Byte Symbol; Byte Freq; - UInt16 SuccessorLow; - UInt16 SuccessorHigh; + UInt16 Successor_0; + UInt16 Successor_1; } CPpmd_State; -#pragma pack(pop) - -typedef - #ifdef PPMD_32BIT - CPpmd_State * - #else - UInt32 - #endif - CPpmd_State_Ref; - -typedef - #ifdef PPMD_32BIT - void * - #else - UInt32 - #endif - CPpmd_Void_Ref; - -typedef - #ifdef PPMD_32BIT - Byte * - #else - UInt32 - #endif - CPpmd_Byte_Ref; +typedef struct CPpmd_State2_ +{ + Byte Symbol; + Byte Freq; +} CPpmd_State2; + +typedef struct CPpmd_State4_ +{ + UInt16 Successor_0; + UInt16 Successor_1; +} CPpmd_State4; + +MY_CPU_pragma_pop + +/* + PPMD code can write full CPpmd_State structure data to CPpmd*_Context + at (byte offset = 2) instead of some fields of original CPpmd*_Context structure. + + If we use pointers to different types, but that point to shared + memory space, we can have aliasing problem (strict aliasing). + + XLC compiler in -O2 mode can change the order of memory write instructions + in relation to read instructions, if we have use pointers to different types. + + To solve that aliasing problem we use combined CPpmd*_Context structure + with unions that contain the fields from both structures: + the original CPpmd*_Context and CPpmd_State. + So we can access the fields from both structures via one pointer, + and the compiler doesn't change the order of write instructions + in relation to read instructions. + + If we don't use memory write instructions to shared memory in + some local code, and we use only reading instructions (read only), + then probably it's safe to use pointers to different types for reading. +*/ + + + +#ifdef PPMD_32BIT + + #define Ppmd_Ref_Type(type) type * + #define Ppmd_GetRef(p, ptr) (ptr) + #define Ppmd_GetPtr(p, ptr) (ptr) + #define Ppmd_GetPtr_Type(p, ptr, note_type) (ptr) + +#else + + #define Ppmd_Ref_Type(type) UInt32 + #define Ppmd_GetRef(p, ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) + #define Ppmd_GetPtr(p, offs) ((void *)((p)->Base + (offs))) + #define Ppmd_GetPtr_Type(p, offs, type) ((type *)Ppmd_GetPtr(p, offs)) + +#endif // PPMD_32BIT + + +typedef Ppmd_Ref_Type(CPpmd_State) CPpmd_State_Ref; +typedef Ppmd_Ref_Type(void) CPpmd_Void_Ref; +typedef Ppmd_Ref_Type(Byte) CPpmd_Byte_Ref; + + +/* +#ifdef MY_CPU_LE_UNALIGN +// the unaligned 32-bit access latency can be too large, if the data is not in L1 cache. +#define Ppmd_GET_SUCCESSOR(p) ((CPpmd_Void_Ref)*(const UInt32 *)(const void *)&(p)->Successor_0) +#define Ppmd_SET_SUCCESSOR(p, v) *(UInt32 *)(void *)(void *)&(p)->Successor_0 = (UInt32)(v) + +#else +*/ + +/* + We can write 16-bit halves to 32-bit (Successor) field in any selected order. + But the native order is more consistent way. + So we use the native order, if LE/BE order can be detected here at compile time. +*/ + +#ifdef MY_CPU_BE + + #define Ppmd_GET_SUCCESSOR(p) \ + ( (CPpmd_Void_Ref) (((UInt32)(p)->Successor_0 << 16) | (p)->Successor_1) ) + + #define Ppmd_SET_SUCCESSOR(p, v) { \ + (p)->Successor_0 = (UInt16)(((UInt32)(v) >> 16) /* & 0xFFFF */); \ + (p)->Successor_1 = (UInt16)((UInt32)(v) /* & 0xFFFF */); } + +#else + + #define Ppmd_GET_SUCCESSOR(p) \ + ( (CPpmd_Void_Ref) ((p)->Successor_0 | ((UInt32)(p)->Successor_1 << 16)) ) + + #define Ppmd_SET_SUCCESSOR(p, v) { \ + (p)->Successor_0 = (UInt16)((UInt32)(v) /* & 0xFFFF */); \ + (p)->Successor_1 = (UInt16)(((UInt32)(v) >> 16) /* & 0xFFFF */); } + +#endif + +// #endif + #define PPMD_SetAllBitsIn256Bytes(p) \ - { unsigned z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ + { size_t z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ p[z+7] = p[z+6] = p[z+5] = p[z+4] = p[z+3] = p[z+2] = p[z+1] = p[z+0] = ~(size_t)0; }} EXTERN_C_END diff --git a/3rdparty/lzma/C/Ppmd7.c b/3rdparty/lzma/C/Ppmd7.c index eda8eb70a22..cf401cb37c6 100644 --- a/3rdparty/lzma/C/Ppmd7.c +++ b/3rdparty/lzma/C/Ppmd7.c @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2016-05-21 : Igor Pavlov : Public domain +2021-04-13 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -8,21 +8,23 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Ppmd7.h" -const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +/* define PPMD7_ORDER_0_SUPPPORT to suport order-0 mode, unsupported by orignal PPMd var.H. code */ +// #define PPMD7_ORDER_0_SUPPPORT + +MY_ALIGN(16) +static const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +MY_ALIGN(16) static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; #define MAX_FREQ 124 #define UNIT_SIZE 12 #define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) -#define U2I(nu) (p->Units2Indx[(nu) - 1]) -#define I2U(indx) (p->Indx2Units[indx]) +#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) +#define I2U(indx) ((unsigned)p->Indx2Units[indx]) +#define I2U_UInt16(indx) ((UInt16)p->Indx2Units[indx]) -#ifdef PPMD_32BIT - #define REF(ptr) (ptr) -#else - #define REF(ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) -#endif +#define REF(ptr) Ppmd_GetRef(p, ptr) #define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) @@ -35,13 +37,7 @@ typedef CPpmd7_Context * CTX_PTR; struct CPpmd7_Node_; -typedef - #ifdef PPMD_32BIT - struct CPpmd7_Node_ * - #else - UInt32 - #endif - CPpmd7_Node_Ref; +typedef Ppmd_Ref_Type(struct CPpmd7_Node_) CPpmd7_Node_Ref; typedef struct CPpmd7_Node_ { @@ -51,17 +47,13 @@ typedef struct CPpmd7_Node_ CPpmd7_Node_Ref Prev; } CPpmd7_Node; -#ifdef PPMD_32BIT - #define NODE(ptr) (ptr) -#else - #define NODE(offs) ((CPpmd7_Node *)(p->Base + (offs))) -#endif +#define NODE(r) Ppmd_GetPtr_Type(p, r, CPpmd7_Node) void Ppmd7_Construct(CPpmd7 *p) { unsigned i, k, m; - p->Base = 0; + p->Base = NULL; for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) { @@ -77,6 +69,7 @@ void Ppmd7_Construct(CPpmd7 *p) for (i = 0; i < 3; i++) p->NS2Indx[i] = (Byte)i; + for (m = i, k = 1; i < 256; i++) { p->NS2Indx[i] = (Byte)m; @@ -84,52 +77,63 @@ void Ppmd7_Construct(CPpmd7 *p) k = (++m) - 2; } - memset(p->HB2Flag, 0, 0x40); - memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); + memcpy(p->ExpEscape, PPMD7_kExpEscape, 16); } -void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc) + +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->Base); + ISzAlloc_Free(alloc, p->Base); p->Size = 0; - p->Base = 0; + p->Base = NULL; } -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) + +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) { - if (p->Base == 0 || p->Size != size) + if (!p->Base || p->Size != size) { Ppmd7_Free(p, alloc); - p->AlignOffset = - #ifdef PPMD_32BIT - (4 - size) & 3; - #else - 4 - (size & 3); - #endif - if ((p->Base = (Byte *)alloc->Alloc(alloc, p->AlignOffset + size - #ifndef PPMD_32BIT - + UNIT_SIZE - #endif - )) == 0) + p->AlignOffset = (4 - size) & 3; + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == NULL) return False; p->Size = size; } return True; } + + +// ---------- Internal Memory Allocator ---------- + +/* We can use CPpmd7_Node in list of free units (as in Ppmd8) + But we still need one additional list walk pass in GlueFreeBlocks(). + So we use simple CPpmd_Void_Ref instead of CPpmd7_Node in InsertNode() / RemoveNode() +*/ + +#define EMPTY_NODE 0 + + static void InsertNode(CPpmd7 *p, void *node, unsigned indx) { *((CPpmd_Void_Ref *)node) = p->FreeList[indx]; + // ((CPpmd7_Node *)node)->Next = (CPpmd7_Node_Ref)p->FreeList[indx]; + p->FreeList[indx] = REF(node); + } + static void *RemoveNode(CPpmd7 *p, unsigned indx) { CPpmd_Void_Ref *node = (CPpmd_Void_Ref *)Ppmd7_GetPtr(p, p->FreeList[indx]); p->FreeList[indx] = *node; + // CPpmd7_Node *node = NODE((CPpmd7_Node_Ref)p->FreeList[indx]); + // p->FreeList[indx] = node->Next; return node; } + static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) { unsigned i, nu = I2U(oldIndx) - I2U(newIndx); @@ -142,123 +146,167 @@ static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) InsertNode(p, ptr, i); } -static void GlueFreeBlocks(CPpmd7 *p) + +/* we use CPpmd7_Node_Union union to solve XLC -O2 strict pointer aliasing problem */ + +typedef union _CPpmd7_Node_Union { - #ifdef PPMD_32BIT - CPpmd7_Node headItem; - CPpmd7_Node_Ref head = &headItem; - #else - CPpmd7_Node_Ref head = p->AlignOffset + p->Size; - #endif - - CPpmd7_Node_Ref n = head; - unsigned i; + CPpmd7_Node Node; + CPpmd7_Node_Ref NextRef; +} CPpmd7_Node_Union; + +/* Original PPmdH (Ppmd7) code uses doubly linked list in GlueFreeBlocks() + we use single linked list similar to Ppmd8 code */ + +static void GlueFreeBlocks(CPpmd7 *p) +{ + /* + we use first UInt16 field of 12-bytes UNITs as record type stamp + CPpmd_State { Byte Symbol; Byte Freq; : Freq != 0 + CPpmd7_Context { UInt16 NumStats; : NumStats != 0 + CPpmd7_Node { UInt16 Stamp : Stamp == 0 for free record + : Stamp == 1 for head record and guard + Last 12-bytes UNIT in array is always contains 12-bytes order-0 CPpmd7_Context record. + */ + CPpmd7_Node_Ref head, n = 0; + p->GlueCount = 255; - /* create doubly-linked list of free blocks */ - for (i = 0; i < PPMD_NUM_INDEXES; i++) + + /* we set guard NODE at LoUnit */ + if (p->LoUnit != p->HiUnit) + ((CPpmd7_Node *)(void *)p->LoUnit)->Stamp = 1; + { - UInt16 nu = I2U(i); - CPpmd7_Node_Ref next = (CPpmd7_Node_Ref)p->FreeList[i]; - p->FreeList[i] = 0; - while (next != 0) + /* Create list of free blocks. + We still need one additional list walk pass before Glue. */ + unsigned i; + for (i = 0; i < PPMD_NUM_INDEXES; i++) { - CPpmd7_Node *node = NODE(next); - node->Next = n; - n = NODE(n)->Prev = next; - next = *(const CPpmd7_Node_Ref *)node; - node->Stamp = 0; - node->NU = (UInt16)nu; + const UInt16 nu = I2U_UInt16(i); + CPpmd7_Node_Ref next = (CPpmd7_Node_Ref)p->FreeList[i]; + p->FreeList[i] = 0; + while (next != 0) + { + /* Don't change the order of the following commands: */ + CPpmd7_Node_Union *un = (CPpmd7_Node_Union *)NODE(next); + const CPpmd7_Node_Ref tmp = next; + next = un->NextRef; + un->Node.Stamp = EMPTY_NODE; + un->Node.NU = nu; + un->Node.Next = n; + n = tmp; + } } } - NODE(head)->Stamp = 1; - NODE(head)->Next = n; - NODE(n)->Prev = head; - if (p->LoUnit != p->HiUnit) - ((CPpmd7_Node *)p->LoUnit)->Stamp = 1; - - /* Glue free blocks */ - while (n != head) + + head = n; + /* Glue and Fill must walk the list in same direction */ { - CPpmd7_Node *node = NODE(n); - UInt32 nu = (UInt32)node->NU; - for (;;) + /* Glue free blocks */ + CPpmd7_Node_Ref *prev = &head; + while (n) { - CPpmd7_Node *node2 = NODE(n) + nu; - nu += node2->NU; - if (node2->Stamp != 0 || nu >= 0x10000) - break; - NODE(node2->Prev)->Next = node2->Next; - NODE(node2->Next)->Prev = node2->Prev; - node->NU = (UInt16)nu; + CPpmd7_Node *node = NODE(n); + UInt32 nu = node->NU; + n = node->Next; + if (nu == 0) + { + *prev = n; + continue; + } + prev = &node->Next; + for (;;) + { + CPpmd7_Node *node2 = node + nu; + nu += node2->NU; + if (node2->Stamp != EMPTY_NODE || nu >= 0x10000) + break; + node->NU = (UInt16)nu; + node2->NU = 0; + } } - n = node->Next; } - + /* Fill lists of free blocks */ - for (n = NODE(head)->Next; n != head;) + for (n = head; n != 0;) { CPpmd7_Node *node = NODE(n); - unsigned nu; - CPpmd7_Node_Ref next = node->Next; - for (nu = node->NU; nu > 128; nu -= 128, node += 128) + UInt32 nu = node->NU; + unsigned i; + n = node->Next; + if (nu == 0) + continue; + for (; nu > 128; nu -= 128, node += 128) InsertNode(p, node, PPMD_NUM_INDEXES - 1); if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, node + k, nu - k - 1); + InsertNode(p, node + k, (unsigned)nu - k - 1); } InsertNode(p, node, i); - n = next; } } + +MY_NO_INLINE static void *AllocUnitsRare(CPpmd7 *p, unsigned indx) { unsigned i; - void *retVal; + if (p->GlueCount == 0) { GlueFreeBlocks(p); if (p->FreeList[indx] != 0) return RemoveNode(p, indx); } + i = indx; + do { if (++i == PPMD_NUM_INDEXES) { UInt32 numBytes = U2B(I2U(indx)); + Byte *us = p->UnitsStart; p->GlueCount--; - return ((UInt32)(p->UnitsStart - p->Text) > numBytes) ? (p->UnitsStart -= numBytes) : (NULL); + return ((UInt32)(us - p->Text) > numBytes) ? (p->UnitsStart = us - numBytes) : NULL; } } while (p->FreeList[i] == 0); - retVal = RemoveNode(p, i); - SplitBlock(p, retVal, i, indx); - return retVal; + + { + void *block = RemoveNode(p, i); + SplitBlock(p, block, i, indx); + return block; + } } + static void *AllocUnits(CPpmd7 *p, unsigned indx) { - UInt32 numBytes; if (p->FreeList[indx] != 0) return RemoveNode(p, indx); - numBytes = U2B(I2U(indx)); - if (numBytes <= (UInt32)(p->HiUnit - p->LoUnit)) { - void *retVal = p->LoUnit; - p->LoUnit += numBytes; - return retVal; + UInt32 numBytes = U2B(I2U(indx)); + Byte *lo = p->LoUnit; + if ((UInt32)(p->HiUnit - lo) >= numBytes) + { + p->LoUnit = lo + numBytes; + return lo; + } } return AllocUnitsRare(p, indx); } + #define MyMem12Cpy(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *s = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; s += 3; d += 3; } while (--n); } + { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ + do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } + +/* static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU) { unsigned i0 = U2I(oldNU); @@ -275,20 +323,25 @@ static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU SplitBlock(p, oldPtr, i0, i1); return oldPtr; } +*/ -#define SUCCESSOR(p) ((CPpmd_Void_Ref)((p)->SuccessorLow | ((UInt32)(p)->SuccessorHigh << 16))) +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) { - (p)->SuccessorLow = (UInt16)((UInt32)(v) & 0xFFFF); - (p)->SuccessorHigh = (UInt16)(((UInt32)(v) >> 16) & 0xFFFF); + Ppmd_SET_SUCCESSOR(p, v); } -static void RestartModel(CPpmd7 *p) + + +MY_NO_INLINE +static +void RestartModel(CPpmd7 *p) { - unsigned i, k, m; + unsigned i, k; memset(p->FreeList, 0, sizeof(p->FreeList)); + p->Text = p->Base + p->AlignOffset; p->HiUnit = p->Text + p->Size; p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; @@ -298,57 +351,110 @@ static void RestartModel(CPpmd7 *p) p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; p->PrevSuccess = 0; - p->MinContext = p->MaxContext = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ - p->MinContext->Suffix = 0; - p->MinContext->NumStats = 256; - p->MinContext->SummFreq = 256 + 1; - p->FoundState = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ - p->LoUnit += U2B(256 / 2); - p->MinContext->Stats = REF(p->FoundState); - for (i = 0; i < 256; i++) { - CPpmd_State *s = &p->FoundState[i]; - s->Symbol = (Byte)i; - s->Freq = 1; - SetSuccessor(s, 0); + CPpmd7_Context *mc = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + + p->LoUnit += U2B(256 / 2); + p->MaxContext = p->MinContext = mc; + p->FoundState = s; + + mc->NumStats = 256; + mc->Union2.SummFreq = 256 + 1; + mc->Union4.Stats = REF(s); + mc->Suffix = 0; + + for (i = 0; i < 256; i++, s++) + { + s->Symbol = (Byte)i; + s->Freq = 1; + SetSuccessor(s, 0); + } + + #ifdef PPMD7_ORDER_0_SUPPPORT + if (p->MaxOrder == 0) + { + CPpmd_Void_Ref r = REF(mc); + s = p->FoundState; + for (i = 0; i < 256; i++, s++) + SetSuccessor(s, r); + return; + } + #endif } for (i = 0; i < 128; i++) + + + for (k = 0; k < 8; k++) { + unsigned m; UInt16 *dest = p->BinSumm[i] + k; UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 2)); for (m = 0; m < 64; m += 8) dest[m] = val; } - + + for (i = 0; i < 25; i++) - for (k = 0; k < 16; k++) + { + + CPpmd_See *s = p->See[i]; + + + + unsigned summ = ((5 * i + 10) << (PPMD_PERIOD_BITS - 4)); + for (k = 0; k < 16; k++, s++) { - CPpmd_See *s = &p->See[i][k]; - s->Summ = (UInt16)((5 * i + 10) << (s->Shift = PPMD_PERIOD_BITS - 4)); + s->Summ = (UInt16)summ; + s->Shift = (PPMD_PERIOD_BITS - 4); s->Count = 4; } + } + + p->DummySee.Summ = 0; /* unused */ + p->DummySee.Shift = PPMD_PERIOD_BITS; + p->DummySee.Count = 64; /* unused */ } + void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) { p->MaxOrder = maxOrder; + RestartModel(p); - p->DummySee.Shift = PPMD_PERIOD_BITS; - p->DummySee.Summ = 0; /* unused */ - p->DummySee.Count = 64; /* unused */ } -static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) + + +/* + CreateSuccessors() + It's called when (FoundState->Successor) is RAW-Successor, + that is the link to position in Raw text. + So we create Context records and write the links to + FoundState->Successor and to identical RAW-Successors in suffix + contexts of MinContex. + + The function returns: + if (OrderFall == 0) then MinContext is already at MAX order, + { return pointer to new or existing context of same MAX order } + else + { return pointer to new real context that will be (Order+1) in comparison with MinContext + + also it can return pointer to real context of same order, +*/ + +MY_NO_INLINE +static CTX_PTR CreateSuccessors(CPpmd7 *p) { - CPpmd_State upState; CTX_PTR c = p->MinContext; CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); - CPpmd_State *ps[PPMD7_MAX_ORDER]; + Byte newSym, newFreq; unsigned numPs = 0; - - if (!skip) + CPpmd_State *ps[PPMD7_MAX_ORDER]; + + if (p->OrderFall != 0) ps[numPs++] = p->FoundState; while (c->Suffix) @@ -356,44 +462,70 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) CPpmd_Void_Ref successor; CPpmd_State *s; c = SUFFIX(c); + + if (c->NumStats != 1) { - for (s = STATS(c); s->Symbol != p->FoundState->Symbol; s++); + Byte sym = p->FoundState->Symbol; + for (s = STATS(c); s->Symbol != sym; s++); + } else + { s = ONE_STATE(c); + + } successor = SUCCESSOR(s); if (successor != upBranch) { + // (c) is real record Context here, c = CTX(successor); if (numPs == 0) + { + // (c) is real record MAX Order Context here, + // So we don't need to create any new contexts. return c; + } break; } ps[numPs++] = s; } - upState.Symbol = *(const Byte *)Ppmd7_GetPtr(p, upBranch); - SetSuccessor(&upState, upBranch + 1); + // All created contexts will have single-symbol with new RAW-Successor + // All new RAW-Successors will point to next position in RAW text + // after FoundState->Successor + + newSym = *(const Byte *)Ppmd7_GetPtr(p, upBranch); + upBranch++; + if (c->NumStats == 1) - upState.Freq = ONE_STATE(c)->Freq; + newFreq = ONE_STATE(c)->Freq; else { UInt32 cf, s0; CPpmd_State *s; - for (s = STATS(c); s->Symbol != upState.Symbol; s++); - cf = s->Freq - 1; - s0 = c->SummFreq - c->NumStats - cf; - upState.Freq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((2 * cf + 3 * s0 - 1) / (2 * s0)))); + for (s = STATS(c); s->Symbol != newSym; s++); + cf = (UInt32)s->Freq - 1; + s0 = (UInt32)c->Union2.SummFreq - c->NumStats - cf; + /* + cf - is frequency of symbol that will be Successor in new context records. + s0 - is commulative frequency sum of another symbols from parent context. + max(newFreq)= (s->Freq + 1), when (s0 == 1) + we have requirement (Ppmd7Context_OneState()->Freq <= 128) in BinSumm[] + so (s->Freq < 128) - is requirement for multi-symbol contexts + */ + newFreq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : (2 * cf + s0 - 1) / (2 * s0) + 1)); } + // Create new single-symbol contexts from low order to high order in loop + do { - /* Create Child */ - CTX_PTR c1; /* = AllocContext(p); */ + CTX_PTR c1; + /* = AllocContext(p); */ if (p->HiUnit != p->LoUnit) - c1 = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); + c1 = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); else if (p->FreeList[0] != 0) c1 = (CTX_PTR)RemoveNode(p, 0); else @@ -402,8 +534,11 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) if (!c1) return NULL; } + c1->NumStats = 1; - *ONE_STATE(c1) = upState; + ONE_STATE(c1)->Symbol = newSym; + ONE_STATE(c1)->Freq = newFreq; + SetSuccessor(ONE_STATE(c1), upBranch); c1->Suffix = REF(c); SetSuccessor(ps[--numPs], REF(c1)); c = c1; @@ -413,21 +548,26 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) return c; } -static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) -{ - CPpmd_State tmp = *t1; - *t1 = *t2; - *t2 = tmp; -} -static void UpdateModel(CPpmd7 *p) + +#define SwapStates(s) \ + { CPpmd_State tmp = s[0]; s[0] = s[-1]; s[-1] = tmp; } + + +void Ppmd7_UpdateModel(CPpmd7 *p); +MY_NO_INLINE +void Ppmd7_UpdateModel(CPpmd7 *p) { - CPpmd_Void_Ref successor, fSuccessor = SUCCESSOR(p->FoundState); - CTX_PTR c; + CPpmd_Void_Ref maxSuccessor, minSuccessor; + CTX_PTR c, mc; unsigned s0, ns; - + + + if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) { + /* Update Freqs in Suffix Context */ + c = SUFFIX(p->MinContext); if (c->NumStats == 1) @@ -439,27 +579,39 @@ static void UpdateModel(CPpmd7 *p) else { CPpmd_State *s = STATS(c); - if (s->Symbol != p->FoundState->Symbol) + Byte sym = p->FoundState->Symbol; + + if (s->Symbol != sym) { - do { s++; } while (s->Symbol != p->FoundState->Symbol); + do + { + // s++; if (s->Symbol == sym) break; + s++; + } + while (s->Symbol != sym); + if (s[0].Freq >= s[-1].Freq) { - SwapStates(&s[0], &s[-1]); + SwapStates(s); s--; } } + if (s->Freq < MAX_FREQ - 9) { - s->Freq += 2; - c->SummFreq += 2; + s->Freq = (Byte)(s->Freq + 2); + c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2); } } } + if (p->OrderFall == 0) { - p->MinContext = p->MaxContext = CreateSuccessors(p, True); - if (p->MinContext == 0) + /* MAX ORDER context */ + /* (FoundState->Successor) is RAW-Successor. */ + p->MaxContext = p->MinContext = CreateSuccessors(p); + if (!p->MinContext) { RestartModel(p); return; @@ -467,45 +619,93 @@ static void UpdateModel(CPpmd7 *p) SetSuccessor(p->FoundState, REF(p->MinContext)); return; } + + + /* NON-MAX ORDER context */ - *p->Text++ = p->FoundState->Symbol; - successor = REF(p->Text); - if (p->Text >= p->UnitsStart) { - RestartModel(p); - return; + Byte *text = p->Text; + *text++ = p->FoundState->Symbol; + p->Text = text; + if (text >= p->UnitsStart) + { + RestartModel(p); + return; + } + maxSuccessor = REF(text); } - if (fSuccessor) + minSuccessor = SUCCESSOR(p->FoundState); + + if (minSuccessor) { - if (fSuccessor <= successor) + // there is Successor for FoundState in MinContext. + // So the next context will be one order higher than MinContext. + + if (minSuccessor <= maxSuccessor) { - CTX_PTR cs = CreateSuccessors(p, False); - if (cs == NULL) + // minSuccessor is RAW-Successor. So we will create real contexts records: + CTX_PTR cs = CreateSuccessors(p); + if (!cs) { RestartModel(p); return; } - fSuccessor = REF(cs); + minSuccessor = REF(cs); } + + // minSuccessor now is real Context pointer that points to existing (Order+1) context + if (--p->OrderFall == 0) { - successor = fSuccessor; + /* + if we move to MaxOrder context, then minSuccessor will be common Succesor for both: + MinContext that is (MaxOrder - 1) + MaxContext that is (MaxOrder) + so we don't need new RAW-Successor, and we can use real minSuccessor + as succssors for both MinContext and MaxContext. + */ + maxSuccessor = minSuccessor; + + /* + if (MaxContext != MinContext) + { + there was order fall from MaxOrder and we don't need current symbol + to transfer some RAW-Succesors to real contexts. + So we roll back pointer in raw data for one position. + } + */ p->Text -= (p->MaxContext != p->MinContext); } } else { - SetSuccessor(p->FoundState, successor); - fSuccessor = REF(p->MinContext); + /* + FoundState has NULL-Successor here. + And only root 0-order context can contain NULL-Successors. + We change Successor in FoundState to RAW-Successor, + And next context will be same 0-order root Context. + */ + SetSuccessor(p->FoundState, maxSuccessor); + minSuccessor = REF(p->MinContext); } - - s0 = p->MinContext->SummFreq - (ns = p->MinContext->NumStats) - (p->FoundState->Freq - 1); - - for (c = p->MaxContext; c != p->MinContext; c = SUFFIX(c)) + + mc = p->MinContext; + c = p->MaxContext; + + p->MaxContext = p->MinContext = CTX(minSuccessor); + + if (c == mc) + return; + + // s0 : is pure Escape Freq + s0 = mc->Union2.SummFreq - (ns = mc->NumStats) - ((unsigned)p->FoundState->Freq - 1); + + do { unsigned ns1; - UInt32 cf, sf; + UInt32 sum; + if ((ns1 = c->NumStats) != 1) { if ((ns1 & 1) == 0) @@ -513,7 +713,7 @@ static void UpdateModel(CPpmd7 *p) /* Expand for one UNIT */ unsigned oldNU = ns1 >> 1; unsigned i = U2I(oldNU); - if (i != U2I(oldNU + 1)) + if (i != U2I((size_t)oldNU + 1)) { void *ptr = AllocUnits(p, i + 1); void *oldPtr; @@ -525,80 +725,127 @@ static void UpdateModel(CPpmd7 *p) oldPtr = STATS(c); MyMem12Cpy(ptr, oldPtr, oldNU); InsertNode(p, oldPtr, i); - c->Stats = STATS_REF(ptr); + c->Union4.Stats = STATS_REF(ptr); } } - c->SummFreq = (UInt16)(c->SummFreq + (2 * ns1 < ns) + 2 * ((4 * ns1 <= ns) & (c->SummFreq <= 8 * ns1))); + sum = c->Union2.SummFreq; + /* max increase of Escape_Freq is 3 here. + total increase of Union2.SummFreq for all symbols is less than 256 here */ + sum += (UInt32)(2 * ns1 < ns) + 2 * ((unsigned)(4 * ns1 <= ns) & (sum <= 8 * ns1)); + /* original PPMdH uses 16-bit variable for (sum) here. + But (sum < 0x9000). So we don't truncate (sum) to 16-bit */ + // sum = (UInt16)sum; } else { + // instead of One-symbol context we create 2-symbol context CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); if (!s) { RestartModel(p); return; } - *s = *ONE_STATE(c); - c->Stats = REF(s); - if (s->Freq < MAX_FREQ / 4 - 1) - s->Freq <<= 1; - else - s->Freq = MAX_FREQ - 4; - c->SummFreq = (UInt16)(s->Freq + p->InitEsc + (ns > 3)); - } - cf = 2 * (UInt32)p->FoundState->Freq * (c->SummFreq + 6); - sf = (UInt32)s0 + c->SummFreq; - if (cf < 6 * sf) - { - cf = 1 + (cf > sf) + (cf >= 4 * sf); - c->SummFreq += 3; - } - else - { - cf = 4 + (cf >= 9 * sf) + (cf >= 12 * sf) + (cf >= 15 * sf); - c->SummFreq = (UInt16)(c->SummFreq + cf); + { + unsigned freq = c->Union2.State2.Freq; + // s = *ONE_STATE(c); + s->Symbol = c->Union2.State2.Symbol; + s->Successor_0 = c->Union4.State4.Successor_0; + s->Successor_1 = c->Union4.State4.Successor_1; + // SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of + // (Successor_0 and Successor_1) in LE/BE. + c->Union4.Stats = REF(s); + if (freq < MAX_FREQ / 4 - 1) + freq <<= 1; + else + freq = MAX_FREQ - 4; + // (max(s->freq) == 120), when we convert from 1-symbol into 2-symbol context + s->Freq = (Byte)freq; + // max(InitEsc = PPMD7_kExpEscape[*]) is 25. So the max(escapeFreq) is 26 here + sum = freq + p->InitEsc + (ns > 3); + } } + { CPpmd_State *s = STATS(c) + ns1; - SetSuccessor(s, successor); + UInt32 cf = 2 * (sum + 6) * (UInt32)p->FoundState->Freq; + UInt32 sf = (UInt32)s0 + sum; s->Symbol = p->FoundState->Symbol; - s->Freq = (Byte)cf; c->NumStats = (UInt16)(ns1 + 1); + SetSuccessor(s, maxSuccessor); + + if (cf < 6 * sf) + { + cf = (UInt32)1 + (cf > sf) + (cf >= 4 * sf); + sum += 3; + /* It can add (0, 1, 2) to Escape_Freq */ + } + else + { + cf = (UInt32)4 + (cf >= 9 * sf) + (cf >= 12 * sf) + (cf >= 15 * sf); + sum += cf; + } + + c->Union2.SummFreq = (UInt16)sum; + s->Freq = (Byte)cf; } + c = SUFFIX(c); } - p->MaxContext = p->MinContext = CTX(fSuccessor); + while (c != mc); } + + +MY_NO_INLINE static void Rescale(CPpmd7 *p) { unsigned i, adder, sumFreq, escFreq; CPpmd_State *stats = STATS(p->MinContext); CPpmd_State *s = p->FoundState; + + /* Sort the list by Freq */ + if (s != stats) { CPpmd_State tmp = *s; - for (; s != stats; s--) + do s[0] = s[-1]; + while (--s != stats); *s = tmp; } - escFreq = p->MinContext->SummFreq - s->Freq; - s->Freq += 4; - adder = (p->OrderFall != 0); - s->Freq = (Byte)((s->Freq + adder) >> 1); + sumFreq = s->Freq; + escFreq = p->MinContext->Union2.SummFreq - sumFreq; + + /* + if (p->OrderFall == 0), adder = 0 : it's allowed to remove symbol from MAX Order context + if (p->OrderFall != 0), adder = 1 : it's NOT allowed to remove symbol from NON-MAX Order context + */ + + adder = (p->OrderFall != 0); + + #ifdef PPMD7_ORDER_0_SUPPPORT + adder |= (p->MaxOrder == 0); // we don't remove symbols from order-0 context + #endif + + sumFreq = (sumFreq + 4 + adder) >> 1; + i = (unsigned)p->MinContext->NumStats - 1; + s->Freq = (Byte)sumFreq; - i = p->MinContext->NumStats - 1; do { - escFreq -= (++s)->Freq; - s->Freq = (Byte)((s->Freq + adder) >> 1); - sumFreq += s->Freq; - if (s[0].Freq > s[-1].Freq) + unsigned freq = (++s)->Freq; + escFreq -= freq; + freq = (freq + adder) >> 1; + sumFreq += freq; + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) { + CPpmd_State tmp = *s; CPpmd_State *s1 = s; - CPpmd_State tmp = *s1; do + { s1[0] = s1[-1]; - while (--s1 != stats && tmp.Freq > s1[-1].Freq); + } + while (--s1 != stats && freq > s1[-1].Freq); *s1 = tmp; } } @@ -606,47 +853,89 @@ static void Rescale(CPpmd7 *p) if (s->Freq == 0) { - unsigned numStats = p->MinContext->NumStats; - unsigned n0, n1; - do { i++; } while ((--s)->Freq == 0); + /* Remove all items with Freq == 0 */ + CPpmd7_Context *mc; + unsigned numStats, numStatsNew, n0, n1; + + i = 0; do { i++; } while ((--s)->Freq == 0); + + /* We increase (escFreq) for the number of removed symbols. + So we will have (0.5) increase for Escape_Freq in avarage per + removed symbol after Escape_Freq halving */ escFreq += i; - p->MinContext->NumStats = (UInt16)(p->MinContext->NumStats - i); - if (p->MinContext->NumStats == 1) + mc = p->MinContext; + numStats = mc->NumStats; + numStatsNew = numStats - i; + mc->NumStats = (UInt16)(numStatsNew); + n0 = (numStats + 1) >> 1; + + if (numStatsNew == 1) { - CPpmd_State tmp = *stats; + /* Create Single-Symbol context */ + unsigned freq = stats->Freq; + do { - tmp.Freq = (Byte)(tmp.Freq - (tmp.Freq >> 1)); escFreq >>= 1; + freq = (freq + 1) >> 1; } while (escFreq > 1); - InsertNode(p, stats, U2I(((numStats + 1) >> 1))); - *(p->FoundState = ONE_STATE(p->MinContext)) = tmp; + + s = ONE_STATE(mc); + *s = *stats; + s->Freq = (Byte)freq; // (freq <= 260 / 4) + p->FoundState = s; + InsertNode(p, stats, U2I(n0)); return; } - n0 = (numStats + 1) >> 1; - n1 = (p->MinContext->NumStats + 1) >> 1; + + n1 = (numStatsNew + 1) >> 1; if (n0 != n1) - p->MinContext->Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + { + // p->MinContext->Union4.Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + unsigned i0 = U2I(n0); + unsigned i1 = U2I(n1); + if (i0 != i1) + { + if (p->FreeList[i1] != 0) + { + void *ptr = RemoveNode(p, i1); + p->MinContext->Union4.Stats = STATS_REF(ptr); + MyMem12Cpy(ptr, (const void *)stats, n1); + InsertNode(p, stats, i0); + } + else + SplitBlock(p, stats, i0, i1); + } + } + } + { + CPpmd7_Context *mc = p->MinContext; + mc->Union2.SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); + // Escape_Freq halving here + p->FoundState = STATS(mc); } - p->MinContext->SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); - p->FoundState = STATS(p->MinContext); } + CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) { CPpmd_See *see; - unsigned nonMasked = p->MinContext->NumStats - numMasked; - if (p->MinContext->NumStats != 256) + const CPpmd7_Context *mc = p->MinContext; + unsigned numStats = mc->NumStats; + if (numStats != 256) { - see = p->See[(unsigned)p->NS2Indx[nonMasked - 1]] + - (nonMasked < (unsigned)SUFFIX(p->MinContext)->NumStats - p->MinContext->NumStats) + - 2 * (unsigned)(p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + - 4 * (unsigned)(numMasked > nonMasked) + + unsigned nonMasked = numStats - numMasked; + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + + (nonMasked < (unsigned)SUFFIX(mc)->NumStats - numStats) + + 2 * (unsigned)(mc->Union2.SummFreq < 11 * numStats) + + 4 * (unsigned)(numMasked > nonMasked) + p->HiBitsFlag; { - unsigned r = (see->Summ >> see->Shift); - see->Summ = (UInt16)(see->Summ - r); + // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ + unsigned summ = (UInt16)see->Summ; // & 0xFFFF + unsigned r = (summ >> see->Shift); + see->Summ = (UInt16)(summ - r); *escFreq = r + (r == 0); } } @@ -658,53 +947,158 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) return see; } + static void NextContext(CPpmd7 *p) { CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); - if (p->OrderFall == 0 && (Byte *)c > p->Text) - p->MinContext = p->MaxContext = c; + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; else - UpdateModel(p); + Ppmd7_UpdateModel(p); } + void Ppmd7_Update1(CPpmd7 *p) { CPpmd_State *s = p->FoundState; - s->Freq += 4; - p->MinContext->SummFreq += 4; - if (s[0].Freq > s[-1].Freq) + unsigned freq = s->Freq; + freq += 4; + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > s[-1].Freq) { - SwapStates(&s[0], &s[-1]); + SwapStates(s); p->FoundState = --s; - if (s->Freq > MAX_FREQ) + if (freq > MAX_FREQ) Rescale(p); } NextContext(p); } + void Ppmd7_Update1_0(CPpmd7 *p) { - p->PrevSuccess = (2 * p->FoundState->Freq > p->MinContext->SummFreq); - p->RunLength += p->PrevSuccess; - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) + CPpmd_State *s = p->FoundState; + CPpmd7_Context *mc = p->MinContext; + unsigned freq = s->Freq; + unsigned summFreq = mc->Union2.SummFreq; + p->PrevSuccess = (2 * freq > summFreq); + p->RunLength += (int)p->PrevSuccess; + mc->Union2.SummFreq = (UInt16)(summFreq + 4); + freq += 4; + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) Rescale(p); NextContext(p); } + +/* void Ppmd7_UpdateBin(CPpmd7 *p) { - p->FoundState->Freq = (Byte)(p->FoundState->Freq + (p->FoundState->Freq < 128 ? 1: 0)); + unsigned freq = p->FoundState->Freq; + p->FoundState->Freq = (Byte)(freq + (freq < 128)); p->PrevSuccess = 1; p->RunLength++; NextContext(p); } +*/ void Ppmd7_Update2(CPpmd7 *p) { - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) - Rescale(p); + CPpmd_State *s = p->FoundState; + unsigned freq = s->Freq; + freq += 4; p->RunLength = p->InitRL; - UpdateModel(p); + p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); + s->Freq = (Byte)freq; + if (freq > MAX_FREQ) + Rescale(p); + Ppmd7_UpdateModel(p); +} + + + +/* +PPMd Memory Map: +{ + [ 0 ] contains subset of original raw text, that is required to create context + records, Some symbols are not written, when max order context was reached + [ Text ] free area + [ UnitsStart ] CPpmd_State vectors and CPpmd7_Context records + [ LoUnit ] free area for CPpmd_State and CPpmd7_Context items +[ HiUnit ] CPpmd7_Context records + [ Size ] end of array } + +These addresses don't cross at any time. +And the following condtions is true for addresses: + (0 <= Text < UnitsStart <= LoUnit <= HiUnit <= Size) + +Raw text is BYTE--aligned. +the data in block [ UnitsStart ... Size ] contains 12-bytes aligned UNITs. + +Last UNIT of array at offset (Size - 12) is root order-0 CPpmd7_Context record. +The code can free UNITs memory blocks that were allocated to store CPpmd_State vectors. +The code doesn't free UNITs allocated for CPpmd7_Context records. + +The code calls RestartModel(), when there is no free memory for allocation. +And RestartModel() changes the state to orignal start state, with full free block. + + +The code allocates UNITs with the following order: + +Allocation of 1 UNIT for Context record + - from free space (HiUnit) down to (LoUnit) + - from FreeList[0] + - AllocUnitsRare() + +AllocUnits() for CPpmd_State vectors: + - from FreeList[i] + - from free space (LoUnit) up to (HiUnit) + - AllocUnitsRare() + +AllocUnitsRare() + - if (GlueCount == 0) + { Glue lists, GlueCount = 255, allocate from FreeList[i]] } + - loop for all higher sized FreeList[...] lists + - from (UnitsStart - Text), GlueCount-- + - ERROR + + +Each Record with Context contains the CPpmd_State vector, where each +CPpmd_State contains the link to Successor. +There are 3 types of Successor: + 1) NULL-Successor - NULL pointer. NULL-Successor links can be stored + only in 0-order Root Context Record. + We use 0 value as NULL-Successor + 2) RAW-Successor - the link to position in raw text, + that "RAW-Successor" is being created after first + occurrence of new symbol for some existing context record. + (RAW-Successor > 0). + 3) RECORD-Successor - the link to CPpmd7_Context record of (Order+1), + that record is being created when we go via RAW-Successor again. + +For any successors at any time: the following condtions are true for Successor links: +(NULL-Successor < RAW-Successor < UnitsStart <= RECORD-Successor) + + +---------- Symbol Frequency, SummFreq and Range in Range_Coder ---------- + +CPpmd7_Context::SummFreq = Sum(Stats[].Freq) + Escape_Freq + +The PPMd code tries to fulfill the condition: + (SummFreq <= (256 * 128 = RC::kBot)) + +We have (Sum(Stats[].Freq) <= 256 * 124), because of (MAX_FREQ = 124) +So (4 = 128 - 124) is average reserve for Escape_Freq for each symbol. +If (CPpmd_State::Freq) is not aligned for 4, the reserve can be 5, 6 or 7. +SummFreq and Escape_Freq can be changed in Rescale() and *Update*() functions. +Rescale() can remove symbols only from max-order contexts. So Escape_Freq can increase after multiple calls of Rescale() for +max-order context. + +When the PPMd code still break (Total <= RC::Range) condition in range coder, +we have two ways to resolve that problem: + 1) we can report error, if we want to keep compatibility with original PPMd code that has no fix for such cases. + 2) we can reduce (Total) value to (RC::Range) by reducing (Escape_Freq) part of (Total) value. +*/ diff --git a/3rdparty/lzma/C/Ppmd7.h b/3rdparty/lzma/C/Ppmd7.h index 87eefde804a..d31809aebdf 100644 --- a/3rdparty/lzma/C/Ppmd7.h +++ b/3rdparty/lzma/C/Ppmd7.h @@ -1,10 +1,8 @@ -/* Ppmd7.h -- PPMdH compression codec -2016-05-21 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -/* This code supports virtual RangeDecoder and includes the implementation -of RangeCoder from 7z, instead of RangeCoder from original PPMd var.H. -If you need the compatibility with original PPMd var.H, you can use external RangeDecoder */ +/* Ppmd7.h -- Ppmd7 (PPMdH) compression codec +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain */ + #ifndef __PPMD7_H #define __PPMD7_H @@ -21,23 +19,56 @@ EXTERN_C_BEGIN struct CPpmd7_Context_; -typedef - #ifdef PPMD_32BIT - struct CPpmd7_Context_ * - #else - UInt32 - #endif - CPpmd7_Context_Ref; +typedef Ppmd_Ref_Type(struct CPpmd7_Context_) CPpmd7_Context_Ref; + +// MY_CPU_pragma_pack_push_1 typedef struct CPpmd7_Context_ { UInt16 NumStats; - UInt16 SummFreq; - CPpmd_State_Ref Stats; + + + union + { + UInt16 SummFreq; + CPpmd_State2 State2; + } Union2; + + union + { + CPpmd_State_Ref Stats; + CPpmd_State4 State4; + } Union4; + CPpmd7_Context_Ref Suffix; } CPpmd7_Context; -#define Ppmd7Context_OneState(p) ((CPpmd_State *)&(p)->SummFreq) +// MY_CPU_pragma_pop + +#define Ppmd7Context_OneState(p) ((CPpmd_State *)&(p)->Union2) + + + + +typedef struct +{ + UInt32 Range; + UInt32 Code; + UInt32 Low; + IByteIn *Stream; +} CPpmd7_RangeDec; + + +typedef struct +{ + UInt32 Range; + Byte Cache; + // Byte _dummy_[3]; + UInt64 Low; + UInt64 CacheSize; + IByteOut *Stream; +} CPpmd7z_RangeEnc; + typedef struct { @@ -48,92 +79,102 @@ typedef struct UInt32 Size; UInt32 GlueCount; - Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; UInt32 AlignOffset; + Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; + - Byte Indx2Units[PPMD_NUM_INDEXES]; + + + union + { + CPpmd7_RangeDec dec; + CPpmd7z_RangeEnc enc; + } rc; + + Byte Indx2Units[PPMD_NUM_INDEXES + 2]; // +2 for alignment Byte Units2Indx[128]; CPpmd_Void_Ref FreeList[PPMD_NUM_INDEXES]; - Byte NS2Indx[256], NS2BSIndx[256], HB2Flag[256]; + + Byte NS2BSIndx[256], NS2Indx[256]; + Byte ExpEscape[16]; CPpmd_See DummySee, See[25][16]; UInt16 BinSumm[128][64]; + // int LastSymbol; } CPpmd7; + void Ppmd7_Construct(CPpmd7 *p); -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc); -void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc); +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL) /* ---------- Internal Functions ---------- */ -extern const Byte PPMD7_kExpEscape[16]; - -#ifdef PPMD_32BIT - #define Ppmd7_GetPtr(p, ptr) (ptr) - #define Ppmd7_GetContext(p, ptr) (ptr) - #define Ppmd7_GetStats(p, ctx) ((ctx)->Stats) -#else - #define Ppmd7_GetPtr(p, offs) ((void *)((p)->Base + (offs))) - #define Ppmd7_GetContext(p, offs) ((CPpmd7_Context *)Ppmd7_GetPtr((p), (offs))) - #define Ppmd7_GetStats(p, ctx) ((CPpmd_State *)Ppmd7_GetPtr((p), ((ctx)->Stats))) -#endif +#define Ppmd7_GetPtr(p, ptr) Ppmd_GetPtr(p, ptr) +#define Ppmd7_GetContext(p, ptr) Ppmd_GetPtr_Type(p, ptr, CPpmd7_Context) +#define Ppmd7_GetStats(p, ctx) Ppmd_GetPtr_Type(p, (ctx)->Union4.Stats, CPpmd_State) void Ppmd7_Update1(CPpmd7 *p); void Ppmd7_Update1_0(CPpmd7 *p); void Ppmd7_Update2(CPpmd7 *p); -void Ppmd7_UpdateBin(CPpmd7 *p); + +#define PPMD7_HiBitsFlag_3(sym) ((((unsigned)sym + 0xC0) >> (8 - 3)) & (1 << 3)) +#define PPMD7_HiBitsFlag_4(sym) ((((unsigned)sym + 0xC0) >> (8 - 4)) & (1 << 4)) +// #define PPMD7_HiBitsFlag_3(sym) ((sym) < 0x40 ? 0 : (1 << 3)) +// #define PPMD7_HiBitsFlag_4(sym) ((sym) < 0x40 ? 0 : (1 << 4)) #define Ppmd7_GetBinSumm(p) \ - &p->BinSumm[(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ - p->NS2BSIndx[Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ - (p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]) + \ - 2 * p->HB2Flag[(unsigned)Ppmd7Context_OneState(p->MinContext)->Symbol] + \ - ((p->RunLength >> 26) & 0x20)] + &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1] \ + [ p->PrevSuccess + ((p->RunLength >> 26) & 0x20) \ + + p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] \ + + PPMD7_HiBitsFlag_4(Ppmd7Context_OneState(p->MinContext)->Symbol) \ + + (p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol)) ] CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale); -/* ---------- Decode ---------- */ +/* +We support two versions of Ppmd7 (PPMdH) methods that use same CPpmd7 structure: + 1) Ppmd7a_*: original PPMdH + 2) Ppmd7z_*: modified PPMdH with 7z Range Coder +Ppmd7_*: the structures and functions that are common for both versions of PPMd7 (PPMdH) +*/ -typedef struct -{ - UInt32 (*GetThreshold)(void *p, UInt32 total); - void (*Decode)(void *p, UInt32 start, UInt32 size); - UInt32 (*DecodeBit)(void *p, UInt32 size0); -} IPpmd7_RangeDec; +/* ---------- Decode ---------- */ -typedef struct -{ - IPpmd7_RangeDec p; - UInt32 Range; - UInt32 Code; - IByteIn *Stream; -} CPpmd7z_RangeDec; +#define PPMD7_SYM_END (-1) +#define PPMD7_SYM_ERROR (-2) -void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); -#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +/* +You must set (CPpmd7::rc.dec.Stream) before Ppmd7*_RangeDec_Init() -int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc); +Ppmd7*_DecodeSymbol() +out: + >= 0 : decoded byte + -1 : PPMD7_SYM_END : End of payload marker + -2 : PPMD7_SYM_ERROR : Data error +*/ +/* Ppmd7a_* : original PPMdH */ +BoolInt Ppmd7a_RangeDec_Init(CPpmd7_RangeDec *p); +#define Ppmd7a_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +int Ppmd7a_DecodeSymbol(CPpmd7 *p); -/* ---------- Encode ---------- */ +/* Ppmd7z_* : modified PPMdH with 7z Range Coder */ +BoolInt Ppmd7z_RangeDec_Init(CPpmd7_RangeDec *p); +#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) +int Ppmd7z_DecodeSymbol(CPpmd7 *p); +// Byte *Ppmd7z_DecodeSymbols(CPpmd7 *p, Byte *buf, const Byte *lim); -typedef struct -{ - UInt64 Low; - UInt32 Range; - Byte Cache; - UInt64 CacheSize; - IByteOut *Stream; -} CPpmd7z_RangeEnc; -void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p); -void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p); +/* ---------- Encode ---------- */ -void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol); +void Ppmd7z_Init_RangeEnc(CPpmd7 *p); +void Ppmd7z_Flush_RangeEnc(CPpmd7 *p); +// void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol); +void Ppmd7z_EncodeSymbols(CPpmd7 *p, const Byte *buf, const Byte *lim); EXTERN_C_END diff --git a/3rdparty/lzma/C/Ppmd7Dec.c b/3rdparty/lzma/C/Ppmd7Dec.c index 04b4b09e330..55d74ff9dc0 100644 --- a/3rdparty/lzma/C/Ppmd7Dec.c +++ b/3rdparty/lzma/C/Ppmd7Dec.c @@ -1,6 +1,8 @@ -/* Ppmd7Dec.c -- PPMdH Decoder -2010-03-12 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ +/* Ppmd7Dec.c -- Ppmd7z (PPMdH with 7z Range Coder) Decoder +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain */ + #include "Precomp.h" @@ -8,182 +10,288 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) + +#define READ_BYTE(p) IByteIn_Read((p)->Stream) + +BoolInt Ppmd7z_RangeDec_Init(CPpmd7_RangeDec *p) { unsigned i; p->Code = 0; p->Range = 0xFFFFFFFF; - if (p->Stream->Read((void *)p->Stream) != 0) + if (READ_BYTE(p) != 0) return False; for (i = 0; i < 4; i++) - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | READ_BYTE(p); return (p->Code < 0xFFFFFFFF); } -static UInt32 Range_GetThreshold(void *pp, UInt32 total) -{ - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; - return (p->Code) / (p->Range /= total); -} +#define RC_NORM_BASE(p) if ((p)->Range < kTopValue) \ + { (p)->Code = ((p)->Code << 8) | READ_BYTE(p); (p)->Range <<= 8; -static void Range_Normalize(CPpmd7z_RangeDec *p) -{ - if (p->Range < kTopValue) - { - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); - p->Range <<= 8; - if (p->Range < kTopValue) - { - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); - p->Range <<= 8; - } - } -} +#define RC_NORM_1(p) RC_NORM_BASE(p) } +#define RC_NORM(p) RC_NORM_BASE(p) RC_NORM_BASE(p) }} -static void Range_Decode(void *pp, UInt32 start, UInt32 size) -{ - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; - p->Code -= start * p->Range; - p->Range *= size; - Range_Normalize(p); -} +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) -static UInt32 Range_DecodeBit(void *pp, UInt32 size0) -{ - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; - UInt32 newBound = (p->Range >> 14) * size0; - UInt32 symbol; - if (p->Code < newBound) - { - symbol = 0; - p->Range = newBound; - } - else - { - symbol = 1; - p->Code -= newBound; - p->Range -= newBound; - } - Range_Normalize(p); - return symbol; -} +#define R (&p->rc.dec) -void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p) +MY_FORCE_INLINE +// MY_NO_INLINE +static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) { - p->p.GetThreshold = Range_GetThreshold; - p->p.Decode = Range_Decode; - p->p.DecodeBit = Range_DecodeBit; + + + R->Code -= start * R->Range; + R->Range *= size; + RC_NORM_LOCAL(R) } +#define RC_Decode(start, size) RangeDec_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) + -#define MASK(sym) ((signed char *)charMask)[sym] +#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) +typedef CPpmd7_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) +void Ppmd7_UpdateModel(CPpmd7 *p); -int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc) +#define MASK(sym) ((unsigned char *)charMask)[sym] +// MY_FORCE_INLINE +// static +int Ppmd7z_DecodeSymbol(CPpmd7 *p) { size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) { CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); unsigned i; UInt32 count, hiCnt; - if ((count = rc->GetThreshold(rc, p->MinContext->SummFreq)) < (hiCnt = s->Freq)) + UInt32 summFreq = p->MinContext->Union2.SummFreq; + + + + + count = RC_GetThreshold(summFreq); + hiCnt = count; + + if ((Int32)(count -= s->Freq) < 0) { - Byte symbol; - rc->Decode(rc, 0, s->Freq); + Byte sym; + RC_DecodeFinal(0, s->Freq); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd7_Update1_0(p); - return symbol; + return sym; } + p->PrevSuccess = 0; - i = p->MinContext->NumStats - 1; + i = (unsigned)p->MinContext->NumStats - 1; + do { - if ((hiCnt += (++s)->Freq) > count) + if ((Int32)(count -= (++s)->Freq) < 0) { - Byte symbol; - rc->Decode(rc, hiCnt - s->Freq, s->Freq); + Byte sym; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd7_Update1(p); - return symbol; + return sym; } } while (--i); - if (count >= p->MinContext->SummFreq) - return -2; - p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; - rc->Decode(rc, hiCnt, p->MinContext->SummFreq - hiCnt); + + if (hiCnt >= summFreq) + return PPMD7_SYM_ERROR; + + hiCnt -= count; + RC_Decode(hiCnt, summFreq - hiCnt); + + p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats - 1; - do { MASK((--s)->Symbol) = 0; } while (--i); + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } else { + CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); UInt16 *prob = Ppmd7_GetBinSumm(p); - if (rc->DecodeBit(rc, *prob) == 0) + UInt32 pr = *prob; + UInt32 size0 = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); + + if (R->Code < size0) { - Byte symbol; - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - symbol = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; - Ppmd7_UpdateBin(p); - return symbol; + Byte sym; + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + + // RangeDec_DecodeBit0(size0); + R->Range = size0; + RC_NORM_1(R) + /* we can use single byte normalization here because of + (min(BinSumm[][]) = 95) > (1 << (14 - 8)) */ + + // sym = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; + // Ppmd7_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + sym = s->Symbol; + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 128)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; + else + Ppmd7_UpdateModel(p); + } + return sym; } - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + + // RangeDec_DecodeBit1(size0); + + R->Code -= size0; + R->Range -= size0; + RC_NORM_LOCAL(R) + PPMD_SetAllBitsIn256Bytes(charMask); MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; p->PrevSuccess = 0; } + for (;;) { - CPpmd_State *ps[256], *s; + CPpmd_State *s, *s2; UInt32 freqSum, count, hiCnt; + CPpmd_See *see; - unsigned i, num, numMasked = p->MinContext->NumStats; + CPpmd7_Context *mc; + unsigned numMasked; + RC_NORM_REMOTE(R) + mc = p->MinContext; + numMasked = mc->NumStats; + do { p->OrderFall++; - if (!p->MinContext->Suffix) - return -1; - p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + if (!mc->Suffix) + return PPMD7_SYM_END; + mc = Ppmd7_GetContext(p, mc->Suffix); } - while (p->MinContext->NumStats == numMasked); - hiCnt = 0; - s = Ppmd7_GetStats(p, p->MinContext); - i = 0; - num = p->MinContext->NumStats - numMasked; - do + while (mc->NumStats == numMasked); + + s = Ppmd7_GetStats(p, mc); + { - int k = (int)(MASK(s->Symbol)); - hiCnt += (s->Freq & k); - ps[i] = s++; - i -= k; + unsigned num = mc->NumStats; + unsigned num2 = num / 2; + + num &= 1; + hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + s += num; + p->MinContext = mc; + + do + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); + hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + } + while (--num2); } - while (i != num); - + see = Ppmd7_MakeEscFreq(p, numMasked, &freqSum); freqSum += hiCnt; - count = rc->GetThreshold(rc, freqSum); + + + + + count = RC_GetThreshold(freqSum); if (count < hiCnt) { - Byte symbol; - CPpmd_State **pps = ps; - for (hiCnt = 0; (hiCnt += (*pps)->Freq) <= count; pps++); - s = *pps; - rc->Decode(rc, hiCnt - s->Freq, s->Freq); + Byte sym; + + s = Ppmd7_GetStats(p, p->MinContext); + hiCnt = count; + // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // if ((Int32)count >= 0) + { + for (;;) + { + count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + }; + } + s--; + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + + // new (see->Summ) value can overflow over 16-bits in some rare cases Ppmd_See_Update(see); p->FoundState = s; - symbol = s->Symbol; + sym = s->Symbol; Ppmd7_Update2(p); - return symbol; + return sym; } + if (count >= freqSum) - return -2; - rc->Decode(rc, hiCnt, freqSum - hiCnt); + return PPMD7_SYM_ERROR; + + RC_Decode(hiCnt, freqSum - hiCnt); + + // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. + // new (see->Summ) value can overflow over 16-bits in some rare cases see->Summ = (UInt16)(see->Summ + freqSum); - do { MASK(ps[--i]->Symbol) = 0; } while (i != 0); + + s = Ppmd7_GetStats(p, p->MinContext); + s2 = s + p->MinContext->NumStats; + do + { + MASK(s->Symbol) = 0; + s++; + } + while (s != s2); + } +} + +/* +Byte *Ppmd7z_DecodeSymbols(CPpmd7 *p, Byte *buf, const Byte *lim) +{ + int sym = 0; + if (buf != lim) + do + { + sym = Ppmd7z_DecodeSymbol(p); + if (sym < 0) + break; + *buf = (Byte)sym; } + while (++buf < lim); + p->LastSymbol = sym; + return buf; } +*/ diff --git a/3rdparty/lzma/C/Ppmd7Enc.c b/3rdparty/lzma/C/Ppmd7Enc.c index e6dd3d2fc29..62139c5b66a 100644 --- a/3rdparty/lzma/C/Ppmd7Enc.c +++ b/3rdparty/lzma/C/Ppmd7Enc.c @@ -1,6 +1,8 @@ -/* Ppmd7Enc.c -- PPMdH Encoder -2015-09-28 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ +/* Ppmd7Enc.c -- Ppmd7z (PPMdH with 7z Range Coder) Encoder +2021-04-13 : Igor Pavlov : Public domain +This code is based on: + PPMd var.H (2001): Dmitry Shkarin : Public domain */ + #include "Precomp.h" @@ -8,65 +10,60 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p) +#define R (&p->rc.enc) + +void Ppmd7z_Init_RangeEnc(CPpmd7 *p) { - p->Low = 0; - p->Range = 0xFFFFFFFF; - p->Cache = 0; - p->CacheSize = 1; + R->Low = 0; + R->Range = 0xFFFFFFFF; + R->Cache = 0; + R->CacheSize = 1; } -static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p) +MY_NO_INLINE +static void RangeEnc_ShiftLow(CPpmd7 *p) { - if ((UInt32)p->Low < (UInt32)0xFF000000 || (unsigned)(p->Low >> 32) != 0) + if ((UInt32)R->Low < (UInt32)0xFF000000 || (unsigned)(R->Low >> 32) != 0) { - Byte temp = p->Cache; + Byte temp = R->Cache; do { - p->Stream->Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); + IByteOut_Write(R->Stream, (Byte)(temp + (Byte)(R->Low >> 32))); temp = 0xFF; } - while (--p->CacheSize != 0); - p->Cache = (Byte)((UInt32)p->Low >> 24); + while (--R->CacheSize != 0); + R->Cache = (Byte)((UInt32)R->Low >> 24); } - p->CacheSize++; - p->Low = (UInt32)p->Low << 8; + R->CacheSize++; + R->Low = (UInt32)((UInt32)R->Low << 8); } -static void RangeEnc_Encode(CPpmd7z_RangeEnc *p, UInt32 start, UInt32 size, UInt32 total) -{ - p->Low += start * (p->Range /= total); - p->Range *= size; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} +#define RC_NORM_BASE(p) if (R->Range < kTopValue) { R->Range <<= 8; RangeEnc_ShiftLow(p); +#define RC_NORM_1(p) RC_NORM_BASE(p) } +#define RC_NORM(p) RC_NORM_BASE(p) RC_NORM_BASE(p) }} -static void RangeEnc_EncodeBit_0(CPpmd7z_RangeEnc *p, UInt32 size0) -{ - p->Range = (p->Range >> 14) * size0; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} +// we must use only one type of Normalization from two: LOCAL or REMOTE +#define RC_NORM_LOCAL(p) // RC_NORM(p) +#define RC_NORM_REMOTE(p) RC_NORM(p) + +/* +#define RangeEnc_Encode(p, start, _size_) \ + { UInt32 size = _size_; \ + R->Low += start * R->Range; \ + R->Range *= size; \ + RC_NORM_LOCAL(p); } +*/ -static void RangeEnc_EncodeBit_1(CPpmd7z_RangeEnc *p, UInt32 size0) +MY_FORCE_INLINE +// MY_NO_INLINE +static void RangeEnc_Encode(CPpmd7 *p, UInt32 start, UInt32 size) { - UInt32 newBound = (p->Range >> 14) * size0; - p->Low += newBound; - p->Range -= newBound; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } + R->Low += start * R->Range; + R->Range *= size; + RC_NORM_LOCAL(p); } -void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) +void Ppmd7z_Flush_RangeEnc(CPpmd7 *p) { unsigned i; for (i = 0; i < 5; i++) @@ -74,31 +71,53 @@ void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) } -#define MASK(sym) ((signed char *)charMask)[sym] -void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) +#define RC_Encode(start, size) RangeEnc_Encode(p, start, size); +#define RC_EncodeFinal(start, size) RC_Encode(start, size); RC_NORM_REMOTE(p); + +#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) +#define SUFFIX(ctx) CTX((ctx)->Suffix) +typedef CPpmd7_Context * CTX_PTR; +#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) + +void Ppmd7_UpdateModel(CPpmd7 *p); + +#define MASK(sym) ((unsigned char *)charMask)[sym] + +MY_FORCE_INLINE +static +void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) { size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) { CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); UInt32 sum; unsigned i; + + + + + R->Range /= p->MinContext->Union2.SummFreq; + if (s->Symbol == symbol) { - RangeEnc_Encode(rc, 0, s->Freq, p->MinContext->SummFreq); + // R->Range /= p->MinContext->Union2.SummFreq; + RC_EncodeFinal(0, s->Freq); p->FoundState = s; Ppmd7_Update1_0(p); return; } p->PrevSuccess = 0; sum = s->Freq; - i = p->MinContext->NumStats - 1; + i = (unsigned)p->MinContext->NumStats - 1; do { if ((++s)->Symbol == symbol) { - RangeEnc_Encode(rc, sum, s->Freq, p->MinContext->SummFreq); + // R->Range /= p->MinContext->Union2.SummFreq; + RC_EncodeFinal(sum, s->Freq); p->FoundState = s; Ppmd7_Update1(p); return; @@ -106,82 +125,199 @@ void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) sum += s->Freq; } while (--i); + + // R->Range /= p->MinContext->Union2.SummFreq; + RC_Encode(sum, p->MinContext->Union2.SummFreq - sum); - p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; + p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats - 1; - do { MASK((--s)->Symbol) = 0; } while (--i); - RangeEnc_Encode(rc, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq); + // MASK(s->Symbol) = 0; + // i = p->MinContext->NumStats - 1; + // do { MASK((--s)->Symbol) = 0; } while (--i); + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } } else { UInt16 *prob = Ppmd7_GetBinSumm(p); CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); + UInt32 pr = *prob; + UInt32 bound = (R->Range >> 14) * pr; + pr = PPMD_UPDATE_PROB_1(pr); if (s->Symbol == symbol) { - RangeEnc_EncodeBit_0(rc, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - p->FoundState = s; - Ppmd7_UpdateBin(p); + *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); + // RangeEnc_EncodeBit_0(p, bound); + R->Range = bound; + RC_NORM_1(p); + + // p->FoundState = s; + // Ppmd7_UpdateBin(p); + { + unsigned freq = s->Freq; + CTX_PTR c = CTX(SUCCESSOR(s)); + p->FoundState = s; + p->PrevSuccess = 1; + p->RunLength++; + s->Freq = (Byte)(freq + (freq < 128)); + // NextContext(p); + if (p->OrderFall == 0 && (const Byte *)c > p->Text) + p->MaxContext = p->MinContext = c; + else + Ppmd7_UpdateModel(p); + } return; } - else - { - RangeEnc_EncodeBit_1(rc, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - p->PrevSuccess = 0; - } + + *prob = (UInt16)pr; + p->InitEsc = p->ExpEscape[pr >> 10]; + // RangeEnc_EncodeBit_1(p, bound); + R->Low += bound; + R->Range -= bound; + RC_NORM_LOCAL(p) + + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + p->PrevSuccess = 0; } + for (;;) { - UInt32 escFreq; CPpmd_See *see; CPpmd_State *s; - UInt32 sum; - unsigned i, numMasked = p->MinContext->NumStats; + UInt32 sum, escFreq; + CPpmd7_Context *mc; + unsigned i, numMasked; + + RC_NORM_REMOTE(p) + + mc = p->MinContext; + numMasked = mc->NumStats; + do { p->OrderFall++; - if (!p->MinContext->Suffix) + if (!mc->Suffix) return; /* EndMarker (symbol = -1) */ - p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + mc = Ppmd7_GetContext(p, mc->Suffix); + i = mc->NumStats; } - while (p->MinContext->NumStats == numMasked); + while (i == numMasked); + + p->MinContext = mc; - see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); - s = Ppmd7_GetStats(p, p->MinContext); + // see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); + { + if (i != 256) + { + unsigned nonMasked = i - numMasked; + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + + p->HiBitsFlag + + (nonMasked < (unsigned)SUFFIX(mc)->NumStats - i) + + 2 * (unsigned)(mc->Union2.SummFreq < 11 * i) + + 4 * (unsigned)(numMasked > nonMasked); + { + // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ + unsigned summ = (UInt16)see->Summ; // & 0xFFFF + unsigned r = (summ >> see->Shift); + see->Summ = (UInt16)(summ - r); + escFreq = r + (r == 0); + } + } + else + { + see = &p->DummySee; + escFreq = 1; + } + } + + s = Ppmd7_GetStats(p, mc); sum = 0; - i = p->MinContext->NumStats; + // i = mc->NumStats; + do { - int cur = s->Symbol; - if (cur == symbol) + unsigned cur = s->Symbol; + if ((int)cur == symbol) { UInt32 low = sum; - CPpmd_State *s1 = s; - do + UInt32 freq = s->Freq; + unsigned num2; + + Ppmd_See_Update(see); + p->FoundState = s; + sum += escFreq; + + num2 = i / 2; + i &= 1; + sum += freq & (0 - (UInt32)i); + if (num2 != 0) { - sum += (s->Freq & (int)(MASK(s->Symbol))); - s++; + s += i; + for (;;) + { + unsigned sym0 = s[0].Symbol; + unsigned sym1 = s[1].Symbol; + s += 2; + sum += (s[-2].Freq & (unsigned)(MASK(sym0))); + sum += (s[-1].Freq & (unsigned)(MASK(sym1))); + if (--num2 == 0) + break; + } } - while (--i); - RangeEnc_Encode(rc, low, s1->Freq, sum + escFreq); - Ppmd_See_Update(see); - p->FoundState = s1; + + + R->Range /= sum; + RC_EncodeFinal(low, freq); Ppmd7_Update2(p); return; } - sum += (s->Freq & (int)(MASK(cur))); - MASK(cur) = 0; + sum += (s->Freq & (unsigned)(MASK(cur))); s++; } while (--i); - RangeEnc_Encode(rc, sum, escFreq, sum + escFreq); - see->Summ = (UInt16)(see->Summ + sum + escFreq); + { + UInt32 total = sum + escFreq; + see->Summ = (UInt16)(see->Summ + total); + + R->Range /= total; + RC_Encode(sum, escFreq); + } + + { + CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + s--; + MASK(s->Symbol) = 0; + do + { + unsigned sym0 = s2[0].Symbol; + unsigned sym1 = s2[1].Symbol; + s2 += 2; + MASK(sym0) = 0; + MASK(sym1) = 0; + } + while (s2 < s); + } + } +} + + +void Ppmd7z_EncodeSymbols(CPpmd7 *p, const Byte *buf, const Byte *lim) +{ + for (; buf < lim; buf++) + { + Ppmd7z_EncodeSymbol(p, *buf); } } diff --git a/3rdparty/lzma/C/Sha256.c b/3rdparty/lzma/C/Sha256.c index b5bc19b9602..8b3983ea732 100644 --- a/3rdparty/lzma/C/Sha256.c +++ b/3rdparty/lzma/C/Sha256.c @@ -1,5 +1,5 @@ -/* Crypto/Sha256.c -- SHA-256 Hash -2015-11-14 : Igor Pavlov : Public domain +/* Sha256.c -- SHA-256 Hash +2021-04-01 : Igor Pavlov : Public domain This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -10,16 +10,107 @@ This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "RotateDefs.h" #include "Sha256.h" +#if defined(_MSC_VER) && (_MSC_VER < 1900) +// #define USE_MY_MM +#endif + +#ifdef MY_CPU_X86_OR_AMD64 + #ifdef _MSC_VER + #if _MSC_VER >= 1200 + #define _SHA_SUPPORTED + #endif + #elif defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1800) // fix that check + #define _SHA_SUPPORTED + #endif + #endif +#elif defined(MY_CPU_ARM_OR_ARM64) + #ifdef _MSC_VER + #if _MSC_VER >= 1910 + #define _SHA_SUPPORTED + #endif + #elif defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define _SHA_SUPPORTED + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define _SHA_SUPPORTED + #endif + #endif +#endif + +void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); + +#ifdef _SHA_SUPPORTED + void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); + + static SHA256_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS = Sha256_UpdateBlocks; + static SHA256_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS_HW; + + #define UPDATE_BLOCKS(p) p->func_UpdateBlocks +#else + #define UPDATE_BLOCKS(p) Sha256_UpdateBlocks +#endif + + +BoolInt Sha256_SetFunction(CSha256 *p, unsigned algo) +{ + SHA256_FUNC_UPDATE_BLOCKS func = Sha256_UpdateBlocks; + + #ifdef _SHA_SUPPORTED + if (algo != SHA256_ALGO_SW) + { + if (algo == SHA256_ALGO_DEFAULT) + func = g_FUNC_UPDATE_BLOCKS; + else + { + if (algo != SHA256_ALGO_HW) + return False; + func = g_FUNC_UPDATE_BLOCKS_HW; + if (!func) + return False; + } + } + #else + if (algo > 1) + return False; + #endif + + p->func_UpdateBlocks = func; + return True; +} + + /* define it for speed optimization */ -#ifndef _SFX -#define _SHA256_UNROLL -#define _SHA256_UNROLL2 + +#ifdef _SFX + #define STEP_PRE 1 + #define STEP_MAIN 1 +#else + #define STEP_PRE 2 + #define STEP_MAIN 4 + // #define _SHA256_UNROLL #endif -/* #define _SHA256_UNROLL2 */ +#if STEP_MAIN != 16 + #define _SHA256_BIG_W +#endif -void Sha256_Init(CSha256 *p) + + + +void Sha256_InitState(CSha256 *p) { + p->count = 0; p->state[0] = 0x6a09e667; p->state[1] = 0xbb67ae85; p->state[2] = 0x3c6ef372; @@ -28,7 +119,17 @@ void Sha256_Init(CSha256 *p) p->state[5] = 0x9b05688c; p->state[6] = 0x1f83d9ab; p->state[7] = 0x5be0cd19; - p->count = 0; +} + +void Sha256_Init(CSha256 *p) +{ + p->func_UpdateBlocks = + #ifdef _SHA_SUPPORTED + g_FUNC_UPDATE_BLOCKS; + #else + NULL; + #endif + Sha256_InitState(p); } #define S0(x) (rotrFixed(x, 2) ^ rotrFixed(x,13) ^ rotrFixed(x, 22)) @@ -36,61 +137,100 @@ void Sha256_Init(CSha256 *p) #define s0(x) (rotrFixed(x, 7) ^ rotrFixed(x,18) ^ (x >> 3)) #define s1(x) (rotrFixed(x,17) ^ rotrFixed(x,19) ^ (x >> 10)) -#define blk0(i) (W[i]) -#define blk2(i) (W[i] += s1(W[((i)-2)&15]) + W[((i)-7)&15] + s0(W[((i)-15)&15])) - #define Ch(x,y,z) (z^(x&(y^z))) #define Maj(x,y,z) ((x&y)|(z&(x|y))) -#ifdef _SHA256_UNROLL2 - -#define R(a,b,c,d,e,f,g,h, i) \ - h += S1(e) + Ch(e,f,g) + K[(i)+(j)] + (j ? blk2(i) : blk0(i)); \ - d += h; \ - h += S0(a) + Maj(a, b, c) -#define RX_8(i) \ - R(a,b,c,d,e,f,g,h, i); \ - R(h,a,b,c,d,e,f,g, i+1); \ - R(g,h,a,b,c,d,e,f, i+2); \ - R(f,g,h,a,b,c,d,e, i+3); \ - R(e,f,g,h,a,b,c,d, i+4); \ - R(d,e,f,g,h,a,b,c, i+5); \ - R(c,d,e,f,g,h,a,b, i+6); \ - R(b,c,d,e,f,g,h,a, i+7) +#define W_PRE(i) (W[(i) + (size_t)(j)] = GetBe32(data + ((size_t)(j) + i) * 4)) -#define RX_16 RX_8(0); RX_8(8); +#define blk2_main(j, i) s1(w(j, (i)-2)) + w(j, (i)-7) + s0(w(j, (i)-15)) +#ifdef _SHA256_BIG_W + // we use +i instead of +(i) to change the order to solve CLANG compiler warning for signed/unsigned. + #define w(j, i) W[(size_t)(j) + i] + #define blk2(j, i) (w(j, i) = w(j, (i)-16) + blk2_main(j, i)) #else + #if STEP_MAIN == 16 + #define w(j, i) W[(i) & 15] + #else + #define w(j, i) W[((size_t)(j) + (i)) & 15] + #endif + #define blk2(j, i) (w(j, i) += blk2_main(j, i)) +#endif -#define a(i) T[(0-(i))&7] -#define b(i) T[(1-(i))&7] -#define c(i) T[(2-(i))&7] -#define d(i) T[(3-(i))&7] -#define e(i) T[(4-(i))&7] -#define f(i) T[(5-(i))&7] -#define g(i) T[(6-(i))&7] -#define h(i) T[(7-(i))&7] - -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(j)] + (j ? blk2(i) : blk0(i)); \ - d(i) += h(i); \ - h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) \ +#define W_MAIN(i) blk2(j, i) -#ifdef _SHA256_UNROLL -#define RX_8(i) R(i+0); R(i+1); R(i+2); R(i+3); R(i+4); R(i+5); R(i+6); R(i+7); -#define RX_16 RX_8(0); RX_8(8); +#define T1(wx, i) \ + tmp = h + S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ + h = g; \ + g = f; \ + f = e; \ + e = d + tmp; \ + tmp += S0(a) + Maj(a, b, c); \ + d = c; \ + c = b; \ + b = a; \ + a = tmp; \ -#else +#define R1_PRE(i) T1( W_PRE, i) +#define R1_MAIN(i) T1( W_MAIN, i) -#define RX_16 unsigned i; for (i = 0; i < 16; i++) { R(i); } +#if (!defined(_SHA256_UNROLL) || STEP_MAIN < 8) && (STEP_MAIN >= 4) +#define R2_MAIN(i) \ + R1_MAIN(i) \ + R1_MAIN(i + 1) \ #endif + + +#if defined(_SHA256_UNROLL) && STEP_MAIN >= 8 + +#define T4( a,b,c,d,e,f,g,h, wx, i) \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ + tmp = h; \ + h += d; \ + d = tmp + S0(a) + Maj(a, b, c); \ + +#define R4( wx, i) \ + T4 ( a,b,c,d,e,f,g,h, wx, (i )); \ + T4 ( d,a,b,c,h,e,f,g, wx, (i+1)); \ + T4 ( c,d,a,b,g,h,e,f, wx, (i+2)); \ + T4 ( b,c,d,a,f,g,h,e, wx, (i+3)); \ + +#define R4_PRE(i) R4( W_PRE, i) +#define R4_MAIN(i) R4( W_MAIN, i) + + +#define T8( a,b,c,d,e,f,g,h, wx, i) \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ + d += h; \ + h += S0(a) + Maj(a, b, c); \ + +#define R8( wx, i) \ + T8 ( a,b,c,d,e,f,g,h, wx, i ); \ + T8 ( h,a,b,c,d,e,f,g, wx, i+1); \ + T8 ( g,h,a,b,c,d,e,f, wx, i+2); \ + T8 ( f,g,h,a,b,c,d,e, wx, i+3); \ + T8 ( e,f,g,h,a,b,c,d, wx, i+4); \ + T8 ( d,e,f,g,h,a,b,c, wx, i+5); \ + T8 ( c,d,e,f,g,h,a,b, wx, i+6); \ + T8 ( b,c,d,e,f,g,h,a, wx, i+7); \ + +#define R8_PRE(i) R8( W_PRE, i) +#define R8_MAIN(i) R8( W_MAIN, i) + #endif -static const UInt32 K[64] = { +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); + +// static +extern MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64]; + +MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, @@ -109,30 +249,27 @@ static const UInt32 K[64] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 }; -static void Sha256_WriteByteBlock(CSha256 *p) -{ - UInt32 W[16]; - unsigned j; - UInt32 *state; +#define K SHA256_K_ARRAY - #ifdef _SHA256_UNROLL2 - UInt32 a,b,c,d,e,f,g,h; + +MY_NO_INLINE +void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + UInt32 W + #ifdef _SHA256_BIG_W + [64]; #else - UInt32 T[8]; + [16]; #endif - for (j = 0; j < 16; j += 4) - { - const Byte *ccc = p->buffer + j * 4; - W[j ] = GetBe32(ccc); - W[j + 1] = GetBe32(ccc + 4); - W[j + 2] = GetBe32(ccc + 8); - W[j + 3] = GetBe32(ccc + 12); - } + unsigned j; - state = p->state; + UInt32 a,b,c,d,e,f,g,h; - #ifdef _SHA256_UNROLL2 + #if !defined(_SHA256_UNROLL) || (STEP_MAIN <= 4) || (STEP_PRE <= 4) + UInt32 tmp; + #endif + a = state[0]; b = state[1]; c = state[2]; @@ -141,39 +278,96 @@ static void Sha256_WriteByteBlock(CSha256 *p) f = state[5]; g = state[6]; h = state[7]; - #else - for (j = 0; j < 8; j++) - T[j] = state[j]; - #endif - for (j = 0; j < 64; j += 16) + while (numBlocks) { - RX_16 + + for (j = 0; j < 16; j += STEP_PRE) + { + #if STEP_PRE > 4 + + #if STEP_PRE < 8 + R4_PRE(0); + #else + R8_PRE(0); + #if STEP_PRE == 16 + R8_PRE(8); + #endif + #endif + + #else + + R1_PRE(0); + #if STEP_PRE >= 2 + R1_PRE(1); + #if STEP_PRE >= 4 + R1_PRE(2); + R1_PRE(3); + #endif + #endif + + #endif + } + + for (j = 16; j < 64; j += STEP_MAIN) + { + #if defined(_SHA256_UNROLL) && STEP_MAIN >= 8 + + #if STEP_MAIN < 8 + R4_MAIN(0); + #else + R8_MAIN(0); + #if STEP_MAIN == 16 + R8_MAIN(8); + #endif + #endif + + #else + + R1_MAIN(0); + #if STEP_MAIN >= 2 + R1_MAIN(1); + #if STEP_MAIN >= 4 + R2_MAIN(2); + #if STEP_MAIN >= 8 + R2_MAIN(4); + R2_MAIN(6); + #if STEP_MAIN >= 16 + R2_MAIN(8); + R2_MAIN(10); + R2_MAIN(12); + R2_MAIN(14); + #endif + #endif + #endif + #endif + #endif + } + + a += state[0]; state[0] = a; + b += state[1]; state[1] = b; + c += state[2]; state[2] = c; + d += state[3]; state[3] = d; + e += state[4]; state[4] = e; + f += state[5]; state[5] = f; + g += state[6]; state[6] = g; + h += state[7]; state[7] = h; + + data += 64; + numBlocks--; } - #ifdef _SHA256_UNROLL2 - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; - state[5] += f; - state[6] += g; - state[7] += h; - #else - for (j = 0; j < 8; j++) - state[j] += T[j]; - #endif - /* Wipe variables */ /* memset(W, 0, sizeof(W)); */ - /* memset(T, 0, sizeof(T)); */ } #undef S0 #undef S1 #undef s0 #undef s1 +#undef K + +#define Sha256_UpdateBlock(p) UPDATE_BLOCKS(p)(p->state, p->buffer, 1) void Sha256_Update(CSha256 *p, const Byte *data, size_t size) { @@ -193,25 +387,26 @@ void Sha256_Update(CSha256 *p, const Byte *data, size_t size) return; } - size -= num; - memcpy(p->buffer + pos, data, num); - data += num; + if (pos != 0) + { + size -= num; + memcpy(p->buffer + pos, data, num); + data += num; + Sha256_UpdateBlock(p); + } } - - for (;;) { - Sha256_WriteByteBlock(p); - if (size < 64) - break; - size -= 64; - memcpy(p->buffer, data, 64); - data += 64; - } - - if (size != 0) + size_t numBlocks = size >> 6; + UPDATE_BLOCKS(p)(p->state, data, numBlocks); + size &= 0x3F; + if (size == 0) + return; + data += (numBlocks << 6); memcpy(p->buffer, data, size); + } } + void Sha256_Final(CSha256 *p, Byte *digest) { unsigned pos = (unsigned)p->count & 0x3F; @@ -219,13 +414,30 @@ void Sha256_Final(CSha256 *p, Byte *digest) p->buffer[pos++] = 0x80; - while (pos != (64 - 8)) + if (pos > (64 - 8)) + { + while (pos != 64) { p->buffer[pos++] = 0; } + // memset(&p->buf.buffer[pos], 0, 64 - pos); + Sha256_UpdateBlock(p); + pos = 0; + } + + /* + if (pos & 3) { - pos &= 0x3F; - if (pos == 0) - Sha256_WriteByteBlock(p); - p->buffer[pos++] = 0; + p->buffer[pos] = 0; + p->buffer[pos + 1] = 0; + p->buffer[pos + 2] = 0; + pos += 3; + pos &= ~3; } + { + for (; pos < 64 - 8; pos += 4) + *(UInt32 *)(&p->buffer[pos]) = 0; + } + */ + + memset(&p->buffer[pos], 0, (64 - 8) - pos); { UInt64 numBits = (p->count << 3); @@ -233,16 +445,42 @@ void Sha256_Final(CSha256 *p, Byte *digest) SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); } - Sha256_WriteByteBlock(p); + Sha256_UpdateBlock(p); for (i = 0; i < 8; i += 2) { UInt32 v0 = p->state[i]; - UInt32 v1 = p->state[i + 1]; + UInt32 v1 = p->state[(size_t)i + 1]; SetBe32(digest , v0); SetBe32(digest + 4, v1); digest += 8; } - Sha256_Init(p); + Sha256_InitState(p); +} + + +void Sha256Prepare() +{ + #ifdef _SHA_SUPPORTED + SHA256_FUNC_UPDATE_BLOCKS f, f_hw; + f = Sha256_UpdateBlocks; + f_hw = NULL; + #ifdef MY_CPU_X86_OR_AMD64 + #ifndef USE_MY_MM + if (CPU_IsSupported_SHA() + && CPU_IsSupported_SSSE3() + // && CPU_IsSupported_SSE41() + ) + #endif + #else + if (CPU_IsSupported_SHA2()) + #endif + { + // printf("\n========== HW SHA256 ======== \n"); + f = f_hw = Sha256_UpdateBlocks_HW; + } + g_FUNC_UPDATE_BLOCKS = f; + g_FUNC_UPDATE_BLOCKS_HW = f_hw; + #endif } diff --git a/3rdparty/lzma/C/Sha256.h b/3rdparty/lzma/C/Sha256.h index 3f455dbc0d8..aa38501e579 100644 --- a/3rdparty/lzma/C/Sha256.h +++ b/3rdparty/lzma/C/Sha256.h @@ -1,26 +1,76 @@ /* Sha256.h -- SHA-256 Hash -2013-01-18 : Igor Pavlov : Public domain */ +2021-01-01 : Igor Pavlov : Public domain */ -#ifndef __CRYPTO_SHA256_H -#define __CRYPTO_SHA256_H +#ifndef __7Z_SHA256_H +#define __7Z_SHA256_H #include "7zTypes.h" EXTERN_C_BEGIN -#define SHA256_DIGEST_SIZE 32 +#define SHA256_NUM_BLOCK_WORDS 16 +#define SHA256_NUM_DIGEST_WORDS 8 + +#define SHA256_BLOCK_SIZE (SHA256_NUM_BLOCK_WORDS * 4) +#define SHA256_DIGEST_SIZE (SHA256_NUM_DIGEST_WORDS * 4) + +typedef void (MY_FAST_CALL *SHA256_FUNC_UPDATE_BLOCKS)(UInt32 state[8], const Byte *data, size_t numBlocks); + +/* + if (the system supports different SHA256 code implementations) + { + (CSha256::func_UpdateBlocks) will be used + (CSha256::func_UpdateBlocks) can be set by + Sha256_Init() - to default (fastest) + Sha256_SetFunction() - to any algo + } + else + { + (CSha256::func_UpdateBlocks) is ignored. + } +*/ typedef struct { - UInt32 state[8]; + SHA256_FUNC_UPDATE_BLOCKS func_UpdateBlocks; UInt64 count; - Byte buffer[64]; + UInt64 __pad_2[2]; + UInt32 state[SHA256_NUM_DIGEST_WORDS]; + + Byte buffer[SHA256_BLOCK_SIZE]; } CSha256; + +#define SHA256_ALGO_DEFAULT 0 +#define SHA256_ALGO_SW 1 +#define SHA256_ALGO_HW 2 + +/* +Sha256_SetFunction() +return: + 0 - (algo) value is not supported, and func_UpdateBlocks was not changed + 1 - func_UpdateBlocks was set according (algo) value. +*/ + +BoolInt Sha256_SetFunction(CSha256 *p, unsigned algo); + +void Sha256_InitState(CSha256 *p); void Sha256_Init(CSha256 *p); void Sha256_Update(CSha256 *p, const Byte *data, size_t size); void Sha256_Final(CSha256 *p, Byte *digest); + + + +// void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); + +/* +call Sha256Prepare() once at program start. +It prepares all supported implementations, and detects the fastest implementation. +*/ + +void Sha256Prepare(void); + EXTERN_C_END #endif diff --git a/3rdparty/lzma/C/Sha256Opt.c b/3rdparty/lzma/C/Sha256Opt.c new file mode 100644 index 00000000000..decc1382cd6 --- /dev/null +++ b/3rdparty/lzma/C/Sha256Opt.c @@ -0,0 +1,373 @@ +/* Sha256Opt.c -- SHA-256 optimized code for SHA-256 hardware instructions +2021-04-01 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#if defined(_MSC_VER) +#if (_MSC_VER < 1900) && (_MSC_VER >= 1200) +// #define USE_MY_MM +#endif +#endif + +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_SHA + #ifndef __SHA__ + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + #if defined(_MSC_VER) + // SSSE3: for clang-cl: + #include + #define __SHA__ + #endif + #endif + + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 8) // fix that check + #define USE_HW_SHA + #ifndef __SHA__ + #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) + // #pragma GCC target("sha,ssse3") + #endif + #endif + #elif defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1800) // fix that check + #define USE_HW_SHA + #endif + #elif defined(_MSC_VER) + #ifdef USE_MY_MM + #define USE_VER_MIN 1300 + #else + #define USE_VER_MIN 1910 + #endif + #if _MSC_VER >= USE_VER_MIN + #define USE_HW_SHA + #endif + #endif +// #endif // MY_CPU_X86_OR_AMD64 + +#ifdef USE_HW_SHA + +// #pragma message("Sha256 HW") +// #include + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +#include +#else +#include + +#if defined(_MSC_VER) && (_MSC_VER >= 1600) +// #include +#endif + +#ifdef USE_MY_MM +#include "My_mm.h" +#endif + +#endif + +/* +SHA256 uses: +SSE2: + _mm_loadu_si128 + _mm_storeu_si128 + _mm_set_epi32 + _mm_add_epi32 + _mm_shuffle_epi32 / pshufd + + + +SSSE3: + _mm_shuffle_epi8 / pshufb + _mm_alignr_epi8 +SHA: + _mm_sha256* +*/ + +// K array must be aligned for 16-bytes at least. +// The compiler can look align attribute and selects +// movdqu - for code without align attribute +// movdqa - for code with align attribute +extern +MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64]; + +#define K SHA256_K_ARRAY + + +#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); +#define SHA256_MSG1(dest, src) dest = _mm_sha256msg1_epu32(dest, src); +#define SHA25G_MSG2(dest, src) dest = _mm_sha256msg2_epu32(dest, src); + + +#define LOAD_SHUFFLE(m, k) \ + m = _mm_loadu_si128((const __m128i *)(const void *)(data + (k) * 16)); \ + m = _mm_shuffle_epi8(m, mask); \ + +#define SM1(g0, g1, g2, g3) \ + SHA256_MSG1(g3, g0); \ + +#define SM2(g0, g1, g2, g3) \ + tmp = _mm_alignr_epi8(g1, g0, 4); \ + ADD_EPI32(g2, tmp); \ + SHA25G_MSG2(g2, g1); \ + +// #define LS0(k, g0, g1, g2, g3) LOAD_SHUFFLE(g0, k) +// #define LS1(k, g0, g1, g2, g3) LOAD_SHUFFLE(g1, k+1) + + +#define NNN(g0, g1, g2, g3) + + +#define RND2(t0, t1) \ + t0 = _mm_sha256rnds2_epu32(t0, t1, msg); + +#define RND2_0(m, k) \ + msg = _mm_add_epi32(m, *(const __m128i *) (const void *) &K[(k) * 4]); \ + RND2(state0, state1); \ + msg = _mm_shuffle_epi32(msg, 0x0E); \ + + +#define RND2_1 \ + RND2(state1, state0); \ + + +// We use scheme with 3 rounds ahead for SHA256_MSG1 / 2 rounds ahead for SHA256_MSG2 + +#define R4(k, g0, g1, g2, g3, OP0, OP1) \ + RND2_0(g0, k); \ + OP0(g0, g1, g2, g3); \ + RND2_1; \ + OP1(g0, g1, g2, g3); \ + +#define R16(k, OP0, OP1, OP2, OP3, OP4, OP5, OP6, OP7) \ + R4 ( (k)*4+0, m0, m1, m2, m3, OP0, OP1 ) \ + R4 ( (k)*4+1, m1, m2, m3, m0, OP2, OP3 ) \ + R4 ( (k)*4+2, m2, m3, m0, m1, OP4, OP5 ) \ + R4 ( (k)*4+3, m3, m0, m1, m2, OP6, OP7 ) \ + +#define PREPARE_STATE \ + tmp = _mm_shuffle_epi32(state0, 0x1B); /* abcd */ \ + state0 = _mm_shuffle_epi32(state1, 0x1B); /* efgh */ \ + state1 = state0; \ + state0 = _mm_unpacklo_epi64(state0, tmp); /* cdgh */ \ + state1 = _mm_unpackhi_epi64(state1, tmp); /* abef */ \ + + +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + const __m128i mask = _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203); + __m128i tmp; + __m128i state0, state1; + + if (numBlocks == 0) + return; + + state0 = _mm_loadu_si128((const __m128i *) (const void *) &state[0]); + state1 = _mm_loadu_si128((const __m128i *) (const void *) &state[4]); + + PREPARE_STATE + + do + { + __m128i state0_save, state1_save; + __m128i m0, m1, m2, m3; + __m128i msg; + // #define msg tmp + + state0_save = state0; + state1_save = state1; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + + + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + + ADD_EPI32(state0, state0_save); + ADD_EPI32(state1, state1_save); + + data += 64; + } + while (--numBlocks); + + PREPARE_STATE + + _mm_storeu_si128((__m128i *) (void *) &state[0], state0); + _mm_storeu_si128((__m128i *) (void *) &state[4], state1); +} + +#endif // USE_HW_SHA + +#elif defined(MY_CPU_ARM_OR_ARM64) + + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_SHA + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_SHA + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_SHA + #endif + #endif + +#ifdef USE_HW_SHA + +// #pragma message("=== Sha256 HW === ") + +#if defined(__clang__) || defined(__GNUC__) + #ifdef MY_CPU_ARM64 + #define ATTRIB_SHA __attribute__((__target__("+crypto"))) + #else + #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #endif +#else + // _MSC_VER + // for arm32 + #define _ARM_USE_NEW_NEON_INTRINSICS +#endif + +#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#include +#else +#include +#endif + +typedef uint32x4_t v128; +// typedef __n128 v128; // MSVC + +#ifdef MY_CPU_BE + #define MY_rev32_for_LE(x) +#else + #define MY_rev32_for_LE(x) x = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x))) +#endif + +#define LOAD_128(_p) (*(const v128 *)(const void *)(_p)) +#define STORE_128(_p, _v) *(v128 *)(void *)(_p) = (_v) + +#define LOAD_SHUFFLE(m, k) \ + m = LOAD_128((data + (k) * 16)); \ + MY_rev32_for_LE(m); \ + +// K array must be aligned for 16-bytes at least. +extern +MY_ALIGN(64) +const UInt32 SHA256_K_ARRAY[64]; + +#define K SHA256_K_ARRAY + + +#define SHA256_SU0(dest, src) dest = vsha256su0q_u32(dest, src); +#define SHA25G_SU1(dest, src2, src3) dest = vsha256su1q_u32(dest, src2, src3); + +#define SM1(g0, g1, g2, g3) SHA256_SU0(g3, g0) +#define SM2(g0, g1, g2, g3) SHA25G_SU1(g2, g0, g1) +#define NNN(g0, g1, g2, g3) + + +#define R4(k, g0, g1, g2, g3, OP0, OP1) \ + msg = vaddq_u32(g0, *(const v128 *) (const void *) &K[(k) * 4]); \ + tmp = state0; \ + state0 = vsha256hq_u32( state0, state1, msg ); \ + state1 = vsha256h2q_u32( state1, tmp, msg ); \ + OP0(g0, g1, g2, g3); \ + OP1(g0, g1, g2, g3); \ + + +#define R16(k, OP0, OP1, OP2, OP3, OP4, OP5, OP6, OP7) \ + R4 ( (k)*4+0, m0, m1, m2, m3, OP0, OP1 ) \ + R4 ( (k)*4+1, m1, m2, m3, m0, OP2, OP3 ) \ + R4 ( (k)*4+2, m2, m3, m0, m1, OP4, OP5 ) \ + R4 ( (k)*4+3, m3, m0, m1, m2, OP6, OP7 ) \ + + +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef ATTRIB_SHA +ATTRIB_SHA +#endif +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + v128 state0, state1; + + if (numBlocks == 0) + return; + + state0 = LOAD_128(&state[0]); + state1 = LOAD_128(&state[4]); + + do + { + v128 state0_save, state1_save; + v128 m0, m1, m2, m3; + v128 msg, tmp; + + state0_save = state0; + state1_save = state1; + + LOAD_SHUFFLE (m0, 0) + LOAD_SHUFFLE (m1, 1) + LOAD_SHUFFLE (m2, 2) + LOAD_SHUFFLE (m3, 3) + + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + + state0 = vaddq_u32(state0, state0_save); + state1 = vaddq_u32(state1, state1_save); + + data += 64; + } + while (--numBlocks); + + STORE_128(&state[0], state0); + STORE_128(&state[4], state1); +} + +#endif // USE_HW_SHA + +#endif // MY_CPU_ARM_OR_ARM64 + + +#ifndef USE_HW_SHA + +// #error Stop_Compiling_UNSUPPORTED_SHA +// #include + +// #include "Sha256.h" +void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); + +#pragma message("Sha256 HW-SW stub was used") + +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +{ + Sha256_UpdateBlocks(state, data, numBlocks); + /* + UNUSED_VAR(state); + UNUSED_VAR(data); + UNUSED_VAR(numBlocks); + exit(1); + return; + */ +} + +#endif diff --git a/3rdparty/lzma/C/Threads.c b/3rdparty/lzma/C/Threads.c index d3d0912da42..58eb90ffa06 100644 --- a/3rdparty/lzma/C/Threads.c +++ b/3rdparty/lzma/C/Threads.c @@ -1,9 +1,11 @@ /* Threads.c -- multithreading library -2014-09-21 : Igor Pavlov : Public domain */ +2021-12-21 : Igor Pavlov : Public domain */ #include "Precomp.h" -#ifndef UNDER_CE +#ifdef _WIN32 + +#ifndef USE_THREADS_CreateThread #include #endif @@ -12,44 +14,121 @@ static WRes GetError() { DWORD res = GetLastError(); - return (res) ? (WRes)(res) : 1; + return res ? (WRes)res : 1; } -WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); } -WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } +static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); } +static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } WRes HandlePtr_Close(HANDLE *p) { if (*p != NULL) + { if (!CloseHandle(*p)) return GetError(); - *p = NULL; + *p = NULL; + } return 0; } -WRes Handle_WaitObject(HANDLE h) { return (WRes)WaitForSingleObject(h, INFINITE); } +WRes Handle_WaitObject(HANDLE h) +{ + DWORD dw = WaitForSingleObject(h, INFINITE); + /* + (dw) result: + WAIT_OBJECT_0 // 0 + WAIT_ABANDONED // 0x00000080 : is not compatible with Win32 Error space + WAIT_TIMEOUT // 0x00000102 : is compatible with Win32 Error space + WAIT_FAILED // 0xFFFFFFFF + */ + if (dw == WAIT_FAILED) + { + dw = GetLastError(); + if (dw == 0) + return WAIT_FAILED; + } + return (WRes)dw; +} + +#define Thread_Wait(p) Handle_WaitObject(*(p)) + +WRes Thread_Wait_Close(CThread *p) +{ + WRes res = Thread_Wait(p); + WRes res2 = Thread_Close(p); + return (res != 0 ? res : res2); +} WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) { /* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */ - - #ifdef UNDER_CE - - DWORD threadId; - *p = CreateThread(0, 0, func, param, 0, &threadId); - #else + #ifdef USE_THREADS_CreateThread + DWORD threadId; + *p = CreateThread(NULL, 0, func, param, 0, &threadId); + + #else + unsigned threadId; - *p = (HANDLE)_beginthreadex(NULL, 0, func, param, 0, &threadId); - + *p = (HANDLE)(_beginthreadex(NULL, 0, func, param, 0, &threadId)); + #endif /* maybe we must use errno here, but probably GetLastError() is also OK. */ return HandleToWRes(*p); } -WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) + +WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity) +{ + #ifdef USE_THREADS_CreateThread + + UNUSED_VAR(affinity) + return Thread_Create(p, func, param); + + #else + + /* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */ + HANDLE h; + WRes wres; + unsigned threadId; + h = (HANDLE)(_beginthreadex(NULL, 0, func, param, CREATE_SUSPENDED, &threadId)); + *p = h; + wres = HandleToWRes(h); + if (h) + { + { + // DWORD_PTR prevMask = + SetThreadAffinityMask(h, (DWORD_PTR)affinity); + /* + if (prevMask == 0) + { + // affinity change is non-critical error, so we can ignore it + // wres = GetError(); + } + */ + } + { + DWORD prevSuspendCount = ResumeThread(h); + /* ResumeThread() returns: + 0 : was_not_suspended + 1 : was_resumed + -1 : error + */ + if (prevSuspendCount == (DWORD)-1) + wres = GetError(); + } + } + + /* maybe we must use errno here, but probably GetLastError() is also OK. */ + return wres; + + #endif +} + + +static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) { *p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL); return HandleToWRes(*p); @@ -66,10 +145,22 @@ WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) { return AutoResetEven WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) { + // negative ((LONG)maxCount) is not supported in WIN32::CreateSemaphore() *p = CreateSemaphore(NULL, (LONG)initCount, (LONG)maxCount, NULL); return HandleToWRes(*p); } +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount) +{ + // if (Semaphore_IsCreated(p)) + { + WRes wres = Semaphore_Close(p); + if (wres != 0) + return wres; + } + return Semaphore_Create(p, initCount, maxCount); +} + static WRes Semaphore_Release(CSemaphore *p, LONG releaseCount, LONG *previousCount) { return BOOLToWRes(ReleaseSemaphore(*p, releaseCount, previousCount)); } WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num) @@ -78,7 +169,9 @@ WRes Semaphore_Release1(CSemaphore *p) { return Semaphore_ReleaseN(p, 1); } WRes CriticalSection_Init(CCriticalSection *p) { - /* InitializeCriticalSection can raise only STATUS_NO_MEMORY exception */ + /* InitializeCriticalSection() can raise exception: + Windows XP, 2003 : can raise a STATUS_NO_MEMORY exception + Windows Vista+ : no exceptions */ #ifdef _MSC_VER __try #endif @@ -87,7 +180,361 @@ WRes CriticalSection_Init(CCriticalSection *p) /* InitializeCriticalSectionAndSpinCount(p, 0); */ } #ifdef _MSC_VER - __except (EXCEPTION_EXECUTE_HANDLER) { return 1; } + __except (EXCEPTION_EXECUTE_HANDLER) { return ERROR_NOT_ENOUGH_MEMORY; } #endif return 0; } + + + + +#else // _WIN32 + +// ---------- POSIX ---------- + +#ifndef __APPLE__ +#ifndef _7ZIP_AFFINITY_DISABLE +// _GNU_SOURCE can be required for pthread_setaffinity_np() / CPU_ZERO / CPU_SET +#define _GNU_SOURCE +#endif +#endif + +#include "Threads.h" + +#include +#include +#include +#ifdef _7ZIP_AFFINITY_SUPPORTED +// #include +#endif + + +// #include +// #define PRF(p) p +#define PRF(p) + +#define Print(s) PRF(printf("\n%s\n", s)) + +// #include + +WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, const CCpuSet *cpuSet) +{ + // new thread in Posix probably inherits affinity from parrent thread + Print("Thread_Create_With_CpuSet"); + + pthread_attr_t attr; + int ret; + // int ret2; + + p->_created = 0; + + RINOK(pthread_attr_init(&attr)); + + ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + if (!ret) + { + if (cpuSet) + { + #ifdef _7ZIP_AFFINITY_SUPPORTED + + /* + printf("\n affinity :"); + unsigned i; + for (i = 0; i < sizeof(*cpuSet) && i < 8; i++) + { + Byte b = *((const Byte *)cpuSet + i); + char temp[32]; + #define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + temp[0] = GET_HEX_CHAR((b & 0xF)); + temp[1] = GET_HEX_CHAR((b >> 4)); + // temp[0] = GET_HEX_CHAR((b >> 4)); // big-endian + // temp[1] = GET_HEX_CHAR((b & 0xF)); // big-endian + temp[2] = 0; + printf("%s", temp); + } + printf("\n"); + */ + + // ret2 = + pthread_attr_setaffinity_np(&attr, sizeof(*cpuSet), cpuSet); + // if (ret2) ret = ret2; + #endif + } + + ret = pthread_create(&p->_tid, &attr, func, param); + + if (!ret) + { + p->_created = 1; + /* + if (cpuSet) + { + // ret2 = + pthread_setaffinity_np(p->_tid, sizeof(*cpuSet), cpuSet); + // if (ret2) ret = ret2; + } + */ + } + } + // ret2 = + pthread_attr_destroy(&attr); + // if (ret2 != 0) ret = ret2; + return ret; +} + + +WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) +{ + return Thread_Create_With_CpuSet(p, func, param, NULL); +} + + +WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity) +{ + Print("Thread_Create_WithAffinity"); + CCpuSet cs; + unsigned i; + CpuSet_Zero(&cs); + for (i = 0; i < sizeof(affinity) * 8; i++) + { + if (affinity == 0) + break; + if (affinity & 1) + { + CpuSet_Set(&cs, i); + } + affinity >>= 1; + } + return Thread_Create_With_CpuSet(p, func, param, &cs); +} + + +WRes Thread_Close(CThread *p) +{ + // Print("Thread_Close"); + int ret; + if (!p->_created) + return 0; + + ret = pthread_detach(p->_tid); + p->_tid = 0; + p->_created = 0; + return ret; +} + + +WRes Thread_Wait_Close(CThread *p) +{ + // Print("Thread_Wait_Close"); + void *thread_return; + int ret; + if (!p->_created) + return EINVAL; + + ret = pthread_join(p->_tid, &thread_return); + // probably we can't use that (_tid) after pthread_join(), so we close thread here + p->_created = 0; + p->_tid = 0; + return ret; +} + + + +static WRes Event_Create(CEvent *p, int manualReset, int signaled) +{ + RINOK(pthread_mutex_init(&p->_mutex, NULL)); + RINOK(pthread_cond_init(&p->_cond, NULL)); + p->_manual_reset = manualReset; + p->_state = (signaled ? True : False); + p->_created = 1; + return 0; +} + +WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled) + { return Event_Create(p, True, signaled); } +WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p) + { return ManualResetEvent_Create(p, 0); } +WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled) + { return Event_Create(p, False, signaled); } +WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) + { return AutoResetEvent_Create(p, 0); } + + +WRes Event_Set(CEvent *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + p->_state = True; + int res1 = pthread_cond_broadcast(&p->_cond); + int res2 = pthread_mutex_unlock(&p->_mutex); + return (res2 ? res2 : res1); +} + +WRes Event_Reset(CEvent *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + p->_state = False; + return pthread_mutex_unlock(&p->_mutex); +} + +WRes Event_Wait(CEvent *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + while (p->_state == False) + { + // ETIMEDOUT + // ret = + pthread_cond_wait(&p->_cond, &p->_mutex); + // if (ret != 0) break; + } + if (p->_manual_reset == False) + { + p->_state = False; + } + return pthread_mutex_unlock(&p->_mutex); +} + +WRes Event_Close(CEvent *p) +{ + if (!p->_created) + return 0; + p->_created = 0; + { + int res1 = pthread_mutex_destroy(&p->_mutex); + int res2 = pthread_cond_destroy(&p->_cond); + return (res1 ? res1 : res2); + } +} + + +WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) +{ + if (initCount > maxCount || maxCount < 1) + return EINVAL; + RINOK(pthread_mutex_init(&p->_mutex, NULL)); + RINOK(pthread_cond_init(&p->_cond, NULL)); + p->_count = initCount; + p->_maxCount = maxCount; + p->_created = 1; + return 0; +} + + +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount) +{ + if (Semaphore_IsCreated(p)) + { + /* + WRes wres = Semaphore_Close(p); + if (wres != 0) + return wres; + */ + if (initCount > maxCount || maxCount < 1) + return EINVAL; + // return EINVAL; // for debug + p->_count = initCount; + p->_maxCount = maxCount; + return 0; + } + return Semaphore_Create(p, initCount, maxCount); +} + + +WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 releaseCount) +{ + UInt32 newCount; + int ret; + + if (releaseCount < 1) + return EINVAL; + + RINOK(pthread_mutex_lock(&p->_mutex)); + + newCount = p->_count + releaseCount; + if (newCount > p->_maxCount) + ret = ERROR_TOO_MANY_POSTS; // EINVAL; + else + { + p->_count = newCount; + ret = pthread_cond_broadcast(&p->_cond); + } + RINOK(pthread_mutex_unlock(&p->_mutex)); + return ret; +} + +WRes Semaphore_Wait(CSemaphore *p) +{ + RINOK(pthread_mutex_lock(&p->_mutex)); + while (p->_count < 1) + { + pthread_cond_wait(&p->_cond, &p->_mutex); + } + p->_count--; + return pthread_mutex_unlock(&p->_mutex); +} + +WRes Semaphore_Close(CSemaphore *p) +{ + if (!p->_created) + return 0; + p->_created = 0; + { + int res1 = pthread_mutex_destroy(&p->_mutex); + int res2 = pthread_cond_destroy(&p->_cond); + return (res1 ? res1 : res2); + } +} + + + +WRes CriticalSection_Init(CCriticalSection *p) +{ + // Print("CriticalSection_Init"); + if (!p) + return EINTR; + return pthread_mutex_init(&p->_mutex, NULL); +} + +void CriticalSection_Enter(CCriticalSection *p) +{ + // Print("CriticalSection_Enter"); + if (p) + { + // int ret = + pthread_mutex_lock(&p->_mutex); + } +} + +void CriticalSection_Leave(CCriticalSection *p) +{ + // Print("CriticalSection_Leave"); + if (p) + { + // int ret = + pthread_mutex_unlock(&p->_mutex); + } +} + +void CriticalSection_Delete(CCriticalSection *p) +{ + // Print("CriticalSection_Delete"); + if (p) + { + // int ret = + pthread_mutex_destroy(&p->_mutex); + } +} + +LONG InterlockedIncrement(LONG volatile *addend) +{ + // Print("InterlockedIncrement"); + #ifdef USE_HACK_UNSAFE_ATOMIC + LONG val = *addend + 1; + *addend = val; + return val; + #else + return __sync_add_and_fetch(addend, 1); + #endif +} + +#endif // _WIN32 diff --git a/3rdparty/lzma/C/Threads.h b/3rdparty/lzma/C/Threads.h index 9b3e1c556d2..89ecb92be90 100644 --- a/3rdparty/lzma/C/Threads.h +++ b/3rdparty/lzma/C/Threads.h @@ -1,38 +1,139 @@ /* Threads.h -- multithreading library -2013-11-12 : Igor Pavlov : Public domain */ +2021-12-21 : Igor Pavlov : Public domain */ #ifndef __7Z_THREADS_H #define __7Z_THREADS_H #ifdef _WIN32 -#include +#include +#else + +#if defined(__linux__) +#if !defined(__APPLE__) && !defined(_AIX) && !defined(__ANDROID__) +#ifndef _7ZIP_AFFINITY_DISABLE +#define _7ZIP_AFFINITY_SUPPORTED +// #pragma message(" ==== _7ZIP_AFFINITY_SUPPORTED") +// #define _GNU_SOURCE +#endif +#endif +#endif + +#include + #endif #include "7zTypes.h" EXTERN_C_BEGIN +#ifdef _WIN32 + WRes HandlePtr_Close(HANDLE *h); WRes Handle_WaitObject(HANDLE h); typedef HANDLE CThread; -#define Thread_Construct(p) *(p) = NULL + +#define Thread_Construct(p) { *(p) = NULL; } #define Thread_WasCreated(p) (*(p) != NULL) #define Thread_Close(p) HandlePtr_Close(p) -#define Thread_Wait(p) Handle_WaitObject(*(p)) +// #define Thread_Wait(p) Handle_WaitObject(*(p)) -typedef #ifdef UNDER_CE - DWORD + // if (USE_THREADS_CreateThread is defined), we use _beginthreadex() + // if (USE_THREADS_CreateThread is not definned), we use CreateThread() + #define USE_THREADS_CreateThread +#endif + +typedef + #ifdef USE_THREADS_CreateThread + DWORD + #else + unsigned + #endif + THREAD_FUNC_RET_TYPE; + +typedef DWORD_PTR CAffinityMask; +typedef DWORD_PTR CCpuSet; + +#define CpuSet_Zero(p) { *(p) = 0; } +#define CpuSet_Set(p, cpu) { *(p) |= ((DWORD_PTR)1 << (cpu)); } + +#else // _WIN32 + +typedef struct _CThread +{ + pthread_t _tid; + int _created; +} CThread; + +#define Thread_Construct(p) { (p)->_tid = 0; (p)->_created = 0; } +#define Thread_WasCreated(p) ((p)->_created != 0) +WRes Thread_Close(CThread *p); +// #define Thread_Wait Thread_Wait_Close + +typedef void * THREAD_FUNC_RET_TYPE; + +typedef UInt64 CAffinityMask; + +#ifdef _7ZIP_AFFINITY_SUPPORTED + +typedef cpu_set_t CCpuSet; +#define CpuSet_Zero(p) CPU_ZERO(p) +#define CpuSet_Set(p, cpu) CPU_SET(cpu, p) +#define CpuSet_IsSet(p, cpu) CPU_ISSET(cpu, p) + #else - unsigned + +typedef UInt64 CCpuSet; +#define CpuSet_Zero(p) { *(p) = 0; } +#define CpuSet_Set(p, cpu) { *(p) |= ((UInt64)1 << (cpu)); } +#define CpuSet_IsSet(p, cpu) ((*(p) & ((UInt64)1 << (cpu))) != 0) + #endif - THREAD_FUNC_RET_TYPE; + + +#endif // _WIN32 + #define THREAD_FUNC_CALL_TYPE MY_STD_CALL -#define THREAD_FUNC_DECL THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE + +#if defined(_WIN32) && defined(__GNUC__) +/* GCC compiler for x86 32-bit uses the rule: + the stack is 16-byte aligned before CALL instruction for function calling. + But only root function main() contains instructions that + set 16-byte alignment for stack pointer. And another functions + just keep alignment, if it was set in some parent function. + + The problem: + if we create new thread in MinGW (GCC) 32-bit x86 via _beginthreadex() or CreateThread(), + the root function of thread doesn't set 16-byte alignment. + And stack frames in all child functions also will be unaligned in that case. + + Here we set (force_align_arg_pointer) attribute for root function of new thread. + Do we need (force_align_arg_pointer) also for another systems? */ + + #define THREAD_FUNC_ATTRIB_ALIGN_ARG __attribute__((force_align_arg_pointer)) + // #define THREAD_FUNC_ATTRIB_ALIGN_ARG // for debug : bad alignment in SSE functions +#else + #define THREAD_FUNC_ATTRIB_ALIGN_ARG +#endif + +#define THREAD_FUNC_DECL THREAD_FUNC_ATTRIB_ALIGN_ARG THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE + typedef THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE * THREAD_FUNC_TYPE)(void *); WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param); +WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity); +WRes Thread_Wait_Close(CThread *p); + +#ifdef _WIN32 +#define Thread_Create_With_CpuSet(p, func, param, cs) \ + Thread_Create_With_Affinity(p, func, param, *cs) +#else +WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, const CCpuSet *cpuSet); +#endif + + +#ifdef _WIN32 typedef HANDLE CEvent; typedef CEvent CAutoResetEvent; @@ -49,10 +150,12 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled); WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); typedef HANDLE CSemaphore; -#define Semaphore_Construct(p) (*p) = NULL +#define Semaphore_Construct(p) *(p) = NULL +#define Semaphore_IsCreated(p) (*(p) != NULL) #define Semaphore_Close(p) HandlePtr_Close(p) #define Semaphore_Wait(p) Handle_WaitObject(*(p)) WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount); WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num); WRes Semaphore_Release1(CSemaphore *p); @@ -62,6 +165,68 @@ WRes CriticalSection_Init(CCriticalSection *p); #define CriticalSection_Enter(p) EnterCriticalSection(p) #define CriticalSection_Leave(p) LeaveCriticalSection(p) + +#else // _WIN32 + +typedef struct _CEvent +{ + int _created; + int _manual_reset; + int _state; + pthread_mutex_t _mutex; + pthread_cond_t _cond; +} CEvent; + +typedef CEvent CAutoResetEvent; +typedef CEvent CManualResetEvent; + +#define Event_Construct(p) (p)->_created = 0 +#define Event_IsCreated(p) ((p)->_created) + +WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled); +WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p); +WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled); +WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); +WRes Event_Set(CEvent *p); +WRes Event_Reset(CEvent *p); +WRes Event_Wait(CEvent *p); +WRes Event_Close(CEvent *p); + + +typedef struct _CSemaphore +{ + int _created; + UInt32 _count; + UInt32 _maxCount; + pthread_mutex_t _mutex; + pthread_cond_t _cond; +} CSemaphore; + +#define Semaphore_Construct(p) (p)->_created = 0 +#define Semaphore_IsCreated(p) ((p)->_created) + +WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount); +WRes Semaphore_OptCreateInit(CSemaphore *p, UInt32 initCount, UInt32 maxCount); +WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num); +#define Semaphore_Release1(p) Semaphore_ReleaseN(p, 1) +WRes Semaphore_Wait(CSemaphore *p); +WRes Semaphore_Close(CSemaphore *p); + + +typedef struct _CCriticalSection +{ + pthread_mutex_t _mutex; +} CCriticalSection; + +WRes CriticalSection_Init(CCriticalSection *p); +void CriticalSection_Delete(CCriticalSection *cs); +void CriticalSection_Enter(CCriticalSection *cs); +void CriticalSection_Leave(CCriticalSection *cs); + +LONG InterlockedIncrement(LONG volatile *addend); + +#endif // _WIN32 + EXTERN_C_END #endif diff --git a/3rdparty/lzma/C/Util/7z/7zMain.c b/3rdparty/lzma/C/Util/7z/7zMain.c index 1837e9d606a..9d555098e29 100644 --- a/3rdparty/lzma/C/Util/7z/7zMain.c +++ b/3rdparty/lzma/C/Util/7z/7zMain.c @@ -1,11 +1,13 @@ /* 7zMain.c - Test application for 7z Decoder -2016-05-16 : Igor Pavlov : Public domain */ +2021-04-29 : Igor Pavlov : Public domain */ #include "Precomp.h" #include #include +#include "../../CpuArch.h" + #include "../../7z.h" #include "../../7zAlloc.h" #include "../../7zBuf.h" @@ -18,12 +20,29 @@ #ifdef _WIN32 #include #else +#include +#include +#ifdef __GNUC__ +#include +#endif +#include +// #include #include #include #endif #endif -static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +#define kInputBufSize ((size_t)1 << 18) + +static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + + +static void Print(const char *s) +{ + fputs(s, stdout); +} + static int Buf_EnsureSize(CBuf *dest, size_t size) { @@ -96,7 +115,7 @@ static Byte *Utf16_To_Utf8(Byte *dest, const UInt16 *src, const UInt16 *srcLim) if (val < 0x80) { - *dest++ = (char)val; + *dest++ = (Byte)val; continue; } @@ -150,21 +169,21 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s ) { unsigned len = 0; - for (len = 0; s[len] != 0; len++); + for (len = 0; s[len] != 0; len++) {} #ifndef _USE_UTF8 { - unsigned size = len * 3 + 100; + const unsigned size = len * 3 + 100; if (!Buf_EnsureSize(buf, size)) return SZ_ERROR_MEM; { buf->data[0] = 0; if (len != 0) { - char defaultChar = '_'; + const char defaultChar = '_'; BOOL defUsed; - unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, s, len, (char *)buf->data, size, &defaultChar, &defUsed); + const unsigned numChars = (unsigned)WideCharToMultiByte( + codePage, 0, (LPCWSTR)s, (int)len, (char *)buf->data, (int)size, &defaultChar, &defUsed); if (numChars == 0 || numChars >= size) return SZ_ERROR_FAIL; buf->data[numChars] = 0; @@ -180,8 +199,8 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s #ifdef _WIN32 #ifndef USE_WINDOWS_FILE static UINT g_FileCodePage = CP_ACP; + #define MY_FILE_CODE_PAGE_PARAM ,g_FileCodePage #endif - #define MY_FILE_CODE_PAGE_PARAM ,g_FileCodePage #else #define MY_FILE_CODE_PAGE_PARAM #endif @@ -190,7 +209,7 @@ static WRes MyCreateDir(const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return CreateDirectoryW(name, NULL) ? 0 : GetLastError(); + return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); #else @@ -215,7 +234,7 @@ static WRes MyCreateDir(const UInt16 *name) static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, name); + return OutFile_OpenW(p, (LPCWSTR)name); #else CBuf buf; WRes res; @@ -227,6 +246,7 @@ static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) #endif } + static SRes PrintString(const UInt16 *s) { CBuf buf; @@ -238,12 +258,12 @@ static SRes PrintString(const UInt16 *s) #endif ); if (res == SZ_OK) - fputs((const char *)buf.data, stdout); + Print((const char *)buf.data); Buf_Free(&buf, &g_Alloc); return res; } -static void UInt64ToStr(UInt64 value, char *s) +static void UInt64ToStr(UInt64 value, char *s, int numDigits) { char temp[32]; int pos = 0; @@ -253,6 +273,10 @@ static void UInt64ToStr(UInt64 value, char *s) value /= 10; } while (value != 0); + + for (numDigits -= pos; numDigits > 0; numDigits--) + *s++ = ' '; + do *s++ = temp[--pos]; while (pos); @@ -266,8 +290,10 @@ static char *UIntToStr(char *s, unsigned value, int numDigits) do temp[pos++] = (char)('0' + (value % 10)); while (value /= 10); + for (numDigits -= pos; numDigits > 0; numDigits--) *s++ = '0'; + do *s++ = temp[--pos]; while (pos); @@ -281,17 +307,142 @@ static void UIntToStr_2(char *s, unsigned value) s[1] = (char)('0' + (value % 10)); } + #define PERIOD_4 (4 * 365 + 1) #define PERIOD_100 (PERIOD_4 * 25 - 1) #define PERIOD_400 (PERIOD_100 * 4 + 1) -static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) + + +#ifndef _WIN32 + +// MS uses long for BOOL, but long is 32-bit in MS. So we use int. +// typedef long BOOL; +typedef int BOOL; + +typedef struct _FILETIME +{ + DWORD dwLowDateTime; + DWORD dwHighDateTime; +} FILETIME; + +static LONG TIME_GetBias() +{ + time_t utc = time(NULL); + struct tm *ptm = localtime(&utc); + int localdaylight = ptm->tm_isdst; /* daylight for local timezone */ + ptm = gmtime(&utc); + ptm->tm_isdst = localdaylight; /* use local daylight, not that of Greenwich */ + LONG bias = (int)(mktime(ptm)-utc); + return bias; +} + +#define TICKS_PER_SEC 10000000 + +#define GET_TIME_64(pft) ((pft)->dwLowDateTime | ((UInt64)(pft)->dwHighDateTime << 32)) + +#define SET_FILETIME(ft, v64) \ + (ft)->dwLowDateTime = (DWORD)v64; \ + (ft)->dwHighDateTime = (DWORD)(v64 >> 32); + +#define WINAPI +#define TRUE 1 + +static BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *localFileTime) +{ + UInt64 v = GET_TIME_64(fileTime); + v = (UInt64)((Int64)v - (Int64)TIME_GetBias() * TICKS_PER_SEC); + SET_FILETIME(localFileTime, v); + return TRUE; +} + +static const UInt32 kNumTimeQuantumsInSecond = 10000000; +static const UInt32 kFileTimeStartYear = 1601; +static const UInt32 kUnixTimeStartYear = 1970; +static const UInt64 kUnixTimeOffset = + (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); + +static Int64 Time_FileTimeToUnixTime64(const FILETIME *ft) +{ + UInt64 winTime = GET_TIME_64(ft); + return (Int64)(winTime / kNumTimeQuantumsInSecond) - (Int64)kUnixTimeOffset; +} + +#if defined(_AIX) + #define MY_ST_TIMESPEC st_timespec +#else + #define MY_ST_TIMESPEC timespec +#endif + +static void FILETIME_To_timespec(const FILETIME *ft, struct MY_ST_TIMESPEC *ts) +{ + if (ft) + { + const Int64 sec = Time_FileTimeToUnixTime64(ft); + // time_t is long + const time_t sec2 = (time_t)sec; + if (sec2 == sec) + { + ts->tv_sec = sec2; + UInt64 winTime = GET_TIME_64(ft); + ts->tv_nsec = (long)((winTime % 10000000) * 100);; + return; + } + } + // else + { + ts->tv_sec = 0; + // ts.tv_nsec = UTIME_NOW; // set to the current time + ts->tv_nsec = UTIME_OMIT; // keep old timesptamp + } +} + +static WRes Set_File_FILETIME(const UInt16 *name, const FILETIME *mTime) +{ + struct timespec times[2]; + + const int flags = 0; // follow link + // = AT_SYMLINK_NOFOLLOW; // don't follow link + + CBuf buf; + int res; + Buf_Init(&buf); + RINOK(Utf16_To_Char(&buf, name MY_FILE_CODE_PAGE_PARAM)); + FILETIME_To_timespec(NULL, ×[0]); + FILETIME_To_timespec(mTime, ×[1]); + res = utimensat(AT_FDCWD, (const char *)buf.data, times, flags); + Buf_Free(&buf, &g_Alloc); + if (res == 0) + return 0; + return errno; +} + +#endif + +static void NtfsFileTime_to_FILETIME(const CNtfsFileTime *t, FILETIME *ft) +{ + ft->dwLowDateTime = (DWORD)(t->Low); + ft->dwHighDateTime = (DWORD)(t->High); +} + +static void ConvertFileTimeToString(const CNtfsFileTime *nTime, char *s) { unsigned year, mon, hour, min, sec; Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; unsigned t; UInt32 v; - UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); + // UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); + UInt64 v64; + { + FILETIME fileTime, locTime; + NtfsFileTime_to_FILETIME(nTime, &fileTime); + if (!FileTimeToLocalFileTime(&fileTime, &locTime)) + { + locTime.dwHighDateTime = + locTime.dwLowDateTime = 0; + } + v64 = locTime.dwLowDateTime | ((UInt64)locTime.dwHighDateTime << 32); + } v64 /= 10000000; sec = (unsigned)(v64 % 60); v64 /= 60; min = (unsigned)(v64 % 60); v64 /= 60; @@ -323,12 +474,56 @@ static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) UIntToStr_2(s, sec); s[2] = 0; } -void PrintError(char *sz) +static void PrintLF() +{ + Print("\n"); +} + +static void PrintError(char *s) +{ + Print("\nERROR: "); + Print(s); + PrintLF(); +} + +static void PrintError_WRes(const char *message, WRes wres) { - printf("\nERROR: %s\n", sz); + Print("\nERROR: "); + Print(message); + PrintLF(); + { + char s[32]; + UIntToStr(s, (unsigned)wres, 1); + Print("System error code: "); + Print(s); + } + // sprintf(buffer + strlen(buffer), "\nSystem error code: %d", (unsigned)wres); + #ifdef _WIN32 + { + char *s = NULL; + if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, wres, 0, (LPSTR) &s, 0, NULL) != 0 && s) + { + Print(" : "); + Print(s); + LocalFree(s); + } + } + #else + { + const char *s = strerror(wres); + if (s) + { + Print(" : "); + Print(s); + } + } + #endif + PrintLF(); } -static void GetAttribString(UInt32 wa, Bool isDir, char *s) +static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) { #ifdef USE_WINDOWS_FILE s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.'); @@ -343,25 +538,27 @@ static void GetAttribString(UInt32 wa, Bool isDir, char *s) #endif } + // #define NUM_PARENTS_MAX 128 int MY_CDECL main(int numargs, char *args[]) { + ISzAlloc allocImp; + ISzAlloc allocTempImp; + CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; SRes res; - ISzAlloc allocImp; - ISzAlloc allocTempImp; UInt16 *temp = NULL; size_t tempSize = 0; // UInt32 parents[NUM_PARENTS_MAX]; - printf("\n7z ANSI-C Decoder " MY_VERSION_COPYRIGHT_DATE "\n\n"); + Print("\n7z Decoder " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"); if (numargs == 1) { - printf( + Print( "Usage: 7zDec \n\n" "\n" " e: Extract files from archive (without using directory names)\n" @@ -370,7 +567,7 @@ int MY_CDECL main(int numargs, char *args[]) " x: eXtract files with full paths\n"); return 0; } - + if (numargs < 3) { PrintError("incorrect command"); @@ -381,33 +578,51 @@ int MY_CDECL main(int numargs, char *args[]) g_FileCodePage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; #endif - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - allocTempImp.Alloc = SzAllocTemp; - allocTempImp.Free = SzFreeTemp; + allocImp = g_Alloc; + allocTempImp = g_Alloc; - #ifdef UNDER_CE - if (InFile_OpenW(&archiveStream.file, L"\test.7z")) - #else - if (InFile_Open(&archiveStream.file, args[2])) - #endif { - PrintError("can not open input file"); - return 1; + WRes wres = + #ifdef UNDER_CE + InFile_OpenW(&archiveStream.file, L"\test.7z"); // change it + #else + InFile_Open(&archiveStream.file, args[2]); + #endif + if (wres != 0) + { + PrintError_WRes("cannot open input file", wres); + return 1; + } } FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); - - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); + archiveStream.wres = 0; + LookToRead2_CreateVTable(&lookStream, False); + lookStream.buf = NULL; - CrcGenerateTable(); + res = SZ_OK; + { + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); + if (!lookStream.buf) + res = SZ_ERROR_MEM; + else + { + lookStream.bufSize = kInputBufSize; + lookStream.realStream = &archiveStream.vt; + LookToRead2_Init(&lookStream); + } + } + + CrcGenerateTable(); + SzArEx_Init(&db); - - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + + if (res == SZ_OK) + { + res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); + } if (res == SZ_OK) { @@ -442,7 +657,7 @@ int MY_CDECL main(int numargs, char *args[]) size_t outSizeProcessed = 0; // const CSzFileItem *f = db.Files + i; size_t len; - unsigned isDir = SzArEx_IsDir(&db, i); + const BoolInt isDir = SzArEx_IsDir(&db, i); if (listCommand == 0 && isDir && !fullPaths) continue; len = SzArEx_GetFileNameUtf16(&db, i, NULL); @@ -477,7 +692,7 @@ int MY_CDECL main(int numargs, char *args[]) GetAttribString(SzBitWithVals_Check(&db.Attribs, i) ? db.Attribs.Vals[i] : 0, isDir, attr); fileSize = SzArEx_GetFileSize(&db, i); - UInt64ToStr(fileSize, s); + UInt64ToStr(fileSize, s, 10); if (SzBitWithVals_Check(&db.MTime, i)) ConvertFileTimeToString(&db.MTime.Vals[i], t); @@ -489,29 +704,33 @@ int MY_CDECL main(int numargs, char *args[]) t[j] = '\0'; } - printf("%s %s %10s ", t, attr, s); + Print(t); + Print(" "); + Print(attr); + Print(" "); + Print(s); + Print(" "); res = PrintString(temp); if (res != SZ_OK) break; if (isDir) - printf("/"); - printf("\n"); + Print("/"); + PrintLF(); continue; } - fputs(testCommand ? - "Testing ": - "Extracting ", - stdout); + Print(testCommand ? + "T ": + "- "); res = PrintString(temp); if (res != SZ_OK) break; if (isDir) - printf("/"); + Print("/"); else { - res = SzArEx_Extract(&db, &lookStream.s, i, + res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp); @@ -543,62 +762,126 @@ int MY_CDECL main(int numargs, char *args[]) if (isDir) { MyCreateDir(destPath); - printf("\n"); + PrintLF(); continue; } - else if (OutFile_OpenUtf16(&outFile, destPath)) + else { - PrintError("can not open output file"); - res = SZ_ERROR_FAIL; - break; + WRes wres = OutFile_OpenUtf16(&outFile, destPath); + if (wres != 0) + { + PrintError_WRes("cannot open output file", wres); + res = SZ_ERROR_FAIL; + break; + } } processedSize = outSizeProcessed; - if (File_Write(&outFile, outBuffer + offset, &processedSize) != 0 || processedSize != outSizeProcessed) { - PrintError("can not write output file"); - res = SZ_ERROR_FAIL; - break; + WRes wres = File_Write(&outFile, outBuffer + offset, &processedSize); + if (wres != 0 || processedSize != outSizeProcessed) + { + PrintError_WRes("cannot write output file", wres); + res = SZ_ERROR_FAIL; + break; + } } - - if (File_Close(&outFile)) + { - PrintError("can not close output file"); - res = SZ_ERROR_FAIL; - break; + FILETIME mtime; + FILETIME *mtimePtr = NULL; + + #ifdef USE_WINDOWS_FILE + FILETIME ctime; + FILETIME *ctimePtr = NULL; + #endif + + if (SzBitWithVals_Check(&db.MTime, i)) + { + const CNtfsFileTime *t = &db.MTime.Vals[i]; + mtime.dwLowDateTime = (DWORD)(t->Low); + mtime.dwHighDateTime = (DWORD)(t->High); + mtimePtr = &mtime; + } + + #ifdef USE_WINDOWS_FILE + if (SzBitWithVals_Check(&db.CTime, i)) + { + const CNtfsFileTime *t = &db.CTime.Vals[i]; + ctime.dwLowDateTime = (DWORD)(t->Low); + ctime.dwHighDateTime = (DWORD)(t->High); + ctimePtr = &ctime; + } + + if (mtimePtr || ctimePtr) + SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); + #endif + + { + WRes wres = File_Close(&outFile); + if (wres != 0) + { + PrintError_WRes("cannot close output file", wres); + res = SZ_ERROR_FAIL; + break; + } + } + + #ifndef USE_WINDOWS_FILE + #ifdef _WIN32 + mtimePtr = mtimePtr; + #else + if (mtimePtr) + Set_File_FILETIME(destPath, mtimePtr); + #endif + #endif } #ifdef USE_WINDOWS_FILE if (SzBitWithVals_Check(&db.Attribs, i)) - SetFileAttributesW(destPath, db.Attribs.Vals[i]); + { + UInt32 attrib = db.Attribs.Vals[i]; + /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker. + We remove posix bits, if we detect posix mode field */ + if ((attrib & 0xF0000000) != 0) + attrib &= 0x7FFF; + SetFileAttributesW((LPCWSTR)destPath, attrib); + } #endif } - printf("\n"); + PrintLF(); } - IAlloc_Free(&allocImp, outBuffer); + ISzAlloc_Free(&allocImp, outBuffer); } } - SzArEx_Free(&db, &allocImp); SzFree(NULL, temp); + SzArEx_Free(&db, &allocImp); + ISzAlloc_Free(&allocImp, lookStream.buf); File_Close(&archiveStream.file); if (res == SZ_OK) { - printf("\nEverything is Ok\n"); + Print("\nEverything is Ok\n"); return 0; } if (res == SZ_ERROR_UNSUPPORTED) PrintError("decoder doesn't support this archive"); else if (res == SZ_ERROR_MEM) - PrintError("can not allocate memory"); + PrintError("cannot allocate memory"); else if (res == SZ_ERROR_CRC) PrintError("CRC error"); + else if (res == SZ_ERROR_READ /* || archiveStream.Res != 0 */) + PrintError_WRes("Read Error", archiveStream.wres); else - printf("\nERROR #%d\n", res); + { + char s[32]; + UInt64ToStr((unsigned)res, s, 0); + PrintError(s); + } return 1; } diff --git a/3rdparty/lzma/C/Util/7z/makefile.gcc b/3rdparty/lzma/C/Util/7z/makefile.gcc index 51053bad65c..4263d67558d 100644 --- a/3rdparty/lzma/C/Util/7z/makefile.gcc +++ b/3rdparty/lzma/C/Util/7z/makefile.gcc @@ -1,75 +1,34 @@ -PROG = 7zDec -CXX = gcc -LIB = -RM = rm -f -CFLAGS = -c -O2 -Wall - -OBJS = 7zMain.o 7zAlloc.o 7zArcIn.o 7zBuf.o 7zBuf2.o 7zCrc.o 7zCrcOpt.o 7zDec.o CpuArch.o Delta.o LzmaDec.o Lzma2Dec.o Bra.o Bra86.o BraIA64.o Bcj2.o Ppmd7.o Ppmd7Dec.o 7zFile.o 7zStream.o - -all: $(PROG) - -$(PROG): $(OBJS) - $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) - -7zMain.o: 7zMain.c - $(CXX) $(CFLAGS) 7zMain.c - -7zAlloc.o: ../../7zAlloc.c - $(CXX) $(CFLAGS) ../../7zAlloc.c - -7zArcIn.o: ../../7zArcIn.c - $(CXX) $(CFLAGS) ../../7zArcIn.c - -7zBuf.o: ../../7zBuf.c - $(CXX) $(CFLAGS) ../../7zBuf.c - -7zBuf2.o: ../../7zBuf2.c - $(CXX) $(CFLAGS) ../../7zBuf2.c - -7zCrc.o: ../../7zCrc.c - $(CXX) $(CFLAGS) ../../7zCrc.c - -7zCrcOpt.o: ../../7zCrc.c - $(CXX) $(CFLAGS) ../../7zCrcOpt.c - -7zDec.o: ../../7zDec.c - $(CXX) $(CFLAGS) -D_7ZIP_PPMD_SUPPPORT ../../7zDec.c - -CpuArch.o: ../../CpuArch.c - $(CXX) $(CFLAGS) ../../CpuArch.c - -Delta.o: ../../Delta.c - $(CXX) $(CFLAGS) ../../Delta.c - -LzmaDec.o: ../../LzmaDec.c - $(CXX) $(CFLAGS) ../../LzmaDec.c - -Lzma2Dec.o: ../../Lzma2Dec.c - $(CXX) $(CFLAGS) ../../Lzma2Dec.c - -Bra.o: ../../Bra.c - $(CXX) $(CFLAGS) ../../Bra.c - -Bra86.o: ../../Bra86.c - $(CXX) $(CFLAGS) ../../Bra86.c - -BraIA64.o: ../../BraIA64.c - $(CXX) $(CFLAGS) ../../BraIA64.c - -Bcj2.o: ../../Bcj2.c - $(CXX) $(CFLAGS) ../../Bcj2.c - -Ppmd7.o: ../../Ppmd7.c - $(CXX) $(CFLAGS) ../../Ppmd7.c - -Ppmd7Dec.o: ../../Ppmd7Dec.c - $(CXX) $(CFLAGS) ../../Ppmd7Dec.c - -7zFile.o: ../../7zFile.c - $(CXX) $(CFLAGS) ../../7zFile.c - -7zStream.o: ../../7zStream.c - $(CXX) $(CFLAGS) ../../7zStream.c - -clean: - -$(RM) $(PROG) $(OBJS) +PROG = 7zdec + +LOCAL_FLAGS = -D_7ZIP_PPMD_SUPPPORT + +include ../../../CPP/7zip/LzmaDec_gcc.mak + + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $O/Bcj2.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/CpuArch.o \ + $O/Delta.o \ + $O/Lzma2Dec.o \ + $O/LzmaDec.o \ + $O/Ppmd7.o \ + $O/Ppmd7Dec.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/7zAlloc.o \ + $O/7zArcIn.o \ + $O/7zBuf.o \ + $O/7zBuf2.o \ + $O/7zDec.o \ + $O/7zMain.o \ + $O/7zFile.o \ + $O/7zStream.o \ + + +include ../../7zip_gcc_c.mak diff --git a/3rdparty/lzma/C/Util/Lzma/LzmaUtil.c b/3rdparty/lzma/C/Util/Lzma/LzmaUtil.c index e5857ceff09..62a59079f3f 100644 --- a/3rdparty/lzma/C/Util/Lzma/LzmaUtil.c +++ b/3rdparty/lzma/C/Util/Lzma/LzmaUtil.c @@ -1,5 +1,5 @@ /* LzmaUtil.c -- Test application for LZMA compression -2015-11-08 : Igor Pavlov : Public domain */ +2021-11-01 : Igor Pavlov : Public domain */ #include "../../Precomp.h" @@ -7,26 +7,30 @@ #include #include +#include "../../CpuArch.h" + #include "../../Alloc.h" #include "../../7zFile.h" #include "../../7zVersion.h" +#include "../../LzFind.h" #include "../../LzmaDec.h" #include "../../LzmaEnc.h" -const char *kCantReadMessage = "Can not read input file"; -const char *kCantWriteMessage = "Can not write output file"; -const char *kCantAllocateMessage = "Can not allocate memory"; -const char *kDataErrorMessage = "Data error"; +static const char * const kCantReadMessage = "Cannot read input file"; +static const char * const kCantWriteMessage = "Cannot write output file"; +static const char * const kCantAllocateMessage = "Cannot allocate memory"; +static const char * const kDataErrorMessage = "Data error"; -void PrintHelp(char *buffer) +static void PrintHelp(char *buffer) { - strcat(buffer, "\nLZMA Utility " MY_VERSION_COPYRIGHT_DATE "\n" - "\nUsage: lzma inputFile outputFile\n" - " e: encode file\n" - " d: decode file\n"); + strcat(buffer, + "\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n" + "Usage: lzma inputFile outputFile\n" + " e: encode file\n" + " d: decode file\n"); } -int PrintError(char *buffer, const char *message) +static int PrintError(char *buffer, const char *message) { strcat(buffer, "\nError: "); strcat(buffer, message); @@ -34,20 +38,38 @@ int PrintError(char *buffer, const char *message) return 1; } -int PrintErrorNumber(char *buffer, SRes val) +static int PrintError_WRes(char *buffer, const char *message, WRes wres) +{ + strcat(buffer, "\nError: "); + strcat(buffer, message); + sprintf(buffer + strlen(buffer), "\nSystem error code: %d", (unsigned)wres); + #ifndef _WIN32 + { + const char *s = strerror(wres); + if (s) + sprintf(buffer + strlen(buffer), " : %s", s); + } + #endif + strcat(buffer, "\n"); + return 1; +} + +static int PrintErrorNumber(char *buffer, SRes val) { - sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val); + sprintf(buffer + strlen(buffer), "\n7-Zip error code: %d\n", (unsigned)val); return 1; } -int PrintUserError(char *buffer) +static int PrintUserError(char *buffer) { return PrintError(buffer, "Incorrect command"); } + #define IN_BUF_SIZE (1 << 16) #define OUT_BUF_SIZE (1 << 16) + static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 unpackSize) { @@ -101,6 +123,7 @@ static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inS } } + static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream) { UInt64 unpackSize; @@ -163,20 +186,25 @@ static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 file return res; } -int main2(int numArgs, const char *args[], char *rs) + +static int main2(int numArgs, const char *args[], char *rs) { CFileSeqInStream inStream; CFileOutStream outStream; char c; int res; int encodeMode; - Bool useOutFile = False; + BoolInt useOutFile = False; + + LzFindPrepare(); FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file); + inStream.wres = 0; FileOutStream_CreateVTable(&outStream); File_Construct(&outStream.file); + outStream.wres = 0; if (numArgs == 1) { @@ -199,14 +227,19 @@ int main2(int numArgs, const char *args[], char *rs) return PrintError(rs, "Incorrect UInt32 or UInt64"); } - if (InFile_Open(&inStream.file, args[2]) != 0) - return PrintError(rs, "Can not open input file"); + { + WRes wres = InFile_Open(&inStream.file, args[2]); + if (wres != 0) + return PrintError_WRes(rs, "Cannot open input file", wres); + } if (numArgs > 3) { + WRes wres; useOutFile = True; - if (OutFile_Open(&outStream.file, args[3]) != 0) - return PrintError(rs, "Can not open output file"); + wres = OutFile_Open(&outStream.file, args[3]); + if (wres != 0) + return PrintError_WRes(rs, "Cannot open output file", wres); } else if (encodeMode) PrintUserError(rs); @@ -214,12 +247,14 @@ int main2(int numArgs, const char *args[], char *rs) if (encodeMode) { UInt64 fileSize; - File_GetLength(&inStream.file, &fileSize); - res = Encode(&outStream.s, &inStream.s, fileSize, rs); + WRes wres = File_GetLength(&inStream.file, &fileSize); + if (wres != 0) + return PrintError_WRes(rs, "Cannot get file length", wres); + res = Encode(&outStream.vt, &inStream.vt, fileSize, rs); } else { - res = Decode(&outStream.s, useOutFile ? &inStream.s : NULL); + res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL); } if (useOutFile) @@ -233,17 +268,18 @@ int main2(int numArgs, const char *args[], char *rs) else if (res == SZ_ERROR_DATA) return PrintError(rs, kDataErrorMessage); else if (res == SZ_ERROR_WRITE) - return PrintError(rs, kCantWriteMessage); + return PrintError_WRes(rs, kCantWriteMessage, outStream.wres); else if (res == SZ_ERROR_READ) - return PrintError(rs, kCantReadMessage); + return PrintError_WRes(rs, kCantReadMessage, inStream.wres); return PrintErrorNumber(rs, res); } return 0; } + int MY_CDECL main(int numArgs, const char *args[]) { - char rs[800] = { 0 }; + char rs[1000] = { 0 }; int res = main2(numArgs, args, rs); fputs(rs, stdout); return res; diff --git a/3rdparty/lzma/C/Util/Lzma/LzmaUtil.dsp b/3rdparty/lzma/C/Util/Lzma/LzmaUtil.dsp index 43b8e176b22..4e38e4a65b6 100644 --- a/3rdparty/lzma/C/Util/Lzma/LzmaUtil.dsp +++ b/3rdparty/lzma/C/Util/Lzma/LzmaUtil.dsp @@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c # End Source File # Begin Source File +SOURCE=..\..\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\7zVersion.h # End Source File # Begin Source File @@ -130,6 +134,10 @@ SOURCE=..\..\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\LzFindOpt.c +# End Source File +# Begin Source File + SOURCE=..\..\LzHash.h # End Source File # Begin Source File @@ -160,9 +168,5 @@ SOURCE=..\..\Threads.c SOURCE=..\..\Threads.h # End Source File -# Begin Source File - -SOURCE=..\..\Types.h -# End Source File # End Target # End Project diff --git a/3rdparty/lzma/C/Util/Lzma/makefile b/3rdparty/lzma/C/Util/Lzma/makefile index 4795322786b..7813bdb059d 100644 --- a/3rdparty/lzma/C/Util/Lzma/makefile +++ b/3rdparty/lzma/C/Util/Lzma/makefile @@ -8,8 +8,10 @@ LIB_OBJS = \ C_OBJS = \ $O\Alloc.obj \ + $O\CpuArch.obj \ $O\LzFind.obj \ $O\LzFindMt.obj \ + $O\LzFindOpt.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\7zFile.obj \ diff --git a/3rdparty/lzma/C/Util/Lzma/makefile.gcc b/3rdparty/lzma/C/Util/Lzma/makefile.gcc index 67aa8b17957..2acb0b808d2 100644 --- a/3rdparty/lzma/C/Util/Lzma/makefile.gcc +++ b/3rdparty/lzma/C/Util/Lzma/makefile.gcc @@ -1,44 +1,21 @@ -PROG = lzma -CXX = g++ -LIB = -RM = rm -f -CFLAGS = -c -O2 -Wall -D_7ZIP_ST +PROG = 7lzma -OBJS = \ - LzmaUtil.o \ - Alloc.o \ - LzFind.o \ - LzmaDec.o \ - LzmaEnc.o \ - 7zFile.o \ - 7zStream.o \ - - -all: $(PROG) - -$(PROG): $(OBJS) - $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB) $(LIB2) - -LzmaUtil.o: LzmaUtil.c - $(CXX) $(CFLAGS) LzmaUtil.c - -Alloc.o: ../../Alloc.c - $(CXX) $(CFLAGS) ../../Alloc.c +include ../../../CPP/7zip/LzmaDec_gcc.mak -LzFind.o: ../../LzFind.c - $(CXX) $(CFLAGS) ../../LzFind.c -LzmaDec.o: ../../LzmaDec.c - $(CXX) $(CFLAGS) ../../LzmaDec.c - -LzmaEnc.o: ../../LzmaEnc.c - $(CXX) $(CFLAGS) ../../LzmaEnc.c - -7zFile.o: ../../7zFile.c - $(CXX) $(CFLAGS) ../../7zFile.c - -7zStream.o: ../../7zStream.c - $(CXX) $(CFLAGS) ../../7zStream.c - -clean: - -$(RM) $(PROG) $(OBJS) +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $O/7zFile.o \ + $O/7zStream.o \ + $O/Alloc.o \ + $O/CpuArch.o \ + $O/LzFind.o \ + $O/LzFindMt.o \ + $O/LzFindOpt.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/LzmaUtil.o \ + $O/Threads.o \ + + +include ../../7zip_gcc_c.mak diff --git a/3rdparty/lzma/C/Util/LzmaLib/LzmaLib.dsp b/3rdparty/lzma/C/Util/LzmaLib/LzmaLib.dsp index 3421de83a1d..6ce91dcd2af 100644 --- a/3rdparty/lzma/C/Util/LzmaLib/LzmaLib.dsp +++ b/3rdparty/lzma/C/Util/LzmaLib/LzmaLib.dsp @@ -136,6 +136,10 @@ SOURCE=..\..\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\LzFindOpt.c +# End Source File +# Begin Source File + SOURCE=..\..\LzHash.h # End Source File # Begin Source File diff --git a/3rdparty/lzma/C/Util/LzmaLib/makefile b/3rdparty/lzma/C/Util/LzmaLib/makefile index 74103bb0540..b36f1de09f9 100644 --- a/3rdparty/lzma/C/Util/LzmaLib/makefile +++ b/3rdparty/lzma/C/Util/LzmaLib/makefile @@ -11,8 +11,10 @@ LIB_OBJS = \ C_OBJS = \ $O\Alloc.obj \ + $O\CpuArch.obj \ $O\LzFind.obj \ $O\LzFindMt.obj \ + $O\LzFindOpt.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\LzmaLib.obj \ diff --git a/3rdparty/lzma/C/Util/SfxSetup/SfxSetup.c b/3rdparty/lzma/C/Util/SfxSetup/SfxSetup.c index 57e73d86528..ef19aeac545 100644 --- a/3rdparty/lzma/C/Util/SfxSetup/SfxSetup.c +++ b/3rdparty/lzma/C/Util/SfxSetup/SfxSetup.c @@ -1,5 +1,5 @@ /* SfxSetup.c - 7z SFX Setup -2016-05-16 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -24,6 +24,8 @@ #define k_EXE_ExtIndex 2 +#define kInputBufSize ((size_t)1 << 18) + static const char * const kExts[] = { "bat" @@ -125,7 +127,7 @@ static WRes MyCreateDir(const WCHAR *name) #define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufferSize]; size_t numPrevBytes = 0; @@ -161,7 +163,7 @@ static Bool FindSignature(CSzFile *stream, UInt64 *resPos) } } -static Bool DoesFileOrDirExist(const WCHAR *path) +static BoolInt DoesFileOrDirExist(const WCHAR *path) { WIN32_FIND_DATAW fd; HANDLE handle; @@ -238,7 +240,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, #endif { CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; SRes res = SZ_OK; ISzAlloc allocImp; @@ -252,7 +254,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, DWORD winRes; const wchar_t *cmdLineParams; const char *errorMessage = NULL; - Bool useShellExecute = True; + BoolInt useShellExecute = True; DWORD exitCode = 0; LoadSecurityDlls(); @@ -275,7 +277,8 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, allocTempImp.Free = SzFreeTemp; FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); + LookToRead2_CreateVTable(&lookStream, False); + lookStream.buf = NULL; winRes = GetModuleFileNameW(NULL, sfxPath, MAX_PATH); if (winRes == 0 || winRes > MAX_PATH) @@ -284,7 +287,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, cmdLineParams = GetCommandLineW(); #ifndef UNDER_CE { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; cmdLineParams++) { wchar_t c = *cmdLineParams; @@ -376,14 +379,22 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (res == SZ_OK) { - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); + if (!lookStream.buf) + res = SZ_ERROR_MEM; + else + { + lookStream.bufSize = kInputBufSize; + lookStream.realStream = &archiveStream.vt; + LookToRead2_Init(&lookStream); + } } SzArEx_Init(&db); + if (res == SZ_OK) { - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); } if (res == SZ_OK) @@ -409,9 +420,9 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); { - res = SzArEx_Extract(&db, &lookStream.s, i, + res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp); @@ -516,16 +527,19 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { WCHAR *temp = path + pathLen; UInt32 j; - SzArEx_GetFileNameUtf16(&db, executeFileIndex, temp); + SzArEx_GetFileNameUtf16(&db, executeFileIndex, (UInt16 *)temp); for (j = 0; temp[j] != 0; j++) if (temp[j] == '/') temp[j] = CHAR_PATH_SEPARATOR; } } - IAlloc_Free(&allocImp, outBuffer); + ISzAlloc_Free(&allocImp, outBuffer); } + SzArEx_Free(&db, &allocImp); + ISzAlloc_Free(&allocImp, lookStream.buf); + File_Close(&archiveStream.file); if (res == SZ_OK) diff --git a/3rdparty/lzma/C/Util/SfxSetup/makefile b/3rdparty/lzma/C/Util/SfxSetup/makefile index 76cd9112464..544da67dfb5 100644 --- a/3rdparty/lzma/C/Util/SfxSetup/makefile +++ b/3rdparty/lzma/C/Util/SfxSetup/makefile @@ -1,4 +1,5 @@ PROG = 7zS2.sfx +MY_FIXED = 1 C_OBJS = \ $O\7zAlloc.obj \ diff --git a/3rdparty/lzma/C/Util/SfxSetup/makefile_con b/3rdparty/lzma/C/Util/SfxSetup/makefile_con index ed9174d3b20..d0f8352541e 100644 --- a/3rdparty/lzma/C/Util/SfxSetup/makefile_con +++ b/3rdparty/lzma/C/Util/SfxSetup/makefile_con @@ -1,4 +1,5 @@ PROG = 7zS2con.sfx +MY_FIXED = 1 CFLAGS = $(CFLAGS) -D_CONSOLE C_OBJS = \ diff --git a/3rdparty/lzma/C/Xz.c b/3rdparty/lzma/C/Xz.c index 1ef6878252c..7c53b600737 100644 --- a/3rdparty/lzma/C/Xz.c +++ b/3rdparty/lzma/C/Xz.c @@ -1,5 +1,5 @@ /* Xz.c - Xz -2015-05-01 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,7 +9,7 @@ #include "XzCrc64.h" const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 }; -const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; +/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) { @@ -20,28 +20,28 @@ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) v >>= 7; } while (v != 0); - buf[i - 1] &= 0x7F; + buf[(size_t)i - 1] &= 0x7F; return i; } void Xz_Construct(CXzStream *p) { - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + p->numBlocks = 0; + p->blocks = NULL; p->flags = 0; } -void Xz_Free(CXzStream *p, ISzAlloc *alloc) +void Xz_Free(CXzStream *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->blocks); - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + ISzAlloc_Free(alloc, p->blocks); + p->numBlocks = 0; + p->blocks = NULL; } unsigned XzFlags_GetCheckSize(CXzStreamFlags f) { unsigned t = XzFlags_GetCheckType(f); - return (t == 0) ? 0 : (4 << ((t - 1) / 3)); + return (t == 0) ? 0 : ((unsigned)4 << ((t - 1) / 3)); } void XzCheck_Init(CXzCheck *p, unsigned mode) diff --git a/3rdparty/lzma/C/Xz.h b/3rdparty/lzma/C/Xz.h index be3a1c36220..849b944bf5e 100644 --- a/3rdparty/lzma/C/Xz.h +++ b/3rdparty/lzma/C/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2015-05-01 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -47,12 +47,13 @@ typedef struct CXzFilter filters[XZ_NUM_FILTERS_MAX]; } CXzBlock; -#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) +#define XzBlock_GetNumFilters(p) (((unsigned)(p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) #define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) +#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); /* ---------- xz stream ---------- */ @@ -60,7 +61,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt #define XZ_FOOTER_SIG_SIZE 2 extern const Byte XZ_SIG[XZ_SIG_SIZE]; + +/* extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE]; +*/ + +#define XZ_FOOTER_SIG_0 'Y' +#define XZ_FOOTER_SIG_1 'Z' #define XZ_STREAM_FLAGS_SIZE 2 #define XZ_STREAM_CRC_SIZE 4 @@ -106,13 +113,12 @@ typedef struct { CXzStreamFlags flags; size_t numBlocks; - size_t numBlocksAllocated; CXzBlockSizes *blocks; UInt64 startOffset; } CXzStream; void Xz_Construct(CXzStream *p); -void Xz_Free(CXzStream *p, ISzAlloc *alloc); +void Xz_Free(CXzStream *p, ISzAllocPtr alloc); #define XZ_SIZE_OVERFLOW ((UInt64)(Int64)-1) @@ -127,12 +133,15 @@ typedef struct } CXzs; void Xzs_Construct(CXzs *p); -void Xzs_Free(CXzs *p, ISzAlloc *alloc); -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAlloc *alloc); +void Xzs_Free(CXzs *p, ISzAllocPtr alloc); +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc); UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + typedef enum { CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ @@ -141,43 +150,55 @@ typedef enum CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ } ECoderStatus; + +// ECoderFinishMode values are identical to ELzmaFinishMode + typedef enum { CODER_FINISH_ANY, /* finish at any point */ CODER_FINISH_END /* block must be finished at the end */ } ECoderFinishMode; + typedef struct _IStateCoder { void *p; - void (*Free)(void *p, ISzAlloc *alloc); - SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc); + void (*Free)(void *p, ISzAllocPtr alloc); + SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); - SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); } IStateCoder; + + #define MIXCODER_NUM_FILTERS_MAX 4 typedef struct { - ISzAlloc *alloc; + ISzAllocPtr alloc; Byte *buf; unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + BoolInt wasFinished; + SRes res; + ECoderStatus status; + // BoolInt SingleBufMode; + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; } CMixCoder; -void MixCoder_Construct(CMixCoder *p, ISzAlloc *alloc); -void MixCoder_Free(CMixCoder *p); -void MixCoder_Init(CMixCoder *p); -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId); -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status); typedef enum { @@ -191,6 +212,7 @@ typedef enum XZ_STATE_BLOCK_FOOTER } EXzState; + typedef struct { EXzState state; @@ -204,7 +226,7 @@ typedef struct UInt64 packSize; UInt64 unpackSize; - UInt64 numBlocks; + UInt64 numBlocks; // number of finished blocks in current stream UInt64 indexSize; UInt64 indexPos; UInt64 padSize; @@ -218,14 +240,72 @@ typedef struct CXzBlock block; CXzCheck check; CSha256 sha; + + BoolInt parseMode; + BoolInt headerParsedOk; + BoolInt decodeToStreamSignature; + unsigned decodeOnlyOneBlock; + + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; -void XzUnpacker_Construct(CXzUnpacker *p, ISzAlloc *alloc); +/* alloc : aligned for cache line allocation is better */ +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); void XzUnpacker_Free(CXzUnpacker *p); +/* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + { + XzUnpacker_Code(); + } + XzUnpacker_IsStreamWasFinished() + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + { + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + XzUnpacker_IsStreamWasFinished() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + XzUnpacker_IsStreamWasFinished() + } +*/ + /* finishMode: It has meaning only if the decoding reaches output limit (*destLen). @@ -236,8 +316,12 @@ Returns: SZ_OK status: CODER_STATUS_NOT_FINISHED, - CODER_STATUS_NEEDS_MORE_INPUT - maybe there are more xz streams, - call XzUnpacker_IsStreamWasFinished to check that current stream was finished + CODER_STATUS_NEEDS_MORE_INPUT - the decoder can return it in two cases: + 1) it needs more input data to finish current xz stream + 2) xz stream was finished successfully. But the decoder supports multiple + concatented xz streams. So it expects more input data for new xz streams. + Call XzUnpacker_IsStreamWasFinished() to check that latest xz stream was finished successfully. + SZ_ERROR_MEM - Memory allocation error SZ_ERROR_DATA - Data error SZ_ERROR_UNSUPPORTED - Unsupported method or method properties @@ -255,20 +339,178 @@ Returns: SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, - ECoderStatus *status); + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); + +/* +If you decode full xz stream(s), then you can call XzUnpacker_IsStreamWasFinished() +after successful XzUnpacker_CodeFull() or after last call of XzUnpacker_Code(). +*/ -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p); +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of -xz stream in two cases: -XzUnpacker_Code() returns: +XzUnpacker_GetExtraSize() returns then number of unconfirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some data after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT res == SZ_ERROR_NO_ARCHIVE */ -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p); +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); + + +/* + for random block decoding: + XzUnpacker_Init(); + set CXzUnpacker::streamFlags + XzUnpacker_PrepareToRandomBlockDecoding() + loop + { + XzUnpacker_Code() + XzUnpacker_IsBlockFinished() + } +*/ + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); + +#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + + + + + +/* ---- Single-Thread and Multi-Thread xz Decoding with Input/Output Streams ---- */ + +/* + if (CXzDecMtProps::numThreads > 1), the decoder can try to use + Multi-Threading. The decoder analyses xz block header, and if + there are pack size and unpack size values stored in xz block header, + the decoder reads compressed data of block to internal buffers, + and then it can start parallel decoding, if there are another blocks. + The decoder can switch back to Single-Thread decoding after some conditions. + + The sequence of calls for xz decoding with in/out Streams: + { + XzDecMt_Create() + XzDecMtProps_Init(XzDecMtProps) to set default values of properties + // then you can change some XzDecMtProps parameters with required values + // here you can set the number of threads and (memUseMax) - the maximum + Memory usage for multithreading decoding. + for() + { + XzDecMt_Decode() // one call per one file + } + XzDecMt_Destroy() + } +*/ + + +typedef struct +{ + size_t inBufSize_ST; // size of input buffer for Single-Thread decoding + size_t outStep_ST; // size of output buffer for Single-Thread decoding + BoolInt ignoreErrors; // if set to 1, the decoder can ignore some errors and it skips broken parts of data. + + #ifndef _7ZIP_ST + unsigned numThreads; // the number of threads for Multi-Thread decoding. if (umThreads == 1) it will use Single-thread decoding + size_t inBufSize_MT; // size of small input data buffers for Multi-Thread decoding. Big number of such small buffers can be created + size_t memUseMax; // the limit of total memory usage for Multi-Thread decoding. + // it's recommended to set (memUseMax) manually to value that is smaller of total size of RAM in computer. + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc internally for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; // there are some additional data after good xz streams, and that data is not new xz stream. + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed. That value doesn't include the data after + // end of xz stream, if that data was not correct + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; // the error code of xz streams data decoding + SRes ReadRes; // error code from ISeqInStream:Read() + SRes ProgressRes; // error code from ICompressProgress:Progress() + + SRes CombinedRes; // Combined result error code that shows main rusult + // = S_OK, if there is no error. + // but check also (DataAfterEnd) that can show additional minor errors. + + SRes CombinedRes_Type; // = SZ_ERROR_READ, if error from ISeqInStream + // = SZ_ERROR_PROGRESS, if error from ICompressProgress + // = SZ_ERROR_WRITE, if error from ISeqOutStream + // = SZ_ERROR_* codes for decoding +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* + +XzDecMt_Decode() +SRes: it's combined decoding result. It also is equal to stat->CombinedRes. + + SZ_OK - no error + check also output value in (stat->DataAfterEnd) + that can show additional possible error + + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_UNSUPPORTED - Unsupported method or method properties + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, // out: decoding results and statistics + int *isMT, // out: 0 means that ST (Single-Thread) version was used + // 1 means that MT (Multi-Thread) version was used + ICompressProgress *progress); EXTERN_C_END diff --git a/3rdparty/lzma/C/XzCrc64.c b/3rdparty/lzma/C/XzCrc64.c index e2da63b6ebe..b6d02cbebd5 100644 --- a/3rdparty/lzma/C/XzCrc64.c +++ b/3rdparty/lzma/C/XzCrc64.c @@ -1,5 +1,5 @@ /* XzCrc64.c -- CRC64 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-05-23 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,9 +9,9 @@ #define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) #ifdef MY_CPU_LE - #define CRC_NUM_TABLES 4 + #define CRC64_NUM_TABLES 4 #else - #define CRC_NUM_TABLES 5 + #define CRC64_NUM_TABLES 5 #define CRC_UINT64_SWAP(v) \ ((v >> 56) \ | ((v >> 40) & ((UInt64)0xFF << 8)) \ @@ -29,10 +29,10 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); #endif -typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); +typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); -static CRC_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC_NUM_TABLES]; +static CRC64_FUNC g_Crc64Update; +UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) { @@ -52,12 +52,12 @@ void MY_FAST_CALL Crc64GenerateTable() UInt64 r = i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrc64Poly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); g_Crc64Table[i] = r; } - for (; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) { - UInt64 r = g_Crc64Table[i - 256]; + UInt64 r = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); } @@ -74,9 +74,9 @@ void MY_FAST_CALL Crc64GenerateTable() else #endif { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) { - UInt64 x = g_Crc64Table[i - 256]; + UInt64 x = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = CRC_UINT64_SWAP(x); } g_Crc64Update = XzCrc64UpdateT1_BeT4; diff --git a/3rdparty/lzma/C/XzCrc64Opt.c b/3rdparty/lzma/C/XzCrc64Opt.c index c4fde65a10b..93a9ffff5f8 100644 --- a/3rdparty/lzma/C/XzCrc64Opt.c +++ b/3rdparty/lzma/C/XzCrc64Opt.c @@ -1,5 +1,5 @@ /* XzCrc64Opt.c -- CRC64 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2021-02-09 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,24 +7,25 @@ #ifndef MY_CPU_BE -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { - UInt32 d = (UInt32)v ^ *(const UInt32 *)p; + UInt32 d = (UInt32)v ^ *(const UInt32 *)(const void *)p; v = (v >> 32) - ^ table[0x300 + ((d ) & 0xFF)] - ^ table[0x200 + ((d >> 8) & 0xFF)] - ^ table[0x100 + ((d >> 16) & 0xFF)] - ^ table[0x000 + ((d >> 24))]; + ^ (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); return v; } @@ -43,26 +44,27 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con | ((v << 40) & ((UInt64)0xFF << 48)) \ | ((v << 56))) -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; table += 0x100; v = CRC_UINT64_SWAP(v); for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { - UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; + UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)(const void *)p; v = (v << 32) - ^ table[0x000 + ((d ) & 0xFF)] - ^ table[0x100 + ((d >> 8) & 0xFF)] - ^ table[0x200 + ((d >> 16) & 0xFF)] - ^ table[0x300 + ((d >> 24))]; + ^ (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); return CRC_UINT64_SWAP(v); } diff --git a/3rdparty/lzma/C/XzDec.c b/3rdparty/lzma/C/XzDec.c index 1edc0245637..3f96a37f94f 100644 --- a/3rdparty/lzma/C/XzDec.c +++ b/3rdparty/lzma/C/XzDec.c @@ -1,14 +1,33 @@ /* XzDec.c -- Xz Decode -2015-11-09 : Igor Pavlov : Public domain */ +2021-09-04 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include + +// #define XZ_DUMP + /* #define XZ_DUMP */ #ifdef XZ_DUMP #include #endif +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + #include #include @@ -19,16 +38,18 @@ #include "Delta.h" #include "Lzma2Dec.h" +// #define USE_SUBBLOCK + #ifdef USE_SUBBLOCK #include "Bcj3Dec.c" -#include "SbDec.c" +#include "SbDec.h" #endif #include "Xz.h" #define XZ_CHECK_SIZE_MAX 64 -#define CODER_BUF_SIZE (1 << 17) +#define CODER_BUF_SIZE ((size_t)1 << 17) unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value) { @@ -56,8 +77,9 @@ typedef struct size_t bufConv; size_t bufTotal; - UInt32 methodId; int encodeMode; + + UInt32 methodId; UInt32 delta; UInt32 ip; UInt32 x86State; @@ -66,12 +88,12 @@ typedef struct Byte buf[BRA_BUF_SIZE]; } CBraState; -static void BraState_Free(void *pp, ISzAlloc *alloc) +static void BraState_Free(void *pp, ISzAllocPtr alloc) { - alloc->Free(alloc, pp); + ISzAlloc_Free(alloc, pp); } -static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { CBraState *p = ((CBraState *)pp); UNUSED_VAR(alloc); @@ -121,69 +143,84 @@ static void BraState_Init(void *pp) Delta_Init(p->deltaState); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: p->bufConv = isa ## _Convert(p->buf, p->bufTotal, p->ip, p->encodeMode); break; -static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; + +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) { CBraState *p = ((CBraState *)pp); - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; UNUSED_VAR(finishMode); + *destLen = 0; *srcLen = 0; - *wasFinished = 0; - while (destLenOrig > 0) + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) { if (p->bufPos != p->bufConv) { - size_t curSize = p->bufConv - p->bufPos; - if (curSize > destLenOrig) - curSize = destLenOrig; - memcpy(dest, p->buf + p->bufPos, curSize); - p->bufPos += curSize; - *destLen += curSize; - dest += curSize; - destLenOrig -= curSize; + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; continue; } + p->bufTotal -= p->bufPos; memmove(p->buf, p->buf + p->bufPos, p->bufTotal); p->bufPos = 0; p->bufConv = 0; { - size_t curSize = BRA_BUF_SIZE - p->bufTotal; - if (curSize > srcLenOrig) - curSize = srcLenOrig; - memcpy(p->buf + p->bufTotal, src, curSize); - *srcLen += curSize; - src += curSize; - srcLenOrig -= curSize; - p->bufTotal += curSize; + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; } if (p->bufTotal == 0) break; - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, p->buf, p->bufTotal); - else - Delta_Decode(p->deltaState, p->delta, p->buf, p->bufTotal); - p->bufConv = p->bufTotal; - break; - case XZ_ID_X86: - p->bufConv = x86_Convert(p->buf, p->bufTotal, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - default: - return SZ_ERROR_UNSUPPORTED; - } - p->ip += (UInt32)p->bufConv; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); if (p->bufConv == 0) { @@ -192,48 +229,55 @@ static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, p->bufConv = p->bufTotal; } } - if (p->bufTotal == p->bufPos && srcLenOrig == 0 && srcWasFinished) - *wasFinished = 1; + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + return SZ_OK; } -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc) + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; - if (id != XZ_ID_Delta && - id != XZ_ID_X86 && - id != XZ_ID_PPC && - id != XZ_ID_IA64 && - id != XZ_ID_ARM && - id != XZ_ID_ARMT && - id != XZ_ID_SPARC) + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - p->p = 0; - decoder = (CBraState *)alloc->Alloc(alloc, sizeof(CBraState)); - if (decoder == 0) - return SZ_ERROR_MEM; + decoder = (CBraState *)p->p; + if (!decoder) + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } decoder->methodId = (UInt32)id; decoder->encodeMode = encodeMode; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code = BraState_Code; return SZ_OK; } + + /* ---------- SbState ---------- */ #ifdef USE_SUBBLOCK -static void SbState_Free(void *pp, ISzAlloc *alloc) +static void SbState_Free(void *pp, ISzAllocPtr alloc) { CSbDec *p = (CSbDec *)pp; SbDec_Free(p); - alloc->Free(alloc, pp); + ISzAlloc_Free(alloc, pp); } -static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { UNUSED_VAR(pp); UNUSED_VAR(props); @@ -246,8 +290,10 @@ static void SbState_Init(void *pp) SbDec_Init((CSbDec *)pp); } -static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) { CSbDec *p = (CSbDec *)pp; SRes res; @@ -260,102 +306,185 @@ static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, res = SbDec_Decode((CSbDec *)pp); *destLen -= p->destLen; *srcLen -= p->srcLen; - *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; return res; } -SRes SbState_SetFromMethod(IStateCoder *p, ISzAlloc *alloc) +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) { - CSbDec *decoder; - p->p = 0; - decoder = alloc->Alloc(alloc, sizeof(CSbDec)); - if (decoder == 0) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code = SbState_Code; + CSbDec *decoder = (CSbDec *)p->p; + if (!decoder) + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } SbDec_Construct(decoder); SbDec_SetAlloc(decoder, alloc); return SZ_OK; } + #endif -/* ---------- Lzma2State ---------- */ -static void Lzma2State_Free(void *pp, ISzAlloc *alloc) + +/* ---------- Lzma2 ---------- */ + +typedef struct +{ + CLzma2Dec decoder; + BoolInt outBufMode; +} CLzma2Dec_Spec; + + +static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) { - Lzma2Dec_Free((CLzma2Dec *)pp, alloc); - alloc->Free(alloc, pp); + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); + ISzAlloc_Free(alloc, pp); } -static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - return Lzma2Dec_Allocate((CLzma2Dec *)pp, props[0], alloc); + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } } static void Lzma2State_Init(void *pp) { - Lzma2Dec_Init((CLzma2Dec *)pp); + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); } -static SRes Lzma2State_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) { - ELzmaStatus status; + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res = Lzma2Dec_DecodeToBuf((CLzma2Dec *)pp, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status); + SRes res; UNUSED_VAR(srcWasFinished); - *wasFinished = (status == LZMA_STATUS_FINISHED_WITH_MARK); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = (ECoderStatus)status2; return res; } -static SRes Lzma2State_SetFromMethod(IStateCoder *p, ISzAlloc *alloc) + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) { - CLzma2Dec *decoder = (CLzma2Dec *)alloc->Alloc(alloc, sizeof(CLzma2Dec)); - p->p = decoder; - if (decoder == 0) - return SZ_ERROR_MEM; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code = Lzma2State_Code; - Lzma2Dec_Construct(decoder); + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } return SZ_OK; } -void MixCoder_Construct(CMixCoder *p, ISzAlloc *alloc) + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) { unsigned i; p->alloc = alloc; p->buf = NULL; p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) p->coders[i].p = NULL; } -void MixCoder_Free(CMixCoder *p) + +static void MixCoder_Free(CMixCoder *p) { unsigned i; - for (i = 0; i < p->numCoders; i++) + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) { IStateCoder *sc = &p->coders[i]; - if (p->alloc && sc->p) + if (sc->p) + { sc->Free(sc->p, p->alloc); + sc->p = NULL; + } } - p->numCoders = 0; if (p->buf) { - p->alloc->Free(p->alloc, p->buf); + ISzAlloc_Free(p->alloc, p->buf); p->buf = NULL; /* 9.31: the BUG was fixed */ } } -void MixCoder_Init(CMixCoder *p) +static void MixCoder_Init(CMixCoder *p) { unsigned i; for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) @@ -368,16 +497,22 @@ void MixCoder_Init(CMixCoder *p) { IStateCoder *coder = &p->coders[i]; coder->Init(coder->p); + p->results[i] = SZ_OK; } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; } -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; switch (methodId) { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, p->alloc); + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); #ifdef USE_SUBBLOCK case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); #endif @@ -387,31 +522,128 @@ SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) return BraState_SetFromMethod(sc, methodId, 0, p->alloc); } -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status) + ECoderFinishMode finishMode) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; - Bool allFinished = True; + *destLen = 0; *srcLen = 0; - *status = CODER_STATUS_NOT_FINISHED; - if (!p->buf) + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; + + // if (p->SingleBufMode) + if (p->outBuf) { - p->buf = (Byte *)p->alloc->Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; } + PRF_STR("standard mix"); + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + finishMode = CODER_FINISH_ANY; + } for (;;) { - Bool processed = False; + BoolInt processed = False; + BoolInt allFinished = True; + SRes resMain = SZ_OK; unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) break; @@ -421,79 +653,106 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, { SRes res; IStateCoder *coder = &p->coders[i]; - Byte *destCur; - SizeT destLenCur, srcLenCur; - const Byte *srcCur; - int srcFinishedCur; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; int encodingWasFinished; + ECoderStatus status2; if (i == 0) { - srcCur = src; - srcLenCur = srcLenOrig - *srcLen; - srcFinishedCur = srcWasFinished; + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; } else { - srcCur = p->buf + (CODER_BUF_SIZE * (i - 1)) + p->pos[i - 1]; - srcLenCur = p->size[i - 1] - p->pos[i - 1]; - srcFinishedCur = p->finished[i - 1]; + size_t k = i - 1; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; } if (i == p->numCoders - 1) { - destCur = dest; - destLenCur = destLenOrig - *destLen; + dest2 = dest; + destLen2 = destLenOrig - *destLen; } else { if (p->pos[i] != p->size[i]) continue; - destCur = p->buf + (CODER_BUF_SIZE * i); - destLenCur = CODER_BUF_SIZE; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; } - res = coder->Code(coder->p, destCur, &destLenCur, srcCur, &srcLenCur, srcFinishedCur, finishMode, &encodingWasFinished); + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + if (!encodingWasFinished) + { allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } if (i == 0) { - *srcLen += srcLenCur; - src += srcLenCur; + *srcLen += srcLen2; + src += srcLen2; } else - { - p->pos[i - 1] += srcLenCur; - } + p->pos[(size_t)i - 1] += srcLen2; if (i == p->numCoders - 1) { - *destLen += destLenCur; - dest += destLenCur; + *destLen += destLen2; + dest += destLen2; } else { - p->size[i] = destLenCur; + p->size[i] = destLen2; p->pos[i] = 0; p->finished[i] = encodingWasFinished; } - if (res != SZ_OK) - return res; - - if (destLenCur != 0 || srcLenCur != 0) + if (destLen2 != 0 || srcLen2 != 0) processed = True; } + if (!processed) - break; + { + if (allFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; + } } - if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; } + SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) { *p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE); @@ -503,18 +762,46 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; } -static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) +static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) { - return - indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) && - (GetUi32(buf) == CrcCalc(buf + 4, 6) && - flags == GetBe16(buf + 8) && - memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0); + return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) + && GetUi32(buf) == CrcCalc(buf + 4, 6) + && flags == GetBe16(buf + 8) + && buf[10] == XZ_FOOTER_SIG_0 + && buf[11] == XZ_FOOTER_SIG_1; } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ - if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } + if (s == 0) return SZ_ERROR_ARCHIVE; \ + pos += s; } + + +static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} SRes XzBlock_Parse(CXzBlock *p, const Byte *header) @@ -523,14 +810,15 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) unsigned numFilters, i; unsigned headerSize = (unsigned)header[0] << 2; + /* (headerSize != 0) : another code checks */ + if (CrcCalc(header, headerSize) != GetUi32(header + headerSize)) return SZ_ERROR_ARCHIVE; pos = 1; - if (pos == headerSize) - return SZ_ERROR_ARCHIVE; p->flags = header[pos++]; + p->packSize = (UInt64)(Int64)-1; if (XzBlock_HasPackSize(p)) { READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize); @@ -538,6 +826,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) return SZ_ERROR_ARCHIVE; } + p->unpackSize = (UInt64)(Int64)-1; if (XzBlock_HasUnpackSize(p)) READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize); @@ -564,37 +853,56 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) #endif } + if (XzBlock_HasUnsupportedFlags(p)) + return SZ_ERROR_UNSUPPORTED; + while (pos < headerSize) if (header[pos++] != 0) return SZ_ERROR_ARCHIVE; return SZ_OK; } -SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; - Bool needReInit = True; + BoolInt needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders) + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) { + needReInit = False; for (i = 0; i < numFilters; i++) if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; break; - needReInit = (i != numFilters); + } } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; if (needReInit) { MixCoder_Free(p); - p->numCoders = numFilters; for (i = 0; i < numFilters; i++) { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - RINOK(MixCoder_SetFromMethod(p, i, f->id)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); } + p->numCoders = numFilters; } - + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i]; @@ -606,6 +914,8 @@ SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) return SZ_OK; } + + void XzUnpacker_Init(CXzUnpacker *p) { p->state = XZ_STATE_STREAM_HEADER; @@ -614,75 +924,171 @@ void XzUnpacker_Init(CXzUnpacker *p) p->numFinishedStreams = 0; p->numTotalBlocks = 0; p->padSize = 0; + p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; +} + + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; } -void XzUnpacker_Construct(CXzUnpacker *p, ISzAlloc *alloc) + +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) { MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; XzUnpacker_Init(p); } + void XzUnpacker_Free(CXzUnpacker *p) { MixCoder_Free(&p->decoder); } + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) +{ + p->indexSize = 0; + p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + p->decodeOnlyOneBlock = 1; +} + + +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_SPECIFIED; + for (;;) { - SizeT srcRem = srcLenOrig - *srcLen; + SizeT srcRem; if (p->state == XZ_STATE_BLOCK) { SizeT destLen2 = destLenOrig - *destLen; SizeT srcLen2 = srcLenOrig - *srcLen; SRes res; + + ECoderFinishMode finishMode2 = finishMode; + BoolInt srcFinished2 = srcFinished; + BoolInt destFinish = False; + + if (p->block.packSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.packSize - p->packSize; + if (srcLen2 >= rem) + { + srcFinished2 = True; + srcLen2 = (SizeT)rem; + } + if (rem == 0 && p->block.unpackSize == p->unpackSize) + return SZ_ERROR_DATA; + } + + if (p->block.unpackSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.unpackSize - p->unpackSize; + if (destLen2 >= rem) + { + destFinish = True; + finishMode2 = CODER_FINISH_END; + destLen2 = (SizeT)rem; + } + } + + /* if (srcLen2 == 0 && destLen2 == 0) { *status = CODER_STATUS_NOT_FINISHED; return SZ_OK; } + */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode, status); - XzCheck_Update(&p->check, dest, destLen2); + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } (*srcLen) += srcLen2; src += srcLen2; p->packSize += srcLen2; - (*destLen) += destLen2; - dest += destLen2; p->unpackSize += destLen2; - + RINOK(res); - - if (*status == CODER_STATUS_FINISHED_WITH_MARK) - { - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, p->packSize + p->blockHeaderSize + XzFlags_GetCheckSize(p->streamFlags)); - num += Xz_WriteVarInt(temp + num, p->unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; + + if (*status != CODER_STATUS_FINISHED_WITH_MARK) + { + if (p->block.packSize == p->packSize + && *status == CODER_STATUS_NEEDS_MORE_INPUT) + { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); + *status = CODER_STATUS_NOT_SPECIFIED; + return SZ_ERROR_DATA; + } + return SZ_OK; + } + { + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } } - else if (srcLen2 == 0 && destLen2 == 0) - return SZ_OK; - - continue; + // continue; } - if (srcRem == 0) + srcRem = srcLenOrig - *srcLen; + + // XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes + if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER) { *status = CODER_STATUS_NEEDS_MORE_INPUT; return SZ_OK; @@ -696,6 +1102,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; p->buf[p->pos++] = *src++; (*srcLen)++; } @@ -703,10 +1111,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); p->numStartedStreams++; - p->state = XZ_STATE_BLOCK_HEADER; - Sha256_Init(&p->sha); p->indexSize = 0; p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; } break; @@ -720,6 +1128,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, (*srcLen)++; if (p->buf[0] == 0) { + if (p->decodeOnlyOneBlock) + return SZ_ERROR_DATA; p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks); p->indexPos = p->indexPreSize; p->indexSize += p->indexPreSize; @@ -727,10 +1137,13 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, Sha256_Init(&p->sha); p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize); p->state = XZ_STATE_STREAM_INDEX; + break; } p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + break; } - else if (p->pos != p->blockHeaderSize) + + if (p->pos != p->blockHeaderSize) { UInt32 cur = p->blockHeaderSize - p->pos; if (cur > srcRem) @@ -743,20 +1156,32 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, else { RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; p->state = XZ_STATE_BLOCK; p->packSize = 0; p->unpackSize = 0; XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - RINOK(XzDec_Init(&p->decoder, &p->block)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); } break; } case XZ_STATE_BLOCK_FOOTER: { - if (((p->packSize + p->alignPos) & 3) != 0) + if ((((unsigned)p->packSize + p->alignPos) & 3) != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } (*srcLen)++; p->alignPos++; if (*src++ != 0) @@ -768,20 +1193,31 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, UInt32 cur = checkSize - p->pos; if (cur != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } if (cur > srcRem) cur = (UInt32)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; src += cur; + if (checkSize != p->pos) + break; } - else { Byte digest[XZ_CHECK_SIZE_MAX]; p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0) return SZ_ERROR_CRC; + if (p->decodeOnlyOneBlock) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } } } break; @@ -841,9 +1277,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } else { + const Byte *ptr = p->buf; p->state = XZ_STATE_STREAM_FOOTER; p->pos = 0; - if (CRC_GET_DIGEST(p->crc) != GetUi32(p->buf)) + if (CRC_GET_DIGEST(p->crc) != GetUi32(ptr)) return SZ_ERROR_CRC; } break; @@ -897,17 +1334,1504 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, */ } -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p) + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) { - return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); } -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p) + +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); +} + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); +} + +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) - num += p->padSize; + num = p->padSize; else if (p->state == XZ_STATE_STREAM_HEADER) - num += p->padSize + p->pos; + num = p->padSize + p->pos; return num; } + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + BoolInt skipMode; + // BoolInt finishedWithMark; + EMtDecParseState parseState; + BoolInt parsing_Truncated; + BoolInt atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + BoolInt dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + BoolInt readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + BoolInt mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; // it's set to error code, if the size Code() output doesn't patch the size from Parsing stage + // it can be = SZ_ERROR_INPUT_EOF + // it can be = SZ_ERROR_DATA, in some another cases + BoolInt isBlockHeaderState_Parse; + BoolInt isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + BoolInt parsing_Truncated; + + BoolInt mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + XzUnpacker_Free(&p->dec); + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return SZ_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (srcSize > coder->inPreSize - coder->inCodeSize) + return SZ_ERROR_FAIL; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + size_t step = coder->inPreHeaderSize - coder->inCodeSize; + if (step > srcSize) + step = srcSize; + src += step; + srcSize -= step; + coder->inCodeSize += step; + *inCodePos = coder->inCodeSize; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return SZ_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, BoolInt isCross, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = (int)coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = (int)coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = (int)coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + /* + We have processed all xz-blocks of stream, + And xz unpacker is at XZ_STATE_BLOCK_HEADER state, where + (src) is a pointer to xz-Index structure. + We finish reading of current xz-Stream, including Zero padding after xz-Stream. + We exit, if we reach extra byte (first byte of new-Stream or another data). + But we don't update input stream pointer for that new extra byte. + If extra byte is not correct first byte of xz-signature, + we have SZ_ERROR_NO_ARCHIVE error here. + */ + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + // res = SZ_ERROR_ARCHIVE; // for failure test + + me->status = status; + me->codeRes = res; + + if (isCross) + me->mtc.crossStart += srcProcessed; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + srcSize -= srcProcessed; + src += srcProcessed; + + if (res != SZ_OK) + { + return SZ_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + + if (!isCross) + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + if (srcSize != 0) + memcpy(crossBuf, src, srcSize); + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize; + } + + PRF_STR_INT("XZ_STATE_STREAM_HEADER crossEnd = ", (unsigned)me->mtc.crossEnd); + + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT || srcSize != 0) + { + return SZ_ERROR_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + // const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + + // inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)crossBuf, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + crossBuf + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return SZ_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + +/* + XzDecMt_Decode_ST() can return SZ_OK or the following errors + - SZ_ERROR_MEM for memory allocation error + - error from XzUnpacker_Code() function + - SZ_ERROR_WRITE for ISeqOutStream::Write(). stat->CombinedRes_Type = SZ_ERROR_WRITE in that case + - ICompressProgress::Progress() error, stat->CombinedRes_Type = SZ_ERROR_PROGRESS. + But XzDecMt_Decode_ST() doesn't return ISeqInStream::Read() errors. + ISeqInStream::Read() result is set to p->readRes. + also it can set stat->CombinedRes_Type to SZ_ERROR_WRITE or SZ_ERROR_PROGRESS. +*/ + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + BoolInt finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)p->inBuf, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + const size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + // p->outProcessed += written; // 21.01: BUG fixed + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + if (p->inProcessed - inPrev >= (1 << 22) || + p->outProcessed - outPrev >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + { + // p->codeRes is preliminary error from XzUnpacker_Code. + // and it can be corrected later as final result + // so we return SZ_OK here instead of (res); + return SZ_OK; + // return res; + } + } +} + + + +/* +XzStatInfo_SetStat() transforms + CXzUnpacker return code and status to combined CXzStatInfo results. + it can convert SZ_OK to SZ_ERROR_INPUT_EOF + it can convert SZ_ERROR_NO_ARCHIVE to SZ_OK and (DataAfterEnd = 1) +*/ + +static void XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + // UInt64 readProcessed, + UInt64 inProcessed, + SRes res, // it's result from CXzUnpacker unpacker + ECoderStatus status, + BoolInt decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + // any extra data is part of correct data + extraSize = 0; + // if xz stream was not finished, then we need more data + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else + { + // CODER_STATUS_FINISHED_WITH_MARK is not possible for multi stream xz decoding + // so he we have (status == CODER_STATUS_NOT_FINISHED) + // if (status != CODER_STATUS_FINISHED_WITH_MARK) + if (!decodingTruncated || finishMode) + res = SZ_ERROR_DATA; + } + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize and inProcessed don't include "bad" byte + */ + // if (inProcessed == extraSize), there was no any good xz stream header, and we keep error + if (inProcessed != extraSize) // if there were good xz streams before error + { + // if (extraSize != 0 || readProcessed != inProcessed) + { + // he we suppose that all xz streams were finsihed OK, and we have + // some extra data after all streams + stat->DataAfterEnd = True; + res = SZ_OK; + } + } + } + + if (stat->DecodeRes == SZ_OK) + stat->DecodeRes = res; + + stat->InSize -= extraSize; +} + + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + p->readRes = SZ_OK; + + p->codeRes = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; + + XzUnpacker_Init(&p->dec); + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback2 vt; + BoolInt needContinue; + SRes res; + // we just free ST buffers here + // but we still keep state variables, that was set in XzUnpacker_Init() + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + + res = MtDec_Code(&p->mtc); + + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + { + SRes codeRes; + BoolInt truncated = False; + ECoderStatus status; + const CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + const CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return SZ_ERROR_FAIL; + + if (p->mainErrorCode != SZ_OK) + stat->DecodeRes = p->mainErrorCode; + + XzStatInfo_SetStat(dec, p->finishMode, + // p->mtc.readProcessed, + p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + } + + if (res == SZ_OK) + { + stat->ReadRes = p->mtc.readRes; + + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK + // && p->mtc.inProcessed == p->mtc.readProcessed + && stat->DecodeRes == SZ_ERROR_INPUT_EOF) + { + res = p->mtc.readRes; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (stat->DecodeRes != SZ_OK) + res = stat->DecodeRes; + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + #ifndef _7ZIP_ST + // we must set error code from MT decoding at first + if (p->mainErrorCode != SZ_OK) + stat->DecodeRes = p->mainErrorCode; + #endif + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + // p->readProcessed, + p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + stat->ReadRes = p->readRes; + + if (res == SZ_OK) + { + if (p->readRes != SZ_OK + // && p->inProcessed == p->readProcessed + && stat->DecodeRes == SZ_ERROR_INPUT_EOF) + { + // we set read error as combined error, only if that error was the reason + // of decoding problem + res = p->readRes; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (stat->DecodeRes != SZ_OK) + res = stat->DecodeRes; + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +} diff --git a/3rdparty/lzma/C/XzEnc.c b/3rdparty/lzma/C/XzEnc.c index ccac7a8ed4b..be174cccf52 100644 --- a/3rdparty/lzma/C/XzEnc.c +++ b/3rdparty/lzma/C/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2015-09-16 : Igor Pavlov : Public domain */ +2021-04-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,7 +7,6 @@ #include #include "7zCrc.h" -#include "Alloc.h" #include "Bra.h" #include "CpuArch.h" @@ -19,22 +18,41 @@ #include "XzEnc.h" +// #define _7ZIP_ST + +#ifndef _7ZIP_ST +#include "MtCoder.h" +#else +#define MTCODER__THREADS_MAX 1 +#define MTCODER__BLOCKS_MAX 1 +#endif + +#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) + +/* max pack size for LZMA2 block + check-64bytrs: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) + +#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) + + #define XzBlock_ClearFlags(p) (p)->flags = 0; -#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); +#define XzBlock_SetNumFilters(p, n) (p)->flags = (Byte)((p)->flags | ((n) - 1)); #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; -static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size) + +static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) { - return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; + return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; } -static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc) +static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) { *crc = CrcUpdate(*crc, buf, size); return WriteBytes(s, buf, size); } + static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) { UInt32 crc; @@ -78,98 +96,154 @@ static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) } -static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s) + + +typedef struct { - Byte buf[32]; - UInt64 globalPos; - { - UInt32 crc = CRC_INIT_VAL; - unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); - size_t i; + size_t numBlocks; + size_t size; + size_t allocated; + Byte *blocks; +} CXzEncIndex; - globalPos = pos; - buf[0] = 0; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - for (i = 0; i < p->numBlocks; i++) - { - const CXzBlockSizes *block = &p->blocks[i]; - pos = Xz_WriteVarInt(buf, block->totalSize); - pos += Xz_WriteVarInt(buf + pos, block->unpackSize); - globalPos += pos; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - } - - pos = ((unsigned)globalPos & 3); - - if (pos != 0) - { - buf[0] = buf[1] = buf[2] = 0; - RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc)); - globalPos += 4 - pos; - } - { - SetUi32(buf, CRC_GET_DIGEST(crc)); - RINOK(WriteBytes(s, buf, 4)); - globalPos += 4; - } - } +static void XzEncIndex_Construct(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; + p->blocks = NULL; +} +static void XzEncIndex_Init(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; +} + +static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) +{ + if (p->blocks) { - UInt32 indexSize = (UInt32)((globalPos >> 2) - 1); - SetUi32(buf + 4, indexSize); - buf[8] = (Byte)(p->flags >> 8); - buf[9] = (Byte)(p->flags & 0xFF); - SetUi32(buf, CrcCalc(buf + 4, 6)); - memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE); - return WriteBytes(s, buf, 12); + ISzAlloc_Free(alloc, p->blocks); + p->blocks = NULL; } + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; } -static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc) +static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) { - if (!p->blocks || p->numBlocksAllocated == p->numBlocks) + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!blocks) + return SZ_ERROR_MEM; + if (p->size != 0) + memcpy(blocks, p->blocks, p->size); + if (p->blocks) + ISzAlloc_Free(alloc, p->blocks); + p->blocks = blocks; + p->allocated = newSize; + return SZ_OK; +} + + +static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + UInt64 pos; { - size_t num = p->numBlocks * 2 + 1; - size_t newSize = sizeof(CXzBlockSizes) * num; - CXzBlockSizes *blocks; - if (newSize / sizeof(CXzBlockSizes) != num) - return SZ_ERROR_MEM; - blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize); - if (!blocks) + Byte buf[32]; + unsigned pos2 = Xz_WriteVarInt(buf, totalSize); + pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); + pos = numBlocks * pos2; + } + + if (pos <= p->allocated - p->size) + return SZ_OK; + { + UInt64 newSize64 = p->size + pos; + size_t newSize = (size_t)newSize64; + if (newSize != newSize64) return SZ_ERROR_MEM; - if (p->numBlocks != 0) - { - memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes)); - alloc->Free(alloc, p->blocks); - } - p->blocks = blocks; - p->numBlocksAllocated = num; + return XzEncIndex_ReAlloc(p, newSize, alloc); } +} + + +static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + Byte buf[32]; + unsigned pos = Xz_WriteVarInt(buf, totalSize); + pos += Xz_WriteVarInt(buf + pos, unpackSize); + + if (pos > p->allocated - p->size) { - CXzBlockSizes *block = &p->blocks[p->numBlocks++]; - block->unpackSize = unpackSize; - block->totalSize = totalSize; + size_t newSize = p->allocated * 2 + 16 * 2; + if (newSize < p->size + pos) + return SZ_ERROR_MEM; + RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); } + memcpy(p->blocks + p->size, buf, pos); + p->size += pos; + p->numBlocks++; return SZ_OK; } +static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) +{ + Byte buf[32]; + UInt64 globalPos; + UInt32 crc = CRC_INIT_VAL; + unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); + + globalPos = pos; + buf[0] = 0; + RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); + RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); + globalPos += p->size; + + pos = XZ_GET_PAD_SIZE(globalPos); + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + globalPos += pos; + + crc = CrcUpdate(crc, buf + 4 - pos, pos); + SetUi32(buf + 4, CRC_GET_DIGEST(crc)); + + SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); + buf[8 + 8] = (Byte)(flags >> 8); + buf[8 + 9] = (Byte)(flags & 0xFF); + SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); + buf[8 + 10] = XZ_FOOTER_SIG_0; + buf[8 + 11] = XZ_FOOTER_SIG_1; + + return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); +} + + + /* ---------- CSeqCheckInStream ---------- */ typedef struct { - ISeqInStream p; + ISeqInStream vt; ISeqInStream *realStream; + const Byte *data; + UInt64 limit; UInt64 processed; + int realStreamFinished; CXzCheck check; } CSeqCheckInStream; -static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode) +static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) { + p->limit = (UInt64)(Int64)-1; p->processed = 0; - XzCheck_Init(&p->check, mode); + p->realStreamFinished = 0; + XzCheck_Init(&p->check, checkMode); } static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) @@ -177,12 +251,31 @@ static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) XzCheck_Final(&p->check, digest); } -static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) +static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { - CSeqCheckInStream *p = (CSeqCheckInStream *)pp; - SRes res = p->realStream->Read(p->realStream, data, size); - XzCheck_Update(&p->check, data, *size); - p->processed += *size; + CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + if (p->realStream) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->realStreamFinished = (size2 == 0) ? 1 : 0; + } + else + memcpy(data, p->data + (size_t)p->processed, size2); + XzCheck_Update(&p->check, data, size2); + p->processed += size2; + } + *size = size2; return res; } @@ -191,15 +284,24 @@ static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) typedef struct { - ISeqOutStream p; + ISeqOutStream vt; ISeqOutStream *realStream; + Byte *outBuf; + size_t outBufLimit; UInt64 processed; } CSeqSizeOutStream; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp; - size = p->realStream->Write(p->realStream, data, size); + CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); + if (p->realStream) + size = ISeqOutStream_Write(p->realStream, data, size); + else + { + if (size > p->outBufLimit - (size_t)p->processed) + return 0; + memcpy(p->outBuf + (size_t)p->processed, data, size); + } p->processed += size; return size; } @@ -220,9 +322,29 @@ typedef struct int srcWasFinished; } CSeqInFilter; -static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); + +static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) { - CSeqInFilter *p = (CSeqInFilter *)pp; + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + if (!p->buf) + return SZ_ERROR_MEM; + } + p->curPos = p->endPos = 0; + p->srcWasFinished = 0; + RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); + RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); + p->StateCoder.Init(p->StateCoder.p); + return SZ_OK; +} + + +static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); size_t sizeOriginal = *size; if (sizeOriginal == 0) return SZ_OK; @@ -234,19 +356,22 @@ static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) { p->curPos = 0; p->endPos = FILTER_BUF_SIZE; - RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos)); + RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)); if (p->endPos == 0) p->srcWasFinished = 1; } { SizeT srcLen = p->endPos - p->curPos; - int wasFinished; + ECoderStatus status; SRes res; *size = sizeOriginal; - res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); + res = p->StateCoder.Code2(p->StateCoder.p, + (Byte *)data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != 0) + if (*size != 0 || srcLen == 0 || res != SZ_OK) return res; } } @@ -255,34 +380,22 @@ static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) static void SeqInFilter_Construct(CSeqInFilter *p) { p->buf = NULL; + p->StateCoder.p = NULL; p->p.Read = SeqInFilter_Read; } -static void SeqInFilter_Free(CSeqInFilter *p) +static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) { - if (p->buf) + if (p->StateCoder.p) { - g_Alloc.Free(&g_Alloc, p->buf); - p->buf = NULL; + p->StateCoder.Free(p->StateCoder.p, alloc); + p->StateCoder.p = NULL; } -} - -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc); - -static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) -{ - if (!p->buf) + if (p->buf) { - p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE); - if (!p->buf) - return SZ_ERROR_MEM; + ISzAlloc_Free(alloc, p->buf); + p->buf = NULL; } - p->curPos = p->endPos = 0; - p->srcWasFinished = 0; - RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc)); - RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc)); - p->StateCoder.Init(p->StateCoder.p); - return SZ_OK; } @@ -292,17 +405,17 @@ static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) typedef struct { - ISeqInStream p; + ISeqInStream vt; ISeqInStream *inStream; CSbEnc enc; } CSbEncInStream; -static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) +static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { - CSbEncInStream *p = (CSbEncInStream *)pp; + CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); size_t sizeOriginal = *size; if (sizeOriginal == 0) - return S_OK; + return SZ_OK; for (;;) { @@ -322,14 +435,14 @@ static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) *size = sizeOriginal; RINOK(SbEnc_Read(&p->enc, data, size)); if (*size != 0 || !p->enc.needRead) - return S_OK; + return SZ_OK; } } -void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc) +void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc) { SbEnc_Construct(&p->enc, alloc); - p->p.Read = SbEncInStream_Read; + p->vt.Read = SbEncInStream_Read; } SRes SbEncInStream_Init(CSbEncInStream *p) @@ -345,43 +458,233 @@ void SbEncInStream_Free(CSbEncInStream *p) #endif + +/* ---------- CXzProps ---------- */ + + +void XzFilterProps_Init(CXzFilterProps *p) +{ + p->id = 0; + p->delta = 0; + p->ip = 0; + p->ipDefined = False; +} + +void XzProps_Init(CXzProps *p) +{ + p->checkId = XZ_CHECK_CRC32; + p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; + p->numTotalThreads = -1; + p->reduceSize = (UInt64)(Int64)-1; + p->forceWriteSizesInHeader = 0; + // p->forceWriteSizesInHeader = 1; + + XzFilterProps_Init(&p->filterProps); + Lzma2EncProps_Init(&p->lzma2Props); +} + + +static void XzEncProps_Normalize_Fixed(CXzProps *p) +{ + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; + { + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + Lzma2EncProps_Normalize(&tp); + t1n = tp.numTotalThreads; + } + + t1 = p->lzma2Props.numTotalThreads; + t2 = p->numBlockThreads_Max; + t3 = p->numTotalThreads; + + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + + if (t3 <= 0) + { + if (t2 <= 0) + t2 = 1; + t3 = t1n * t2; + } + else if (t2 <= 0) + { + t2 = t3 / t1n; + if (t2 == 0) + { + t1 = 1; + t2 = t3; + } + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + } + else if (t1 <= 0) + { + t1 = t3 / t2; + if (t1 == 0) + t1 = 1; + } + else + t3 = t1n * t2; + + p->lzma2Props.numTotalThreads = t1; + + t2r = t2; + + fileSize = p->reduceSize; + + if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzma2Props.lzmaProps.reduceSize = p->blockSize; + + Lzma2EncProps_Normalize(&p->lzma2Props); + + t1 = p->lzma2Props.numTotalThreads; + + { + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; + if (numBlocks < (unsigned)t2) + { + t2r = (int)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; + } + } + } + + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; + p->numTotalThreads = t3; +} + + +static void XzProps_Normalize(CXzProps *p) +{ + /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. + Lzma2Enc_SetProps() will normalize lzma2Props later. */ + + if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) + { + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + return; + } + else + { + CLzma2EncProps *lzma2 = &p->lzma2Props; + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + // xz-auto + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + // if (xz-auto && lzma2-solid) - we use solid for both + p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + } + else + { + // if (xz-auto && (lzma2-auto || lzma2-fixed_) + // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + + Lzma2EncProps_Normalize(&tp); + + p->blockSize = tp.blockSize; // fixed or solid + p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; + p->numBlockThreads_Max = tp.numBlockThreads_Max; + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->lzmaProps.reduceSize = tp.blockSize; + lzma2->numBlockThreads_Reduced = 1; + lzma2->numBlockThreads_Max = 1; + return; + } + } + else + { + // xz-fixed + // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize + + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + { + UInt64 r = p->reduceSize; + if (r > p->blockSize || r == (UInt64)(Int64)-1) + r = p->blockSize; + lzma2->lzmaProps.reduceSize = r; + } + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->blockSize = p->blockSize; + + XzEncProps_Normalize_Fixed(p); + } + } +} + + +/* ---------- CLzma2WithFilters ---------- */ + typedef struct { CLzma2EncHandle lzma2; + CSeqInFilter filter; + #ifdef USE_SUBBLOCK CSbEncInStream sb; #endif - CSeqInFilter filter; - ISzAlloc *alloc; - ISzAlloc *bigAlloc; } CLzma2WithFilters; -static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc) +static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) { - p->alloc = alloc; - p->bigAlloc = bigAlloc; p->lzma2 = NULL; + SeqInFilter_Construct(&p->filter); + #ifdef USE_SUBBLOCK SbEncInStream_Construct(&p->sb, alloc); #endif - SeqInFilter_Construct(&p->filter); } -static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p) + +static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) { - p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc); if (!p->lzma2) - return SZ_ERROR_MEM; + { + p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); + if (!p->lzma2) + return SZ_ERROR_MEM; + } return SZ_OK; } -static void Lzma2WithFilters_Free(CLzma2WithFilters *p) + +static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) { - SeqInFilter_Free(&p->filter); #ifdef USE_SUBBLOCK SbEncInStream_Free(&p->sb); #endif + + SeqInFilter_Free(&p->filter, alloc); if (p->lzma2) { Lzma2Enc_Destroy(p->lzma2); @@ -390,137 +693,626 @@ static void Lzma2WithFilters_Free(CLzma2WithFilters *p) } -void XzProps_Init(CXzProps *p) +typedef struct { - p->lzma2Props = NULL; - p->filterProps = NULL; - p->checkId = XZ_CHECK_CRC32; + UInt64 unpackSize; + UInt64 totalSize; + size_t headerSize; +} CXzEncBlockInfo; + + +static SRes Xz_CompressBlock( + CLzma2WithFilters *lzmaf, + + ISeqOutStream *outStream, + Byte *outBufHeader, + Byte *outBufData, size_t outBufDataLimit, + + ISeqInStream *inStream, + // UInt64 expectedSize, + const Byte *inBuf, // used if (!inStream) + size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored + + const CXzProps *props, + ICompressProgress *progress, + int *inStreamFinished, /* only for inStream version */ + CXzEncBlockInfo *blockSizes, + ISzAllocPtr alloc, + ISzAllocPtr allocBig) +{ + CSeqCheckInStream checkInStream; + CSeqSizeOutStream seqSizeOutStream; + CXzBlock block; + unsigned filterIndex = 0; + CXzFilter *filter = NULL; + const CXzFilterProps *fp = &props->filterProps; + if (fp->id == 0) + fp = NULL; + + *inStreamFinished = False; + + RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); + + RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); + + XzBlock_ClearFlags(&block); + XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + + if (fp) + { + filter = &block.filters[filterIndex++]; + filter->id = fp->id; + filter->propsSize = 0; + + if (fp->id == XZ_ID_Delta) + { + filter->props[0] = (Byte)(fp->delta - 1); + filter->propsSize = 1; + } + else if (fp->ipDefined) + { + Byte *ptr = filter->props; + SetUi32(ptr, fp->ip); + filter->propsSize = 4; + } + } + + { + CXzFilter *f = &block.filters[filterIndex++]; + f->id = XZ_ID_LZMA2; + f->propsSize = 1; + f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + } + + seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; + seqSizeOutStream.realStream = outStream; + seqSizeOutStream.outBuf = outBufData; + seqSizeOutStream.outBufLimit = outBufDataLimit; + seqSizeOutStream.processed = 0; + + /* + if (expectedSize != (UInt64)(Int64)-1) + { + block.unpackSize = expectedSize; + if (props->blockSize != (UInt64)(Int64)-1) + if (expectedSize > props->blockSize) + block.unpackSize = props->blockSize; + XzBlock_SetHasUnpackSize(&block); + } + */ + + if (outStream) + { + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + } + + checkInStream.vt.Read = SeqCheckInStream_Read; + SeqCheckInStream_Init(&checkInStream, props->checkId); + + checkInStream.realStream = inStream; + checkInStream.data = inBuf; + checkInStream.limit = props->blockSize; + if (!inStream) + checkInStream.limit = inBufSize; + + if (fp) + { + #ifdef USE_SUBBLOCK + if (fp->id == XZ_ID_Subblock) + { + lzmaf->sb.inStream = &checkInStream.vt; + RINOK(SbEncInStream_Init(&lzmaf->sb)); + } + else + #endif + { + lzmaf->filter.realStream = &checkInStream.vt; + RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); + } + } + + { + SRes res; + Byte *outBuf = NULL; + size_t outSize = 0; + BoolInt useStream = (fp || inStream); + // useStream = True; + + if (!useStream) + { + XzCheck_Update(&checkInStream.check, inBuf, inBufSize); + checkInStream.processed = inBufSize; + } + + if (!outStream) + { + outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; + outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; + } + + res = Lzma2Enc_Encode2(lzmaf->lzma2, + outBuf ? NULL : &seqSizeOutStream.vt, + outBuf, + outBuf ? &outSize : NULL, + + useStream ? + (fp ? + ( + #ifdef USE_SUBBLOCK + (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: + #endif + &lzmaf->filter.p) : + &checkInStream.vt) : NULL, + + useStream ? NULL : inBuf, + useStream ? 0 : inBufSize, + + progress); + + if (outBuf) + seqSizeOutStream.processed += outSize; + + RINOK(res); + blockSizes->unpackSize = checkInStream.processed; + } + { + Byte buf[4 + 64]; + unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + UInt64 packSize = seqSizeOutStream.processed; + + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + + SeqCheckInStream_GetDigest(&checkInStream, buf + 4); + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); + + blockSizes->totalSize = seqSizeOutStream.processed - padSize; + + if (!outStream) + { + seqSizeOutStream.outBuf = outBufHeader; + seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; + seqSizeOutStream.processed = 0; + + block.unpackSize = blockSizes->unpackSize; + XzBlock_SetHasUnpackSize(&block); + + block.packSize = packSize; + XzBlock_SetHasPackSize(&block); + + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + + blockSizes->headerSize = (size_t)seqSizeOutStream.processed; + blockSizes->totalSize += seqSizeOutStream.processed; + } + } + + if (inStream) + *inStreamFinished = checkInStream.realStreamFinished; + else + { + *inStreamFinished = False; + if (checkInStream.processed != inBufSize) + return SZ_ERROR_FAIL; + } + + return SZ_OK; } -void XzFilterProps_Init(CXzFilterProps *p) + + +typedef struct { - p->id = 0; - p->delta = 0; - p->ip = 0; - p->ipDefined = False; + ICompressProgress vt; + ICompressProgress *progress; + UInt64 inOffset; + UInt64 outOffset; +} CCompressProgress_XzEncOffset; + + +static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); + inSize += p->inOffset; + outSize += p->outOffset; + return ICompressProgress_Progress(p->progress, inSize, outSize); } -static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf, - ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress) + + +typedef struct { - xz->flags = (Byte)props->checkId; + ISzAllocPtr alloc; + ISzAllocPtr allocBig; + + CXzProps xzProps; + UInt64 expectedDataSize; - RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props)); - RINOK(Xz_WriteHeader(xz->flags, outStream)); + CXzEncIndex xzIndex; + CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; + + size_t outBufSize; /* size of allocated outBufs[i] */ + Byte *outBufs[MTCODER__BLOCKS_MAX]; + + #ifndef _7ZIP_ST + unsigned checkType; + ISeqOutStream *outStream; + BoolInt mtCoder_WasConstructed; + CMtCoder mtCoder; + CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; + #endif + +} CXzEnc; + + +static void XzEnc_Construct(CXzEnc *p) +{ + unsigned i; + + XzEncIndex_Construct(&p->xzIndex); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); + + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; { - CSeqCheckInStream checkInStream; - CSeqSizeOutStream seqSizeOutStream; - CXzBlock block; - unsigned filterIndex = 0; - CXzFilter *filter = NULL; - const CXzFilterProps *fp = props->filterProps; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } + #endif +} + + +static void XzEnc_FreeOutBufs(CXzEnc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + + +static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) +{ + unsigned i; + + XzEncIndex_Free(&p->xzIndex, alloc); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); + + #ifndef _7ZIP_ST + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + XzEnc_FreeOutBufs(p); + #endif +} + + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); + if (!p) + return NULL; + XzEnc_Construct(p); + XzProps_Init(&p->xzProps); + XzProps_Normalize(&p->xzProps); + p->expectedDataSize = (UInt64)(Int64)-1; + p->alloc = alloc; + p->allocBig = allocBig; + return p; +} + + +void XzEnc_Destroy(CXzEncHandle pp) +{ + CXzEnc *p = (CXzEnc *)pp; + XzEnc_Free(p, p->alloc); + ISzAlloc_Free(p->alloc, p); +} + + +SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) +{ + CXzEnc *p = (CXzEnc *)pp; + p->xzProps = *props; + XzProps_Normalize(&p->xzProps); + return SZ_OK; +} + + +void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) +{ + CXzEnc *p = (CXzEnc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + + + +#ifndef _7ZIP_ST + +static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CXzEnc *me = (CXzEnc *)pp; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + UNUSED_VAR(finished) + + { + CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + bInfo->totalSize = 0; + bInfo->unpackSize = 0; + bInfo->headerSize = 0; + } + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + MtProgressThunk_Init(&progressThunk); + + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + res = Xz_CompressBlock( + &me->lzmaf_Items[coderIndex], + + NULL, + dest, + dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, + + NULL, + // srcSize, // expectedSize + src, srcSize, + + &me->xzProps, + &progressThunk.vt, + &inStreamFinished, + &blockSizes, + me->alloc, + me->allocBig); - XzBlock_ClearFlags(&block); - XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + if (res == SZ_OK) + me->EncBlocks[outBufIndex] = blockSizes; + + return res; + } +} + + +static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CXzEnc *me = (CXzEnc *)pp; + + const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); + + { + UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); + } + + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); +} + +#endif + + + +SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +{ + CXzEnc *p = (CXzEnc *)pp; + + const CXzProps *props = &p->xzProps; + + XzEncIndex_Init(&p->xzIndex); + { + UInt64 numBlocks = 1; + UInt64 blockSize = props->blockSize; - if (fp) + if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID + && props->reduceSize != (UInt64)(Int64)-1) { - filter = &block.filters[filterIndex++]; - filter->id = fp->id; - filter->propsSize = 0; - - if (fp->id == XZ_ID_Delta) - { - filter->props[0] = (Byte)(fp->delta - 1); - filter->propsSize = 1; - } - else if (fp->ipDefined) - { - SetUi32(filter->props, fp->ip); - filter->propsSize = 4; - } + numBlocks = props->reduceSize / blockSize; + if (numBlocks * blockSize != props->reduceSize) + numBlocks++; + } + else + blockSize = (UInt64)1 << 62; + + RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); + } + + RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); + + + #ifndef _7ZIP_ST + if (props->numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); } + vt.Code = XzEnc_MtCallback_Code; + vt.Write = XzEnc_MtCallback_Write; + + p->checkType = props->checkId; + p->xzProps = *props; + + p->outStream = outStream; + + p->mtCoder.allocBig = p->allocBig; + p->mtCoder.progress = progress; + p->mtCoder.inStream = inStream; + p->mtCoder.inData = NULL; + p->mtCoder.inDataSize = 0; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID + || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) + return SZ_ERROR_FAIL; + + p->mtCoder.blockSize = (size_t)props->blockSize; + if (p->mtCoder.blockSize != props->blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ + { - CXzFilter *f = &block.filters[filterIndex++]; - f->id = XZ_ID_LZMA2; - f->propsSize = 1; - f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + XzEnc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - seqSizeOutStream.p.Write = MyWrite; - seqSizeOutStream.realStream = outStream; - seqSizeOutStream.processed = 0; + p->mtCoder.numThreadsMax = (unsigned)props->numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p)); + RINOK(MtCoder_Code(&p->mtCoder)); + } + else + #endif + { + int writeStartSizes; + CCompressProgress_XzEncOffset progress2; + Byte *bufData = NULL; + size_t bufSize = 0; + + progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; + progress2.inOffset = 0; + progress2.outOffset = 0; + progress2.progress = progress; - checkInStream.p.Read = SeqCheckInStream_Read; - checkInStream.realStream = inStream; - SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags)); + writeStartSizes = 0; - if (fp) + if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) { - #ifdef USE_SUBBLOCK - if (fp->id == XZ_ID_Subblock) - { - lzmaf->sb.inStream = &checkInStream.p; - RINOK(SbEncInStream_Init(&lzmaf->sb)); - } - else - #endif + writeStartSizes = (props->forceWriteSizesInHeader > 0); + + if (writeStartSizes) { - lzmaf->filter.realStream = &checkInStream.p; - RINOK(SeqInFilter_Init(&lzmaf->filter, filter)); + size_t t2; + size_t t = (size_t)props->blockSize; + if (t != props->blockSize) + return SZ_ERROR_PARAM; + t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); + if (t < props->blockSize) + return SZ_ERROR_PARAM; + t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; + if (!p->outBufs[0] || t2 != p->outBufSize) + { + XzEnc_FreeOutBufs(p); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); + if (!p->outBufs[0]) + return SZ_ERROR_MEM; + p->outBufSize = t2; + } + bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; + bufSize = t; } } - + + for (;;) { - UInt64 packPos = seqSizeOutStream.processed; + CXzEncBlockInfo blockSizes; + int inStreamFinished; - SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p, - fp ? - #ifdef USE_SUBBLOCK - (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p: - #endif - &lzmaf->filter.p: - &checkInStream.p, - progress); + /* + UInt64 rem = (UInt64)(Int64)-1; + if (props->reduceSize != (UInt64)(Int64)-1 + && props->reduceSize >= progress2.inOffset) + rem = props->reduceSize - progress2.inOffset; + */ + + blockSizes.headerSize = 0; // for GCC - RINOK(res); - block.unpackSize = checkInStream.processed; - block.packSize = seqSizeOutStream.processed - packPos; - } + RINOK(Xz_CompressBlock( + &p->lzmaf_Items[0], + + writeStartSizes ? NULL : outStream, + writeStartSizes ? p->outBufs[0] : NULL, + bufData, bufSize, + + inStream, + // rem, + NULL, 0, + + props, + progress ? &progress2.vt : NULL, + &inStreamFinished, + &blockSizes, + p->alloc, + p->allocBig)); - { - unsigned padSize = 0; - Byte buf[128]; - while ((((unsigned)block.packSize + padSize) & 3) != 0) - buf[padSize++] = 0; - SeqCheckInStream_GetDigest(&checkInStream, buf + padSize); - RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags))); - RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc)); + { + UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); + + if (writeStartSizes) + { + RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); + RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); + } + + RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); + + progress2.inOffset += blockSizes.unpackSize; + progress2.outOffset += totalPackFull; + } + + if (inStreamFinished) + break; } } - return Xz_WriteFooter(xz, outStream); + + return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); } +#include "Alloc.h" + SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress) { SRes res; - CXzStream xz; - CLzma2WithFilters lzmaf; - Xz_Construct(&xz); - Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc); - res = Lzma2WithFilters_Create(&lzmaf); + CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!xz) + return SZ_ERROR_MEM; + res = XzEnc_SetProps(xz, props); if (res == SZ_OK) - res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress); - Lzma2WithFilters_Free(&lzmaf); - Xz_Free(&xz, &g_Alloc); + res = XzEnc_Encode(xz, outStream, inStream, progress); + XzEnc_Destroy(xz); return res; } @@ -528,11 +1320,11 @@ SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, SRes Xz_EncodeEmpty(ISeqOutStream *outStream) { SRes res; - CXzStream xz; - Xz_Construct(&xz); - res = Xz_WriteHeader(xz.flags, outStream); + CXzEncIndex xzIndex; + XzEncIndex_Construct(&xzIndex); + res = Xz_WriteHeader((CXzStreamFlags)0, outStream); if (res == SZ_OK) - res = Xz_WriteFooter(&xz, outStream); - Xz_Free(&xz, &g_Alloc); + res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); + XzEncIndex_Free(&xzIndex, NULL); // g_Alloc return res; } diff --git a/3rdparty/lzma/C/XzEnc.h b/3rdparty/lzma/C/XzEnc.h index c3c19eca008..0c29e7e1e12 100644 --- a/3rdparty/lzma/C/XzEnc.h +++ b/3rdparty/lzma/C/XzEnc.h @@ -1,5 +1,5 @@ /* XzEnc.h -- Xz Encode -2011-02-07 : Igor Pavlov : Public domain */ +2017-06-27 : Igor Pavlov : Public domain */ #ifndef __XZ_ENC_H #define __XZ_ENC_H @@ -10,6 +10,11 @@ EXTERN_C_BEGIN + +#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO +#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + + typedef struct { UInt32 id; @@ -20,15 +25,31 @@ typedef struct void XzFilterProps_Init(CXzFilterProps *p); + typedef struct { - const CLzma2EncProps *lzma2Props; - const CXzFilterProps *filterProps; + CLzma2EncProps lzma2Props; + CXzFilterProps filterProps; unsigned checkId; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; + int numTotalThreads; + int forceWriteSizesInHeader; + UInt64 reduceSize; } CXzProps; void XzProps_Init(CXzProps *p); + +typedef void * CXzEncHandle; + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); +void XzEnc_Destroy(CXzEncHandle p); +SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); +void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); + SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress); diff --git a/3rdparty/lzma/C/XzIn.c b/3rdparty/lzma/C/XzIn.c index 07b1cc39516..84f868ec657 100644 --- a/3rdparty/lzma/C/XzIn.c +++ b/3rdparty/lzma/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2015-11-08 : Igor Pavlov : Public domain */ +2021-09-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -9,6 +9,12 @@ #include "CpuArch.h" #include "Xz.h" +/* +#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0) +*/ +#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) + + SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) { Byte sig[XZ_STREAM_HEADER_SIZE]; @@ -20,15 +26,16 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ - if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } + if (s == 0) return SZ_ERROR_ARCHIVE; \ + pos += s; } -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes) +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; unsigned headerSize; *headerSizeRes = 0; RINOK(SeqInStream_ReadByte(inStream, &header[0])); - headerSize = ((unsigned)header[0] << 2) + 4; + headerSize = (unsigned)header[0]; if (headerSize == 0) { *headerSizeRes = 1; @@ -37,12 +44,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt } *isIndex = False; + headerSize = (headerSize << 2) + 4; *headerSizeRes = headerSize; RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); return XzBlock_Parse(p, header); } -#define ADD_SIZE_CHECH(size, val) \ +#define ADD_SIZE_CHECK(size, val) \ { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } UInt64 Xz_GetUnpackSize(const CXzStream *p) @@ -50,7 +58,7 @@ UInt64 Xz_GetUnpackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, p->blocks[i].unpackSize); + ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); return size; } @@ -59,7 +67,7 @@ UInt64 Xz_GetPackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); + ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); return size; } @@ -70,7 +78,7 @@ SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStream *inStream) } */ -static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc *alloc) +static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc) { size_t numBlocks, pos = 1; UInt32 crc; @@ -96,9 +104,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc * { size_t i; p->numBlocks = numBlocks; - p->numBlocksAllocated = numBlocks; - p->blocks = alloc->Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); - if (p->blocks == 0) + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) { @@ -115,7 +122,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc * return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; } -static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAlloc *alloc) +static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAllocPtr alloc) { SRes res; size_t size; @@ -125,13 +132,13 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, size = (size_t)indexSize; if (size != indexSize) return SZ_ERROR_UNSUPPORTED; - buf = alloc->Alloc(alloc, size); - if (buf == 0) + buf = (Byte *)ISzAlloc_Alloc(alloc, size); + if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); if (res == SZ_OK) res = Xz_ReadIndex2(p, buf, size, alloc); - alloc->Free(alloc, buf); + ISzAlloc_Free(alloc, buf); return res; } @@ -142,11 +149,11 @@ static SRes LookInStream_SeekRead_ForArc(ILookInStream *stream, UInt64 offset, v /* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */ } -static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAlloc *alloc) +static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAllocPtr alloc) { UInt64 indexSize; Byte buf[XZ_STREAM_FOOTER_SIZE]; - UInt64 pos = *startOffset; + UInt64 pos = (UInt64)*startOffset; if ((pos & 3) != 0 || pos < XZ_STREAM_FOOTER_SIZE) return SZ_ERROR_NO_ARCHIVE; @@ -154,7 +161,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) { UInt32 total = 0; pos += XZ_STREAM_FOOTER_SIZE; @@ -187,7 +194,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_NO_ARCHIVE; pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) return SZ_ERROR_NO_ARCHIVE; } @@ -196,8 +203,13 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff if (!XzFlags_IsSupported(p->flags)) return SZ_ERROR_UNSUPPORTED; - if (GetUi32(buf) != CrcCalc(buf + 4, 6)) - return SZ_ERROR_ARCHIVE; + { + /* to eliminate GCC 6.3 warning: + dereferencing type-punned pointer will break strict-aliasing rules */ + const Byte *buf_ptr = buf; + if (GetUi32(buf_ptr) != CrcCalc(buf + 4, 6)) + return SZ_ERROR_ARCHIVE; + } indexSize = ((UInt64)GetUi32(buf + 4) + 1) << 2; @@ -216,7 +228,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_ARCHIVE; pos -= (totalSize + XZ_STREAM_HEADER_SIZE); RINOK(LookInStream_SeekTo(stream, pos)); - *startOffset = pos; + *startOffset = (Int64)pos; } { CXzStreamFlags headerFlags; @@ -224,7 +236,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff SecToRead_CreateVTable(&secToRead); secToRead.realStream = stream; - RINOK(Xz_ReadHeader(&headerFlags, &secToRead.s)); + RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)); return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE; } } @@ -238,12 +250,12 @@ void Xzs_Construct(CXzs *p) p->streams = 0; } -void Xzs_Free(CXzs *p, ISzAlloc *alloc) +void Xzs_Free(CXzs *p, ISzAllocPtr alloc) { size_t i; for (i = 0; i < p->num; i++) Xz_Free(&p->streams[i], alloc); - alloc->Free(alloc, p->streams); + ISzAlloc_Free(alloc, p->streams); p->num = p->numAllocated = 0; p->streams = 0; } @@ -262,7 +274,7 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); return size; } @@ -272,15 +284,15 @@ UInt64 Xzs_GetPackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); return size; } */ -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAlloc *alloc) +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc) { Int64 endOffset = 0; - RINOK(stream->Seek(stream, &endOffset, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)); *startOffset = endOffset; for (;;) { @@ -288,25 +300,25 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr SRes res; Xz_Construct(&st); res = Xz_ReadBackward(&st, stream, startOffset, alloc); - st.startOffset = *startOffset; + st.startOffset = (UInt64)*startOffset; RINOK(res); if (p->num == p->numAllocated) { - size_t newNum = p->num + p->num / 4 + 1; - Byte *data = (Byte *)alloc->Alloc(alloc, newNum * sizeof(CXzStream)); - if (data == 0) + const size_t newNum = p->num + p->num / 4 + 1; + void *data = ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); + if (!data) return SZ_ERROR_MEM; p->numAllocated = newNum; if (p->num != 0) memcpy(data, p->streams, p->num * sizeof(CXzStream)); - alloc->Free(alloc, p->streams); + ISzAlloc_Free(alloc, p->streams); p->streams = (CXzStream *)data; } p->streams[p->num++] = st; if (*startOffset == 0) break; - RINOK(LookInStream_SeekTo(stream, *startOffset)); - if (progress && progress->Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) + RINOK(LookInStream_SeekTo(stream, (UInt64)*startOffset)); + if (progress && ICompressProgress_Progress(progress, (UInt64)(endOffset - *startOffset), (UInt64)(Int64)-1) != SZ_OK) return SZ_ERROR_PROGRESS; } return SZ_OK; diff --git a/3rdparty/lzma/C/var_clang.mak b/3rdparty/lzma/C/var_clang.mak new file mode 100644 index 00000000000..a6df26e77c8 --- /dev/null +++ b/3rdparty/lzma/C/var_clang.mak @@ -0,0 +1,11 @@ +PLATFORM= +O=b/c +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/C/var_clang_arm64.mak b/3rdparty/lzma/C/var_clang_arm64.mak new file mode 100644 index 00000000000..4b354096956 --- /dev/null +++ b/3rdparty/lzma/C/var_clang_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/C/var_clang_x64.mak b/3rdparty/lzma/C/var_clang_x64.mak new file mode 100644 index 00000000000..34e1b49c5e2 --- /dev/null +++ b/3rdparty/lzma/C/var_clang_x64.mak @@ -0,0 +1,11 @@ +PLATFORM=x64 +O=b/c_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/C/var_clang_x86.mak b/3rdparty/lzma/C/var_clang_x86.mak new file mode 100644 index 00000000000..bd2317c299f --- /dev/null +++ b/3rdparty/lzma/C/var_clang_x86.mak @@ -0,0 +1,11 @@ +PLATFORM=x86 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/C/var_gcc.mak b/3rdparty/lzma/C/var_gcc.mak new file mode 100644 index 00000000000..664491cf2fc --- /dev/null +++ b/3rdparty/lzma/C/var_gcc.mak @@ -0,0 +1,12 @@ +PLATFORM= +O=b/g +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/3rdparty/lzma/C/var_gcc_arm64.mak b/3rdparty/lzma/C/var_gcc_arm64.mak new file mode 100644 index 00000000000..4bbb687dbb3 --- /dev/null +++ b/3rdparty/lzma/C/var_gcc_arm64.mak @@ -0,0 +1,12 @@ +PLATFORM=arm64 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-mtune=cortex-a53 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/3rdparty/lzma/C/var_gcc_x64.mak b/3rdparty/lzma/C/var_gcc_x64.mak new file mode 100644 index 00000000000..1acf604f5f2 --- /dev/null +++ b/3rdparty/lzma/C/var_gcc_x64.mak @@ -0,0 +1,10 @@ +PLATFORM=x64 +O=b/g_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ diff --git a/3rdparty/lzma/C/var_gcc_x86.mak b/3rdparty/lzma/C/var_gcc_x86.mak new file mode 100644 index 00000000000..f0718ec710e --- /dev/null +++ b/3rdparty/lzma/C/var_gcc_x86.mak @@ -0,0 +1,10 @@ +PLATFORM=x86 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ diff --git a/3rdparty/lzma/C/var_mac_arm64.mak b/3rdparty/lzma/C/var_mac_arm64.mak new file mode 100644 index 00000000000..adf5fa1d703 --- /dev/null +++ b/3rdparty/lzma/C/var_mac_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/m_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-arch arm64 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/C/var_mac_x64.mak b/3rdparty/lzma/C/var_mac_x64.mak new file mode 100644 index 00000000000..13d7aa7fedf --- /dev/null +++ b/3rdparty/lzma/C/var_mac_x64.mak @@ -0,0 +1,11 @@ +PLATFORM=x64 +O=b/m_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-arch x86_64 +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/C/warn_clang.mak b/3rdparty/lzma/C/warn_clang.mak new file mode 100644 index 00000000000..ed4f908f607 --- /dev/null +++ b/3rdparty/lzma/C/warn_clang.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_1 = \ + -Wno-deprecated-copy-dtor \ + + + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_1) diff --git a/3rdparty/lzma/C/warn_clang_mac.mak b/3rdparty/lzma/C/warn_clang_mac.mak new file mode 100644 index 00000000000..41044a2ce6c --- /dev/null +++ b/3rdparty/lzma/C/warn_clang_mac.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_MAC = \ + -Wno-poison-system-directories \ + -Wno-c++11-long-long \ + -Wno-atomic-implicit-seq-cst \ + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_MAC) diff --git a/3rdparty/lzma/C/warn_gcc.mak b/3rdparty/lzma/C/warn_gcc.mak new file mode 100644 index 00000000000..7aab7a44937 --- /dev/null +++ b/3rdparty/lzma/C/warn_gcc.mak @@ -0,0 +1,51 @@ +CFLAGS_WARN_GCC_4_5 = \ + +CFLAGS_WARN_GCC_6 = \ + -Waddress \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wcast-align \ + -Wcomment \ + -Wdiv-by-zero \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Winit-self \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + +# -Wno-strict-aliasing + +CFLAGS_WARN_GCC_9 = \ + -Waddress \ + -Waddress-of-packed-member \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wbool-operation \ + -Wcast-align \ + -Wcast-align=strict \ + -Wcomment \ + -Wdangling-else \ + -Wdiv-by-zero \ + -Wduplicated-branches \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Wimplicit-fallthrough=5 \ + -Winit-self \ + -Wint-in-bool-context \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + -Wconversion \ + +# -Wno-sign-conversion \ + +CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ + -Wno-strict-aliasing \ + + +CFLAGS_WARN = $(CFLAGS_WARN_GCC_9) \ + +# $(CFLAGS_WARN_GCC_PPMD_UNALIGNED) diff --git a/3rdparty/lzma/CPP/7zip/7zip_gcc.mak b/3rdparty/lzma/CPP/7zip/7zip_gcc.mak new file mode 100644 index 00000000000..f65cff2eddf --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/7zip_gcc.mak @@ -0,0 +1,1259 @@ +# USE_CLANG=1 +# USE_ASM = 1 +# IS_X64 = 1 +# MY_ARCH = +# USE_ASM= +# USE_JWASM=1 + +MY_ARCH_2 = $(MY_ARCH) + +MY_ASM = asmc +ifdef USE_JWASM +MY_ASM = jwasm +endif + + +PROGPATH = $(O)/$(PROG) +PROGPATH_STATIC = $(O)/$(PROG)s + + +ifneq ($(CC), xlc) +CFLAGS_WARN_WALL = -Wall -Werror -Wextra +endif + +# for object file +CFLAGS_BASE_LIST = -c +# CFLAGS_BASE_LIST = -S +CFLAGS_BASE = -O2 $(CFLAGS_BASE_LIST) $(CFLAGS_WARN_WALL) $(CFLAGS_WARN) \ + -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \ + -fPIC + +# -D_7ZIP_AFFINITY_DISABLE + + +ifdef SystemDrive +IS_MINGW = 1 +else +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif +endif + +ifdef IS_MINGW +LDFLAGS_STATIC_2 = -static +else +ifndef DEF_FILE +ifndef IS_NOT_STANDALONE +ifndef MY_DYNAMIC_LINK +ifneq ($(CC), clang) +LDFLAGS_STATIC_2 = +# -static +# -static-libstdc++ -static-libgcc +endif +endif +endif +endif +endif + +LDFLAGS_STATIC = -DNDEBUG $(LDFLAGS_STATIC_2) + +ifndef O + ifdef IS_MINGW + O=_o + else + O=_o + endif +endif + + +ifdef DEF_FILE + + +ifdef IS_MINGW +SHARED_EXT=.dll +LDFLAGS = -shared -DEF $(DEF_FILE) $(LDFLAGS_STATIC) +else +SHARED_EXT=.so +LDFLAGS = -shared -fPIC $(LDFLAGS_STATIC) +CC_SHARED=-fPIC +endif + + +else + +LDFLAGS = $(LDFLAGS_STATIC) +# -s is not required for clang, do we need it for GGC ??? +# -s + +#-static -static-libgcc -static-libstdc++ + +ifdef IS_MINGW +SHARED_EXT=.exe +else +SHARED_EXT= +endif + +endif + + +PROGPATH = $(O)/$(PROG)$(SHARED_EXT) +PROGPATH_STATIC = $(O)/$(PROG)s$(SHARED_EXT) + +ifdef IS_MINGW + +ifdef MSYSTEM +RM = rm -f +MY_MKDIR=mkdir -p +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) +LIB_HTMLHELP=-lhtmlhelp +else +RM = del +MY_MKDIR=mkdir +DEL_OBJ_EXE = -$(RM) $(O)\*.o $(O)\$(PROG).exe $(O)\$(PROG).dll +endif + +LIB2_GUI = -lOle32 -lGdi32 -lComctl32 -lComdlg32 $(LIB_HTMLHELP) +LIB2 = -loleaut32 -luuid -ladvapi32 -lUser32 $(LIB2_GUI) + +CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE +# -Wno-delete-non-virtual-dtor + + +else + +RM = rm -f +MY_MKDIR=mkdir -p +DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) + +# CFLAGS_BASE := $(CFLAGS_BASE) -D_7ZIP_ST +# CXXFLAGS_EXTRA = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE + +# LOCAL_LIBS=-lpthread +# LOCAL_LIBS_DLL=$(LOCAL_LIBS) -ldl +LIB2 = -lpthread -ldl + + +endif + + + +CFLAGS = $(MY_ARCH_2) $(LOCAL_FLAGS) $(CFLAGS_BASE2) $(CFLAGS_BASE) $(CC_SHARED) -o $@ + + +ifdef IS_MINGW + +ifdef IS_X64 +AFLAGS_ABI = -win64 +else +AFLAGS_ABI = -coff -DABI_CDECL +# -DABI_CDECL +# -DABI_LINUX +# -DABI_CDECL +endif +AFLAGS = -nologo $(AFLAGS_ABI) -Fo$(O)/$(basename $( predef_cc_log +# $(CC) $(CFLAGS) -E -dM - < /dev/null +predef_cxx: + $(CXX) $(CFLAGS) -E $(SHOW_PREDEF) ../../../Common/CrcReg.cpp > predef_cxx_log + +predef: predef_cc predef_cxx + + +clean: + -$(DEL_OBJ_EXE) diff --git a/3rdparty/lzma/CPP/7zip/Aes.mak b/3rdparty/lzma/CPP/7zip/Aes.mak index c5a58f28f4a..7d8da2d8101 100644 --- a/3rdparty/lzma/CPP/7zip/Aes.mak +++ b/3rdparty/lzma/CPP/7zip/Aes.mak @@ -1,7 +1,10 @@ C_OBJS = $(C_OBJS) \ $O\Aes.obj -!IF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" && "$(CPU)" != "ARM" +!IF defined(USE_C_AES) || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +C_OBJS = $(C_OBJS) \ + $O\AesOpt.obj +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" ASM_OBJS = $(ASM_OBJS) \ $O\AesOpt.obj !ENDIF diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zCompressionMode.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zCompressionMode.h index 8105ff0413d..9e8463452d6 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -13,7 +13,11 @@ struct CMethodFull: public CMethodProps { CMethodId Id; UInt32 NumStreams; + int CodecIndex; + UInt32 NumThreads; + bool Set_NumThreads; + CMethodFull(): CodecIndex(-1), NumThreads(1), Set_NumThreads(false) {} bool IsSimpleCoder() const { return NumStreams == 1; } }; @@ -51,22 +55,31 @@ struct CCompressionMethodMode #ifndef _7ZIP_ST UInt32 NumThreads; + bool NumThreads_WasForced; bool MultiThreadMixer; #endif + + UInt64 MemoryUsageLimit; + bool MemoryUsageLimit_WasSet; bool PasswordIsDefined; - UString Password; + UString Password; // _Wipe bool IsEmpty() const { return (Methods.IsEmpty() && !PasswordIsDefined); } CCompressionMethodMode(): - DefaultMethod_was_Inserted(false), - Filter_was_Inserted(false), - PasswordIsDefined(false) + DefaultMethod_was_Inserted(false) + , Filter_was_Inserted(false) #ifndef _7ZIP_ST , NumThreads(1) + , NumThreads_WasForced(false) , MultiThreadMixer(true) #endif + , MemoryUsageLimit((UInt64)1 << 30) + , MemoryUsageLimit_WasSet(false) + , PasswordIsDefined(false) {} + + ~CCompressionMethodMode() { Password.Wipe_and_Empty(); } }; }} diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.cpp index b0d6dd838aa..c27c8fbc6ac 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.cpp @@ -158,7 +158,7 @@ STDMETHODIMP CLockedSequentialInStreamMT::Read(void *data, UInt32 size, UInt32 * if (_pos != _glob->Pos) { - RINOK(_glob->Stream->Seek(_pos, STREAM_SEEK_SET, NULL)); + RINOK(_glob->Stream->Seek((Int64)_pos, STREAM_SEEK_SET, NULL)); _glob->Pos = _pos; } @@ -200,7 +200,7 @@ STDMETHODIMP CLockedSequentialInStreamST::Read(void *data, UInt32 size, UInt32 * { if (_pos != _glob->Pos) { - RINOK(_glob->Stream->Seek(_pos, STREAM_SEEK_SET, NULL)); + RINOK(_glob->Stream->Seek((Int64)_pos, STREAM_SEEK_SET, NULL)); _glob->Pos = _pos; } @@ -226,19 +226,23 @@ HRESULT CDecoder::Decode( , ISequentialOutStream *outStream , ICompressProgressInfo *compressProgress + , ISequentialInStream ** + #ifdef USE_MIXER_ST + inStreamMainRes + #endif - #ifdef USE_MIXER_ST - inStreamMainRes - #endif + , bool &dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ) { + dataAfterEnd_Error = false; + const UInt64 *packPositions = &folders.PackPositions[folders.FoStartPackStreamIndex[folderIndex]]; CFolderEx folderInfo; folders.ParseFolderEx(folderIndex, folderInfo); @@ -272,6 +276,7 @@ HRESULT CDecoder::Decode( if (!_bindInfoPrev_Defined || !AreBindInfoExEqual(bindInfo, _bindInfoPrev)) { + _bindInfoPrev_Defined = false; _mixerRef.Release(); #ifdef USE_MIXER_MT @@ -308,7 +313,7 @@ HRESULT CDecoder::Decode( #endif CCreatedCoder cod; - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS coderInfo.MethodID, false, cod)); @@ -344,46 +349,64 @@ HRESULT CDecoder::Decode( _bindInfoPrev_Defined = true; } - _mixer->ReInit(); + RINOK(_mixer->ReInit2()); UInt32 packStreamIndex = 0; UInt32 unpackStreamIndexStart = folders.FoToCoderUnpackSizes[folderIndex]; unsigned i; + #if !defined(_7ZIP_ST) + bool mt_wasUsed = false; + #endif + for (i = 0; i < folderInfo.Coders.Size(); i++) { const CCoderInfo &coderInfo = folderInfo.Coders[i]; IUnknown *decoder = _mixer->GetCoder(i).GetUnknown(); + #if !defined(_7ZIP_ST) + if (!mt_wasUsed) + { + if (mtMode) + { + CMyComPtr setCoderMt; + decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); + if (setCoderMt) + { + mt_wasUsed = true; + RINOK(setCoderMt->SetNumberOfThreads(numThreads)); + } + } + // if (memUsage != 0) + { + CMyComPtr setMemLimit; + decoder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + mt_wasUsed = true; + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } + } + #endif + { CMyComPtr setDecoderProperties; decoder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); if (setDecoderProperties) { const CByteBuffer &props = coderInfo.Props; - size_t size = props.Size(); - if (size > 0xFFFFFFFF) + const UInt32 size32 = (UInt32)props.Size(); + if (props.Size() != size32) return E_NOTIMPL; - HRESULT res = setDecoderProperties->SetDecoderProperties2((const Byte *)props, (UInt32)size); + HRESULT res = setDecoderProperties->SetDecoderProperties2((const Byte *)props, size32); if (res == E_INVALIDARG) res = E_NOTIMPL; RINOK(res); } } - #if !defined(_7ZIP_ST) && !defined(_SFX) - if (mtMode) - { - CMyComPtr setCoderMt; - decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); - if (setCoderMt) - { - RINOK(setCoderMt->SetNumberOfThreads(numThreads)); - } - } - #endif - #ifndef _NO_CRYPTO { CMyComPtr cryptoSetPassword; @@ -393,17 +416,17 @@ HRESULT CDecoder::Decode( isEncrypted = true; if (!getTextPassword) return E_NOTIMPL; - CMyComBSTR passwordBSTR; + CMyComBSTR_Wipe passwordBSTR; RINOK(getTextPassword->CryptoGetTextPassword(&passwordBSTR)); passwordIsDefined = true; - password.Empty(); + password.Wipe_and_Empty(); size_t len = 0; if (passwordBSTR) { password = passwordBSTR; len = password.Len(); } - CByteBuffer buffer(len * 2); + CByteBuffer_Wipe buffer(len * 2); for (size_t k = 0; k < len; k++) { wchar_t c = passwordBSTR[k]; @@ -415,12 +438,14 @@ HRESULT CDecoder::Decode( } #endif + bool finishMode = false; { CMyComPtr setFinishMode; decoder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); if (setFinishMode) { - RINOK(setFinishMode->SetFinishMode(BoolToInt(fullUnpack))); + finishMode = fullUnpack; + RINOK(setFinishMode->SetFinishMode(BoolToUInt(finishMode))); } } @@ -450,7 +475,7 @@ HRESULT CDecoder::Decode( unpackSize : &folders.CoderUnpackSizes[unpackStreamIndexStart + i]; - _mixer->SetCoderInfo(i, unpackSizesPointer, packSizesPointers); + _mixer->SetCoderInfo(i, unpackSizesPointer, packSizesPointers, finishMode); } if (outStream) @@ -463,36 +488,49 @@ HRESULT CDecoder::Decode( CLockedInStream *lockedInStreamSpec = new CLockedInStream; CMyComPtr lockedInStream = lockedInStreamSpec; - bool needMtLock = false; + #ifdef USE_MIXER_MT + #ifdef USE_MIXER_ST + bool needMtLock = _useMixerMT; + #endif + #endif if (folderInfo.PackStreams.Size() > 1) { // lockedInStream.Pos = (UInt64)(Int64)-1; // RINOK(inStream->Seek(0, STREAM_SEEK_CUR, &lockedInStream.Pos)); - RINOK(inStream->Seek(startPos + packPositions[0], STREAM_SEEK_SET, &lockedInStreamSpec->Pos)); + RINOK(inStream->Seek((Int64)(startPos + packPositions[0]), STREAM_SEEK_SET, &lockedInStreamSpec->Pos)); lockedInStreamSpec->Stream = inStream; + #ifdef USE_MIXER_MT #ifdef USE_MIXER_ST - if (_mixer->IsThere_ExternalCoder_in_PackTree(_mixer->MainCoderIndex)) - #endif + /* + For ST-mixer mode: + If parallel input stream reading from pack streams is possible, + we must use MT-lock for packed streams. + Internal decoders in 7-Zip will not read pack streams in parallel in ST-mixer mode. + So we force to needMtLock mode only if there is unknown (external) decoder. + */ + if (!needMtLock && _mixer->IsThere_ExternalCoder_in_PackTree(_mixer->MainCoderIndex)) needMtLock = true; + #endif + #endif } for (unsigned j = 0; j < folderInfo.PackStreams.Size(); j++) { CMyComPtr packStream; - UInt64 packPos = startPos + packPositions[j]; + const UInt64 packPos = startPos + packPositions[j]; if (folderInfo.PackStreams.Size() == 1) { - RINOK(inStream->Seek(packPos, STREAM_SEEK_SET, NULL)); + RINOK(inStream->Seek((Int64)packPos, STREAM_SEEK_SET, NULL)); packStream = inStream; } else { #ifdef USE_MIXER_MT #ifdef USE_MIXER_ST - if (_useMixerMT || needMtLock) + if (needMtLock) #endif { CLockedSequentialInStreamMT *lockedStreamImpSpec = new CLockedSequentialInStreamMT; @@ -518,7 +556,7 @@ HRESULT CDecoder::Decode( streamSpec->Init(packPositions[j + 1] - packPositions[j]); } - unsigned num = inStreams.Size(); + const unsigned num = inStreams.Size(); CObjArray inStreamPointers(num); for (i = 0; i < num; i++) inStreamPointers[i] = inStreams[i]; @@ -530,7 +568,9 @@ HRESULT CDecoder::Decode( progress2 = new CDecProgress(compressProgress); ISequentialOutStream *outStreamPointer = outStream; - return _mixer->Code(inStreamPointers, &outStreamPointer, progress2 ? (ICompressProgressInfo *)progress2 : compressProgress); + return _mixer->Code(inStreamPointers, &outStreamPointer, + progress2 ? (ICompressProgressInfo *)progress2 : compressProgress, + dataAfterEnd_Error); } #ifdef USE_MIXER_ST diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.h index 5b729f6c86d..eeb146e384c 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zDecode.h @@ -53,12 +53,14 @@ public: , ISequentialOutStream *outStream , ICompressProgressInfo *compressProgress + , ISequentialInStream **inStreamMainRes + , bool &dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ); }; diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.cpp index 97e9ad7adb8..83b0f18f4bc 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.cpp @@ -154,9 +154,18 @@ HRESULT CEncoder::CreateMixerCoder( CCreatedCoder cod; - RINOK(CreateCoder( + if (methodFull.CodecIndex >= 0) + { + RINOK(CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + (unsigned)methodFull.CodecIndex, true, cod)); + } + else + { + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodFull.Id, true, cod)); + } if (cod.NumStreams != methodFull.NumStreams) return E_FAIL; @@ -166,12 +175,16 @@ HRESULT CEncoder::CreateMixerCoder( CMyComPtr encoderCommon = cod.Coder ? (IUnknown *)cod.Coder : (IUnknown *)cod.Coder2; #ifndef _7ZIP_ST + if (methodFull.Set_NumThreads) { CMyComPtr setCoderMt; encoderCommon.QueryInterface(IID_ICompressSetCoderMt, &setCoderMt); if (setCoderMt) { - RINOK(setCoderMt->SetNumberOfThreads(_options.NumThreads)); + RINOK(setCoderMt->SetNumberOfThreads( + /* _options.NumThreads */ + methodFull.NumThreads + )); } } #endif @@ -206,7 +219,7 @@ HRESULT CEncoder::CreateMixerCoder( if (cryptoSetPassword) { const unsigned sizeInBytes = _options.Password.Len() * 2; - CByteBuffer buffer(sizeInBytes); + CByteBuffer_Wipe buffer(sizeInBytes); for (unsigned i = 0; i < _options.Password.Len(); i++) { wchar_t c = _options.Password[i]; @@ -240,11 +253,12 @@ public: STDMETHODIMP CSequentialOutTempBufferImp2::Write(const void *data, UInt32 size, UInt32 *processed) { - if (!_buf->Write(data, size)) + HRESULT res = _buf->Write_HRESULT(data, size); + if (res != S_OK) { if (processed) *processed = 0; - return E_FAIL; + return res; } if (processed) *processed = size; @@ -300,7 +314,7 @@ HRESULT CEncoder::Encode( RINOK(CreateMixerCoder(EXTERNAL_CODECS_LOC_VARS inSizeForReduce)); } - _mixer->ReInit(); + RINOK(_mixer->ReInit2()); CMtEncMultiProgress *mtProgressSpec = NULL; CMyComPtr mtProgress; @@ -333,7 +347,7 @@ HRESULT CEncoder::Encode( } for (i = 0; i < numMethods; i++) - _mixer->SetCoderInfo(i, NULL, NULL); + _mixer->SetCoderInfo(i, NULL, NULL, false); /* inStreamSize can be used by BCJ2 to set optimal range of conversion. @@ -370,6 +384,17 @@ HRESULT CEncoder::Encode( resetInitVector->ResetInitVector(); } + { + CMyComPtr optProps; + coder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps); + if (optProps) + { + PROPID propID = NCoderPropID::kExpectedDataSize; + NWindows::NCOM::CPropVariant prop = (UInt64)unpackSize; + RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1)); + } + } + CMyComPtr writeCoderProperties; coder->QueryInterface(IID_ICompressWriteCoderProperties, (void **)&writeCoderProperties); @@ -380,7 +405,7 @@ HRESULT CEncoder::Encode( CDynBufSeqOutStream *outStreamSpec = new CDynBufSeqOutStream; CMyComPtr dynOutStream(outStreamSpec); outStreamSpec->Init(); - writeCoderProperties->WriteCoderProperties(dynOutStream); + RINOK(writeCoderProperties->WriteCoderProperties(dynOutStream)); outStreamSpec->CopyToBuffer(props); } else @@ -429,10 +454,12 @@ HRESULT CEncoder::Encode( for (i = 1; i < _bindInfo.PackStreams.Size(); i++) outStreamPointers.Add(tempBuffers[i - 1]); + bool dataAfterEnd_Error; + RINOK(_mixer->Code( &inStreamPointer, &outStreamPointers.Front(), - mtProgress ? (ICompressProgressInfo *)mtProgress : compressProgress)); + mtProgress ? (ICompressProgressInfo *)mtProgress : compressProgress, dataAfterEnd_Error)); if (_bindInfo.PackStreams.Size() != 0) packSizes.Add(outStreamSizeCountSpec->GetSize()); @@ -456,7 +483,7 @@ HRESULT CEncoder::Encode( unpackSize = streamSize; } else - streamSize = _mixer->GetBondStreamSize(bond); + streamSize = _mixer->GetBondStreamSize((unsigned)bond); coderUnpackSizes.Add(streamSize); } @@ -587,13 +614,13 @@ HRESULT CEncoder::EncoderConstr() int bond = _bindInfo.FindBond_for_PackStream(outIndex); if (bond >= 0) { - ci = _bindInfo.Bonds[bond].UnpackIndex; + ci = _bindInfo.Bonds[(unsigned)bond].UnpackIndex; continue; } int si = _bindInfo.FindStream_in_PackStreams(outIndex); if (si >= 0) - _bindInfo.PackStreams.MoveToFront(si); + _bindInfo.PackStreams.MoveToFront((unsigned)si); break; } } diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.h index f1a9b5ad720..6ea7f276902 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zEncode.h @@ -41,7 +41,7 @@ public: STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; -class CEncoder +class CEncoder MY_UNCOPYABLE { #ifdef USE_MIXER_ST NCoderMixer2::CMixerST *_mixerST; diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zExtract.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zExtract.cpp index 05fd80de3fd..8ca815d4242 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zExtract.cpp @@ -152,6 +152,12 @@ STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *proc if (_fileIsOpen) { UInt32 cur = (size < _rem ? size : (UInt32)_rem); + if (_calcCrc) + { + const UInt32 k_Step = (UInt32)1 << 20; + if (cur > k_Step) + cur = k_Step; + } HRESULT result = S_OK; if (_stream) result = _stream->Write(data, cur, &cur); @@ -211,6 +217,10 @@ HRESULT CFolderOutStream::FlushCorrupted(Int32 callbackOperationResult) STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 testModeSpec, IArchiveExtractCallback *extractCallbackSpec) { + // for GCC + // CFolderOutStream *folderOutStream = new CFolderOutStream; + // CMyComPtr outStream(folderOutStream); + COM_TRY_BEGIN CMyComPtr extractCallback = extractCallbackSpec; @@ -344,10 +354,12 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, #ifndef _NO_CRYPTO bool isEncrypted = false; bool passwordIsDefined = false; - UString password; + UString_Wipe password; #endif + bool dataAfterEnd_Error = false; + HRESULT result = decoder.Decode( EXTERNAL_CODECS_VARS _inStream, @@ -358,20 +370,27 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, outStream, progress, NULL // *inStreamMainRes + , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) - , true, _numThreads + #if !defined(_7ZIP_ST) + , true, _numThreads, _memUsage_Decompress #endif ); - if (result == S_FALSE || result == E_NOTIMPL) + if (result == S_FALSE || result == E_NOTIMPL || dataAfterEnd_Error) { bool wasFinished = folderOutStream->WasWritingFinished(); - - int resOp = (result == S_FALSE ? - NExtract::NOperationResult::kDataError : - NExtract::NOperationResult::kUnsupportedMethod); + + int resOp = NExtract::NOperationResult::kDataError; + + if (result != S_FALSE) + { + if (result == E_NOTIMPL) + resOp = NExtract::NOperationResult::kUnsupportedMethod; + else if (wasFinished && dataAfterEnd_Error) + resOp = NExtract::NOperationResult::kDataAfterEnd; + } RINOK(folderOutStream->FlushCorrupted(resOp)); @@ -396,7 +415,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, { RINOK(folderOutStream->FlushCorrupted(NExtract::NOperationResult::kDataError)); // continue; - return E_FAIL; + // return E_FAIL; + throw; } } diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.cpp index 14cdc436452..cf50e694811 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../Windows/TimeUtils.h" + #include "7zFolderInStream.h" namespace NArchive { @@ -13,17 +15,17 @@ void CFolderInStream::Init(IArchiveUpdateCallback *updateCallback, _updateCallback = updateCallback; _indexes = indexes; _numFiles = numFiles; - _index = 0; Processed.ClearAndReserve(numFiles); CRCs.ClearAndReserve(numFiles); Sizes.ClearAndReserve(numFiles); - - _pos = 0; - _crc = CRC_INIT_VAL; - _size_Defined = false; - _size = 0; + if (Need_CTime) CTimes.ClearAndReserve(numFiles); + if (Need_ATime) ATimes.ClearAndReserve(numFiles); + if (Need_MTime) MTimes.ClearAndReserve(numFiles); + if (Need_Attrib) Attribs.ClearAndReserve(numFiles); + TimesDefined.ClearAndReserve(numFiles); + _stream.Release(); } @@ -32,44 +34,101 @@ HRESULT CFolderInStream::OpenStream() _pos = 0; _crc = CRC_INIT_VAL; _size_Defined = false; + _times_Defined = false; _size = 0; + FILETIME_Clear(_cTime); + FILETIME_Clear(_aTime); + FILETIME_Clear(_mTime); + _attrib = 0; - while (_index < _numFiles) + while (Processed.Size() < _numFiles) { CMyComPtr stream; - HRESULT result = _updateCallback->GetStream(_indexes[_index], &stream); - if (result != S_OK) - { - if (result != S_FALSE) - return result; - } + const HRESULT result = _updateCallback->GetStream(_indexes[Processed.Size()], &stream); + if (result != S_OK && result != S_FALSE) + return result; _stream = stream; if (stream) { - CMyComPtr streamGetSize; - stream.QueryInterface(IID_IStreamGetSize, &streamGetSize); - if (streamGetSize) { - if (streamGetSize->GetSize(&_size) == S_OK) - _size_Defined = true; + CMyComPtr getProps; + stream.QueryInterface(IID_IStreamGetProps, (void **)&getProps); + if (getProps) + { + // access could be changed in first myx pass + if (getProps->GetProps(&_size, + Need_CTime ? &_cTime : NULL, + Need_ATime ? &_aTime : NULL, + Need_MTime ? &_mTime : NULL, + Need_Attrib ? &_attrib : NULL) + == S_OK) + { + _size_Defined = true; + _times_Defined = true; + } + return S_OK; + } + } + { + CMyComPtr streamGetSize; + stream.QueryInterface(IID_IStreamGetSize, &streamGetSize); + if (streamGetSize) + { + if (streamGetSize->GetSize(&_size) == S_OK) + _size_Defined = true; + } + return S_OK; } - return S_OK; } - _index++; - RINOK(_updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); - AddFileInfo(result == S_OK); + RINOK(AddFileInfo(result == S_OK)); } return S_OK; } -void CFolderInStream::AddFileInfo(bool isProcessed) +static void AddFt(CRecordVector &vec, const FILETIME &ft) { - Processed.Add(isProcessed); - Sizes.Add(_pos); - CRCs.Add(CRC_GET_DIGEST(_crc)); + vec.AddInReserved(FILETIME_To_UInt64(ft)); +} + +/* +HRESULT ReportItemProps(IArchiveUpdateCallbackArcProp *reportArcProp, + UInt32 index, UInt64 size, const UInt32 *crc) +{ + PROPVARIANT prop; + prop.vt = VT_EMPTY; + prop.wReserved1 = 0; + + NWindows::NCOM::PropVarEm_Set_UInt64(&prop, size); + RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidSize, &prop)); + if (crc) + { + NWindows::NCOM::PropVarEm_Set_UInt32(&prop, *crc); + RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidCRC, &prop)); + } + return reportArcProp->ReportFinished(NEventIndexType::kOutArcIndex, index, NUpdate::NOperationResult::kOK); +} +*/ + +HRESULT CFolderInStream::AddFileInfo(bool isProcessed) +{ + // const UInt32 index = _indexes[Processed.Size()]; + Processed.AddInReserved(isProcessed); + Sizes.AddInReserved(_pos); + const UInt32 crc = CRC_GET_DIGEST(_crc); + CRCs.AddInReserved(crc); + TimesDefined.AddInReserved(_times_Defined); + if (Need_CTime) AddFt(CTimes, _cTime); + if (Need_ATime) AddFt(ATimes, _aTime); + if (Need_MTime) AddFt(MTimes, _mTime); + if (Need_Attrib) Attribs.AddInReserved(_attrib); + /* + if (isProcessed && _reportArcProp) + RINOK(ReportItemProps(_reportArcProp, index, _pos, &crc)) + */ + return _updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); } STDMETHODIMP CFolderInStream::Read(void *data, UInt32 size, UInt32 *processedSize) @@ -80,30 +139,25 @@ STDMETHODIMP CFolderInStream::Read(void *data, UInt32 size, UInt32 *processedSiz { if (_stream) { - UInt32 processed2; - RINOK(_stream->Read(data, size, &processed2)); - if (processed2 != 0) + UInt32 cur = size; + const UInt32 kMax = (UInt32)1 << 20; + if (cur > kMax) + cur = kMax; + RINOK(_stream->Read(data, cur, &cur)); + if (cur != 0) { - _crc = CrcUpdate(_crc, data, processed2); - _pos += processed2; + _crc = CrcUpdate(_crc, data, cur); + _pos += cur; if (processedSize) - *processedSize = processed2; + *processedSize = cur; return S_OK; } _stream.Release(); - _index++; - AddFileInfo(true); - - _pos = 0; - _crc = CRC_INIT_VAL; - _size_Defined = false; - _size = 0; - - RINOK(_updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); + RINOK(AddFileInfo(true)); } - if (_index >= _numFiles) + if (Processed.Size() >= _numFiles) break; RINOK(OpenStream()); } diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.h index 805db54e9d3..f054e681854 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zFolderInStream.h @@ -23,21 +23,37 @@ class CFolderInStream: UInt64 _pos; UInt32 _crc; bool _size_Defined; + bool _times_Defined; UInt64 _size; + FILETIME _cTime; + FILETIME _aTime; + FILETIME _mTime; + UInt32 _attrib; - const UInt32 *_indexes; unsigned _numFiles; - unsigned _index; + const UInt32 *_indexes; CMyComPtr _updateCallback; HRESULT OpenStream(); - void AddFileInfo(bool isProcessed); + HRESULT AddFileInfo(bool isProcessed); public: CRecordVector Processed; CRecordVector CRCs; CRecordVector Sizes; + CRecordVector CTimes; + CRecordVector ATimes; + CRecordVector MTimes; + CRecordVector Attribs; + CRecordVector TimesDefined; + + bool Need_CTime; + bool Need_ATime; + bool Need_MTime; + bool Need_Attrib; + + // CMyComPtr _reportArcProp; MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize) STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); @@ -45,7 +61,7 @@ public: void Init(IArchiveUpdateCallback *updateCallback, const UInt32 *indexes, unsigned numFiles); - bool WasFinished() const { return _index == _numFiles; } + bool WasFinished() const { return Processed.Size() == _numFiles; } UInt64 GetFullSize() const { @@ -54,6 +70,13 @@ public: size += Sizes[i]; return size; } + + CFolderInStream(): + Need_CTime(false), + Need_ATime(false), + Need_MTime(false), + Need_Attrib(false) + {} }; }} diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.cpp index ccd2c624e66..ca22f8814d9 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.cpp @@ -40,7 +40,6 @@ CHandler::CHandler() _crcSize = 4; #ifdef __7Z_SET_PROPERTIES - _numThreads = NSystem::GetNumberOfProcessors(); _useMultiThreadMixer = true; #endif @@ -108,34 +107,62 @@ static void ConvertMethodIdToString(AString &res, UInt64 id) res += s + len - ConvertMethodIdToString_Back(s + len, id); } -static unsigned GetStringForSizeValue(char *s, UInt32 val) + +static char *GetStringForSizeValue(char *s, UInt32 val) { unsigned i; for (i = 0; i <= 31; i++) if (((UInt32)1 << i) == val) { - if (i < 10) + if (i >= 10) { - s[0] = (char)('0' + i); - s[1] = 0; - return 1; + *s++= (char)('0' + i / 10); + i %= 10; } - if (i < 20) { s[0] = '1'; s[1] = (char)('0' + i - 10); } - else if (i < 30) { s[0] = '2'; s[1] = (char)('0' + i - 20); } - else { s[0] = '3'; s[1] = (char)('0' + i - 30); } - s[2] = 0; - return 2; + *s++ = (char)('0' + i); + *s = 0; + return s; } + char c = 'b'; if ((val & ((1 << 20) - 1)) == 0) { val >>= 20; c = 'm'; } else if ((val & ((1 << 10) - 1)) == 0) { val >>= 10; c = 'k'; } - ::ConvertUInt32ToString(val, s); - unsigned pos = MyStringLen(s); - s[pos++] = c; - s[pos] = 0; - return pos; + s = ConvertUInt32ToString(val, s); + *s++ = c; + *s = 0; + return s; } + +static void GetLzma2String(char *s, unsigned d) +{ + if (d > 40) + { + *s = 0; + return; + // s = MyStpCpy(s, "unsup"); + } + else if ((d & 1) == 0) + d = (d >> 1) + 12; + else + { + // s = GetStringForSizeValue(s, (UInt32)3 << ((d >> 1) + 11)); + d = (d >> 1) + 1; + char c = 'k'; + if (d >= 10) + { + c = 'm'; + d -= 10; + } + s = ConvertUInt32ToString((UInt32)3 << d, s); + *s++ = c; + *s = 0; + return; + } + ConvertUInt32ToString(d, s); +} + + /* static inline void AddHexToString(UString &res, Byte value) { @@ -148,8 +175,7 @@ static char *AddProp32(char *s, const char *name, UInt32 v) { *s++ = ':'; s = MyStpCpy(s, name); - ::ConvertUInt32ToString(v, s); - return s + MyStringLen(s); + return ConvertUInt32ToString(v, s); } void CHandler::AddMethodName(AString &s, UInt64 id) @@ -185,10 +211,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) if (id == k_LZMA2) { s += "LZMA2:"; - if ((pm.Lzma2Prop & 1) == 0) - ConvertUInt32ToString((pm.Lzma2Prop >> 1) + 12, temp); - else - GetStringForSizeValue(temp, 3 << ((pm.Lzma2Prop >> 1) + 11)); + GetLzma2String(temp, pm.Lzma2Prop); s += temp; } else if (id == k_LZMA) @@ -237,19 +260,25 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = v; break; } + + case kpidReadOnly: + { + if (!_db.CanUpdate()) + prop = true; + break; + } } - prop.Detach(value); - return S_OK; + return prop.Detach(value); #ifndef _SFX COM_TRY_END #endif } -static void SetFileTimeProp_From_UInt64Def(PROPVARIANT *prop, const CUInt64DefVector &v, int index) +static void SetFileTimeProp_From_UInt64Def(PROPVARIANT *prop, const CUInt64DefVector &v, unsigned index) { UInt64 value; if (v.GetItem(index, value)) - PropVarEm_Set_FileTime64(prop, value); + PropVarEm_Set_FileTime64_Prec(prop, value, k_PropVar_TimePrec_100ns); } bool CHandler::IsFolderEncrypted(CNum folderIndex) const @@ -410,7 +439,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const if (propsSize == 5) { UInt32 dicSize = GetUi32((const Byte *)props + 1); - char *dest = s + GetStringForSizeValue(s, dicSize); + char *dest = GetStringForSizeValue(s, dicSize); UInt32 d = props[0]; if (d != 0x5D) { @@ -428,24 +457,16 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const { name = "LZMA2"; if (propsSize == 1) - { - Byte d = props[0]; - if ((d & 1) == 0) - ConvertUInt32ToString((UInt32)((d >> 1) + 12), s); - else - GetStringForSizeValue(s, 3 << ((d >> 1) + 11)); - } + GetLzma2String(s, props[0]); } else if (id == k_PPMD) { name = "PPMD"; if (propsSize == 5) { - Byte order = *props; char *dest = s; *dest++ = 'o'; - ConvertUInt32ToString(order, dest); - dest += MyStringLen(dest); + dest = ConvertUInt32ToString(*props, dest); dest = MyStpCpy(dest, ":mem"); GetStringForSizeValue(dest, GetUi32(props + 1)); } @@ -528,7 +549,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value) { - PropVariant_Clear(value); + RINOK(PropVariant_Clear(value)); // COM_TRY_BEGIN // NCOM::CPropVariant prop; @@ -540,7 +561,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val */ const CFileItem &item = _db.Files[index]; - UInt32 index2 = index; + const UInt32 index2 = index; switch (propID) { @@ -575,7 +596,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidCTime: SetFileTimeProp_From_UInt64Def(value, _db.CTime, index2); break; case kpidATime: SetFileTimeProp_From_UInt64Def(value, _db.ATime, index2); break; case kpidMTime: SetFileTimeProp_From_UInt64Def(value, _db.MTime, index2); break; - case kpidAttrib: if (item.AttribDefined) PropVarEm_Set_UInt32(value, item.Attrib); break; + case kpidAttrib: if (_db.Attrib.ValidAndDefined(index2)) PropVarEm_Set_UInt32(value, _db.Attrib.Vals[index2]); break; case kpidCRC: if (item.CrcDefined) PropVarEm_Set_UInt32(value, item.Crc); break; case kpidEncrypted: PropVarEm_Set_Bool(value, IsFolderEncrypted(_db.FileIndexToFolderIndexMap[index2])); break; case kpidIsAnti: PropVarEm_Set_Bool(value, _db.IsItemAnti(index2)); break; @@ -631,7 +652,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val #endif } - // prop.Detach(value); + // return prop.Detach(value); return S_OK; // COM_TRY_END } @@ -702,7 +723,7 @@ STDMETHODIMP CHandler::Close() #ifndef _NO_CRYPTO _isEncrypted = false; _passwordIsDefined = false; - _password.Empty(); + _password.Wipe_and_Empty(); #endif return S_OK; COM_TRY_END @@ -714,8 +735,8 @@ STDMETHODIMP CHandler::Close() STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN - const UInt32 numProcessors = NSystem::GetNumberOfProcessors(); - _numThreads = numProcessors; + + InitCommon(); _useMultiThreadMixer = true; for (UInt32 i = 0; i < numProps; i++) @@ -734,13 +755,15 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(PROPVARIANT_to_bool(value, _useMultiThreadMixer)); continue; } - if (name.IsPrefixedBy_Ascii_NoCase("mt")) { - RINOK(ParseMtProp(name.Ptr(2), value, numProcessors, _numThreads)); - continue; + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres); + continue; + } } - else - return E_INVALIDARG; + return E_INVALIDARG; } } return S_OK; diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.h index d46401afe3c..08bd65407b1 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandler.h @@ -8,16 +8,6 @@ #include "../../Common/CreateCoder.h" -#ifndef EXTRACT_ONLY -#include "../Common/HandlerOut.h" -#endif - -#include "7zCompressionMode.h" -#include "7zIn.h" - -namespace NArchive { -namespace N7z { - #ifndef __7Z_SET_PROPERTIES #ifdef EXTRACT_ONLY @@ -30,6 +20,16 @@ namespace N7z { #endif +// #ifdef __7Z_SET_PROPERTIES +#include "../Common/HandlerOut.h" +// #endif + +#include "7zCompressionMode.h" +#include "7zIn.h" + +namespace NArchive { +namespace N7z { + #ifndef EXTRACT_ONLY @@ -38,8 +38,6 @@ class COutHandler: public CMultiMethodProps HRESULT SetSolidFromString(const UString &s); HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); public: - bool _removeSfxBlock; - UInt64 _numSolidFiles; UInt64 _numSolidBytes; bool _numSolidBytesDefined; @@ -51,12 +49,14 @@ public: bool _encryptHeaders; // bool _useParents; 9.26 - CBoolPair Write_CTime; - CBoolPair Write_ATime; - CBoolPair Write_MTime; + CHandlerTimeOptions TimeOptions; + + CBoolPair Write_Attrib; bool _useMultiThreadMixer; + bool _removeSfxBlock; + // bool _volumeMode; void InitSolidFiles() { _numSolidFiles = (UInt64)(Int64)(-1); } @@ -69,9 +69,10 @@ public: _numSolidBytesDefined = false; } + void InitProps7z(); void InitProps(); - COutHandler() { InitProps(); } + COutHandler() { InitProps7z(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; @@ -81,16 +82,23 @@ public: class CHandler: public IInArchive, public IArchiveGetRawProps, + #ifdef __7Z_SET_PROPERTIES public ISetProperties, #endif + #ifndef EXTRACT_ONLY public IOutArchive, #endif + PUBLIC_ISetCompressCodecsInfo - public CMyUnknownImp + + public CMyUnknownImp, + #ifndef EXTRACT_ONLY - , public COutHandler + public COutHandler + #else + public CCommonMethodProps #endif { public: @@ -120,6 +128,10 @@ public: DECL_ISetCompressCodecsInfo CHandler(); + ~CHandler() + { + Close(); + } private: CMyComPtr _inStream; @@ -128,13 +140,12 @@ private: #ifndef _NO_CRYPTO bool _isEncrypted; bool _passwordIsDefined; - UString _password; + UString _password; // _Wipe #endif #ifdef EXTRACT_ONLY #ifdef __7Z_SET_PROPERTIES - UInt32 _numThreads; bool _useMultiThreadMixer; #endif @@ -146,12 +157,7 @@ private: HRESULT PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m); HRESULT SetHeaderMethod(CCompressionMethodMode &headerMethod); - HRESULT SetMainMethod(CCompressionMethodMode &method - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ); - + HRESULT SetMainMethod(CCompressionMethodMode &method); #endif diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandlerOut.cpp index 2b86ed26ab7..bf4e7a69c27 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -13,16 +13,19 @@ #include "7zOut.h" #include "7zUpdate.h" +#ifndef EXTRACT_ONLY + using namespace NWindows; namespace NArchive { namespace N7z { -static const char *k_LZMA_Name = "LZMA"; -static const char *kDefaultMethodName = "LZMA2"; -static const char *k_Copy_Name = "Copy"; +#define k_LZMA_Name "LZMA" +#define kDefaultMethodName "LZMA2" +#define k_Copy_Name "Copy" + +#define k_MatchFinder_ForHeaders "BT2" -static const char *k_MatchFinder_ForHeaders = "BT2"; static const UInt32 k_NumFastBytes_ForHeaders = 273; static const UInt32 k_Level_ForHeaders = 5; static const UInt32 k_Dictionary_ForHeaders = @@ -40,9 +43,11 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) HRESULT CHandler::PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m) { - if (!FindMethod( + dest.CodecIndex = FindMethod_Index( EXTERNAL_CODECS_VARS - m.MethodName, dest.Id, dest.NumStreams)) + m.MethodName, true, + dest.Id, dest.NumStreams); + if (dest.CodecIndex < 0) return E_INVALIDARG; (CProps &)dest = (CProps &)m; return S_OK; @@ -64,15 +69,12 @@ HRESULT CHandler::SetHeaderMethod(CCompressionMethodMode &headerMethod) return PropsMethod_To_FullMethod(methodFull, m); } -HRESULT CHandler::SetMainMethod( - CCompressionMethodMode &methodMode - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ) + +HRESULT CHandler::SetMainMethod(CCompressionMethodMode &methodMode) { methodMode.Bonds = _bonds; + // we create local copy of _methods. So we can modify it. CObjectVector methods = _methods; { @@ -106,55 +108,155 @@ HRESULT CHandler::SetMainMethod( } const UInt64 kSolidBytes_Min = (1 << 24); - const UInt64 kSolidBytes_Max = ((UInt64)1 << 32) - 1; + const UInt64 kSolidBytes_Max = ((UInt64)1 << 32); bool needSolid = false; FOR_VECTOR (i, methods) { COneMethodInfo &oneMethodInfo = methods[i]; - SetGlobalLevelAndThreads(oneMethodInfo - #ifndef _7ZIP_ST - , numThreads - #endif - ); + + SetGlobalLevelTo(oneMethodInfo); + + #ifndef _7ZIP_ST + const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0); + if (!numThreads_WasSpecifiedInMethod) + { + // here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already + CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, methodMode.NumThreads); + } + #endif CMethodFull &methodFull = methodMode.Methods.AddNew(); RINOK(PropsMethod_To_FullMethod(methodFull, oneMethodInfo)); + #ifndef _7ZIP_ST + methodFull.Set_NumThreads = true; + methodFull.NumThreads = methodMode.NumThreads; + #endif + if (methodFull.Id != k_Copy) needSolid = true; - if (_numSolidBytesDefined) - continue; - - UInt32 dicSize; + UInt64 dicSize; switch (methodFull.Id) { case k_LZMA: case k_LZMA2: dicSize = oneMethodInfo.Get_Lzma_DicSize(); break; case k_PPMD: dicSize = oneMethodInfo.Get_Ppmd_MemSize(); break; case k_Deflate: dicSize = (UInt32)1 << 15; break; + case k_Deflate64: dicSize = (UInt32)1 << 16; break; case k_BZip2: dicSize = oneMethodInfo.Get_BZip2_BlockSize(); break; default: continue; } + + UInt64 numSolidBytes; - _numSolidBytes = (UInt64)dicSize << 7; - if (_numSolidBytes < kSolidBytes_Min) _numSolidBytes = kSolidBytes_Min; - if (_numSolidBytes > kSolidBytes_Max) _numSolidBytes = kSolidBytes_Max; + if (methodFull.Id == k_LZMA2) + { + // he we calculate default chunk Size for LZMA2 as defined in LZMA2 encoder code + /* lzma2 code use dictionary upo to fake 4 GiB to calculate ChunkSize. + So we do same */ + UInt64 cs = (UInt64)dicSize << 2; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + if (cs < kMinSize) cs = kMinSize; + if (cs > kMaxSize) cs = kMaxSize; + if (cs < dicSize) cs = dicSize; + cs += (kMinSize - 1); + cs &= ~(UInt64)(kMinSize - 1); + // we want to use at least 64 chunks (threads) per one solid block. + + // here we don't use chunckSize property + numSolidBytes = cs << 6; + + // here we get real chunckSize + cs = oneMethodInfo.Get_Xz_BlockSize(); + if (dicSize > cs) + dicSize = cs; + + const UInt64 kSolidBytes_Lzma2_Max = ((UInt64)1 << 34); + if (numSolidBytes > kSolidBytes_Lzma2_Max) + numSolidBytes = kSolidBytes_Lzma2_Max; + + methodFull.Set_NumThreads = false; // we don't use ICompressSetCoderMt::SetNumberOfThreads() for LZMA2 encoder + + #ifndef _7ZIP_ST + if (!numThreads_WasSpecifiedInMethod + && !methodMode.NumThreads_WasForced + && methodMode.MemoryUsageLimit_WasSet + ) + { + const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads(); + const UInt32 numBlockThreads_Original = methodMode.NumThreads / lzmaThreads; + + if (numBlockThreads_Original > 1) + { + /* + const UInt32 kNumThreads_Max = 1024; + if (numBlockThreads > kNumMaxThreads) + numBlockThreads = kNumMaxThreads; + */ + + UInt32 numBlockThreads = numBlockThreads_Original; + const UInt64 lzmaMemUsage = oneMethodInfo.Get_Lzma_MemUsage(false); // solid + + for (; numBlockThreads > 1; numBlockThreads--) + { + UInt64 size = numBlockThreads * (lzmaMemUsage + cs); + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (cs < ((UInt32)1 << 26)) numPackChunks++; + if (cs < ((UInt32)1 << 24)) numPackChunks++; + if (cs < ((UInt32)1 << 22)) numPackChunks++; + size += numPackChunks * cs; + // printf("\nnumBlockThreads = %d, size = %d\n", (unsigned)(numBlockThreads), (unsigned)(size >> 20)); + if (size <= methodMode.MemoryUsageLimit) + break; + } + + if (numBlockThreads == 0) + numBlockThreads = 1; + if (numBlockThreads != numBlockThreads_Original) + { + const UInt32 numThreads_New = numBlockThreads * lzmaThreads; + CMultiMethodProps::SetMethodThreadsTo_Replace(methodFull, numThreads_New); + } + } + } + #endif + } + else + { + numSolidBytes = (UInt64)dicSize << 7; + if (numSolidBytes > kSolidBytes_Max) + numSolidBytes = kSolidBytes_Max; + } + + if (_numSolidBytesDefined) + continue; + + if (numSolidBytes < kSolidBytes_Min) + numSolidBytes = kSolidBytes_Min; + _numSolidBytes = numSolidBytes; _numSolidBytesDefined = true; } if (!_numSolidBytesDefined) + { if (needSolid) _numSolidBytes = kSolidBytes_Max; else _numSolidBytes = 0; + } _numSolidBytesDefined = true; + + return S_OK; } -static HRESULT GetTime(IArchiveUpdateCallback *updateCallback, int index, PROPID propID, UInt64 &ft, bool &ftDefined) + + +static HRESULT GetTime(IArchiveUpdateCallback *updateCallback, unsigned index, PROPID propID, UInt64 &ft, bool &ftDefined) { // ft = 0; // ftDefined = false; @@ -262,6 +364,9 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt db = &_db; #endif + if (db && !db->CanUpdate()) + return E_NOTIMPL; + /* CMyComPtr getRawProps; updateCallback->QueryInterface(IID_IArchiveGetRawProps, (void **)&getRawProps); @@ -279,18 +384,21 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CObjectVector updateItems; - bool need_CTime = (Write_CTime.Def && Write_CTime.Val); - bool need_ATime = (Write_ATime.Def && Write_ATime.Val); - bool need_MTime = (Write_MTime.Def && Write_MTime.Val || !Write_MTime.Def); + bool need_CTime = (TimeOptions.Write_CTime.Def && TimeOptions.Write_CTime.Val); + bool need_ATime = (TimeOptions.Write_ATime.Def && TimeOptions.Write_ATime.Val); + bool need_MTime = (TimeOptions.Write_MTime.Def ? TimeOptions.Write_MTime.Val : true); + bool need_Attrib = (Write_Attrib.Def ? Write_Attrib.Val : true); if (db && !db->Files.IsEmpty()) { - if (!Write_CTime.Def) need_CTime = !db->CTime.Defs.IsEmpty(); - if (!Write_ATime.Def) need_ATime = !db->ATime.Defs.IsEmpty(); - if (!Write_MTime.Def) need_MTime = !db->MTime.Defs.IsEmpty(); + if (!TimeOptions.Write_CTime.Def) need_CTime = !db->CTime.Defs.IsEmpty(); + if (!TimeOptions.Write_ATime.Def) need_ATime = !db->ATime.Defs.IsEmpty(); + if (!TimeOptions.Write_MTime.Def) need_MTime = !db->MTime.Defs.IsEmpty(); + if (!Write_Attrib.Def) need_Attrib = !db->Attrib.Defs.IsEmpty(); } - UString s; + // UString s; + UString name; for (UInt32 i = 0; i < numItems; i++) { @@ -302,38 +410,39 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CUpdateItem ui; ui.NewProps = IntToBool(newProps); ui.NewData = IntToBool(newData); - ui.IndexInArchive = indexInArchive; + ui.IndexInArchive = (int)indexInArchive; ui.IndexInClient = i; ui.IsAnti = false; ui.Size = 0; - UString name; + name.Empty(); // bool isAltStream = false; if (ui.IndexInArchive != -1) { - if (db == 0 || (unsigned)ui.IndexInArchive >= db->Files.Size()) + if (!db || (unsigned)ui.IndexInArchive >= db->Files.Size()) return E_INVALIDARG; - const CFileItem &fi = db->Files[ui.IndexInArchive]; + const CFileItem &fi = db->Files[(unsigned)ui.IndexInArchive]; if (!ui.NewProps) { - _db.GetPath(ui.IndexInArchive, name); + _db.GetPath((unsigned)ui.IndexInArchive, name); } ui.IsDir = fi.IsDir; ui.Size = fi.Size; // isAltStream = fi.IsAltStream; - ui.IsAnti = db->IsItemAnti(ui.IndexInArchive); + ui.IsAnti = db->IsItemAnti((unsigned)ui.IndexInArchive); if (!ui.NewProps) { - ui.CTimeDefined = db->CTime.GetItem(ui.IndexInArchive, ui.CTime); - ui.ATimeDefined = db->ATime.GetItem(ui.IndexInArchive, ui.ATime); - ui.MTimeDefined = db->MTime.GetItem(ui.IndexInArchive, ui.MTime); + ui.CTimeDefined = db->CTime.GetItem((unsigned)ui.IndexInArchive, ui.CTime); + ui.ATimeDefined = db->ATime.GetItem((unsigned)ui.IndexInArchive, ui.ATime); + ui.MTimeDefined = db->MTime.GetItem((unsigned)ui.IndexInArchive, ui.MTime); } } if (ui.NewProps) { bool folderStatusIsDefined; + if (need_Attrib) { NCOM::CPropVariant prop; RINOK(updateCallback->GetProperty(i, kpidAttrib, &prop)); @@ -377,7 +486,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt return E_INVALIDARG; else { - name = NItemName::MakeLegalName(prop.bstrVal); + name = prop.bstrVal; + NItemName::ReplaceSlashes_OsToUnix(name); } } { @@ -537,30 +647,36 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt CCompressionMethodMode methodMode, headerMethod; - HRESULT res = SetMainMethod(methodMode - #ifndef _7ZIP_ST - , _numThreads - #endif - ); - RINOK(res); + methodMode.MemoryUsageLimit = _memUsage_Compress; + methodMode.MemoryUsageLimit_WasSet = _memUsage_WasSet; - RINOK(SetHeaderMethod(headerMethod)); - #ifndef _7ZIP_ST - methodMode.NumThreads = _numThreads; - methodMode.MultiThreadMixer = _useMultiThreadMixer; - headerMethod.NumThreads = 1; - headerMethod.MultiThreadMixer = _useMultiThreadMixer; + { + UInt32 numThreads = _numThreads; + const UInt32 kNumThreads_Max = 1024; + if (numThreads > kNumThreads_Max) + numThreads = kNumThreads_Max; + methodMode.NumThreads = numThreads; + methodMode.NumThreads_WasForced = _numThreads_WasForced; + methodMode.MultiThreadMixer = _useMultiThreadMixer; + // headerMethod.NumThreads = 1; + headerMethod.MultiThreadMixer = _useMultiThreadMixer; + } #endif + HRESULT res = SetMainMethod(methodMode); + RINOK(res); + + RINOK(SetHeaderMethod(headerMethod)); + CMyComPtr getPassword2; updateCallback->QueryInterface(IID_ICryptoGetTextPassword2, (void **)&getPassword2); methodMode.PasswordIsDefined = false; - methodMode.Password.Empty(); + methodMode.Password.Wipe_and_Empty(); if (getPassword2) { - CMyComBSTR password; + CMyComBSTR_Wipe password; Int32 passwordIsDefined; RINOK(getPassword2->CryptoGetTextPassword2(&passwordIsDefined, &password)); methodMode.PasswordIsDefined = IntToBool(passwordIsDefined); @@ -603,6 +719,11 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt int level = GetLevel(); CUpdateOptions options; + options.Need_CTime = need_CTime; + options.Need_ATime = need_ATime; + options.Need_MTime = need_MTime; + options.Need_Attrib = need_Attrib; + options.Method = &methodMode; options.HeaderMethod = (_compressHeaders || encryptHeaders) ? &headerMethod : NULL; options.UseFilters = (level != 0 && _autoFilter && !methodMode.Filter_was_Inserted); @@ -614,6 +735,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt options.HeaderOptions.WriteCTime = Write_CTime; options.HeaderOptions.WriteATime = Write_ATime; options.HeaderOptions.WriteMTime = Write_MTime; + options.HeaderOptions.WriteAttrib = Write_Attrib; */ options.NumSolidFiles = _numSolidFiles; @@ -692,19 +814,16 @@ static HRESULT ParseBond(UString &srcString, UInt32 &coder, UInt32 &stream) return S_OK; } -void COutHandler::InitProps() +void COutHandler::InitProps7z() { - CMultiMethodProps::Init(); - _removeSfxBlock = false; _compressHeaders = true; _encryptHeadersSpecified = false; _encryptHeaders = false; // _useParents = false; - Write_CTime.Init(); - Write_ATime.Init(); - Write_MTime.Init(); + TimeOptions.Init(); + Write_Attrib.Init(); _useMultiThreadMixer = true; @@ -714,6 +833,14 @@ void COutHandler::InitProps() _useTypeSorting = false; } +void COutHandler::InitProps() +{ + CMultiMethodProps::Init(); + InitProps7z(); +} + + + HRESULT COutHandler::SetSolidFromString(const UString &s) { UString s2 = s; @@ -730,7 +857,7 @@ HRESULT COutHandler::SetSolidFromString(const UString &s) _solidExtension = true; continue; } - i += (int)(end - start); + i += (unsigned)(end - start); if (i == s2.Len()) return E_INVALIDARG; wchar_t c = s2[i++]; @@ -754,6 +881,10 @@ HRESULT COutHandler::SetSolidFromString(const UString &s) } _numSolidBytes = (v << numBits); _numSolidBytesDefined = true; + /* + if (_numSolidBytes == 0) + _numSolidFiles = 1; + */ } } return S_OK; @@ -802,9 +933,9 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val return E_INVALIDARG; return SetSolidFromString(name); } - + UInt32 number; - int index = ParseStringToUInt32(name, number); + unsigned index = ParseStringToUInt32(name, number); // UString realName = name.Ptr(index); if (index == 0) { @@ -826,9 +957,21 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val return S_OK; } - if (name.IsEqualTo("tc")) return PROPVARIANT_to_BoolPair(value, Write_CTime); - if (name.IsEqualTo("ta")) return PROPVARIANT_to_BoolPair(value, Write_ATime); - if (name.IsEqualTo("tm")) return PROPVARIANT_to_BoolPair(value, Write_MTime); + { + bool processed; + RINOK(TimeOptions.Parse(name, value, processed)); + if (processed) + { + if ( TimeOptions.Prec != (UInt32)(Int32)-1 + && TimeOptions.Prec != k_PropVar_TimePrec_0 + && TimeOptions.Prec != k_PropVar_TimePrec_HighPrec + && TimeOptions.Prec != k_PropVar_TimePrec_100ns) + return E_INVALIDARG; + return S_OK; + } + } + + if (name.IsEqualTo("tr")) return PROPVARIANT_to_BoolPair(value, Write_Attrib); if (name.IsEqualTo("mtf")) return PROPVARIANT_to_bool(value, _useMultiThreadMixer); @@ -911,3 +1054,5 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR } }} + +#endif diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHeader.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHeader.h index d7f0ae365b9..e1bbc0aa2b8 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zHeader.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zHeader.h @@ -108,8 +108,9 @@ const UInt32 k_SWAP4 = 0x20304; const UInt32 k_LZMA = 0x30101; const UInt32 k_PPMD = 0x30401; -const UInt32 k_Deflate = 0x40108; -const UInt32 k_BZip2 = 0x40202; +const UInt32 k_Deflate = 0x40108; +const UInt32 k_Deflate64 = 0x40109; +const UInt32 k_BZip2 = 0x40202; const UInt32 k_BCJ = 0x3030103; const UInt32 k_BCJ2 = 0x303011B; diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.cpp index d8c2717501c..7134595c92d 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.cpp @@ -11,6 +11,8 @@ #include "../../../../C/7zCrc.h" #include "../../../../C/CpuArch.h" +// #include "../../../Common/UTFConvert.h" + #include "../../Common/StreamObjects.h" #include "../../Common/StreamUtils.h" @@ -32,6 +34,22 @@ using namespace NCOM; namespace NArchive { namespace N7z { +unsigned BoolVector_CountSum(const CBoolVector &v); +unsigned BoolVector_CountSum(const CBoolVector &v) +{ + unsigned sum = 0; + const unsigned size = v.Size(); + for (unsigned i = 0; i < size; i++) + if (v[i]) + sum++; + return sum; +} + +static inline bool BoolVector_Item_IsValidAndTrue(const CBoolVector &v, unsigned i) +{ + return (i < v.Size() ? v[i] : false); +} + static void BoolVector_Fill_False(CBoolVector &v, unsigned size) { v.ClearAndSetSize(size); @@ -40,12 +58,17 @@ static void BoolVector_Fill_False(CBoolVector &v, unsigned size) p[i] = false; } + class CInArchiveException {}; class CUnsupportedFeatureException: public CInArchiveException {}; +MY_ATTR_NORETURN static void ThrowException() { throw CInArchiveException(); } +MY_ATTR_NORETURN static inline void ThrowEndOfData() { ThrowException(); } +MY_ATTR_NORETURN static inline void ThrowUnsupported() { throw CUnsupportedFeatureException(); } +MY_ATTR_NORETURN static inline void ThrowIncorrect() { ThrowException(); } class CStreamSwitch @@ -312,7 +335,7 @@ HRESULT CInArchive::FindAndReadSignature(IInStream *stream, const UInt64 *search { memcpy(_header, p, kHeaderSize); _arhiveBeginStreamPosition += offset + pos; - return stream->Seek(_arhiveBeginStreamPosition + kHeaderSize, STREAM_SEEK_SET, NULL); + return stream->Seek((Int64)(_arhiveBeginStreamPosition + kHeaderSize), STREAM_SEEK_SET, NULL); } } @@ -328,7 +351,7 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit) Close(); RINOK(stream->Seek(0, STREAM_SEEK_CUR, &_arhiveBeginStreamPosition)) RINOK(stream->Seek(0, STREAM_SEEK_END, &_fileEndPosition)) - RINOK(stream->Seek(_arhiveBeginStreamPosition, STREAM_SEEK_SET, NULL)) + RINOK(stream->Seek((Int64)_arhiveBeginStreamPosition, STREAM_SEEK_SET, NULL)) RINOK(FindAndReadSignature(stream, searchHeaderSizeLimit)); _stream = stream; return S_OK; @@ -462,7 +485,7 @@ void CDatabase::GetPath(unsigned index, UString &path) const #if defined(_WIN32) && defined(MY_CPU_LE) - wmemcpy(s, (const wchar_t *)p, size); + wmemcpy(s, (const wchar_t *)(const void *)p, size); #else @@ -490,10 +513,27 @@ HRESULT CDatabase::GetPath_Prop(unsigned index, PROPVARIANT *path) const throw() if (size >= (1 << 14)) return S_OK; - RINOK(PropVarEm_Alloc_Bstr(path, (unsigned)size - 1)); + // (size) includes null terminator + + /* + #if WCHAR_MAX > 0xffff + + const Byte *p = ((const Byte *)NamesBuf + offset * 2); + size = Utf16LE__Get_Num_WCHARs(p, size - 1); + // (size) doesn't include null terminator + RINOK(PropVarEm_Alloc_Bstr(path, (unsigned)size)); wchar_t *s = path->bstrVal; + wchar_t *sEnd = Utf16LE__To_WCHARs_Sep(p, size, s); + *sEnd = 0; + if (s + size != sEnd) return E_FAIL; + + #else + */ + RINOK(PropVarEm_Alloc_Bstr(path, (unsigned)size - 1)); + wchar_t *s = path->bstrVal; const Byte *p = ((const Byte *)NamesBuf + offset * 2); + // Utf16LE__To_WCHARs_Sep(p, size, s); for (size_t i = 0; i < size; i++) { @@ -502,10 +542,14 @@ HRESULT CDatabase::GetPath_Prop(unsigned index, PROPVARIANT *path) const throw() #if WCHAR_PATH_SEPARATOR != L'/' if (c == L'/') c = WCHAR_PATH_SEPARATOR; + else if (c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme #endif *s++ = c; } + // #endif + return S_OK; /* @@ -566,21 +610,30 @@ void CInArchive::WaitId(UInt64 id) } } -void CInArchive::ReadHashDigests(unsigned numItems, CUInt32DefVector &crcs) + +void CInArchive::Read_UInt32_Vector(CUInt32DefVector &v) { - ReadBoolVector2(numItems, crcs.Defs); - crcs.Vals.ClearAndSetSize(numItems); - UInt32 *p = &crcs.Vals[0]; - const bool *defs = &crcs.Defs[0]; + unsigned numItems = v.Defs.Size(); + v.Vals.ClearAndSetSize(numItems); + UInt32 *p = &v.Vals[0]; + const bool *defs = &v.Defs[0]; for (unsigned i = 0; i < numItems; i++) { - UInt32 crc = 0; + UInt32 a = 0; if (defs[i]) - crc = ReadUInt32(); - p[i] = crc; + a = ReadUInt32(); + p[i] = a; } } + +void CInArchive::ReadHashDigests(unsigned numItems, CUInt32DefVector &crcs) +{ + ReadBoolVector2(numItems, crcs.Defs); + Read_UInt32_Vector(crcs); +} + + #define k_Scan_NumCoders_MAX 64 #define k_Scan_NumCodersStreams_in_Folder_MAX 64 @@ -648,7 +701,7 @@ void CInArchive::ReadUnpackInfo( { UInt32 indexOfMainStream = 0; UInt32 numPackStreams = 0; - folders.FoCodersDataOffset[fo] = _inByteBack->GetPtr() - startBufPtr; + folders.FoCodersDataOffset[fo] = (size_t)(_inByteBack->GetPtr() - startBufPtr); CNum numInStreams = 0; CNum numCoders = inByte->ReadNum(); @@ -769,10 +822,10 @@ void CInArchive::ReadUnpackInfo( folders.FoToMainUnpackSizeIndex[fo] = (Byte)indexOfMainStream; } - size_t dataSize = _inByteBack->GetPtr() - startBufPtr; + const size_t dataSize = (size_t)(_inByteBack->GetPtr() - startBufPtr); folders.FoToCoderUnpackSizes[fo] = numCodersOutStreams; folders.FoStartPackStreamIndex[fo] = packStreamIndex; - folders.FoCodersDataOffset[fo] = _inByteBack->GetPtr() - startBufPtr; + folders.FoCodersDataOffset[fo] = (size_t)(_inByteBack->GetPtr() - startBufPtr); folders.CodersData.CopyFrom(startBufPtr, dataSize); // if (folders.NumPackStreams != packStreamIndex) ThrowUnsupported(); @@ -937,6 +990,8 @@ void CInArchive::ReadSubStreamsInfo( } } + + void CInArchive::ReadStreamsInfo( const CObjectVector *dataVector, UInt64 &dataOffset, @@ -949,7 +1004,11 @@ void CInArchive::ReadStreamsInfo( if (type == NID::kPackInfo) { dataOffset = ReadNumber(); + if (dataOffset > _rangeLimit) + ThrowIncorrect(); ReadPackInfo(folders); + if (folders.PackPositions[folders.NumPackStreams] > _rangeLimit - dataOffset) + ThrowIncorrect(); type = ReadID(); } @@ -1004,7 +1063,7 @@ void CInArchive::ReadBoolVector(unsigned numItems, CBoolVector &v) mask = 0x80; } p[i] = ((b & mask) != 0); - mask >>= 1; + mask = (Byte)(mask >> 1); } } @@ -1065,8 +1124,8 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( for (CNum i = 0; i < folders.NumFolders; i++) { CByteBuffer &data = dataVector.AddNew(); - UInt64 unpackSize64 = folders.GetFolderUnpackSize(i); - size_t unpackSize = (size_t)unpackSize64; + const UInt64 unpackSize64 = folders.GetFolderUnpackSize(i); + const size_t unpackSize = (size_t)unpackSize64; if (unpackSize != unpackSize64) ThrowUnsupported(); data.Alloc(unpackSize); @@ -1075,24 +1134,36 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( CMyComPtr outStream = outStreamSpec; outStreamSpec->Init(data, unpackSize); + bool dataAfterEnd_Error = false; + HRESULT result = decoder.Decode( EXTERNAL_CODECS_LOC_VARS _stream, baseOffset + dataOffset, folders, i, - NULL, // *unpackSize + NULL, // &unpackSize64 outStream, NULL, // *compressProgress + NULL // **inStreamMainRes + , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) + #if !defined(_7ZIP_ST) , false // mtMode , 1 // numThreads + , 0 // memUsage #endif ); + RINOK(result); + if (dataAfterEnd_Error) + ThereIsHeaderError = true; + + if (unpackSize != outStreamSpec->GetPos()) + ThrowIncorrect(); + if (folders.FolderCRCs.ValidAndDefined(i)) if (CrcCalc(data, unpackSize) != folders.FolderCRCs.Vals[i]) ThrowIncorrect(); @@ -1148,19 +1219,10 @@ HRESULT CInArchive::ReadHeader( type = ReadID(); } - db.Files.Clear(); - if (type == NID::kFilesInfo) { const CNum numFiles = ReadNum(); - db.Files.ClearAndSetSize(numFiles); - /* - db.Files.Reserve(numFiles); - CNum i; - for (i = 0; i < numFiles; i++) - db.Files.Add(CFileItem()); - */ db.ArcInfo.FileInfoPopIDs.Add(NID::kSize); // if (!db.PackSizes.IsEmpty()) @@ -1169,7 +1231,6 @@ HRESULT CInArchive::ReadHeader( db.ArcInfo.FileInfoPopIDs.Add(NID::kCRC); CBoolVector emptyStreamVector; - BoolVector_Fill_False(emptyStreamVector, (unsigned)numFiles); CBoolVector emptyFileVector; CBoolVector antiFileVector; CNum numEmptyStreams = 0; @@ -1197,13 +1258,13 @@ HRESULT CInArchive::ReadHeader( size_t rem = _inByteBack->GetRem(); db.NamesBuf.Alloc(rem); ReadBytes(db.NamesBuf, rem); - db.NameOffsets.Alloc(db.Files.Size() + 1); + db.NameOffsets.Alloc(numFiles + 1); size_t pos = 0; unsigned i; - for (i = 0; i < db.Files.Size(); i++) + for (i = 0; i < numFiles; i++) { - size_t curRem = (rem - pos) / 2; - const UInt16 *buf = (const UInt16 *)(db.NamesBuf + pos); + const size_t curRem = (rem - pos) / 2; + const UInt16 *buf = (const UInt16 *)(const void *)(db.NamesBuf + pos); size_t j; for (j = 0; j < curRem && buf[j] != 0; j++); if (j == curRem) @@ -1216,36 +1277,31 @@ HRESULT CInArchive::ReadHeader( ThereIsHeaderError = true; break; } + case NID::kWinAttrib: { - CBoolVector boolVector; - ReadBoolVector2(db.Files.Size(), boolVector); + ReadBoolVector2(numFiles, db.Attrib.Defs); CStreamSwitch streamSwitch; streamSwitch.Set(this, &dataVector); - for (CNum i = 0; i < numFiles; i++) - { - CFileItem &file = db.Files[i]; - file.AttribDefined = boolVector[i]; - if (file.AttribDefined) - file.Attrib = ReadUInt32(); - } + Read_UInt32_Vector(db.Attrib); break; } + /* case NID::kIsAux: { - ReadBoolVector(db.Files.Size(), db.IsAux); + ReadBoolVector(numFiles, db.IsAux); break; } case NID::kParent: { db.IsTree = true; // CBoolVector boolVector; - // ReadBoolVector2(db.Files.Size(), boolVector); + // ReadBoolVector2(numFiles, boolVector); // CStreamSwitch streamSwitch; // streamSwitch.Set(this, &dataVector); CBoolVector boolVector; - ReadBoolVector2(db.Files.Size(), boolVector); + ReadBoolVector2(numFiles, boolVector); db.ThereAreAltStreams = false; for (i = 0; i < numFiles; i++) @@ -1264,14 +1320,9 @@ HRESULT CInArchive::ReadHeader( case NID::kEmptyStream: { ReadBoolVector(numFiles, emptyStreamVector); - numEmptyStreams = 0; - for (CNum i = 0; i < (CNum)emptyStreamVector.Size(); i++) - if (emptyStreamVector[i]) - numEmptyStreams++; - - BoolVector_Fill_False(emptyFileVector, numEmptyStreams); - BoolVector_Fill_False(antiFileVector, numEmptyStreams); - + numEmptyStreams = BoolVector_CountSum(emptyStreamVector); + emptyFileVector.Clear(); + antiFileVector.Clear(); break; } case NID::kEmptyFile: ReadBoolVector(numEmptyStreams, emptyFileVector); break; @@ -1314,7 +1365,7 @@ HRESULT CInArchive::ReadHeader( ReadBytes(db.SecureBuf + offset, db.SecureOffsets[i + 1] - offset); } db.SecureIDs.Clear(); - for (unsigned i = 0; i < db.Files.Size(); i++) + for (unsigned i = 0; i < numFiles; i++) { db.SecureIDs.Add(ReadNum()); // db.SecureIDs.Add(ReadUInt32()); @@ -1359,22 +1410,21 @@ HRESULT CInArchive::ReadHeader( CNum emptyFileIndex = 0; CNum sizeIndex = 0; - CNum numAntiItems = 0; + const CNum numAntiItems = BoolVector_CountSum(antiFileVector); - CNum i; + if (numAntiItems != 0) + db.IsAnti.ClearAndSetSize(numFiles); - for (i = 0; i < numEmptyStreams; i++) - if (antiFileVector[i]) - numAntiItems++; + db.Files.ClearAndSetSize(numFiles); - for (i = 0; i < numFiles; i++) + for (CNum i = 0; i < numFiles; i++) { CFileItem &file = db.Files[i]; bool isAnti; - file.HasStream = !emptyStreamVector[i]; file.Crc = 0; - if (file.HasStream) + if (!BoolVector_Item_IsValidAndTrue(emptyStreamVector, i)) { + file.HasStream = true; file.IsDir = false; isAnti = false; file.Size = unpackSizes[sizeIndex]; @@ -1385,26 +1435,31 @@ HRESULT CInArchive::ReadHeader( } else { - file.IsDir = !emptyFileVector[emptyFileIndex]; - isAnti = antiFileVector[emptyFileIndex]; + file.HasStream = false; + file.IsDir = !BoolVector_Item_IsValidAndTrue(emptyFileVector, emptyFileIndex); + isAnti = BoolVector_Item_IsValidAndTrue(antiFileVector, emptyFileIndex); emptyFileIndex++; file.Size = 0; file.CrcDefined = false; } if (numAntiItems != 0) - db.IsAnti.Add(isAnti); + db.IsAnti[i] = isAnti; } + } + db.FillLinks(); - /* - if (type != NID::kEnd) - ThrowIncorrect(); - if (_inByteBack->GetRem() != 0) - ThrowIncorrect(); - */ + + if (type != NID::kEnd || _inByteBack->GetRem() != 0) + { + db.UnsupportedFeatureWarning = true; + // ThrowIncorrect(); + } + return S_OK; } + void CDbEx::FillLinks() { FolderStartFileIndex.Alloc(NumFolders); @@ -1447,25 +1502,29 @@ void CDbEx::FillLinks() } if (indexInFolder != 0) + { folderIndex++; - /* - if (indexInFolder != 0) - ThrowIncorrect(); - */ + // 18.06 + ThereIsHeaderError = true; + // ThrowIncorrect(); + } for (;;) { if (folderIndex >= NumFolders) return; FolderStartFileIndex[folderIndex] = i; - /* if (NumUnpackStreamsVector[folderIndex] != 0) - ThrowIncorrect();; - */ + { + // 18.06 + ThereIsHeaderError = true; + // ThrowIncorrect(); + } folderIndex++; } } + HRESULT CInArchive::ReadDatabase2( DECL_EXTERNAL_CODECS_LOC_VARS CDbEx &db @@ -1497,13 +1556,13 @@ HRESULT CInArchive::ReadDatabase2( const unsigned kCheckSize = 512; Byte buf[kCheckSize]; RINOK(_stream->Seek(0, STREAM_SEEK_END, &fileSize)); - UInt64 rem = fileSize - cur; + const UInt64 rem = fileSize - cur; unsigned checkSize = kCheckSize; if (rem < kCheckSize) checkSize = (unsigned)(rem); if (checkSize < 3) return S_FALSE; - RINOK(_stream->Seek(fileSize - checkSize, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)(fileSize - checkSize), STREAM_SEEK_SET, NULL)); RINOK(ReadStream_FALSE(_stream, buf, (size_t)checkSize)); if (buf[checkSize - 1] != 0) @@ -1512,8 +1571,8 @@ HRESULT CInArchive::ReadDatabase2( unsigned i; for (i = checkSize - 2;; i--) { - if (buf[i] == NID::kEncodedHeader && buf[i + 1] == NID::kPackInfo || - buf[i] == NID::kHeader && buf[i + 1] == NID::kMainStreamsInfo) + if ((buf[i] == NID::kEncodedHeader && buf[i + 1] == NID::kPackInfo) || + (buf[i] == NID::kHeader && buf[i + 1] == NID::kMainStreamsInfo)) break; if (i == 0) return S_FALSE; @@ -1521,7 +1580,7 @@ HRESULT CInArchive::ReadDatabase2( nextHeaderSize = checkSize - i; nextHeaderOffset = rem - nextHeaderSize; nextHeaderCRC = CrcCalc(buf + i, (size_t)nextHeaderSize); - RINOK(_stream->Seek(cur, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)cur, STREAM_SEEK_SET, NULL)); db.StartHeaderWasRecovered = true; } else @@ -1538,25 +1597,32 @@ HRESULT CInArchive::ReadDatabase2( if ((Int64)nextHeaderOffset < 0 || nextHeaderSize > ((UInt64)1 << 62)) return S_FALSE; + + HeadersSize = kHeaderSize; + if (nextHeaderSize == 0) { if (nextHeaderOffset != 0) return S_FALSE; db.IsArc = true; + db.HeadersSize = HeadersSize; return S_OK; } if (!db.StartHeaderWasRecovered) db.IsArc = true; - HeadersSize += kHeaderSize + nextHeaderSize; + HeadersSize += nextHeaderSize; + // db.EndHeaderOffset = nextHeaderOffset; + _rangeLimit = nextHeaderOffset; + db.PhySize = kHeaderSize + nextHeaderOffset + nextHeaderSize; if (_fileEndPosition - db.ArcInfo.StartPositionAfterHeader < nextHeaderOffset + nextHeaderSize) { db.UnexpectedEnd = true; return S_FALSE; } - RINOK(_stream->Seek(nextHeaderOffset, STREAM_SEEK_CUR, NULL)); + RINOK(_stream->Seek((Int64)nextHeaderOffset, STREAM_SEEK_CUR, NULL)); size_t nextHeaderSize_t = (size_t)nextHeaderSize; if (nextHeaderSize_t != nextHeaderSize) @@ -1610,6 +1676,7 @@ HRESULT CInArchive::ReadDatabase2( ); } + HRESULT CInArchive::ReadDatabase( DECL_EXTERNAL_CODECS_LOC_VARS CDbEx &db diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.h index 3592e99b530..ffa1e4bc7b1 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zIn.h @@ -115,6 +115,7 @@ struct CDatabase: public CFolders CUInt64DefVector ATime; CUInt64DefVector MTime; CUInt64DefVector StartPos; + CUInt32DefVector Attrib; CBoolVector IsAnti; /* CBoolVector IsAux; @@ -146,6 +147,7 @@ struct CDatabase: public CFolders ATime.Clear(); MTime.Clear(); StartPos.Clear(); + Attrib.Clear(); IsAnti.Clear(); // IsAux.Clear(); } @@ -172,13 +174,14 @@ struct CDatabase: public CFolders HRESULT GetPath_Prop(unsigned index, PROPVARIANT *path) const throw(); }; + struct CInArchiveInfo { CArchiveVersion Version; - UInt64 StartPosition; - UInt64 StartPositionAfterHeader; - UInt64 DataStartPosition; - UInt64 DataStartPosition2; + UInt64 StartPosition; // in stream + UInt64 StartPositionAfterHeader; // in stream + UInt64 DataStartPosition; // in stream + UInt64 DataStartPosition2; // in stream. it's for headers CRecordVector FileInfoPopIDs; void Clear() @@ -191,6 +194,7 @@ struct CInArchiveInfo } }; + struct CDbEx: public CDatabase { CInArchiveInfo ArcInfo; @@ -200,6 +204,7 @@ struct CDbEx: public CDatabase UInt64 HeadersSize; UInt64 PhySize; + // UInt64 EndHeaderOffset; // relative to position after StartHeader (32 bytes) /* CRecordVector SecureOffsets; @@ -253,6 +258,17 @@ struct CDbEx: public CDatabase HeadersSize = 0; PhySize = 0; + // EndHeaderOffset = 0; + } + + bool CanUpdate() const + { + if (ThereIsHeaderError + || UnexpectedEnd + || StartHeaderWasRecovered + || UnsupportedFeatureError) + return false; + return true; } void FillLinks(); @@ -337,6 +353,8 @@ class CInArchive UInt64 _arhiveBeginStreamPosition; UInt64 _fileEndPosition; + UInt64 _rangeLimit; // relative to position after StartHeader (32 bytes) + Byte _header[kHeaderSize]; UInt64 HeadersSize; @@ -369,6 +387,8 @@ class CInArchive void SkipData() { _inByteBack->SkipData(); } void WaitId(UInt64 id); + void Read_UInt32_Vector(CUInt32DefVector &v); + void ReadArchiveProperties(CInArchiveInfo &archiveInfo); void ReadHashDigests(unsigned numItems, CUInt32DefVector &crcs); diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zItem.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zItem.h index 5e2b58f2ef4..0f9fdada687 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zItem.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zItem.h @@ -26,12 +26,14 @@ struct CCoderInfo bool IsSimpleCoder() const { return NumStreams == 1; } }; + struct CBond { UInt32 PackIndex; UInt32 UnpackIndex; }; + struct CFolder { CLASS_NO_COPY(CFolder) @@ -48,7 +50,7 @@ public: { FOR_VECTOR(i, PackStreams) if (PackStreams[i] == packStream) - return i; + return (int)i; return -1; } @@ -56,7 +58,7 @@ public: { FOR_VECTOR(i, Bonds) if (Bonds[i].PackIndex == packStream) - return i; + return (int)i; return -1; } @@ -87,6 +89,7 @@ public: } }; + struct CUInt32DefVector { CBoolVector Defs; @@ -110,9 +113,25 @@ struct CUInt32DefVector Vals.ReserveDown(); } + bool GetItem(unsigned index, UInt32 &value) const + { + if (index < Defs.Size() && Defs[index]) + { + value = Vals[index]; + return true; + } + value = 0; + return false; + } + bool ValidAndDefined(unsigned i) const { return i < Defs.Size() && Defs[i]; } + + bool CheckSize(unsigned size) const { return Defs.Size() == size || Defs.Size() == 0; } + + void SetItem(unsigned index, bool defined, UInt32 value); }; + struct CUInt64DefVector { CBoolVector Defs; @@ -141,15 +160,15 @@ struct CUInt64DefVector return false; } - void SetItem(unsigned index, bool defined, UInt64 value); - bool CheckSize(unsigned size) const { return Defs.Size() == size || Defs.Size() == 0; } + + void SetItem(unsigned index, bool defined, UInt64 value); }; + struct CFileItem { UInt64 Size; - UInt32 Attrib; UInt32 Crc; /* int Parent; @@ -159,23 +178,23 @@ struct CFileItem // stream in some folder. It can be empty stream bool IsDir; bool CrcDefined; - bool AttribDefined; + + /* + void Clear() + { + HasStream = true; + IsDir = false; + CrcDefined = false; + } CFileItem(): - /* - Parent(-1), - IsAltStream(false), - */ + // Parent(-1), + // IsAltStream(false), HasStream(true), IsDir(false), CrcDefined(false), - AttribDefined(false) {} - void SetAttrib(UInt32 attrib) - { - AttribDefined = true; - Attrib = attrib; - } + */ }; }} diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.cpp index 3e70f466e7b..2786bf28772 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.cpp @@ -5,6 +5,7 @@ #include "../../../../C/7zCrc.h" #include "../../../Common/AutoPtr.h" +// #include "../../../Common/UTFConvert.h" #include "../../Common/StreamObjects.h" @@ -196,7 +197,7 @@ void COutArchive::WriteNumber(UInt64 value) break; } firstByte |= mask; - mask >>= 1; + mask = (Byte)(mask >> 1); } WriteByte(firstByte); for (; i > 0; i--) @@ -206,9 +207,9 @@ void COutArchive::WriteNumber(UInt64 value) } } -static UInt32 GetBigNumberSize(UInt64 value) +static unsigned GetBigNumberSize(UInt64 value) { - int i; + unsigned i; for (i = 1; i < 9; i++) if (value < (((UInt64)1 << (i * 7)))) break; @@ -264,18 +265,18 @@ void COutArchive::WriteFolder(const CFolder &folder) for (idSize = 1; idSize < sizeof(id); idSize++) if ((id >> (8 * idSize)) == 0) break; - idSize &= 0xF; + // idSize &= 0xF; // idSize is smaller than 16 already Byte temp[16]; for (unsigned t = idSize; t != 0; t--, id >>= 8) temp[t] = (Byte)(id & 0xFF); - Byte b = (Byte)(idSize); - bool isComplex = !coder.IsSimpleCoder(); + unsigned b = idSize; + const bool isComplex = !coder.IsSimpleCoder(); b |= (isComplex ? 0x10 : 0); - size_t propsSize = coder.Props.Size(); + const size_t propsSize = coder.Props.Size(); b |= ((propsSize != 0) ? 0x20 : 0); - temp[0] = b; + temp[0] = (Byte)b; WriteBytes(temp, idSize + 1); if (isComplex) { @@ -309,7 +310,7 @@ void COutArchive::WriteBoolVector(const CBoolVector &boolVector) { if (boolVector[i]) b |= mask; - mask >>= 1; + mask = (Byte)(mask >> 1); if (mask == 0) { WriteByte(b); @@ -330,13 +331,11 @@ void COutArchive::WritePropBoolVector(Byte id, const CBoolVector &boolVector) WriteBoolVector(boolVector); } +unsigned BoolVector_CountSum(const CBoolVector &v); + void COutArchive::WriteHashDigests(const CUInt32DefVector &digests) { - unsigned numDefined = 0; - unsigned i; - for (i = 0; i < digests.Defs.Size(); i++) - if (digests.Defs[i]) - numDefined++; + const unsigned numDefined = BoolVector_CountSum(digests.Defs); if (numDefined == 0) return; @@ -348,7 +347,8 @@ void COutArchive::WriteHashDigests(const CUInt32DefVector &digests) WriteByte(0); WriteBoolVector(digests.Defs); } - for (i = 0; i < digests.Defs.Size(); i++) + + for (unsigned i = 0; i < digests.Defs.Size(); i++) if (digests.Defs[i]) WriteUInt32(digests.Vals[i]); } @@ -453,10 +453,12 @@ void COutArchive::WriteSubStreamsInfo(const CObjectVector &folders, // 7-Zip 4.50 - 4.58 contain BUG, so they do not support .7z archives with Unknown field. -void COutArchive::SkipAlign(unsigned pos, unsigned alignSize) +void COutArchive::SkipToAligned(unsigned pos, unsigned alignShifts) { if (!_useAlign) return; + + const unsigned alignSize = (unsigned)1 << alignShifts; pos += (unsigned)GetPos(); pos &= (alignSize - 1); if (pos == 0) @@ -471,11 +473,11 @@ void COutArchive::SkipAlign(unsigned pos, unsigned alignSize) WriteByte(0); } -void COutArchive::WriteAlignedBoolHeader(const CBoolVector &v, unsigned numDefined, Byte type, unsigned itemSize) +void COutArchive::WriteAlignedBools(const CBoolVector &v, unsigned numDefined, Byte type, unsigned itemSizeShifts) { const unsigned bvSize = (numDefined == v.Size()) ? 0 : Bv_GetSizeInBytes(v); - const UInt64 dataSize = (UInt64)numDefined * itemSize + bvSize + 2; - SkipAlign(3 + (unsigned)bvSize + (unsigned)GetBigNumberSize(dataSize), itemSize); + const UInt64 dataSize = ((UInt64)numDefined << itemSizeShifts) + bvSize + 2; + SkipToAligned(3 + bvSize + GetBigNumberSize(dataSize), itemSizeShifts); WriteByte(type); WriteNumber(dataSize); @@ -486,24 +488,18 @@ void COutArchive::WriteAlignedBoolHeader(const CBoolVector &v, unsigned numDefin WriteByte(0); WriteBoolVector(v); } - WriteByte(0); + WriteByte(0); // 0 means no switching to external stream } void COutArchive::WriteUInt64DefVector(const CUInt64DefVector &v, Byte type) { - unsigned numDefined = 0; - - unsigned i; - for (i = 0; i < v.Defs.Size(); i++) - if (v.Defs[i]) - numDefined++; - + const unsigned numDefined = BoolVector_CountSum(v.Defs); if (numDefined == 0) return; - WriteAlignedBoolHeader(v.Defs, numDefined, type, 8); + WriteAlignedBools(v.Defs, numDefined, type, 3); - for (i = 0; i < v.Defs.Size(); i++) + for (unsigned i = 0; i < v.Defs.Size(); i++) if (v.Defs[i]) WriteUInt64(v.Vals[i]); } @@ -520,7 +516,7 @@ HRESULT COutArchive::EncodeStream( outFolders.FolderUnpackCRCs.Vals.Add(CrcCalc(data, data.Size())); // outFolders.NumUnpackStreamsVector.Add(1); UInt64 dataSize64 = data.Size(); - UInt64 unpackSize; + UInt64 unpackSize = data.Size(); RINOK(encoder.Encode( EXTERNAL_CODECS_LOC_VARS stream, @@ -550,7 +546,39 @@ void COutArchive::WriteHeader( WriteByte(NID::kHeader); - // Archive Properties + /* + { + // It's example for per archive properies writing + + WriteByte(NID::kArchiveProperties); + + // you must use random 40-bit number that will identify you + // then you can use same kDeveloperID for any properties and methods + const UInt64 kDeveloperID = 0x123456789A; // change that value to real random 40-bit number + + #define GENERATE_7Z_ID(developerID, subID) (((UInt64)0x3F << 56) | ((UInt64)developerID << 16) | subID) + + { + const UInt64 kSubID = 0x1; // you can use small number for subID + const UInt64 kID = GENERATE_7Z_ID(kDeveloperID, kSubID); + WriteNumber(kID); + const unsigned kPropsSize = 3; // it's example size + WriteNumber(kPropsSize); + for (unsigned i = 0; i < kPropsSize; i++) + WriteByte((Byte)(i & 0xFF)); + } + { + const UInt64 kSubID = 0x2; // you can use small number for subID + const UInt64 kID = GENERATE_7Z_ID(kDeveloperID, kSubID); + WriteNumber(kID); + const unsigned kPropsSize = 5; // it's example size + WriteNumber(kPropsSize); + for (unsigned i = 0; i < kPropsSize; i++) + WriteByte((Byte)(i + 16)); + } + WriteByte(NID::kEnd); + } + */ if (db.Folders.Size() > 0) { @@ -642,13 +670,21 @@ void COutArchive::WriteHeader( const UString &name = db.Names[i]; if (!name.IsEmpty()) numDefined++; - namesDataSize += (name.Len() + 1) * 2; + const size_t numUtfChars = + /* + #if WCHAR_MAX > 0xffff + Get_Num_Utf16_chars_from_wchar_string(name.Ptr()); + #else + */ + name.Len(); + // #endif + namesDataSize += (numUtfChars + 1) * 2; } if (numDefined > 0) { namesDataSize++; - SkipAlign(2 + GetBigNumberSize(namesDataSize), 16); + SkipToAligned(2 + GetBigNumberSize(namesDataSize), 4); WriteByte(NID::kName); WriteNumber(namesDataSize); @@ -659,6 +695,25 @@ void COutArchive::WriteHeader( for (unsigned t = 0; t <= name.Len(); t++) { wchar_t c = name[t]; + + /* + #if WCHAR_MAX > 0xffff + if (c >= 0x10000) + { + c -= 0x10000; + if (c < (1 << 20)) + { + unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); + WriteByte((Byte)c0); + WriteByte((Byte)(c0 >> 8)); + c = 0xdc00 + (c & 0x3FF); + } + else + c = '_'; // we change character unsupported by UTF16 + } + #endif + */ + WriteByte((Byte)c); WriteByte((Byte)(c >> 8)); } @@ -673,28 +728,15 @@ void COutArchive::WriteHeader( { /* ---------- Write Attrib ---------- */ - CBoolVector boolVector; - boolVector.ClearAndSetSize(db.Files.Size()); - unsigned numDefined = 0; - - { - FOR_VECTOR (i, db.Files) - { - bool defined = db.Files[i].AttribDefined; - boolVector[i] = defined; - if (defined) - numDefined++; - } - } + const unsigned numDefined = BoolVector_CountSum(db.Attrib.Defs); if (numDefined != 0) { - WriteAlignedBoolHeader(boolVector, numDefined, NID::kWinAttrib, 4); - FOR_VECTOR (i, db.Files) + WriteAlignedBools(db.Attrib.Defs, numDefined, NID::kWinAttrib, 2); + FOR_VECTOR (i, db.Attrib.Defs) { - const CFileItem &file = db.Files[i]; - if (file.AttribDefined) - WriteUInt32(file.Attrib); + if (db.Attrib.Defs[i]) + WriteUInt32(db.Attrib.Vals[i]); } } } @@ -702,18 +744,8 @@ void COutArchive::WriteHeader( /* { // ---------- Write IsAux ---------- - unsigned numAux = 0; - const CBoolVector &isAux = db.IsAux; - for (i = 0; i < isAux.Size(); i++) - if (isAux[i]) - numAux++; - if (numAux > 0) - { - const unsigned bvSize = Bv_GetSizeInBytes(isAux); - WriteByte(NID::kIsAux); - WriteNumber(bvSize); - WriteBoolVector(isAux); - } + if (BoolVector_CountSum(db.IsAux) != 0) + WritePropBoolVector(NID::kIsAux, db.IsAux); } { @@ -734,10 +766,10 @@ void COutArchive::WriteHeader( } if (numParentLinks > 0) { - // WriteAlignedBoolHeader(boolVector, numDefined, NID::kParent, 4); + // WriteAlignedBools(boolVector, numDefined, NID::kParent, 2); const unsigned bvSize = (numIsDir == boolVector.Size()) ? 0 : Bv_GetSizeInBytes(boolVector); const UInt64 dataSize = (UInt64)db.Files.Size() * 4 + bvSize + 1; - SkipAlign(2 + (unsigned)bvSize + (unsigned)GetBigNumberSize(dataSize), 4); + SkipToAligned(2 + (unsigned)bvSize + (unsigned)GetBigNumberSize(dataSize), 2); WriteByte(NID::kParent); WriteNumber(dataSize); @@ -765,7 +797,7 @@ void COutArchive::WriteHeader( // secureDataSize += db.SecureIDs.Size() * 4; for (i = 0; i < db.SecureIDs.Size(); i++) secureDataSize += GetBigNumberSize(db.SecureIDs[i]); - SkipAlign(2 + GetBigNumberSize(secureDataSize), 4); + SkipToAligned(2 + GetBigNumberSize(secureDataSize), 2); WriteByte(NID::kNtSecure); WriteNumber(secureDataSize); WriteByte(0); @@ -883,11 +915,23 @@ HRESULT COutArchive::WriteDatabase( h.NextHeaderSize = headerSize; h.NextHeaderCRC = headerCRC; h.NextHeaderOffset = headerOffset; - RINOK(Stream->Seek(_prefixHeaderPos, STREAM_SEEK_SET, NULL)); + RINOK(Stream->Seek((Int64)_prefixHeaderPos, STREAM_SEEK_SET, NULL)); return WriteStartHeader(h); } } +void CUInt32DefVector::SetItem(unsigned index, bool defined, UInt32 value) +{ + while (index >= Defs.Size()) + Defs.Add(false); + Defs[index] = defined; + if (!defined) + return; + while (index >= Vals.Size()) + Vals.Add(0); + Vals[index] = value; +} + void CUInt64DefVector::SetItem(unsigned index, bool defined, UInt64 value) { while (index >= Defs.Size()) @@ -907,6 +951,7 @@ void CArchiveDatabaseOut::AddFile(const CFileItem &file, const CFileItem2 &file2 ATime.SetItem(index, file2.ATimeDefined, file2.ATime); MTime.SetItem(index, file2.MTimeDefined, file2.MTime); StartPos.SetItem(index, file2.StartPosDefined, file2.StartPos); + Attrib.SetItem(index, file2.AttribDefined, file2.Attrib); SetItem_Anti(index, file2.IsAnti); // SetItem_Aux(index, file2.IsAux); Names.Add(name); diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.h index 6c902668939..1ebad56d271 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zOut.h @@ -45,6 +45,7 @@ public: size_t GetPos() const { return _pos; } }; + struct CHeaderOptions { bool CompressMainHeader; @@ -71,24 +72,31 @@ struct CFileItem2 UInt64 ATime; UInt64 MTime; UInt64 StartPos; + UInt32 Attrib; + bool CTimeDefined; bool ATimeDefined; bool MTimeDefined; bool StartPosDefined; + bool AttribDefined; bool IsAnti; // bool IsAux; + /* void Init() { CTimeDefined = false; ATimeDefined = false; MTimeDefined = false; StartPosDefined = false; + AttribDefined = false; IsAnti = false; // IsAux = false; } + */ }; + struct COutFolders { CUInt32DefVector FolderUnpackCRCs; // Now we use it for headers only. @@ -111,6 +119,7 @@ struct COutFolders } }; + struct CArchiveDatabaseOut: public COutFolders { CRecordVector PackSizes; @@ -123,10 +132,11 @@ struct CArchiveDatabaseOut: public COutFolders CUInt64DefVector ATime; CUInt64DefVector MTime; CUInt64DefVector StartPos; - CRecordVector IsAnti; + CUInt32DefVector Attrib; + CBoolVector IsAnti; /* - CRecordVector IsAux; + CBoolVector IsAux; CByteBuffer SecureBuf; CRecordVector SecureSizes; @@ -154,6 +164,7 @@ struct CArchiveDatabaseOut: public COutFolders ATime.Clear(); MTime.Clear(); StartPos.Clear(); + Attrib.Clear(); IsAnti.Clear(); /* @@ -176,6 +187,7 @@ struct CArchiveDatabaseOut: public COutFolders ATime.ReserveDown(); MTime.ReserveDown(); StartPos.ReserveDown(); + Attrib.ReserveDown(); IsAnti.ReserveDown(); /* @@ -196,11 +208,12 @@ struct CArchiveDatabaseOut: public COutFolders { unsigned size = Files.Size(); return ( - CTime.CheckSize(size) && - ATime.CheckSize(size) && - MTime.CheckSize(size) && - StartPos.CheckSize(size) && - (size == IsAnti.Size() || IsAnti.Size() == 0)); + CTime.CheckSize(size) + && ATime.CheckSize(size) + && MTime.CheckSize(size) + && StartPos.CheckSize(size) + && Attrib.CheckSize(size) + && (size == IsAnti.Size() || IsAnti.Size() == 0)); } bool IsItemAnti(unsigned index) const { return (index < IsAnti.Size() && IsAnti[index]); } @@ -224,6 +237,7 @@ struct CArchiveDatabaseOut: public COutFolders void AddFile(const CFileItem &file, const CFileItem2 &file2, const UString &name); }; + class COutArchive { UInt64 _prefixHeaderPos; @@ -261,8 +275,8 @@ class COutArchive const CRecordVector &unpackSizes, const CUInt32DefVector &digests); - void SkipAlign(unsigned pos, unsigned alignSize); - void WriteAlignedBoolHeader(const CBoolVector &v, unsigned numDefined, Byte type, unsigned itemSize); + void SkipToAligned(unsigned pos, unsigned alignShifts); + void WriteAlignedBools(const CBoolVector &v, unsigned numDefined, Byte type, unsigned itemSizeShifts); void WriteUInt64DefVector(const CUInt64DefVector &v, Byte type); HRESULT EncodeStream( diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zRegister.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zRegister.cpp index 389b54074e4..eac8b4f2d3d 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zRegister.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zRegister.cpp @@ -15,7 +15,13 @@ REGISTER_ARC_IO_DECREMENT_SIG( "7z", "7z", NULL, 7, k_Signature_Dec, 0, - NArcInfoFlags::kFindSignature, - NULL); + NArcInfoFlags::kFindSignature + | NArcInfoFlags::kCTime + | NArcInfoFlags::kATime + | NArcInfoFlags::kMTime + | NArcInfoFlags::kMTime_Default + , TIME_PREC_TO_ARC_FLAGS_MASK(NFileTimeType::kWindows) + | TIME_PREC_TO_ARC_FLAGS_TIME_DEFAULT(NFileTimeType::kWindows) + , NULL); }} diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.cpp b/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.cpp index 68e57f09d5f..b6fd1924e67 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -55,8 +55,8 @@ struct CFilterMode #define PE_SIG 0x00004550 #define PE_OptHeader_Magic_32 0x10B #define PE_OptHeader_Magic_64 0x20B -#define PE_SectHeaderSize 40 -#define PE_SECT_EXECUTE 0x20000000 +// #define PE_SectHeaderSize 40 +// #define PE_SECT_EXECUTE 0x20000000 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) { @@ -124,13 +124,13 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) #define ELF_DATA_2LSB 1 #define ELF_DATA_2MSB 2 -static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } -static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); } -// static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); } +static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } +static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); } +// static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); } static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode) { - Bool /* is32, */ be; + BoolInt /* is32, */ be; UInt32 filterId; if (size < 512 || buf[6] != 1) /* ver */ @@ -200,7 +200,7 @@ static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode if (size < 512) return 0; - Bool /* mode64, */ be; + BoolInt /* mode64, */ be; switch (GetUi32(buf)) { case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break; @@ -239,7 +239,7 @@ static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode #define RIFF_SIG 0x46464952 -static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) +static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) { UInt32 subChunkSize, pos; if (size < 0x2C) @@ -254,10 +254,12 @@ static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1) return False; - unsigned numChannels = GetUi16(buf + 0x16); - unsigned bitsPerSample = GetUi16(buf + 0x22); - - if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256) + const unsigned numChannels = GetUi16(buf + 0x16); + const unsigned bitsPerSample = GetUi16(buf + 0x22); + if ((bitsPerSample & 0x7) != 0) + return False; + const UInt32 delta = (UInt32)numChannels * (bitsPerSample >> 3); + if (delta == 0 || delta > 256) return False; pos = 0x14 + subChunkSize; @@ -271,9 +273,6 @@ static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) subChunkSize = GetUi32(buf + pos + 4); if (GetUi32(buf + pos) == WAV_SUBCHUNK_data) { - unsigned delta = numChannels * (bitsPerSample >> 3); - if (delta >= 256) - return False; filterMode->Id = k_Delta; filterMode->Delta = delta; return True; @@ -285,7 +284,7 @@ static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) return False; } -static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) +static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) { filterMode->Id = 0; filterMode->Delta = 0; @@ -418,7 +417,7 @@ static unsigned Get_FilterGroup_for_Folder( static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream, UInt64 position, UInt64 size, ICompressProgressInfo *progress) { - RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0)); + RINOK(inStream->Seek((Int64)position, STREAM_SEEK_SET, 0)); CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; CMyComPtr inStreamLimited(streamSpec); streamSpec->SetStream(inStream); @@ -642,13 +641,13 @@ struct CRefItem if (sortByType) { int slashPos = ui.Name.ReverseFind_PathSepar(); - NamePos = slashPos + 1; + NamePos = (unsigned)(slashPos + 1); int dotPos = ui.Name.ReverseFind_Dot(); if (dotPos <= slashPos) ExtensionPos = ui.Name.Len(); else { - ExtensionPos = dotPos + 1; + ExtensionPos = (unsigned)(dotPos + 1); if (ExtensionPos != ui.Name.Len()) { AString s; @@ -805,10 +804,20 @@ struct CAnalysis bool ParseExe; bool ParseAll; + /* + bool Need_ATime; + bool ATime_Defined; + FILETIME ATime; + */ + CAnalysis(): ParseWav(true), ParseExe(false), ParseAll(false) + /* + , Need_ATime(false) + , ATime_Defined(false) + */ {} HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode); @@ -836,7 +845,7 @@ HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMo { const wchar_t *ext; if (dotPos > slashPos) - ext = ui.Name.Ptr(dotPos + 1); + ext = ui.Name.Ptr((unsigned)(dotPos + 1)); else ext = ui.Name.RightPtr(0); @@ -888,13 +897,25 @@ HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMo HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze); if (result == S_OK && stream) { + /* + if (Need_ATime) + { + // access time could be changed in analysis pass + CMyComPtr getProps; + stream.QueryInterface(IID_IStreamGetProps, (void **)&getProps); + if (getProps) + if (getProps->GetProps(NULL, NULL, &ATime, NULL, NULL) == S_OK) + ATime_Defined = true; + } + */ + size_t size = kAnalysisBufSize; result = ReadStream(stream, Buffer, &size); stream.Release(); // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK)); if (result == S_OK) { - Bool parseRes = ParseFile(Buffer, size, &filterModeTemp); + BoolInt parseRes = ParseFile(Buffer, size, &filterModeTemp); if (parseRes && filterModeTemp.Delta == 0) { filterModeTemp.SetDelta(); @@ -1071,12 +1092,12 @@ static HRESULT MakeExeMethod(CCompressionMethodMode &mode, if (alignBits != 0) { if (alignBits > 2 || filterMode.Id == k_Delta) - nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits); + nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits); unsigned lc = 0; if (alignBits < 3) - lc = 3 - alignBits; + lc = (unsigned)(3 - alignBits); nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc); - nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits); + nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits); } } } @@ -1088,18 +1109,23 @@ static HRESULT MakeExeMethod(CCompressionMethodMode &mode, } -static void FromUpdateItemToFileItem(const CUpdateItem &ui, - CFileItem &file, CFileItem2 &file2) +static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2) { - if (ui.AttribDefined) - file.SetAttrib(ui.Attrib); - + file2.Attrib = ui.Attrib; file2.AttribDefined = ui.AttribDefined; file2.CTime = ui.CTime; file2.CTimeDefined = ui.CTimeDefined; file2.ATime = ui.ATime; file2.ATimeDefined = ui.ATimeDefined; file2.MTime = ui.MTime; file2.MTimeDefined = ui.MTimeDefined; file2.IsAnti = ui.IsAnti; // file2.IsAux = false; file2.StartPosDefined = false; + // file2.StartPos = 0; +} + + +static void UpdateItem_To_FileItem(const CUpdateItem &ui, + CFileItem &file, CFileItem2 &file2) +{ + UpdateItem_To_FileItem2(ui, file2); file.Size = ui.Size; file.IsDir = ui.IsDir; @@ -1107,6 +1133,8 @@ static void FromUpdateItemToFileItem(const CUpdateItem &ui, // file.IsAltStream = ui.IsAltStream; } + + class CRepackInStreamWithSizes: public ISequentialInStream, public ICompressGetSubStreamSize, @@ -1437,6 +1465,7 @@ public: #ifndef _7ZIP_ST + bool dataAfterEnd_Error; HRESULT Result; CMyComPtr InStream; @@ -1445,7 +1474,7 @@ public: UInt64 StartPos; const CFolders *Folders; - int FolderIndex; + unsigned FolderIndex; // bool send_UnpackSize; // UInt64 UnpackSize; @@ -1479,7 +1508,9 @@ void CThreadDecoder::Execute() bool passwordIsDefined = false; UString password; #endif - + + dataAfterEnd_Error = false; + Result = Decoder.Decode( EXTERNAL_CODECS_LOC_VARS InStream, @@ -1491,12 +1522,16 @@ void CThreadDecoder::Execute() Fos, NULL, // compressProgress + NULL // *inStreamMainRes + , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS #ifndef _7ZIP_ST - , MtMode, NumThreads + , MtMode, NumThreads, + 0 // MemUsage #endif + ); } catch(...) @@ -1541,6 +1576,7 @@ static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFil file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime); file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime); file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos); + file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib); file2.IsAnti = inDb.IsItemAnti(index); // file2.IsAux = inDb.IsItemAux(index); } @@ -1572,6 +1608,11 @@ HRESULT Update( CMyComPtr extractCallback; updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback); + /* + CMyComPtr reportArcProp; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackArcProp, (void **)&reportArcProp); + */ + // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes(); /* @@ -1594,7 +1635,10 @@ HRESULT Update( CRecordVector filters; CObjectVector groups; + + #ifndef _7ZIP_ST bool thereAreRepacks = false; + #endif bool useFilters = options.UseFilters; if (useFilters) @@ -1621,7 +1665,7 @@ HRESULT Update( { int index = updateItems[i].IndexInArchive; if (index != -1) - fileIndexToUpdateIndexMap[(unsigned)index] = i; + fileIndexToUpdateIndexMap[(unsigned)index] = (int)i; } for (i = 0; i < db->NumFolders; i++) @@ -1633,12 +1677,15 @@ HRESULT Update( for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++) { + if (fi >= db->Files.Size()) + return E_FAIL; + const CFileItem &file = db->Files[fi]; if (file.HasStream) { indexInFolder++; int updateIndex = fileIndexToUpdateIndexMap[fi]; - if (updateIndex >= 0 && !updateItems[updateIndex].NewData) + if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData) { numCopyItems++; repackSize += file.Size; @@ -1670,7 +1717,9 @@ HRESULT Update( complexity += db->GetFolderFullPackSize(i); else { + #ifndef _7ZIP_ST thereAreRepacks = true; + #endif complexity += repackSize; if (inSizeForReduce2 < repackSize) inSizeForReduce2 = repackSize; @@ -1682,13 +1731,14 @@ HRESULT Update( UInt64 inSizeForReduce = 0; { + bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); FOR_VECTOR (i, updateItems) { const CUpdateItem &ui = updateItems[i]; if (ui.NewData) { complexity += ui.Size; - if (numSolidFiles != 1) + if (isSolid) inSizeForReduce += ui.Size; else if (inSizeForReduce < ui.Size) inSizeForReduce = ui.Size; @@ -1708,10 +1758,12 @@ HRESULT Update( #ifndef _7ZIP_ST CStreamBinder sb; + /* if (options.MultiThreadMixer) { RINOK(sb.CreateEvents()); } + */ #endif @@ -1723,12 +1775,15 @@ HRESULT Update( #ifdef EXTERNAL_CODECS threadDecoder.__externalCodecs = __externalCodecs; #endif - RINOK(threadDecoder.Create()); + WRes wres = threadDecoder.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } #endif { CAnalysis analysis; + // analysis.Need_ATime = options.Need_ATime; if (options.AnalysisLevel == 0) { analysis.ParseWav = false; @@ -1763,7 +1818,15 @@ HRESULT Update( CFilterMode2 fm; if (useFilters) { + // analysis.ATime_Defined = false; RINOK(analysis.GetFilterGroup(i, ui, fm)); + /* + if (analysis.ATime_Defined) + { + ui.ATime = FILETIME_To_UInt64(analysis.ATime); + ui.ATime_WasReadByAnalysis = true; + } + */ } fm.Encrypted = method.PasswordIsDefined; @@ -1837,7 +1900,7 @@ HRESULT Update( continue; secureID = ui.SecureIndex; if (ui.NewProps) - FromUpdateItemToFileItem(ui, file, file2); + UpdateItem_To_FileItem(ui, file, file2); else GetFile(*db, ui.IndexInArchive, file, file2); } @@ -1868,7 +1931,7 @@ HRESULT Update( if (ui.HasStream()) continue; } - else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream) + else if (ui.IndexInArchive != -1 && db->Files[(unsigned)ui.IndexInArchive].HasStream) continue; /* if (ui.TreeFolderIndex >= 0) @@ -1887,13 +1950,14 @@ HRESULT Update( UString name; if (ui.NewProps) { - FromUpdateItemToFileItem(ui, file, file2); + UpdateItem_To_FileItem(ui, file, file2); + file.CrcDefined = false; name = ui.Name; } else { - GetFile(*db, ui.IndexInArchive, file, file2); - db->GetPath(ui.IndexInArchive, name); + GetFile(*db, (unsigned)ui.IndexInArchive, file, file2); + db->GetPath((unsigned)ui.IndexInArchive, name); } /* @@ -2045,7 +2109,7 @@ HRESULT Update( { indexInFolder++; int updateIndex = fileIndexToUpdateIndexMap[fi]; - if (updateIndex >= 0 && !updateItems[updateIndex].NewData) + if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData) needExtract = true; // decodeSize += file.Size; } @@ -2079,8 +2143,8 @@ HRESULT Update( { repackBase = threadDecoder.FosSpec; CMyComPtr sbOutStream; - sb.CreateStreams(&sbInStream, &sbOutStream); - sb.ReInit(); + sb.CreateStreams2(sbInStream, sbOutStream); + RINOK(sb.Create_ReInit()); threadDecoder.FosSpec->_stream = sbOutStream; @@ -2107,6 +2171,8 @@ HRESULT Update( #endif CMyComPtr decodedStream; + bool dataAfterEnd_Error = false; + HRESULT res = threadDecoder.Decoder.Decode( EXTERNAL_CODECS_LOC_VARS inStream, @@ -2117,12 +2183,15 @@ HRESULT Update( NULL, // *outStream NULL, // *compressProgress + &decodedStream + , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS #ifndef _7ZIP_ST , false // mtMode , 1 // numThreads + , 0 // memUsage #endif ); @@ -2146,11 +2215,14 @@ HRESULT Update( #ifndef _7ZIP_ST if (options.MultiThreadMixer) { - threadDecoder.Start(); + WRes wres = threadDecoder.Start(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } #endif } + curUnpackSize = sizeToEncode; HRESULT encodeRes = encoder.Encode( EXTERNAL_CODECS_LOC_VARS @@ -2171,20 +2243,27 @@ HRESULT Update( inStreamSizeCount.Release(); sbInStream.Release(); - threadDecoder.WaitExecuteFinish(); + { + WRes wres = threadDecoder.WaitExecuteFinish(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } HRESULT decodeRes = threadDecoder.Result; // if (res == k_My_HRESULT_CRC_ERROR) - if (decodeRes == S_FALSE) + if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error) { if (extractCallback) { RINOK(extractCallback->ReportExtractResult( NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex], // NEventIndexType::kBlockIndex, (UInt32)folderIndex, - NExtract::NOperationResult::kDataError)); + (decodeRes != S_OK ? + NExtract::NOperationResult::kDataError : + NExtract::NOperationResult::kDataAfterEnd))); } - return E_FAIL; + if (decodeRes != S_OK) + return E_FAIL; } RINOK(decodeRes); if (encodeRes == S_OK) @@ -2224,31 +2303,30 @@ HRESULT Update( CNum indexInFolder = 0; for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++) { - CFileItem file; - CFileItem2 file2; - GetFile(*db, fi, file, file2); - UString name; - db->GetPath(fi, name); - if (file.HasStream) + if (db->Files[fi].HasStream) { indexInFolder++; int updateIndex = fileIndexToUpdateIndexMap[fi]; if (updateIndex >= 0) { - const CUpdateItem &ui = updateItems[updateIndex]; + const CUpdateItem &ui = updateItems[(unsigned)updateIndex]; if (ui.NewData) continue; + + UString name; + CFileItem file; + CFileItem2 file2; + GetFile(*db, fi, file, file2); + if (ui.NewProps) { - CFileItem uf; - FromUpdateItemToFileItem(ui, uf, file2); - uf.Size = file.Size; - uf.Crc = file.Crc; - uf.CrcDefined = file.CrcDefined; - uf.HasStream = file.HasStream; - file = uf; + UpdateItem_To_FileItem2(ui, file2); + file.IsDir = ui.IsDir; name = ui.Name; } + else + db->GetPath(fi, name); + /* file.Parent = ui.ParentFolderIndex; if (ui.TreeFolderIndex >= 0) @@ -2270,7 +2348,8 @@ HRESULT Update( continue; CRecordVector refItems; refItems.ClearAndSetSize(numFiles); - bool sortByType = (options.UseTypeSorting && numSolidFiles > 1); + // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1 + bool sortByType = options.UseTypeSorting; unsigned i; @@ -2292,7 +2371,7 @@ HRESULT Update( const CUpdateItem &ui = updateItems[index]; CFileItem file; if (ui.NewProps) - FromUpdateItemToFileItem(ui, file); + UpdateItem_To_FileItem(ui, file); else file = db.Files[ui.IndexInArchive]; if (file.IsAnti || file.IsDir) @@ -2318,7 +2397,7 @@ HRESULT Update( { int slashPos = ui.Name.ReverseFind_PathSepar(); int dotPos = ui.Name.ReverseFind_Dot(); - const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1); + const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : (unsigned)(dotPos + 1)); if (numSubFiles == 0) prevExtension = ext; else if (!StringsAreEqualNoCase(ext, prevExtension)) @@ -2331,12 +2410,34 @@ HRESULT Update( RINOK(lps->SetCur()); + /* + const unsigned folderIndex = newDatabase.NumUnpackStreamsVector.Size(); + + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kBlockIndex, (UInt32)folderIndex, + NUpdateNotifyOp::kAdd)); + } + */ + + CFolderInStream *inStreamSpec = new CFolderInStream; CMyComPtr solidInStream(inStreamSpec); + + // inStreamSpec->_reportArcProp = reportArcProp; + + inStreamSpec->Need_CTime = options.Need_CTime; + inStreamSpec->Need_ATime = options.Need_ATime; + inStreamSpec->Need_MTime = options.Need_MTime; + inStreamSpec->Need_Attrib = options.Need_Attrib; + inStreamSpec->Init(updateCallback, &indices[i], numSubFiles); unsigned startPackIndex = newDatabase.PackSizes.Size(); - UInt64 curFolderUnpackSize; + UInt64 curFolderUnpackSize = totalSize; + // curFolderUnpackSize = (UInt64)(Int64)-1; // for debug + RINOK(encoder.Encode( EXTERNAL_CODECS_LOC_VARS solidInStream, @@ -2348,8 +2449,11 @@ HRESULT Update( if (!inStreamSpec->WasFinished()) return E_FAIL; + UInt64 packSize = 0; + // const UInt32 numStreams = newDatabase.PackSizes.Size() - startPackIndex; for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++) - lps->OutSize += newDatabase.PackSizes[startPackIndex]; + packSize += newDatabase.PackSizes[startPackIndex]; + lps->OutSize += packSize; lps->InSize += curFolderUnpackSize; // for () @@ -2358,7 +2462,9 @@ HRESULT Update( CNum numUnpackStreams = 0; UInt64 skippedSize = 0; - + UInt64 procSize = 0; + // unsigned numProcessedFiles = 0; + for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++) { const CUpdateItem &ui = updateItems[indices[i + subIndex]]; @@ -2367,13 +2473,13 @@ HRESULT Update( UString name; if (ui.NewProps) { - FromUpdateItemToFileItem(ui, file, file2); + UpdateItem_To_FileItem(ui, file, file2); name = ui.Name; } else { - GetFile(*db, ui.IndexInArchive, file, file2); - db->GetPath(ui.IndexInArchive, name); + GetFile(*db, (unsigned)ui.IndexInArchive, file, file2); + db->GetPath((unsigned)ui.IndexInArchive, name); } if (file2.IsAnti || file.IsDir) return E_FAIL; @@ -2384,14 +2490,16 @@ HRESULT Update( */ if (!inStreamSpec->Processed[subIndex]) { + // we don't add file here skippedSize += ui.Size; - continue; - // file.Name.AddAscii(".locked"); + continue; // comment it for debug + // name += ".locked"; // for debug } file.Crc = inStreamSpec->CRCs[subIndex]; file.Size = inStreamSpec->Sizes[subIndex]; + procSize += file.Size; // if (file.Size >= 0) // test purposes if (file.Size != 0) { @@ -2405,6 +2513,23 @@ HRESULT Update( file.HasStream = false; } + if (inStreamSpec->TimesDefined[subIndex]) + { + if (inStreamSpec->Need_CTime) + { file2.CTimeDefined = true; file2.CTime = inStreamSpec->CTimes[subIndex]; } + if (inStreamSpec->Need_ATime + // && !ui.ATime_WasReadByAnalysis + ) + { file2.ATimeDefined = true; file2.ATime = inStreamSpec->ATimes[subIndex]; } + if (inStreamSpec->Need_MTime) + { file2.MTimeDefined = true; file2.MTime = inStreamSpec->MTimes[subIndex]; } + if (inStreamSpec->Need_Attrib) + { + file2.AttribDefined = true; + file2.Attrib = inStreamSpec->Attribs[subIndex]; + } + } + /* file.Parent = ui.ParentFolderIndex; if (ui.TreeFolderIndex >= 0) @@ -2412,9 +2537,22 @@ HRESULT Update( if (totalSecureDataSize != 0) newDatabase.SecureIDs.Add(ui.SecureIndex); */ + /* + if (reportArcProp) + { + RINOK(ReportItemProps(reportArcProp, ui.IndexInClient, file.Size, + file.CrcDefined ? &file.Crc : NULL)) + } + */ + + // numProcessedFiles++; newDatabase.AddFile(file, file2, name); } + // it's optional check to ensure that sizes are correct + if (procSize != curFolderUnpackSize) + return E_FAIL; + // numUnpackStreams = 0 is very bad case for locked files // v3.13 doesn't understand it. newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams); @@ -2425,6 +2563,44 @@ HRESULT Update( complexity -= skippedSize; RINOK(updateCallback->SetTotal(complexity)); } + + /* + if (reportArcProp) + { + PROPVARIANT prop; + prop.vt = VT_EMPTY; + prop.wReserved1 = 0; + { + NWindows::NCOM::PropVarEm_Set_UInt32(&prop, numProcessedFiles); + RINOK(reportArcProp->ReportProp( + NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidNumSubFiles, &prop)); + } + { + NWindows::NCOM::PropVarEm_Set_UInt64(&prop, curFolderUnpackSize); + RINOK(reportArcProp->ReportProp( + NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidSize, &prop)); + } + { + NWindows::NCOM::PropVarEm_Set_UInt64(&prop, packSize); + RINOK(reportArcProp->ReportProp( + NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidPackSize, &prop)); + } + { + NWindows::NCOM::PropVarEm_Set_UInt32(&prop, numStreams); + RINOK(reportArcProp->ReportProp( + NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidNumStreams, &prop)); + } + RINOK(reportArcProp->ReportFinished(NEventIndexType::kBlockIndex, (UInt32)folderIndex, NUpdate::NOperationResult::kOK)); + } + */ + /* + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kBlockIndex, (UInt32)folderIndex, + NUpdateNotifyOp::kOpFinished)); + } + */ } } diff --git a/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.h b/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.h index a7abf779a61..e6c48caee01 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.h +++ b/3rdparty/lzma/CPP/7zip/Archive/7z/7zUpdate.h @@ -31,7 +31,7 @@ struct CTreeFolder struct CUpdateItem { int IndexInArchive; - int IndexInClient; + unsigned IndexInClient; UInt64 CTime; UInt64 ATime; @@ -62,6 +62,8 @@ struct CUpdateItem bool ATimeDefined; bool MTimeDefined; + // bool ATime_WasReadByAnalysis; + // int SecureIndex; // 0 means (no_security) bool HasStream() const { return !IsDir && !IsAnti && Size != 0; } @@ -76,6 +78,7 @@ struct CUpdateItem CTimeDefined(false), ATimeDefined(false), MTimeDefined(false) + // , ATime_WasReadByAnalysis(false) // SecureIndex(0) {} void SetDirStatusFromAttrib() { IsDir = ((Attrib & FILE_ATTRIBUTE_DIRECTORY) != 0); } @@ -103,6 +106,11 @@ struct CUpdateOptions bool RemoveSfxBlock; bool MultiThreadMixer; + bool Need_CTime; + bool Need_ATime; + bool Need_MTime; + bool Need_Attrib; + CUpdateOptions(): Method(NULL), HeaderMethod(NULL), @@ -114,7 +122,11 @@ struct CUpdateOptions SolidExtension(false), UseTypeSorting(true), RemoveSfxBlock(false), - MultiThreadMixer(true) + MultiThreadMixer(true), + Need_CTime(false), + Need_ATime(false), + Need_MTime(false), + Need_Attrib(false) {} }; diff --git a/3rdparty/lzma/CPP/7zip/Archive/ArchiveExports.cpp b/3rdparty/lzma/CPP/7zip/Archive/ArchiveExports.cpp index 28e9946d5f6..8a441bc2801 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/ArchiveExports.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/ArchiveExports.cpp @@ -46,7 +46,7 @@ static inline HRESULT SetPropGUID(const GUID &guid, PROPVARIANT *value) return SetPropStrFromBin((const char *)&guid, sizeof(guid), value); } -int FindFormatCalssId(const GUID *clsid) +static int FindFormatCalssId(const GUID *clsid) { GUID cls = *clsid; CLS_ARC_ID_ITEM(cls) = 0; @@ -59,6 +59,7 @@ int FindFormatCalssId(const GUID *clsid) return -1; } +STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject) { COM_TRY_BEGIN @@ -89,6 +90,7 @@ STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject) return S_OK; } +STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value); STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN @@ -113,6 +115,7 @@ STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value case NArchive::NHandlerPropID::kAltStreams: prop = ((arc.Flags & NArcInfoFlags::kAltStreams) != 0); break; case NArchive::NHandlerPropID::kNtSecure: prop = ((arc.Flags & NArcInfoFlags::kNtSecure) != 0); break; case NArchive::NHandlerPropID::kFlags: prop = (UInt32)arc.Flags; break; + case NArchive::NHandlerPropID::kTimeFlags: prop = (UInt32)arc.TimeFlags; break; case NArchive::NHandlerPropID::kSignatureOffset: prop = (UInt32)arc.SignatureOffset; break; // case NArchive::NHandlerPropID::kVersion: prop = (UInt32)MY_VER_MIX; break; @@ -130,17 +133,20 @@ STDAPI GetHandlerProperty2(UInt32 formatIndex, PROPID propID, PROPVARIANT *value COM_TRY_END } +STDAPI GetHandlerProperty(PROPID propID, PROPVARIANT *value); STDAPI GetHandlerProperty(PROPID propID, PROPVARIANT *value) { return GetHandlerProperty2(g_DefaultArcIndex, propID, value); } +STDAPI GetNumberOfFormats(UINT32 *numFormats); STDAPI GetNumberOfFormats(UINT32 *numFormats) { *numFormats = g_NumArcs; return S_OK; } +STDAPI GetIsArc(UInt32 formatIndex, Func_IsArc *isArc); STDAPI GetIsArc(UInt32 formatIndex, Func_IsArc *isArc) { *isArc = NULL; diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.cpp b/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.cpp index 41b5805c115..c8b67bd4707 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.cpp @@ -60,6 +60,63 @@ static void BoolVector_Fill_False(CBoolVector &v, unsigned size) p[i] = false; } + +HRESULT CCoder::CheckDataAfterEnd(bool &dataAfterEnd_Error /* , bool &InternalPackSizeError */) const +{ + if (Coder) + { + if (PackSizePointers.IsEmpty() || !PackSizePointers[0]) + return S_OK; + CMyComPtr getInStreamProcessedSize; + Coder.QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); + // if (!getInStreamProcessedSize) return E_FAIL; + if (getInStreamProcessedSize) + { + UInt64 processed; + RINOK(getInStreamProcessedSize->GetInStreamProcessedSize(&processed)); + if (processed != (UInt64)(Int64)-1) + { + const UInt64 size = PackSizes[0]; + if (processed < size && Finish) + dataAfterEnd_Error = true; + if (processed > size) + { + // InternalPackSizeError = true; + // return S_FALSE; + } + } + } + } + else if (Coder2) + { + CMyComPtr getInStreamProcessedSize2; + Coder2.QueryInterface(IID_ICompressGetInStreamProcessedSize2, (void **)&getInStreamProcessedSize2); + if (getInStreamProcessedSize2) + FOR_VECTOR (i, PackSizePointers) + { + if (!PackSizePointers[i]) + continue; + UInt64 processed; + RINOK(getInStreamProcessedSize2->GetInStreamProcessedSize2(i, &processed)); + if (processed != (UInt64)(Int64)-1) + { + const UInt64 size = PackSizes[i]; + if (processed < size && Finish) + dataAfterEnd_Error = true; + else if (processed > size) + { + // InternalPackSizeError = true; + // return S_FALSE; + } + } + } + } + + return S_OK; +} + + + class CBondsChecks { CBoolVector _coderUsed; @@ -80,7 +137,7 @@ bool CBondsChecks::CheckCoder(unsigned coderIndex) return false; _coderUsed[coderIndex] = true; - UInt32 start = BindInfo->Coder_to_Stream[coderIndex]; + const UInt32 start = BindInfo->Coder_to_Stream[coderIndex]; for (unsigned i = 0; i < coder.NumStreams; i++) { @@ -89,10 +146,10 @@ bool CBondsChecks::CheckCoder(unsigned coderIndex) if (BindInfo->IsStream_in_PackStreams(ind)) continue; - int bond = BindInfo->FindBond_for_PackStream(ind); + const int bond = BindInfo->FindBond_for_PackStream(ind); if (bond < 0) return false; - if (!CheckCoder(BindInfo->Bonds[bond].UnpackIndex)) + if (!CheckCoder(BindInfo->Bonds[(unsigned)bond].UnpackIndex)) return false; } @@ -151,8 +208,10 @@ bool CBindInfo::CalcMapsAndCheck() } -void CCoder::SetCoderInfo(const UInt64 *unpackSize, const UInt64 * const *packSizes) +void CCoder::SetCoderInfo(const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) { + Finish = finish; + if (unpackSize) { UnpackSize = *unpackSize; @@ -187,15 +246,15 @@ bool CMixer::Is_UnpackSize_Correct_for_Coder(UInt32 coderIndex) if (coderIndex == _bi.UnpackCoder) return true; - int bond = _bi.FindBond_for_UnpackStream(coderIndex); + const int bond = _bi.FindBond_for_UnpackStream(coderIndex); if (bond < 0) throw 20150213; /* UInt32 coderIndex, coderStreamIndex; - _bi.GetCoder_for_Stream(_bi.Bonds[bond].PackIndex, coderIndex, coderStreamIndex); + _bi.GetCoder_for_Stream(_bi.Bonds[(unsigned)bond].PackIndex, coderIndex, coderStreamIndex); */ - UInt32 nextCoder = _bi.Stream_to_Coder[_bi.Bonds[bond].PackIndex]; + const UInt32 nextCoder = _bi.Stream_to_Coder[_bi.Bonds[(unsigned)bond].PackIndex]; if (!IsFilter_Vector[nextCoder]) return false; @@ -208,11 +267,11 @@ bool CMixer::Is_PackSize_Correct_for_Stream(UInt32 streamIndex) if (_bi.IsStream_in_PackStreams(streamIndex)) return true; - int bond = _bi.FindBond_for_PackStream(streamIndex); + const int bond = _bi.FindBond_for_PackStream(streamIndex); if (bond < 0) throw 20150213; - UInt32 nextCoder = _bi.Bonds[bond].UnpackIndex; + const UInt32 nextCoder = _bi.Bonds[(unsigned)bond].UnpackIndex; if (!IsFilter_Vector[nextCoder]) return false; @@ -222,8 +281,8 @@ bool CMixer::Is_PackSize_Correct_for_Stream(UInt32 streamIndex) bool CMixer::Is_PackSize_Correct_for_Coder(UInt32 coderIndex) { - UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; - UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; + const UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; + const UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; for (UInt32 i = 0; i < numStreams; i++) if (!Is_PackSize_Correct_for_Stream(startIndex + i)) return false; @@ -234,19 +293,19 @@ bool CMixer::IsThere_ExternalCoder_in_PackTree(UInt32 coderIndex) { if (IsExternal_Vector[coderIndex]) return true; - UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; - UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; + const UInt32 startIndex = _bi.Coder_to_Stream[coderIndex]; + const UInt32 numStreams = _bi.Coders[coderIndex].NumStreams; for (UInt32 i = 0; i < numStreams; i++) { - UInt32 si = startIndex + i; + const UInt32 si = startIndex + i; if (_bi.IsStream_in_PackStreams(si)) continue; - int bond = _bi.FindBond_for_PackStream(si); + const int bond = _bi.FindBond_for_PackStream(si); if (bond < 0) throw 20150213; - if (IsThere_ExternalCoder_in_PackTree(_bi.Bonds[bond].UnpackIndex)) + if (IsThere_ExternalCoder_in_PackTree(_bi.Bonds[(unsigned)bond].UnpackIndex)) return true; } return false; @@ -301,7 +360,7 @@ CCoder &CMixerST::GetCoder(unsigned index) return _coders[index]; } -void CMixerST::ReInit() {} +HRESULT CMixerST::ReInit2() { return S_OK; } HRESULT CMixerST::GetInStream2( ISequentialInStream * const *inStreams, /* const UInt64 * const *inSizes, */ @@ -385,18 +444,18 @@ HRESULT CMixerST::GetInStream( } } - int bond = FindBond_for_Stream( + const int bond = FindBond_for_Stream( true, // forInputStream inStreamIndex); if (bond < 0) return E_INVALIDARG; RINOK(GetInStream2(inStreams, /* inSizes, */ - _bi.Bonds[bond].Get_OutIndex(EncodeMode), &seqInStream)); + _bi.Bonds[(unsigned)bond].Get_OutIndex(EncodeMode), &seqInStream)); while (_binderStreams.Size() <= (unsigned)bond) _binderStreams.AddNew(); - CStBinderStream &bs = _binderStreams[bond]; + CStBinderStream &bs = _binderStreams[(unsigned)bond]; if (bs.StreamRef || bs.InStreamSpec) return E_NOTIMPL; @@ -439,13 +498,13 @@ HRESULT CMixerST::GetOutStream( } } - int bond = FindBond_for_Stream( + const int bond = FindBond_for_Stream( false, // forInputStream outStreamIndex); if (bond < 0) return E_INVALIDARG; - UInt32 inStreamIndex = _bi.Bonds[bond].Get_InIndex(EncodeMode); + UInt32 inStreamIndex = _bi.Bonds[(unsigned)bond].Get_InIndex(EncodeMode); UInt32 coderIndex = inStreamIndex; UInt32 coderStreamIndex = 0; @@ -501,7 +560,7 @@ HRESULT CMixerST::GetOutStream( while (_binderStreams.Size() <= (unsigned)bond) _binderStreams.AddNew(); - CStBinderStream &bs = _binderStreams[bond]; + CStBinderStream &bs = _binderStreams[(unsigned)bond]; if (bs.StreamRef || bs.OutStreamSpec) return E_NOTIMPL; @@ -551,13 +610,13 @@ HRESULT CMixerST::FinishStream(UInt32 streamIndex) return S_OK; } - int bond = FindBond_for_Stream( + const int bond = FindBond_for_Stream( false, // forInputStream streamIndex); if (bond < 0) return E_INVALIDARG; - UInt32 inStreamIndex = _bi.Bonds[bond].Get_InIndex(EncodeMode); + UInt32 inStreamIndex = _bi.Bonds[(unsigned)bond].Get_InIndex(EncodeMode); UInt32 coderIndex = inStreamIndex; UInt32 coderStreamIndex = 0; @@ -595,7 +654,7 @@ void CMixerST::SelectMainCoder(bool useFirst) unsigned ci = _bi.UnpackCoder; int firstNonFilter = -1; - int firstAllowed = ci; + unsigned firstAllowed = ci; for (;;) { @@ -615,7 +674,7 @@ void CMixerST::SelectMainCoder(bool useFirst) UInt32 st = _bi.Coder_to_Stream[ci]; if (_bi.IsStream_in_PackStreams(st)) break; - int bond = _bi.FindBond_for_PackStream(st); + const int bond = _bi.FindBond_for_PackStream(st); if (bond < 0) throw 20150213; @@ -623,15 +682,15 @@ void CMixerST::SelectMainCoder(bool useFirst) break; if (firstNonFilter == -1 && !IsFilter_Vector[ci]) - firstNonFilter = ci; + firstNonFilter = (int)ci; - ci = _bi.Bonds[bond].UnpackIndex; + ci = _bi.Bonds[(unsigned)bond].UnpackIndex; } if (useFirst) ci = firstAllowed; else if (firstNonFilter >= 0) - ci = firstNonFilter; + ci = (unsigned)firstNonFilter; MainCoderIndex = ci; } @@ -640,8 +699,12 @@ void CMixerST::SelectMainCoder(bool useFirst) HRESULT CMixerST::Code( ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams, - ICompressProgressInfo *progress) + ICompressProgressInfo *progress, + bool &dataAfterEnd_Error) { + // InternalPackSizeError = false; + dataAfterEnd_Error = false; + _binderStreams.Clear(); unsigned ci = MainCoderIndex; @@ -742,7 +805,16 @@ HRESULT CMixerST::Code( if (res == k_My_HRESULT_WritingWasCut) res = S_OK; - return res; + + if (res != S_OK) + return res; + + for (i = 0; i < _coders.Size(); i++) + { + RINOK(_coders[i].CheckDataAfterEnd(dataAfterEnd_Error /*, InternalPackSizeError */)); + } + + return S_OK; } @@ -847,7 +919,8 @@ HRESULT CMixerMT::SetBindInfo(const CBindInfo &bindInfo) _streamBinders.Clear(); FOR_VECTOR (i, _bi.Bonds) { - RINOK(_streamBinders.AddNew().CreateEvents()); + // RINOK(_streamBinders.AddNew().CreateEvents()); + _streamBinders.AddNew(); } return S_OK; } @@ -869,10 +942,13 @@ CCoder &CMixerMT::GetCoder(unsigned index) return _coders[index]; } -void CMixerMT::ReInit() +HRESULT CMixerMT::ReInit2() { FOR_VECTOR (i, _streamBinders) - _streamBinders[i].ReInit(); + { + RINOK(_streamBinders[i].Create_ReInit()); + } + return S_OK; } void CMixerMT::SelectMainCoder(bool useFirst) @@ -890,10 +966,10 @@ void CMixerMT::SelectMainCoder(bool useFirst) UInt32 st = _bi.Coder_to_Stream[ci]; if (_bi.IsStream_in_PackStreams(st)) break; - int bond = _bi.FindBond_for_PackStream(st); + const int bond = _bi.FindBond_for_PackStream(st); if (bond < 0) throw 20150213; - ci = _bi.Bonds[bond].UnpackIndex; + ci = _bi.Bonds[(unsigned)bond].UnpackIndex; } MainCoderIndex = ci; @@ -940,9 +1016,9 @@ HRESULT CMixerMT::Init(ISequentialInStream * const *inStreams, ISequentialOutStr outCoderStreamIndex = EncodeMode ? coderStreamIndex : 0; } - _streamBinders[i].CreateStreams( - &_coders[inCoderIndex].InStreams[inCoderStreamIndex], - &_coders[outCoderIndex].OutStreams[outCoderStreamIndex]); + _streamBinders[i].CreateStreams2( + _coders[inCoderIndex].InStreams[inCoderStreamIndex], + _coders[outCoderIndex].OutStreams[outCoderStreamIndex]); CMyComPtr inSetSize, outSetSize; _coders[inCoderIndex].QueryInterface(IID_ICompressSetBufSize, (void **)&inSetSize); @@ -988,26 +1064,43 @@ HRESULT CMixerMT::ReturnIfError(HRESULT code) HRESULT CMixerMT::Code( ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams, - ICompressProgressInfo *progress) + ICompressProgressInfo *progress, + bool &dataAfterEnd_Error) { + // InternalPackSizeError = false; + dataAfterEnd_Error = false; + Init(inStreams, outStreams); unsigned i; for (i = 0; i < _coders.Size(); i++) if (i != MainCoderIndex) { - RINOK(_coders[i].Create()); + const WRes wres = _coders[i].Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } for (i = 0; i < _coders.Size(); i++) if (i != MainCoderIndex) - _coders[i].Start(); + { + const WRes wres = _coders[i].Start(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } _coders[MainCoderIndex].Code(progress); + WRes wres = 0; for (i = 0; i < _coders.Size(); i++) if (i != MainCoderIndex) - _coders[i].WaitExecuteFinish(); + { + WRes wres2 = _coders[i].WaitExecuteFinish(); + if (wres == 0) + wres = wres2; + } + if (wres != 0) + return HRESULT_FROM_WIN32(wres); RINOK(ReturnIfError(E_ABORT)); RINOK(ReturnIfError(E_OUTOFMEMORY)); @@ -1031,6 +1124,11 @@ HRESULT CMixerMT::Code( return result; } + for (i = 0; i < _coders.Size(); i++) + { + RINOK(_coders[i].CheckDataAfterEnd(dataAfterEnd_Error /* , InternalPackSizeError */)); + } + return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.h b/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.h index e63f2ff0efa..f099ac3e220 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.h +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/CoderMixer2.h @@ -107,7 +107,7 @@ struct CBindInfo { FOR_VECTOR (i, Bonds) if (Bonds[i].PackIndex == packStream) - return i; + return (int)i; return -1; } @@ -115,7 +115,7 @@ struct CBindInfo { FOR_VECTOR (i, Bonds) if (Bonds[i].UnpackIndex == unpackStream) - return i; + return (int)i; return -1; } @@ -144,7 +144,7 @@ struct CBindInfo { FOR_VECTOR(i, PackStreams) if (PackStreams[i] == streamIndex) - return i; + return (int)i; return -1; } @@ -201,9 +201,13 @@ public: CRecordVector PackSizes; CRecordVector PackSizePointers; - CCoder() {} + bool Finish; - void SetCoderInfo(const UInt64 *unpackSize, const UInt64 * const *packSizes); + CCoder(): Finish(false) {} + + void SetCoderInfo(const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish); + + HRESULT CheckDataAfterEnd(bool &dataAfterEnd_Error /* , bool &InternalPackSizeError */) const; IUnknown *GetUnknown() const { @@ -239,11 +243,15 @@ protected: public: unsigned MainCoderIndex; + // bool InternalPackSizeError; + CMixer(bool encodeMode): EncodeMode(encodeMode), MainCoderIndex(0) + // , InternalPackSizeError(false) {} + virtual ~CMixer() {}; /* Sequence of calling: @@ -272,12 +280,13 @@ public: virtual void AddCoder(const CCreatedCoder &cod) = 0; virtual CCoder &GetCoder(unsigned index) = 0; virtual void SelectMainCoder(bool useFirst) = 0; - virtual void ReInit() = 0; - virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes) = 0; + virtual HRESULT ReInit2() = 0; + virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) = 0; virtual HRESULT Code( ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams, - ICompressProgressInfo *progress) = 0; + ICompressProgressInfo *progress, + bool &dataAfterEnd_Error) = 0; virtual UInt64 GetBondStreamSize(unsigned bondIndex) const = 0; bool Is_UnpackSize_Correct_for_Coder(UInt32 coderIndex); @@ -314,6 +323,8 @@ class CMixerST: public CMixer, public CMyUnknownImp { + CLASS_NO_COPY(CMixerST) + HRESULT GetInStream2(ISequentialInStream * const *inStreams, /* const UInt64 * const *inSizes, */ UInt32 outStreamIndex, ISequentialInStream **inStreamRes); HRESULT GetInStream(ISequentialInStream * const *inStreams, /* const UInt64 * const *inSizes, */ @@ -337,13 +348,14 @@ public: virtual void AddCoder(const CCreatedCoder &cod); virtual CCoder &GetCoder(unsigned index); virtual void SelectMainCoder(bool useFirst); - virtual void ReInit(); - virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes) - { _coders[coderIndex].SetCoderInfo(unpackSize, packSizes); } + virtual HRESULT ReInit2(); + virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) + { _coders[coderIndex].SetCoderInfo(unpackSize, packSizes, finish); } virtual HRESULT Code( ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams, - ICompressProgressInfo *progress); + ICompressProgressInfo *progress, + bool &dataAfterEnd_Error); virtual UInt64 GetBondStreamSize(unsigned bondIndex) const; HRESULT GetMainUnpackStream( @@ -393,7 +405,7 @@ public: }; CCoderMT(): EncodeMode(false) {} - ~CCoderMT() { CVirtThread::WaitThreadFinish(); } + virtual ~CCoderMT() { CVirtThread::WaitThreadFinish(); } void Code(ICompressProgressInfo *progress); }; @@ -404,11 +416,14 @@ class CMixerMT: public CMixer, public CMyUnknownImp { + CLASS_NO_COPY(CMixerMT) + CObjectVector _streamBinders; HRESULT Init(ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams); HRESULT ReturnIfError(HRESULT code); + // virtual ~CMixerMT() {}; public: CObjectVector _coders; @@ -418,13 +433,14 @@ public: virtual void AddCoder(const CCreatedCoder &cod); virtual CCoder &GetCoder(unsigned index); virtual void SelectMainCoder(bool useFirst); - virtual void ReInit(); - virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes) - { _coders[coderIndex].SetCoderInfo(unpackSize, packSizes); } + virtual HRESULT ReInit2(); + virtual void SetCoderInfo(unsigned coderIndex, const UInt64 *unpackSize, const UInt64 * const *packSizes, bool finish) + { _coders[coderIndex].SetCoderInfo(unpackSize, packSizes, finish); } virtual HRESULT Code( ISequentialInStream * const *inStreams, ISequentialOutStream * const *outStreams, - ICompressProgressInfo *progress); + ICompressProgressInfo *progress, + bool &dataAfterEnd_Error); virtual UInt64 GetBondStreamSize(unsigned bondIndex) const; CMixerMT(bool encodeMode): CMixer(encodeMode) {} diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.cpp b/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.cpp index 30ca73bd023..1b9a93ebfbb 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -2,54 +2,175 @@ #include "StdAfx.h" -#ifndef _7ZIP_ST -#include "../../../Windows/System.h" -#endif +#include "../../../Common/StringToInt.h" #include "../Common/ParseProperties.h" #include "HandlerOut.h" -using namespace NWindows; - namespace NArchive { -static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value) +bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res) { - if (m.FindProp(propID) < 0) - m.AddProp32(propID, value); + if (*s == 0) + { + switch (prop.vt) + { + case VT_UI4: res = prop.ulVal; return true; + case VT_UI8: res = prop.uhVal.QuadPart; return true; + case VT_BSTR: + s = prop.bstrVal; + break; + default: return false; + } + } + else if (prop.vt != VT_EMPTY) + return false; + + bool percentMode = false; + { + const wchar_t c = *s; + if (MyCharLower_Ascii(c) == 'p') + { + percentMode = true; + s++; + } + } + + const wchar_t *end; + const UInt64 v = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + const wchar_t c = *end; + + if (percentMode) + { + if (c != 0) + return false; + res = Calc_From_Val_Percents(percentsBase, v); + return true; + } + + if (c == 0) + { + res = v; + return true; + } + if (end[1] != 0) + return false; + + if (c == '%') + { + res = Calc_From_Val_Percents(percentsBase, v); + return true; + } + + unsigned numBits; + switch (MyCharLower_Ascii(c)) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + const UInt64 val2 = v << numBits; + if ((val2 >> numBits) != v) + return false; + res = val2; + return true; } -void CMultiMethodProps::SetGlobalLevelAndThreads(COneMethodInfo &oneMethodInfo +bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres) +{ + hres = S_OK; + + if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { #ifndef _7ZIP_ST - , UInt32 numThreads + _numThreads = _numProcessors; + _numThreads_WasForced = false; + hres = ParseMtProp2(name.Ptr(2), value, _numThreads, _numThreads_WasForced); + // "mt" means "_numThreads_WasForced = false" here #endif - ) + return true; + } + + if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + UInt64 v; + if (!ParseSizeString(name.Ptr(6), value, _memAvail, v)) + hres = E_INVALIDARG; + _memUsage_Decompress = v; + _memUsage_Compress = v; + _memUsage_WasSet = true; + return true; + } + + return false; +} + + +#ifndef EXTRACT_ONLY + +static void SetMethodProp32(CMethodProps &m, PROPID propID, UInt32 value) +{ + if (m.FindProp(propID) < 0) + m.AddProp32(propID, value); +} + +void CMultiMethodProps::SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const { UInt32 level = _level; if (level != (UInt32)(Int32)-1) SetMethodProp32(oneMethodInfo, NCoderPropID::kLevel, (UInt32)level); - - #ifndef _7ZIP_ST +} + +#ifndef _7ZIP_ST + +static void SetMethodProp32_Replace(CMethodProps &m, PROPID propID, UInt32 value) +{ + const int i = m.FindProp(propID); + if (i >= 0) + { + NWindows::NCOM::CPropVariant &val = m.Props[(unsigned)i].Value; + val = (UInt32)value; + return; + } + m.AddProp32(propID, value); +} + +void CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(CMethodProps &oneMethodInfo, UInt32 numThreads) +{ SetMethodProp32(oneMethodInfo, NCoderPropID::kNumThreads, numThreads); - #endif } -void CMultiMethodProps::Init() +void CMultiMethodProps::SetMethodThreadsTo_Replace(CMethodProps &oneMethodInfo, UInt32 numThreads) +{ + SetMethodProp32_Replace(oneMethodInfo, NCoderPropID::kNumThreads, numThreads); +} + +#endif // _7ZIP_ST + + +void CMultiMethodProps::InitMulti() { - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NSystem::GetNumberOfProcessors(); - #endif - _level = (UInt32)(Int32)-1; _analysisLevel = -1; - - _autoFilter = true; _crcSize = 4; - _filterMethod.Clear(); + _autoFilter = true; +} + +void CMultiMethodProps::Init() +{ + InitCommon(); + InitMulti(); _methods.Clear(); + _filterMethod.Clear(); } + HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { UString name = nameSpec; @@ -73,26 +194,24 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN return S_OK; } - if (name.IsEqualTo("crc")) + if (name.IsPrefixedBy_Ascii_NoCase("crc")) { name.Delete(0, 3); _crcSize = 4; return ParsePropToUInt32(name, value, _crcSize); } + + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; + } UInt32 number; unsigned index = ParseStringToUInt32(name, number); UString realName = name.Ptr(index); if (index == 0) { - if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - #endif - - return S_OK; - } if (name.IsEqualTo("f")) { HRESULT res = PROPVARIANT_to_bool(value, _autoFilter); @@ -106,51 +225,86 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN } if (number > 64) return E_FAIL; - for (int j = _methods.Size(); j <= (int)number; j++) - _methods.Add(COneMethodInfo()); + for (unsigned j = _methods.Size(); j <= number; j++) + _methods.AddNew(); return _methods[number].ParseMethodFromPROPVARIANT(realName, value); } + + void CSingleMethodProps::Init() { + InitCommon(); + InitSingle(); Clear(); - _level = (UInt32)(Int32)-1; - - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); - AddProp_NumThreads(_numThreads); - #endif } + +HRESULT CSingleMethodProps::SetProperty(const wchar_t *name2, const PROPVARIANT &value) +{ + // processed = false; + UString name = name2; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + if (name.IsPrefixedBy_Ascii_NoCase("x")) + { + UInt32 a = 9; + RINOK(ParsePropToUInt32(name.Ptr(1), value, a)); + _level = a; + AddProp_Level(a); + // processed = true; + return S_OK; + } + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + // processed = true; + return S_OK; + } + } + RINOK(ParseMethodFromPROPVARIANT(name, value)); + return S_OK; +} + + HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { Init(); + for (UInt32 i = 0; i < numProps; i++) { - UString name = names[i]; - name.MakeLower_Ascii(); - if (name.IsEmpty()) - return E_INVALIDARG; - const PROPVARIANT &value = values[i]; - if (name[0] == L'x') - { - UInt32 a = 9; - RINOK(ParsePropToUInt32(name.Ptr(1), value, a)); - _level = a; - AddProp_Level(a); - } - else if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - AddProp_NumThreads(_numThreads); - #endif - } - else - { - RINOK(ParseMethodFromPROPVARIANT(names[i], value)); - } + RINOK(SetProperty(names[i], values[i])); + } + + return S_OK; +} + +#endif + + +static HRESULT PROPVARIANT_to_BoolPair(const PROPVARIANT &prop, CBoolPair &dest) +{ + RINOK(PROPVARIANT_to_bool(prop, dest.Val)); + dest.Def = true; + return S_OK; +} + +HRESULT CHandlerTimeOptions::Parse(const UString &name, const PROPVARIANT &prop, bool &processed) +{ + processed = true; + if (name.IsEqualTo_Ascii_NoCase("tm")) { return PROPVARIANT_to_BoolPair(prop, Write_MTime); } + if (name.IsEqualTo_Ascii_NoCase("ta")) { return PROPVARIANT_to_BoolPair(prop, Write_ATime); } + if (name.IsEqualTo_Ascii_NoCase("tc")) { return PROPVARIANT_to_BoolPair(prop, Write_CTime); } + if (name.IsPrefixedBy_Ascii_NoCase("tp")) + { + UInt32 v = 0; + RINOK(ParsePropToUInt32(name.Ptr(2), prop, v)); + Prec = v; + return S_OK; } + processed = false; return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.h b/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.h index 5a18d9802b3..41ee189d1f0 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.h +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/HandlerOut.h @@ -3,30 +3,88 @@ #ifndef __HANDLER_OUT_H #define __HANDLER_OUT_H +#include "../../../Windows/System.h" + #include "../../Common/MethodProps.h" namespace NArchive { -class CMultiMethodProps +bool ParseSizeString(const wchar_t *name, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res); + +class CCommonMethodProps { - UInt32 _level; - int _analysisLevel; +protected: + void InitCommon() + { + // _Write_MTime = true; + #ifndef _7ZIP_ST + _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); + _numThreads_WasForced = false; + #endif + + UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28; + _memAvail = memAvail; + _memUsage_Compress = memAvail; + _memUsage_Decompress = memAvail; + _memUsage_WasSet = NWindows::NSystem::GetRamSize(memAvail); + if (_memUsage_WasSet) + { + _memAvail = memAvail; + unsigned bits = sizeof(size_t) * 8; + if (bits == 32) + { + const UInt32 limit2 = (UInt32)7 << 28; + if (memAvail > limit2) + memAvail = limit2; + } + // 80% - is auto usage limit in handlers + // _memUsage_Compress = memAvail * 4 / 5; + // _memUsage_Compress = Calc_From_Val_Percents(memAvail, 80); + _memUsage_Compress = Calc_From_Val_Percents_Less100(memAvail, 80); + _memUsage_Decompress = memAvail / 32 * 17; + } + } + public: #ifndef _7ZIP_ST UInt32 _numThreads; UInt32 _numProcessors; + bool _numThreads_WasForced; #endif + bool _memUsage_WasSet; + UInt64 _memUsage_Compress; + UInt64 _memUsage_Decompress; + UInt64 _memAvail; + + bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres); + + CCommonMethodProps() { InitCommon(); } +}; + + +#ifndef EXTRACT_ONLY + +class CMultiMethodProps: public CCommonMethodProps +{ + UInt32 _level; + int _analysisLevel; + + void InitMulti(); +public: UInt32 _crcSize; CObjectVector _methods; COneMethodInfo _filterMethod; bool _autoFilter; - void SetGlobalLevelAndThreads(COneMethodInfo &oneMethodInfo - #ifndef _7ZIP_ST - , UInt32 numThreads - #endif - ); + + void SetGlobalLevelTo(COneMethodInfo &oneMethodInfo) const; + + #ifndef _7ZIP_ST + static void SetMethodThreadsTo_IfNotFinded(CMethodProps &props, UInt32 numThreads); + static void SetMethodThreadsTo_Replace(CMethodProps &props, UInt32 numThreads); + #endif + unsigned GetNumEmptyMethods() const { @@ -41,27 +99,56 @@ public: int GetAnalysisLevel() const { return _analysisLevel; } void Init(); + CMultiMethodProps() { InitMulti(); } - CMultiMethodProps() { Init(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; -class CSingleMethodProps: public COneMethodInfo + +class CSingleMethodProps: public COneMethodInfo, public CCommonMethodProps { UInt32 _level; - -public: - #ifndef _7ZIP_ST - UInt32 _numThreads; - UInt32 _numProcessors; - #endif + void InitSingle() + { + _level = (UInt32)(Int32)-1; + } + +public: void Init(); - CSingleMethodProps() { Init(); } + CSingleMethodProps() { InitSingle(); } + int GetLevel() const { return _level == (UInt32)(Int32)-1 ? 5 : (int)_level; } + HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &values); HRESULT SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); }; +#endif + +struct CHandlerTimeOptions +{ + CBoolPair Write_MTime; + CBoolPair Write_ATime; + CBoolPair Write_CTime; + UInt32 Prec; + + void Init() + { + Write_MTime.Init(); + Write_MTime.Val = true; + Write_ATime.Init(); + Write_CTime.Init(); + Prec = (UInt32)(Int32)-1; + } + + CHandlerTimeOptions() + { + Init(); + } + + HRESULT Parse(const UString &name, const PROPVARIANT &prop, bool &processed); +}; + } #endif diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.cpp b/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.cpp index 7cd3037be50..8caf1d141b1 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.cpp @@ -7,58 +7,101 @@ namespace NArchive { namespace NItemName { -static const wchar_t kOSDirDelimiter = WCHAR_PATH_SEPARATOR; -static const wchar_t kDirDelimiter = L'/'; +static const wchar_t kOsPathSepar = WCHAR_PATH_SEPARATOR; -void ReplaceToOsPathSeparator(wchar_t *s) -{ - #ifdef _WIN32 - for (;;) +#if WCHAR_PATH_SEPARATOR != L'/' +static const wchar_t kUnixPathSepar = L'/'; +#endif + +void ReplaceSlashes_OsToUnix +#if WCHAR_PATH_SEPARATOR != L'/' + (UString &name) { - wchar_t c = *s; - if (c == 0) - break; - if (c == kDirDelimiter) - *s = kOSDirDelimiter; - s++; + name.Replace(kOsPathSepar, kUnixPathSepar); } - #endif -} +#else + (UString &) {} +#endif -UString MakeLegalName(const UString &name) -{ - UString zipName = name; - zipName.Replace(kOSDirDelimiter, kDirDelimiter); - return zipName; -} -UString GetOSName(const UString &name) +UString GetOsPath(const UString &name) { - UString newName = name; - newName.Replace(kDirDelimiter, kOSDirDelimiter); - return newName; + #if WCHAR_PATH_SEPARATOR != L'/' + UString newName = name; + newName.Replace(kUnixPathSepar, kOsPathSepar); + return newName; + #else + return name; + #endif } -UString GetOSName2(const UString &name) + +UString GetOsPath_Remove_TailSlash(const UString &name) { if (name.IsEmpty()) return UString(); - UString newName = GetOSName(name); - if (newName.Back() == kOSDirDelimiter) + UString newName = GetOsPath(name); + if (newName.Back() == kOsPathSepar) newName.DeleteBack(); return newName; } -void ConvertToOSName2(UString &name) + +void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool + #if WCHAR_PATH_SEPARATOR != L'/' + useBackslashReplacement + #endif + ) { - if (!name.IsEmpty()) + if (name.IsEmpty()) + return; + + #if WCHAR_PATH_SEPARATOR != L'/' { - name.Replace(kDirDelimiter, kOSDirDelimiter); - if (name.Back() == kOSDirDelimiter) - name.DeleteBack(); + // name.Replace(kUnixPathSepar, kOsPathSepar); + const unsigned len = name.Len(); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = name[i]; + if (c == L'/') + c = WCHAR_PATH_SEPARATOR; + else if (useBackslashReplacement && c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme + else + continue; + name.ReplaceOneCharAtPos(i, c); + } + } + #endif + + if (name.Back() == kOsPathSepar) + name.DeleteBack(); +} + + +void NormalizeSlashes_in_FileName_for_OsPath(wchar_t *name, unsigned len) +{ + for (unsigned i = 0; i < len; i++) + { + wchar_t c = name[i]; + if (c == L'/') + c = L'_'; + #if WCHAR_PATH_SEPARATOR != L'/' + else if (c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme + #endif + else + continue; + name[i] = c; } } +void NormalizeSlashes_in_FileName_for_OsPath(UString &name) +{ + NormalizeSlashes_in_FileName_for_OsPath(name.GetBuf(), name.Len()); +} + + bool HasTailSlash(const AString &name, UINT #if defined(_WIN32) && !defined(UNDER_CE) codePage @@ -67,20 +110,24 @@ bool HasTailSlash(const AString &name, UINT { if (name.IsEmpty()) return false; - LPCSTR prev = - #if defined(_WIN32) && !defined(UNDER_CE) - CharPrevExA((WORD)codePage, name, &name[name.Len()], 0); - #else - (LPCSTR)(name) + (name.Len() - 1); - #endif - return (*prev == '/'); + char c; + #if defined(_WIN32) && !defined(UNDER_CE) + if (codePage != CP_UTF8) + c = *CharPrevExA((WORD)codePage, name, name.Ptr(name.Len()), 0); + else + #endif + { + c = name.Back(); + } + return (c == '/'); } + #ifndef _WIN32 -UString WinNameToOSName(const UString &name) +UString WinPathToOsPath(const UString &name) { UString newName = name; - newName.Replace(L'\\', kOSDirDelimiter); + newName.Replace(L'\\', WCHAR_PATH_SEPARATOR); return newName; } #endif diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.h b/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.h index d0dc76a4109..3f5f4e8a4d5 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.h +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/ItemNameUtils.h @@ -8,19 +8,22 @@ namespace NArchive { namespace NItemName { - void ReplaceToOsPathSeparator(wchar_t *s); - - UString MakeLegalName(const UString &name); - UString GetOSName(const UString &name); - UString GetOSName2(const UString &name); - void ConvertToOSName2(UString &name); - bool HasTailSlash(const AString &name, UINT codePage); - - #ifdef _WIN32 - inline UString WinNameToOSName(const UString &name) { return name; } - #else - UString WinNameToOSName(const UString &name); - #endif +void ReplaceSlashes_OsToUnix(UString &name); + +UString GetOsPath(const UString &name); +UString GetOsPath_Remove_TailSlash(const UString &name); + +void ReplaceToOsSlashes_Remove_TailSlash(UString &name, bool useBackslashReplacement = false); +void NormalizeSlashes_in_FileName_for_OsPath(wchar_t *s, unsigned len); +void NormalizeSlashes_in_FileName_for_OsPath(UString &name); + +bool HasTailSlash(const AString &name, UINT codePage); + +#ifdef _WIN32 + inline UString WinPathToOsPath(const UString &name) { return name; } +#else + UString WinPathToOsPath(const UString &name); +#endif }} diff --git a/3rdparty/lzma/CPP/7zip/Archive/Common/MultiStream.cpp b/3rdparty/lzma/CPP/7zip/Archive/Common/MultiStream.cpp index 1de74afe43e..162fc928c26 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/Common/MultiStream.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/Common/MultiStream.cpp @@ -36,7 +36,7 @@ STDMETHODIMP CMultiStream::Read(void *data, UInt32 size, UInt32 *processedSize) UInt64 localPos = _pos - s.GlobalOffset; if (localPos != s.LocalPos) { - RINOK(s.Stream->Seek(localPos, STREAM_SEEK_SET, &s.LocalPos)); + RINOK(s.Stream->Seek((Int64)localPos, STREAM_SEEK_SET, &s.LocalPos)); } UInt64 rem = s.Size - localPos; if (size > rem) @@ -60,9 +60,9 @@ STDMETHODIMP CMultiStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosi } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/Archive/DllExports2.cpp b/3rdparty/lzma/CPP/7zip/Archive/DllExports2.cpp index 10889e75521..1f71486154a 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/DllExports2.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/DllExports2.cpp @@ -22,11 +22,24 @@ #include "IArchive.h" -HINSTANCE g_hInstance; +#ifdef _WIN32 + +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION return FALSE; +#endif + +HINSTANCE g_hInstance; + +extern "C" +BOOL WINAPI DllMain( + #ifdef UNDER_CE + HANDLE + #else + HINSTANCE + #endif + hInstance, DWORD dwReason, LPVOID /*lpReserved*/); -#ifdef _WIN32 extern "C" BOOL WINAPI DllMain( #ifdef UNDER_CE @@ -50,7 +63,22 @@ BOOL WINAPI DllMain( */ return TRUE; } -#endif + +#else // _WIN32 + +#include "../../Common/StringConvert.h" +// #include + +// STDAPI LibStartup(); +static __attribute__((constructor)) void Init_ForceToUTF8(); +static __attribute__((constructor)) void Init_ForceToUTF8() +{ + g_ForceToUTF8 = IsNativeUTF8(); + // printf("\nDLLExports2.cpp::Init_ForceToUTF8 =%d\n", g_ForceToUTF8 ? 1 : 0); +} + +#endif // _WIN32 + DEFINE_GUID(CLSID_CArchiveHandler, k_7zip_GUID_Data1, @@ -62,6 +90,7 @@ STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateHasher(const GUID *clsid, IHasher **hasher); STDAPI CreateArchiver(const GUID *clsid, const GUID *iid, void **outObject); +STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject) { // COM_TRY_BEGIN @@ -76,26 +105,49 @@ STDAPI CreateObject(const GUID *clsid, const GUID *iid, void **outObject) // COM_TRY_END } +STDAPI SetLargePageMode(); STDAPI SetLargePageMode() { #if defined(_7ZIP_LARGE_PAGES) + #ifdef _WIN32 SetLargePageSize(); #endif + #endif return S_OK; } extern bool g_CaseSensitive; +STDAPI SetCaseSensitive(Int32 caseSensitive); STDAPI SetCaseSensitive(Int32 caseSensitive) { g_CaseSensitive = (caseSensitive != 0); return S_OK; } +/* +UInt32 g_ClientVersion; +STDAPI SetClientVersion(UInt32 version); +STDAPI SetClientVersion(UInt32 version) +{ + g_ClientVersion = version; + return S_OK; +} +*/ + +/* +STDAPI SetProperty(Int32 id, const PROPVARIANT *value); +STDAPI SetProperty(Int32 id, const PROPVARIANT *value) +{ + return S_OK; +} +*/ + #ifdef EXTERNAL_CODECS CExternalCodecs g_ExternalCodecs; +STDAPI SetCodecs(ICompressCodecsInfo *compressCodecsInfo); STDAPI SetCodecs(ICompressCodecsInfo *compressCodecsInfo) { COM_TRY_BEGIN @@ -114,6 +166,7 @@ STDAPI SetCodecs(ICompressCodecsInfo *compressCodecsInfo) #else +STDAPI SetCodecs(ICompressCodecsInfo *); STDAPI SetCodecs(ICompressCodecsInfo *) { return S_OK; diff --git a/3rdparty/lzma/CPP/7zip/Archive/IArchive.h b/3rdparty/lzma/CPP/7zip/Archive/IArchive.h index 0028d76246b..9551dfaf784 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/IArchive.h +++ b/3rdparty/lzma/CPP/7zip/Archive/IArchive.h @@ -10,13 +10,39 @@ #define ARCHIVE_INTERFACE_SUB(i, base, x) DECL_INTERFACE_SUB(i, base, 6, x) #define ARCHIVE_INTERFACE(i, x) ARCHIVE_INTERFACE_SUB(i, IUnknown, x) +/* +How the function in 7-Zip returns object for output parameter via pointer + +1) The caller sets the value of variable before function call: + PROPVARIANT : vt = VT_EMPTY + BSTR : NULL + IUnknown* and derived interfaces : NULL + another scalar types : any non-initialized value is allowed + +2) The callee in current 7-Zip code now can free input object for output parameter: + PROPVARIANT : the callee calls VariantClear(propvaiant_ptr) for input + value stored in variable + another types : the callee ignores stored value. + +3) The callee writes new value to variable for output parameter and + returns execution to caller. + +4) The caller must free or release object returned by the callee: + PROPVARIANT : VariantClear(&propvaiant) + BSTR : SysFreeString(bstr) + IUnknown* and derived interfaces : if (ptr) ptr->Relase() +*/ + + namespace NFileTimeType { enum EEnum { - kWindows, + kNotDefined = -1, + kWindows = 0, kUnix, - kDOS + kDOS, + k1ns }; } @@ -34,8 +60,33 @@ namespace NArcInfoFlags const UInt32 kPreArc = 1 << 9; // such archive can be stored before real archive (like SFX stub) const UInt32 kSymLinks = 1 << 10; // the handler supports symbolic links const UInt32 kHardLinks = 1 << 11; // the handler supports hard links + const UInt32 kByExtOnlyOpen = 1 << 12; // call handler only if file extension matches + const UInt32 kHashHandler = 1 << 13; // the handler contains the hashes (checksums) + const UInt32 kCTime = 1 << 14; + const UInt32 kCTime_Default = 1 << 15; + const UInt32 kATime = 1 << 16; + const UInt32 kATime_Default = 1 << 17; + const UInt32 kMTime = 1 << 18; + const UInt32 kMTime_Default = 1 << 19; + // const UInt32 kTTime_Reserved = 1 << 20; + // const UInt32 kTTime_Reserved_Default = 1 << 21; +} + +namespace NArcInfoTimeFlags +{ + const unsigned kTime_Prec_Mask_bit_index = 0; + const unsigned kTime_Prec_Mask_num_bits = 26; + + const unsigned kTime_Prec_Default_bit_index = 27; + const unsigned kTime_Prec_Default_num_bits = 5; } +#define TIME_PREC_TO_ARC_FLAGS_MASK(x) \ + ((UInt32)1 << (NArcInfoTimeFlags::kTime_Prec_Mask_bit_index + (x))) + +#define TIME_PREC_TO_ARC_FLAGS_TIME_DEFAULT(x) \ + ((UInt32)(x) << NArcInfoTimeFlags::kTime_Prec_Default_bit_index) + namespace NArchive { namespace NHandlerPropID @@ -53,8 +104,8 @@ namespace NArchive kSignatureOffset, // VT_UI4 kAltStreams, // VT_BOOL kNtSecure, // VT_BOOL - kFlags // VT_UI4 - // kVersion // VT_UI4 ((VER_MAJOR << 8) | VER_MINOR) + kFlags, // VT_UI4 + kTimeFlags // VT_UI4 }; } @@ -66,7 +117,8 @@ namespace NArchive { kExtract = 0, kTest, - kSkip + kSkip, + kReadExternal }; } @@ -96,6 +148,7 @@ namespace NArchive kInArcIndex, kBlockIndex, kOutArcIndex + // kArcProp }; } @@ -106,7 +159,8 @@ namespace NArchive enum { kOK = 0 - , // kError + // kError = 1, + // kError_FileChanged }; } } @@ -137,13 +191,13 @@ IArchiveExtractCallback::GetStream() Int32 askExtractMode (Extract::NAskMode) if (askMode != NExtract::NAskMode::kExtract) { - then the callee can not real stream: (*inStream == NULL) + then the callee doesn't write data to stream: (*outStream == NULL) } Out: - (*inStream == NULL) - for directories - (*inStream == NULL) - if link (hard link or symbolic link) was created - if (*inStream == NULL && askMode == NExtract::NAskMode::kExtract) + (*outStream == NULL) - for directories + (*outStream == NULL) - if link (hard link or symbolic link) was created + if (*outStream == NULL && askMode == NExtract::NAskMode::kExtract) { then the caller must skip extracting of that file. } @@ -433,9 +487,11 @@ namespace NUpdateNotifyOp kRepack, kSkip, kDelete, - kHeader - - // kNumDefined + kHeader, + kHashRead, + kInFileChanged + // , kOpFinished + // , kNumDefined }; }; @@ -456,6 +512,28 @@ ARCHIVE_INTERFACE(IArchiveUpdateCallbackFile, 0x83) }; +#define INTERFACE_IArchiveGetDiskProperty(x) \ + STDMETHOD(GetDiskProperty)(UInt32 index, PROPID propID, PROPVARIANT *value) x; \ + +ARCHIVE_INTERFACE(IArchiveGetDiskProperty, 0x84) +{ + INTERFACE_IArchiveGetDiskProperty(PURE); +}; + +/* +#define INTERFACE_IArchiveUpdateCallbackArcProp(x) \ + STDMETHOD(ReportProp)(UInt32 indexType, UInt32 index, PROPID propID, const PROPVARIANT *value) x; \ + STDMETHOD(ReportRawProp)(UInt32 indexType, UInt32 index, PROPID propID, const void *data, UInt32 dataSize, UInt32 propType) x; \ + STDMETHOD(ReportFinished)(UInt32 indexType, UInt32 index, Int32 opRes) x; \ + STDMETHOD(DoNeedArcProp)(PROPID propID, Int32 *answer) x; \ + + +ARCHIVE_INTERFACE(IArchiveUpdateCallbackArcProp, 0x85) +{ + INTERFACE_IArchiveUpdateCallbackArcProp(PURE); +}; +*/ + /* UpdateItems() ------------- @@ -488,6 +566,16 @@ ARCHIVE_INTERFACE(IOutArchive, 0xA0) }; +/* +ISetProperties::SetProperties() + PROPVARIANT values[i].vt: + VT_EMPTY + VT_BOOL + VT_UI4 - if 32-bit number + VT_UI8 - if 64-bit number + VT_BSTR +*/ + ARCHIVE_INTERFACE(ISetProperties, 0x03) { STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) PURE; @@ -590,9 +678,40 @@ extern "C" typedef HRESULT (WINAPI *Func_SetCaseSensitive)(Int32 caseSensitive); typedef HRESULT (WINAPI *Func_SetLargePageMode)(); + // typedef HRESULT (WINAPI *Func_SetClientVersion)(UInt32 version); typedef IOutArchive * (*Func_CreateOutArchive)(); typedef IInArchive * (*Func_CreateInArchive)(); } + +/* + if there is no time in archive, external MTime of archive + will be used instead of _item.Time from archive. + For 7-zip before 22.00 we need to return some supported value. + But (kpidTimeType > kDOS) is not allowed in 7-Zip before 22.00. + So we return highest precision value supported by old 7-Zip. + new 7-Zip 22.00 doesn't use that value in usual cases. +*/ + + +#define DECLARE_AND_SET_CLIENT_VERSION_VAR +#define GET_FileTimeType_NotDefined_for_GetFileTimeType \ + NFileTimeType::kWindows + +/* +extern UInt32 g_ClientVersion; + +#define GET_CLIENT_VERSION(major, minor) \ + ((UInt32)(((UInt32)(major) << 16) | (UInt32)(minor))) + +#define DECLARE_AND_SET_CLIENT_VERSION_VAR \ + UInt32 g_ClientVersion = GET_CLIENT_VERSION(MY_VER_MAJOR, MY_VER_MINOR); + +#define GET_FileTimeType_NotDefined_for_GetFileTimeType \ + ((UInt32)(g_ClientVersion >= GET_CLIENT_VERSION(22, 0) ? \ + (UInt32)(Int32)NFileTimeType::kNotDefined : \ + NFileTimeType::kWindows)) +*/ + #endif diff --git a/3rdparty/lzma/CPP/7zip/Archive/LzmaHandler.cpp b/3rdparty/lzma/CPP/7zip/Archive/LzmaHandler.cpp index 121cd67cee5..ba547c8364e 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/LzmaHandler.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/LzmaHandler.cpp @@ -44,7 +44,8 @@ static const Byte kProps[] = static const Byte kArcProps[] = { - kpidNumStreams + kpidNumStreams, + kpidMethod }; struct CHeader @@ -53,6 +54,7 @@ struct CHeader Byte FilterID; Byte LzmaProps[5]; + Byte GetProp() const { return LzmaProps[0]; } UInt32 GetDicSize() const { return GetUi32(LzmaProps + 1); } bool HasSize() const { return (Size != (UInt64)(Int64)-1); } bool Parse(const Byte *buf, bool isThereFilter); @@ -129,13 +131,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, if (header.FilterID > 1) return E_NOTIMPL; - { - CMyComPtr setDecoderProperties; - _lzmaDecoder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecoderProperties); - if (!setDecoderProperties) - return E_NOTIMPL; - RINOK(setDecoderProperties->SetDecoderProperties2(header.LzmaProps, 5)); - } + RINOK(_lzmaDecoderSpec->SetDecoderProperties2(header.LzmaProps, 5)); bool filteredMode = (header.FilterID == 1); @@ -197,6 +193,8 @@ class CHandler: UInt64 _unpackSize; UInt64 _numStreams; + void GetMethod(NCOM::CPropVariant &prop); + public: MY_UNKNOWN_IMP2(IInArchive, IArchiveOpenSeq) @@ -220,6 +218,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidPhySize: if (_packSize_Defined) prop = _packSize; break; case kpidNumStreams: if (_numStreams_Defined) prop = _numStreams; break; case kpidUnpackSize: if (_unpackSize_Defined) prop = _unpackSize; break; + case kpidMethod: GetMethod(prop); break; case kpidErrorFlags: { UInt32 v = 0; @@ -229,6 +228,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) if (_unsupported) v |= kpv_ErrorFlags_UnsupportedMethod; if (_dataError) v |= kpv_ErrorFlags_DataError; prop = v; + break; } } prop.Detach(value); @@ -241,23 +241,55 @@ STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) return S_OK; } -static void DictSizeToString(UInt32 value, char *s) + +static char * DictSizeToString(UInt32 val, char *s) { - for (int i = 0; i <= 31; i++) - if (((UInt32)1 << i) == value) - { - ::ConvertUInt32ToString(i, s); - return; - } + for (unsigned i = 0; i <= 31; i++) + if (((UInt32)1 << i) == val) + return ::ConvertUInt32ToString(i, s); char c = 'b'; - if ((value & ((1 << 20) - 1)) == 0) { value >>= 20; c = 'm'; } - else if ((value & ((1 << 10) - 1)) == 0) { value >>= 10; c = 'k'; } - ::ConvertUInt32ToString(value, s); - s += MyStringLen(s); + if ((val & ((1 << 20) - 1)) == 0) { val >>= 20; c = 'm'; } + else if ((val & ((1 << 10) - 1)) == 0) { val >>= 10; c = 'k'; } + s = ::ConvertUInt32ToString(val, s); *s++ = c; *s = 0; + return s; +} + +static char *AddProp32(char *s, const char *name, UInt32 v) +{ + *s++ = ':'; + s = MyStpCpy(s, name); + return ::ConvertUInt32ToString(v, s); +} + +void CHandler::GetMethod(NCOM::CPropVariant &prop) +{ + if (!_stream) + return; + + char sz[64]; + char *s = sz; + if (_header.FilterID != 0) + s = MyStpCpy(s, "BCJ "); + s = MyStpCpy(s, "LZMA:"); + s = DictSizeToString(_header.GetDicSize(), s); + + UInt32 d = _header.GetProp(); + // if (d != 0x5D) + { + UInt32 lc = d % 9; + d /= 9; + UInt32 pb = d / 5; + UInt32 lp = d % 5; + if (lc != 3) s = AddProp32(s, "lc", lc); + if (lp != 0) s = AddProp32(s, "lp", lp); + if (pb != 2) s = AddProp32(s, "pb", pb); + } + prop = sz; } + STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) { NCOM::CPropVariant prop; @@ -265,18 +297,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIAN { case kpidSize: if (_stream && _header.HasSize()) prop = _header.Size; break; case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; - case kpidMethod: - if (_stream) - { - char sz[64]; - char *s = sz; - if (_header.FilterID != 0) - s = MyStpCpy(s, "BCJ "); - s = MyStpCpy(s, "LZMA:"); - DictSizeToString(_header.GetDicSize(), s); - prop = sz; - } - break; + case kpidMethod: GetMethod(prop); break; } prop.Detach(value); return S_OK; @@ -289,10 +310,10 @@ API_FUNC_static_IsArc IsArc_Lzma(const Byte *p, size_t size) return k_IsArc_Res_NEED_MORE; if (p[0] >= 5 * 5 * 9) return k_IsArc_Res_NO; - UInt64 unpackSize = GetUi64(p + 1 + 4); + const UInt64 unpackSize = GetUi64(p + 1 + 4); if (unpackSize != (UInt64)(Int64)-1) { - if (size >= ((UInt64)1 << 56)) + if (unpackSize >= ((UInt64)1 << 56)) return k_IsArc_Res_NO; } if (unpackSize != 0) @@ -325,24 +346,54 @@ API_FUNC_static_IsArc IsArc_Lzma86(const Byte *p, size_t size) } } + + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *) { Close(); - const UInt32 kBufSize = 1 + 5 + 8 + 2; + const unsigned headerSize = GetHeaderSize(); + const UInt32 kBufSize = 1 << 7; Byte buf[kBufSize]; - - RINOK(ReadStream_FALSE(inStream, buf, kBufSize)); - + size_t processedSize = kBufSize; + RINOK(ReadStream(inStream, buf, &processedSize)); + if (processedSize < headerSize + 2) + return S_FALSE; if (!_header.Parse(buf, _lzma86)) return S_FALSE; - const Byte *start = buf + GetHeaderSize(); + const Byte *start = buf + headerSize; if (start[0] != 0 /* || (start[1] & 0x80) != 0 */ ) // empty stream with EOS is not 0x80 return S_FALSE; - + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_packSize)); - if (_packSize >= 24 && _header.Size == 0 && _header.FilterID == 0 && _header.LzmaProps[0] == 0) + + SizeT srcLen = processedSize - headerSize; + + if (srcLen > 10 + && _header.Size == 0 + // && _header.FilterID == 0 + && _header.LzmaProps[0] == 0 + ) return S_FALSE; + + CDecoder state; + const UInt32 outLimit = 1 << 11; + Byte outBuf[outLimit]; + + SizeT outSize = outLimit; + if (outSize > _header.Size) + outSize = (SizeT)_header.Size; + SizeT destLen = outSize; + ELzmaStatus status; + + SRes res = LzmaDecode(outBuf, &destLen, start, &srcLen, + _header.LzmaProps, 5, LZMA_FINISH_ANY, + &status, &g_Alloc); + + if (res != SZ_OK) + if (res != SZ_ERROR_INPUT_EOF) + return S_FALSE; + _isArc = true; _stream = inStream; _seqStream = inStream; @@ -396,9 +447,9 @@ STDMETHODIMP CCompressProgressInfoImp::SetRatioInfo(const UInt64 *inSize, const { if (Callback) { - UInt64 files = 0; - UInt64 value = Offset + *inSize; - return Callback->SetCompleted(&files, &value); + const UInt64 files = 0; + const UInt64 val = Offset + *inSize; + return Callback->SetCompleted(&files, &val); } return S_OK; } @@ -510,7 +561,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, { if (dataAfterEnd) _dataAfterEnd = true; - else if (decoder._lzmaDecoderSpec->NeedMoreInput) + else if (decoder._lzmaDecoderSpec->NeedsMoreInput()) _needMoreInput = true; _packSize = packSize; diff --git a/3rdparty/lzma/CPP/7zip/Archive/SplitHandler.cpp b/3rdparty/lzma/CPP/7zip/Archive/SplitHandler.cpp index 72b52fe732d..6bddfb836d7 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/SplitHandler.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/SplitHandler.cpp @@ -142,8 +142,8 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) } int dotPos = name.ReverseFind_Dot(); - const UString prefix = name.Left(dotPos + 1); - const UString ext = name.Ptr(dotPos + 1); + const UString prefix = name.Left((unsigned)(dotPos + 1)); + const UString ext = name.Ptr((unsigned)(dotPos + 1)); UString ext2 = ext; ext2.MakeLower_Ascii(); @@ -162,7 +162,10 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) numLetters++; } } - else if (ext.Len() >= 2 && StringsAreEqual_Ascii(ext2.RightPtr(2), "01")) + else if (ext2.Len() >= 2 && ( + StringsAreEqual_Ascii(ext2.RightPtr(2), "01") + || StringsAreEqual_Ascii(ext2.RightPtr(2), "00") + )) { while (numLetters < ext2.Len()) { @@ -170,7 +173,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) break; numLetters++; } - if (numLetters != ext.Len()) + if (numLetters != ext2.Len()) return S_FALSE; } else @@ -181,7 +184,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) seqName._splitStyle = splitStyle; if (prefix.Len() < 1) - _subName.SetFromAscii("file"); + _subName = "file"; else _subName.SetFrom(prefix, prefix.Len() - 1); diff --git a/3rdparty/lzma/CPP/7zip/Archive/XzHandler.cpp b/3rdparty/lzma/CPP/7zip/Archive/XzHandler.cpp index 318be190a79..d358ca5655c 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/XzHandler.cpp +++ b/3rdparty/lzma/CPP/7zip/Archive/XzHandler.cpp @@ -3,16 +3,15 @@ #include "StdAfx.h" #include "../../../C/Alloc.h" -#include "../../../C/XzCrc64.h" -#include "../../../C/XzEnc.h" #include "../../Common/ComTry.h" #include "../../Common/Defs.h" #include "../../Common/IntToString.h" +#include "../../Common/MyBuffer.h" +#include "../../Common/StringToInt.h" #include "../../Windows/PropVariant.h" - -#include "../ICoder.h" +#include "../../Windows/System.h" #include "../Common/CWrappers.h" #include "../Common/ProgressUtils.h" @@ -20,126 +19,178 @@ #include "../Common/StreamUtils.h" #include "../Compress/CopyCoder.h" +#include "../Compress/XzDecoder.h" +#include "../Compress/XzEncoder.h" #include "IArchive.h" -#ifndef EXTRACT_ONLY #include "Common/HandlerOut.h" -#endif - -#include "XzHandler.h" using namespace NWindows; -namespace NCompress { -namespace NLzma2 { - -HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props); - -}} - namespace NArchive { namespace NXz { -struct CCrc64Gen { CCrc64Gen() { Crc64GenerateTable(); } } g_Crc64TableInit; +#define k_LZMA2_Name "LZMA2" -static const char *k_LZMA2_Name = "LZMA2"; -void CStatInfo::Clear() +struct CBlockInfo { - InSize = 0; - OutSize = 0; - PhySize = 0; - - NumStreams = 0; - NumBlocks = 0; - - UnpackSize_Defined = false; - - NumStreams_Defined = false; - NumBlocks_Defined = false; - - IsArc = false; - UnexpectedEnd = false; - DataAfterEnd = false; - Unsupported = false; - HeadersError = false; - DataError = false; - CrcError = false; -} + unsigned StreamFlags; + UInt64 PackPos; + UInt64 PackSize; // pure value from Index record, it doesn't include pad zeros + UInt64 UnpackPos; +}; + class CHandler: public IInArchive, public IArchiveOpenSeq, + public IInArchiveGetStream, + public ISetProperties, + #ifndef EXTRACT_ONLY public IOutArchive, - public ISetProperties, - public CMultiMethodProps, #endif - public CMyUnknownImp + + public CMyUnknownImp, + + #ifndef EXTRACT_ONLY + public CMultiMethodProps + #else + public CCommonMethodProps + #endif { - CStatInfo _stat; + CXzStatInfo _stat; // it's stat from backward parsing + CXzStatInfo _stat2; // it's data from forward parsing, if the decoder was called + SRes _stat2_decode_SRes; + bool _stat_defined; + bool _stat2_defined; + + const CXzStatInfo *GetStat() const + { + if (_stat_defined) return &_stat; + if (_stat2_defined) return &_stat2; + return NULL; + } bool _isArc; bool _needSeekToStart; - bool _phySize_Defined; - - CMyComPtr _stream; - CMyComPtr _seqStream; + bool _firstBlockWasRead; AString _methodsString; + #ifndef EXTRACT_ONLY UInt32 _filterId; + UInt64 _numSolidBytes; - void Init() + void InitXz() { _filterId = 0; - CMultiMethodProps::Init(); + _numSolidBytes = XZ_PROPS__BLOCK_SIZE__AUTO; } - + #endif + + void Init() + { + #ifndef EXTRACT_ONLY + InitXz(); + CMultiMethodProps::Init(); + #else + CCommonMethodProps::InitCommon(); + #endif + } + + HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); + HRESULT Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCallback *callback); - HRESULT Decode2(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, - CDecoder &decoder, ICompressProgressInfo *progress) + HRESULT Decode(NCompress::NXz::CDecoder &decoder, + ISequentialInStream *seqInStream, + ISequentialOutStream *outStream, + ICompressProgressInfo *progress) { - RINOK(decoder.Decode(seqInStream, outStream, progress)); - _stat = decoder; - _phySize_Defined = true; - return S_OK; + #ifndef _7ZIP_ST + decoder._numThreads = _numThreads; + #endif + decoder._memUsage = _memUsage_Decompress; + + HRESULT hres = decoder.Decode(seqInStream, outStream, + NULL, // *outSizeLimit + true, // finishStream + progress); + + if (decoder.MainDecodeSRes_wasUsed + && decoder.MainDecodeSRes != SZ_ERROR_MEM + && decoder.MainDecodeSRes != SZ_ERROR_UNSUPPORTED) + { + // if (!_stat2_defined) + { + _stat2_decode_SRes = decoder.MainDecodeSRes; + _stat2 = decoder.Stat; + _stat2_defined = true; + } + } + + return hres; } public: MY_QUERYINTERFACE_BEGIN2(IInArchive) MY_QUERYINTERFACE_ENTRY(IArchiveOpenSeq) + MY_QUERYINTERFACE_ENTRY(IInArchiveGetStream) + MY_QUERYINTERFACE_ENTRY(ISetProperties) #ifndef EXTRACT_ONLY MY_QUERYINTERFACE_ENTRY(IOutArchive) - MY_QUERYINTERFACE_ENTRY(ISetProperties) #endif MY_QUERYINTERFACE_END MY_ADDREF_RELEASE INTERFACE_IInArchive(;) STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(GetStream)(UInt32 index, ISequentialInStream **stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #ifndef EXTRACT_ONLY INTERFACE_IOutArchive(;) - STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #endif + CBlockInfo *_blocks; + size_t _blocksArraySize; + UInt64 _maxBlocksSize; + CMyComPtr _stream; + CMyComPtr _seqStream; + + CXzBlock _firstBlock; + CHandler(); + ~CHandler(); + + HRESULT SeekToPackPos(UInt64 pos) + { + return _stream->Seek((Int64)pos, STREAM_SEEK_SET, NULL); + } }; -CHandler::CHandler() + +CHandler::CHandler(): + _blocks(NULL), + _blocksArraySize(0) { #ifndef EXTRACT_ONLY - Init(); + InitXz(); #endif } +CHandler::~CHandler() +{ + MyFree(_blocks); +} + static const Byte kProps[] = { @@ -152,7 +203,9 @@ static const Byte kArcProps[] = { kpidMethod, kpidNumStreams, - kpidNumBlocks + kpidNumBlocks, + kpidClusterSize, + kpidCharacts }; IMP_IInArchive_Props @@ -169,13 +222,6 @@ static inline void AddHexToString(AString &s, Byte value) s += GetHex(value & 0xF); } -static void AddUInt32ToString(AString &s, UInt32 value) -{ - char temp[16]; - ConvertUInt32ToString(value, temp); - s += temp; -} - static void Lzma2PropToString(AString &s, unsigned prop) { char c = 0; @@ -192,7 +238,7 @@ static void Lzma2PropToString(AString &s, unsigned prop) c = 'm'; } } - AddUInt32ToString(s, size); + s.Add_UInt32(size); if (c != 0) s += c; } @@ -216,7 +262,7 @@ static const CMethodNamePair g_NamePairs[] = { XZ_ID_LZMA2, "LZMA2" } }; -static AString GetMethodString(const CXzFilter &f) +static void AddMethodString(AString &s, const CXzFilter &f) { const char *p = NULL; for (unsigned i = 0; i < ARRAY_SIZE(g_NamePairs); i++) @@ -232,7 +278,7 @@ static AString GetMethodString(const CXzFilter &f) p = temp; } - AString s = p; + s += p; if (f.propsSize > 0) { @@ -240,7 +286,7 @@ static AString GetMethodString(const CXzFilter &f) if (f.id == XZ_ID_LZMA2 && f.propsSize == 1) Lzma2PropToString(s, f.props[0]); else if (f.id == XZ_ID_Delta && f.propsSize == 1) - AddUInt32ToString(s, (UInt32)f.props[0] + 1); + s.Add_UInt32((UInt32)f.props[0] + 1); else { s += '['; @@ -249,13 +295,6 @@ static AString GetMethodString(const CXzFilter &f) s += ']'; } } - return s; -} - -static void AddString(AString &dest, const AString &src) -{ - dest.Add_Space_if_NotEmpty(); - dest += src; } static const char * const kChecks[] = @@ -278,51 +317,76 @@ static const char * const kChecks[] = , NULL }; -static AString GetCheckString(const CXzs &xzs) +static void AddCheckString(AString &s, const CXzs &xzs) { size_t i; UInt32 mask = 0; for (i = 0; i < xzs.num; i++) mask |= ((UInt32)1 << XzFlags_GetCheckType(xzs.streams[i].flags)); - AString s; for (i = 0; i <= XZ_CHECK_MASK; i++) if (((mask >> i) & 1) != 0) { - AString s2; + s.Add_Space_if_NotEmpty(); if (kChecks[i]) - s2 = kChecks[i]; + s += kChecks[i]; else { - s2 = "Check-"; - AddUInt32ToString(s2, (UInt32)i); + s += "Check-"; + s.Add_UInt32((UInt32)i); } - AddString(s, s2); } - return s; } STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN NCOM::CPropVariant prop; + + const CXzStatInfo *stat = GetStat(); + switch (propID) { - case kpidPhySize: if (_phySize_Defined) prop = _stat.PhySize; break; - case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break; - case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break; - case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; + case kpidPhySize: if (stat) prop = stat->InSize; break; + case kpidNumStreams: if (stat && stat->NumStreams_Defined) prop = stat->NumStreams; break; + case kpidNumBlocks: if (stat && stat->NumBlocks_Defined) prop = stat->NumBlocks; break; + case kpidUnpackSize: if (stat && stat->UnpackSize_Defined) prop = stat->OutSize; break; + case kpidClusterSize: if (_stat_defined && _stat.NumBlocks_Defined && stat->NumBlocks > 1) prop = _maxBlocksSize; break; + case kpidCharacts: + if (_firstBlockWasRead) + { + AString s; + if (XzBlock_HasPackSize(&_firstBlock)) + s.Add_OptSpaced("BlockPackSize"); + if (XzBlock_HasUnpackSize(&_firstBlock)) + s.Add_OptSpaced("BlockUnpackSize"); + if (!s.IsEmpty()) + prop = s; + } + break; + + case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; case kpidErrorFlags: { UInt32 v = 0; - if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;; - if (_stat.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd; - if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; - if (_stat.HeadersError) v |= kpv_ErrorFlags_HeadersError; - if (_stat.Unsupported) v |= kpv_ErrorFlags_UnsupportedMethod; - if (_stat.DataError) v |= kpv_ErrorFlags_DataError; - if (_stat.CrcError) v |= kpv_ErrorFlags_CrcError; - prop = v; + SRes sres = _stat2_decode_SRes; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + if (sres == SZ_ERROR_INPUT_EOF) v |= kpv_ErrorFlags_UnexpectedEnd; + if (_stat2_defined && _stat2.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; + if (sres == SZ_ERROR_ARCHIVE) v |= kpv_ErrorFlags_HeadersError; + if (sres == SZ_ERROR_UNSUPPORTED) v |= kpv_ErrorFlags_UnsupportedMethod; + if (sres == SZ_ERROR_DATA) v |= kpv_ErrorFlags_DataError; + if (sres == SZ_ERROR_CRC) v |= kpv_ErrorFlags_CrcError; + if (v != 0) + prop = v; + break; + } + + case kpidMainSubfile: + { + // debug only, comment it: + // if (_blocks) prop = (UInt32)0; + break; } } prop.Detach(value); @@ -339,11 +403,12 @@ STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) STDMETHODIMP CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN + const CXzStatInfo *stat = GetStat(); NCOM::CPropVariant prop; switch (propID) { - case kpidSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; - case kpidPackSize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidSize: if (stat && stat->UnpackSize_Defined) prop = stat->OutSize; break; + case kpidPackSize: if (stat) prop = stat->InSize; break; case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; } prop.Detach(value); @@ -354,27 +419,30 @@ STDMETHODIMP CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value) struct COpenCallbackWrap { - ICompressProgress p; + ICompressProgress vt; IArchiveOpenCallback *OpenCallback; HRESULT Res; - COpenCallbackWrap(IArchiveOpenCallback *progress); + + // new clang shows "non-POD" warning for offsetof(), if we use constructor instead of Init() + void Init(IArchiveOpenCallback *progress); }; -static SRes OpenCallbackProgress(void *pp, UInt64 inSize, UInt64 /* outSize */) +static SRes OpenCallbackProgress(const ICompressProgress *pp, UInt64 inSize, UInt64 /* outSize */) { - COpenCallbackWrap *p = (COpenCallbackWrap *)pp; + COpenCallbackWrap *p = CONTAINER_FROM_VTBL(pp, COpenCallbackWrap, vt); if (p->OpenCallback) p->Res = p->OpenCallback->SetCompleted(NULL, &inSize); - return (SRes)p->Res; + return HRESULT_To_SRes(p->Res, SZ_ERROR_PROGRESS); } -COpenCallbackWrap::COpenCallbackWrap(IArchiveOpenCallback *callback) +void COpenCallbackWrap::Init(IArchiveOpenCallback *callback) { - p.Progress = OpenCallbackProgress; + vt.Progress = OpenCallbackProgress; OpenCallback = callback; Res = SZ_OK; } + struct CXzsCPP { CXzs p; @@ -382,6 +450,30 @@ struct CXzsCPP ~CXzsCPP() { Xzs_Free(&p, &g_Alloc); } }; +#define kInputBufSize ((size_t)1 << 10) + +struct CLookToRead2_CPP: public CLookToRead2 +{ + CLookToRead2_CPP() + { + buf = NULL; + LookToRead2_CreateVTable(this, + True // Lookahead ? + ); + } + void Alloc(size_t allocSize) + { + buf = (Byte *)MyAlloc(allocSize); + if (buf) + this->bufSize = allocSize; + } + ~CLookToRead2_CPP() + { + MyFree(buf); + } +}; + + static HRESULT SRes_to_Open_HRESULT(SRes res) { switch (res) @@ -401,49 +493,90 @@ static HRESULT SRes_to_Open_HRESULT(SRes res) return S_FALSE; } + + HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCallback *callback) { _needSeekToStart = true; { CXzStreamFlags st; - CSeqInStreamWrap inStreamWrap(inStream); - SRes res = Xz_ReadHeader(&st, &inStreamWrap.p); + CSeqInStreamWrap inStreamWrap; + + inStreamWrap.Init(inStream); + + SRes res = Xz_ReadHeader(&st, &inStreamWrap.vt); + + if (inStreamWrap.Res != S_OK) + return inStreamWrap.Res; if (res != SZ_OK) return SRes_to_Open_HRESULT(res); { CXzBlock block; - Bool isIndex; + BoolInt isIndex; UInt32 headerSizeRes; - SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.p, &isIndex, &headerSizeRes); - if (res2 == SZ_OK && !isIndex) + + SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.vt, &isIndex, &headerSizeRes); + + if (inStreamWrap.Res != S_OK) + return inStreamWrap.Res; + + if (res2 != SZ_OK) + { + if (res2 == SZ_ERROR_INPUT_EOF) + { + _stat2_decode_SRes = res2; + _stream = inStream; + _seqStream = inStream; + _isArc = true; + return S_OK; + } + + if (res2 == SZ_ERROR_ARCHIVE) + return S_FALSE; + } + else if (!isIndex) { + _firstBlockWasRead = true; + _firstBlock = block; + unsigned numFilters = XzBlock_GetNumFilters(&block); for (unsigned i = 0; i < numFilters; i++) - AddString(_methodsString, GetMethodString(block.filters[i])); + { + _methodsString.Add_Space_if_NotEmpty(); + AddMethodString(_methodsString, block.filters[i]); + } } } } - RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.PhySize)); + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.InSize)); if (callback) { - RINOK(callback->SetTotal(NULL, &_stat.PhySize)); + RINOK(callback->SetTotal(NULL, &_stat.InSize)); } - CSeekInStreamWrap inStreamImp(inStream); + CSeekInStreamWrap inStreamImp; + + inStreamImp.Init(inStream); + + CLookToRead2_CPP lookStream; - CLookToRead lookStream; - LookToRead_CreateVTable(&lookStream, True); - lookStream.realStream = &inStreamImp.p; - LookToRead_Init(&lookStream); + lookStream.Alloc(kInputBufSize); + + if (!lookStream.buf) + return E_OUTOFMEMORY; - COpenCallbackWrap openWrap(callback); + lookStream.realStream = &inStreamImp.vt; + LookToRead2_Init(&lookStream); + + COpenCallbackWrap openWrap; + openWrap.Init(callback); CXzsCPP xzs; Int64 startPosition; - SRes res = Xzs_ReadBackward(&xzs.p, &lookStream.s, &startPosition, &openWrap.p, &g_Alloc); + SRes res = Xzs_ReadBackward(&xzs.p, &lookStream.vt, &startPosition, &openWrap.vt, &g_Alloc); if (res == SZ_ERROR_PROGRESS) return (openWrap.Res == S_OK) ? E_FAIL : openWrap.Res; /* @@ -452,7 +585,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal */ if (res == SZ_OK && startPosition == 0) { - _phySize_Defined = true; + _stat_defined = true; _stat.OutSize = Xzs_GetUnpackSize(&xzs.p); _stat.UnpackSize_Defined = true; @@ -463,7 +596,64 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal _stat.NumBlocks = Xzs_GetNumBlocks(&xzs.p); _stat.NumBlocks_Defined = true; - AddString(_methodsString, GetCheckString(xzs.p)); + AddCheckString(_methodsString, xzs.p); + + const size_t numBlocks = (size_t)_stat.NumBlocks + 1; + const size_t bytesAlloc = numBlocks * sizeof(CBlockInfo); + + if (bytesAlloc / sizeof(CBlockInfo) == _stat.NumBlocks + 1) + { + _blocks = (CBlockInfo *)MyAlloc(bytesAlloc); + if (_blocks) + { + unsigned blockIndex = 0; + UInt64 unpackPos = 0; + + for (size_t si = xzs.p.num; si != 0;) + { + si--; + const CXzStream &str = xzs.p.streams[si]; + UInt64 packPos = str.startOffset + XZ_STREAM_HEADER_SIZE; + + for (size_t bi = 0; bi < str.numBlocks; bi++) + { + const CXzBlockSizes &bs = str.blocks[bi]; + const UInt64 packSizeAligned = bs.totalSize + ((0 - (unsigned)bs.totalSize) & 3); + + if (bs.unpackSize != 0) + { + if (blockIndex >= _stat.NumBlocks) + return E_FAIL; + + CBlockInfo &block = _blocks[blockIndex++]; + block.StreamFlags = str.flags; + block.PackSize = bs.totalSize; // packSizeAligned; + block.PackPos = packPos; + block.UnpackPos = unpackPos; + } + packPos += packSizeAligned; + unpackPos += bs.unpackSize; + if (_maxBlocksSize < bs.unpackSize) + _maxBlocksSize = bs.unpackSize; + } + } + + /* + if (blockIndex != _stat.NumBlocks) + { + // there are Empty blocks; + } + */ + if (_stat.OutSize != unpackPos) + return E_FAIL; + CBlockInfo &block = _blocks[blockIndex++]; + block.StreamFlags = 0; + block.PackSize = 0; + block.PackPos = 0; + block.UnpackPos = unpackPos; + _blocksArraySize = blockIndex; + } + } } else { @@ -471,12 +661,15 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal } RINOK(SRes_to_Open_HRESULT(res)); + _stream = inStream; _seqStream = inStream; _isArc = true; return S_OK; } + + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *callback) { COM_TRY_BEGIN @@ -498,202 +691,339 @@ STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) STDMETHODIMP CHandler::Close() { - _stat.Clear(); + XzStatInfo_Clear(&_stat); + XzStatInfo_Clear(&_stat2); + _stat_defined = false; + _stat2_defined = false; + _stat2_decode_SRes = SZ_OK; _isArc = false; _needSeekToStart = false; + _firstBlockWasRead = false; - _phySize_Defined = false; - _methodsString.Empty(); _stream.Release(); _seqStream.Release(); + + MyFree(_blocks); + _blocks = NULL; + _blocksArraySize = 0; + _maxBlocksSize = 0; + return S_OK; } -class CSeekToSeqStream: + +struct CXzUnpackerCPP2 +{ + Byte *InBuf; + // Byte *OutBuf; + CXzUnpacker p; + + CXzUnpackerCPP2(); + ~CXzUnpackerCPP2(); +}; + +CXzUnpackerCPP2::CXzUnpackerCPP2(): InBuf(NULL) + // , OutBuf(NULL) +{ + XzUnpacker_Construct(&p, &g_Alloc); +} + +CXzUnpackerCPP2::~CXzUnpackerCPP2() +{ + XzUnpacker_Free(&p); + MidFree(InBuf); + // MidFree(OutBuf); +} + + +class CInStream: public IInStream, public CMyUnknownImp { public: - CMyComPtr Stream; + UInt64 _virtPos; + UInt64 Size; + UInt64 _cacheStartPos; + size_t _cacheSize; + CByteBuffer _cache; + // UInt64 _startPos; + CXzUnpackerCPP2 xz; + + void InitAndSeek() + { + _virtPos = 0; + _cacheStartPos = 0; + _cacheSize = 0; + // _startPos = startPos; + } + + CHandler *_handlerSpec; + CMyComPtr _handler; + MY_UNKNOWN_IMP1(IInStream) STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); + + ~CInStream(); }; -STDMETHODIMP CSeekToSeqStream::Read(void *data, UInt32 size, UInt32 *processedSize) + +CInStream::~CInStream() { - return Stream->Read(data, size, processedSize); + // _cache.Free(); } -STDMETHODIMP CSeekToSeqStream::Seek(Int64, UInt32, UInt64 *) { return E_NOTIMPL; } -CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(0), OutBuf(0) +static size_t FindBlock(const CBlockInfo *blocks, size_t numBlocks, UInt64 pos) { - XzUnpacker_Construct(&p, &g_Alloc); + size_t left = 0, right = numBlocks; + for (;;) + { + size_t mid = (left + right) / 2; + if (mid == left) + return left; + if (pos < blocks[mid].UnpackPos) + right = mid; + else + left = mid; + } } -CXzUnpackerCPP::~CXzUnpackerCPP() -{ - XzUnpacker_Free(&p); - MyFree(InBuf); - MyFree(OutBuf); -} -HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress) -{ - const size_t kInBufSize = 1 << 15; - const size_t kOutBufSize = 1 << 21; - Clear(); - DecodeRes = SZ_OK; +static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, + ISequentialInStream *seqInStream, + unsigned streamFlags, + UInt64 packSize, // pure size from Index record, it doesn't include pad zeros + size_t unpackSize, Byte *dest + // , ICompressProgressInfo *progress + ) +{ + const size_t kInBufSize = (size_t)1 << 16; XzUnpacker_Init(&xzu.p); + if (!xzu.InBuf) - xzu.InBuf = (Byte *)MyAlloc(kInBufSize); - if (!xzu.OutBuf) - xzu.OutBuf = (Byte *)MyAlloc(kOutBufSize); + { + xzu.InBuf = (Byte *)MidAlloc(kInBufSize); + if (!xzu.InBuf) + return E_OUTOFMEMORY; + } + xzu.p.streamFlags = (UInt16)streamFlags; + XzUnpacker_PrepareToRandomBlockDecoding(&xzu.p); + + XzUnpacker_SetOutBuf(&xzu.p, dest, unpackSize); + + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt64 packRem = packSizeAligned; + UInt32 inSize = 0; SizeT inPos = 0; SizeT outPos = 0; + HRESULT readRes = S_OK; + for (;;) { - if (inPos == inSize) + if (inPos == inSize && readRes == S_OK) { - inPos = inSize = 0; - RINOK(seqInStream->Read(xzu.InBuf, kInBufSize, &inSize)); + inPos = 0; + inSize = 0; + UInt32 rem = kInBufSize; + if (rem > packRem) + rem = (UInt32)packRem; + if (rem != 0) + readRes = seqInStream->Read(xzu.InBuf, rem, &inSize); } SizeT inLen = inSize - inPos; - SizeT outLen = kOutBufSize - outPos; - ECoderStatus status; + SizeT outLen = unpackSize - outPos; + ECoderStatus status; + SRes res = XzUnpacker_Code(&xzu.p, - xzu.OutBuf + outPos, &outLen, + // dest + outPos, + NULL, + &outLen, xzu.InBuf + inPos, &inLen, - (inSize == 0 ? CODER_FINISH_END : CODER_FINISH_ANY), &status); + (inLen == 0), // srcFinished + CODER_FINISH_END, &status); - inPos += inLen; - outPos += outLen; + // return E_OUTOFMEMORY; + // res = SZ_ERROR_CRC; - InSize += inLen; - OutSize += outLen; + if (res != SZ_OK) + { + if (res == SZ_ERROR_CRC) + return S_FALSE; + return SResToHRESULT(res); + } - DecodeRes = res; + inPos += inLen; + outPos += outLen; - bool finished = ((inLen == 0 && outLen == 0) || res != SZ_OK); + packRem -= inLen; + + BoolInt blockFinished = XzUnpacker_IsBlockFinished(&xzu.p); - if (outStream) + if ((inLen == 0 && outLen == 0) || blockFinished) { - if (outPos == kOutBufSize || finished) - { - if (outPos != 0) - { - RINOK(WriteStream(outStream, xzu.OutBuf, outPos)); - outPos = 0; - } - } + if (packRem != 0 || !blockFinished || unpackSize != outPos) + return S_FALSE; + if (XzUnpacker_GetPackSizeForIndex(&xzu.p) != packSize) + return S_FALSE; + return S_OK; } - else - outPos = 0; - - if (progress) + } +} + + +STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) +{ + COM_TRY_BEGIN + + if (processedSize) + *processedSize = 0; + if (size == 0) + return S_OK; + + { + if (_virtPos >= Size) + return S_OK; // (Size == _virtPos) ? S_OK: E_FAIL; { - RINOK(progress->SetRatioInfo(&InSize, &OutSize)); + UInt64 rem = Size - _virtPos; + if (size > rem) + size = (UInt32)rem; } - - if (finished) - { - PhySize = InSize; - NumStreams = xzu.p.numStartedStreams; - if (NumStreams > 0) - IsArc = true; - NumBlocks = xzu.p.numTotalBlocks; + } - UnpackSize_Defined = true; - NumStreams_Defined = true; - NumBlocks_Defined = true; + if (size == 0) + return S_OK; - UInt64 extraSize = XzUnpacker_GetExtraSize(&xzu.p); + if (_virtPos < _cacheStartPos || _virtPos >= _cacheStartPos + _cacheSize) + { + size_t bi = FindBlock(_handlerSpec->_blocks, _handlerSpec->_blocksArraySize, _virtPos); + const CBlockInfo &block = _handlerSpec->_blocks[bi]; + const UInt64 unpackSize = _handlerSpec->_blocks[bi + 1].UnpackPos - block.UnpackPos; + if (_cache.Size() < unpackSize) + return E_FAIL; - if (res == SZ_OK) - { - if (status == CODER_STATUS_NEEDS_MORE_INPUT) - { - extraSize = 0; - if (!XzUnpacker_IsStreamWasFinished(&xzu.p)) - { - // finished at padding bytes, but padding is not aligned for 4 - UnexpectedEnd = true; - res = SZ_ERROR_DATA; - } - } - else // status == CODER_STATUS_NOT_FINISHED - res = SZ_ERROR_DATA; - } - else if (res == SZ_ERROR_NO_ARCHIVE) - { - if (InSize == extraSize) - IsArc = false; - else - { - if (extraSize != 0 || inPos != inSize) - { - DataAfterEnd = true; - res = SZ_OK; - } - } - } + _cacheSize = 0; - DecodeRes = res; - PhySize -= extraSize; + RINOK(_handlerSpec->SeekToPackPos(block.PackPos)); + RINOK(DecodeBlock(xz, _handlerSpec->_seqStream, block.StreamFlags, block.PackSize, + (size_t)unpackSize, _cache)); + _cacheStartPos = block.UnpackPos; + _cacheSize = (size_t)unpackSize; + } - switch (res) - { - case SZ_OK: break; - case SZ_ERROR_NO_ARCHIVE: IsArc = false; break; - case SZ_ERROR_ARCHIVE: HeadersError = true; break; - case SZ_ERROR_UNSUPPORTED: Unsupported = true; break; - case SZ_ERROR_CRC: CrcError = true; break; - case SZ_ERROR_DATA: DataError = true; break; - default: DataError = true; break; - } + { + size_t offset = (size_t)(_virtPos - _cacheStartPos); + size_t rem = _cacheSize - offset; + if (size > rem) + size = (UInt32)rem; + memcpy(data, _cache + offset, size); + _virtPos += size; + if (processedSize) + *processedSize = size; + return S_OK; + } - break; - } + COM_TRY_END +} + + +STDMETHODIMP CInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) +{ + switch (seekOrigin) + { + case STREAM_SEEK_SET: break; + case STREAM_SEEK_CUR: offset += _virtPos; break; + case STREAM_SEEK_END: offset += Size; break; + default: return STG_E_INVALIDFUNCTION; + } + if (offset < 0) + return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; + _virtPos = (UInt64)offset; + if (newPosition) + *newPosition = (UInt64)offset; + return S_OK; +} + + + +static const UInt64 kMaxBlockSize_for_GetStream = (UInt64)1 << 40; + +STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) +{ + COM_TRY_BEGIN + + *stream = NULL; + + if (index != 0) + return E_INVALIDARG; + + if (!_stat.UnpackSize_Defined + || _maxBlocksSize == 0 // 18.02 + || _maxBlocksSize > kMaxBlockSize_for_GetStream + || _maxBlocksSize != (size_t)_maxBlocksSize) + return S_FALSE; + + UInt64 memSize; + if (!NSystem::GetRamSize(memSize)) + memSize = (UInt64)(sizeof(size_t)) << 28; + { + if (_maxBlocksSize > memSize / 4) + return S_FALSE; } + CInStream *spec = new CInStream; + CMyComPtr specStream = spec; + spec->_cache.Alloc((size_t)_maxBlocksSize); + spec->_handlerSpec = this; + spec->_handler = (IInArchive *)this; + spec->Size = _stat.OutSize; + spec->InitAndSeek(); + + *stream = specStream.Detach(); return S_OK; + + COM_TRY_END } -Int32 CDecoder::Get_Extract_OperationResult() const + +static Int32 Get_Extract_OperationResult(const NCompress::NXz::CDecoder &decoder) { Int32 opRes; - if (!IsArc) + SRes sres = decoder.MainDecodeSRes; + if (sres == SZ_ERROR_NO_ARCHIVE) // (!IsArc) opRes = NExtract::NOperationResult::kIsNotArc; - else if (UnexpectedEnd) + else if (sres == SZ_ERROR_INPUT_EOF) // (UnexpectedEnd) opRes = NExtract::NOperationResult::kUnexpectedEnd; - else if (DataAfterEnd) + else if (decoder.Stat.DataAfterEnd) opRes = NExtract::NOperationResult::kDataAfterEnd; - else if (CrcError) + else if (sres == SZ_ERROR_CRC) // (CrcError) opRes = NExtract::NOperationResult::kCRCError; - else if (Unsupported) + else if (sres == SZ_ERROR_UNSUPPORTED) // (Unsupported) opRes = NExtract::NOperationResult::kUnsupportedMethod; - else if (HeadersError) + else if (sres == SZ_ERROR_ARCHIVE) // (HeadersError) opRes = NExtract::NOperationResult::kDataError; - else if (DataError) + else if (sres == SZ_ERROR_DATA) // (DataError) opRes = NExtract::NOperationResult::kDataError; - else if (DecodeRes != SZ_OK) + else if (sres != SZ_OK) opRes = NExtract::NOperationResult::kDataError; else opRes = NExtract::NOperationResult::kOK; return opRes; } + + + STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 testMode, IArchiveExtractCallback *extractCallback) { @@ -703,8 +1033,10 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) return E_INVALIDARG; - if (_phySize_Defined) - extractCallback->SetTotal(_stat.PhySize); + const CXzStatInfo *stat = GetStat(); + + if (stat) + extractCallback->SetTotal(stat->InSize); UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)); @@ -733,33 +1065,46 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, else _needSeekToStart = true; - CDecoder decoder; - RINOK(Decode2(_seqStream, realOutStream, decoder, lpsRef)); - Int32 opRes = decoder.Get_Extract_OperationResult(); + + NCompress::NXz::CDecoder decoder; + + HRESULT hres = Decode(decoder, _seqStream, realOutStream, lpsRef); + + if (!decoder.MainDecodeSRes_wasUsed) + return hres == S_OK ? E_FAIL : hres; + + Int32 opRes = Get_Extract_OperationResult(decoder); + if (opRes == NExtract::NOperationResult::kOK + && hres != S_OK) + opRes = NExtract::NOperationResult::kDataError; realOutStream.Release(); return extractCallback->SetOperationResult(opRes); COM_TRY_END } + + #ifndef EXTRACT_ONLY STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType) { - *timeType = NFileTimeType::kUnix; + *timeType = GET_FileTimeType_NotDefined_for_GetFileTimeType; + // *timeType = NFileTimeType::kUnix; return S_OK; } + STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, IArchiveUpdateCallback *updateCallback) { COM_TRY_BEGIN - CSeqOutStreamWrap seqOutStream(outStream); - if (numItems == 0) { - SRes res = Xz_EncodeEmpty(&seqOutStream.p); + CSeqOutStreamWrap seqOutStream; + seqOutStream.Init(outStream); + SRes res = Xz_EncodeEmpty(&seqOutStream.vt); return SResToHRESULT(res); } @@ -785,94 +1130,165 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (IntToBool(newData)) { - UInt64 size; + UInt64 dataSize; { NCOM::CPropVariant prop; RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); if (prop.vt != VT_UI8) return E_INVALIDARG; - size = prop.uhVal.QuadPart; - RINOK(updateCallback->SetTotal(size)); + dataSize = prop.uhVal.QuadPart; } - CLzma2EncProps lzma2Props; - Lzma2EncProps_Init(&lzma2Props); + NCompress::NXz::CEncoder *encoderSpec = new NCompress::NXz::CEncoder; + CMyComPtr encoder = encoderSpec; - lzma2Props.lzmaProps.level = GetLevel(); - - CMyComPtr fileInStream; - RINOK(updateCallback->GetStream(0, &fileInStream)); + CXzProps &xzProps = encoderSpec->xzProps; + CLzma2EncProps &lzma2Props = xzProps.lzma2Props; - CSeqInStreamWrap seqInStream(fileInStream); + lzma2Props.lzmaProps.level = GetLevel(); + xzProps.reduceSize = dataSize; + /* { - NCOM::CPropVariant prop = (UInt64)size; - RINOK(NCompress::NLzma2::SetLzma2Prop(NCoderPropID::kReduceSize, prop, lzma2Props)); + NCOM::CPropVariant prop = (UInt64)dataSize; + RINOK(encoderSpec->SetCoderProp(NCoderPropID::kReduceSize, prop)); } + */ - FOR_VECTOR (i, _methods) + #ifndef _7ZIP_ST + + UInt32 numThreads = _numThreads; + + const UInt32 kNumThreads_Max = 1024; + if (numThreads > kNumThreads_Max) + numThreads = kNumThreads_Max; + + if (!_numThreads_WasForced + && _numThreads >= 1 + && _memUsage_WasSet) { - COneMethodInfo &m = _methods[i]; - SetGlobalLevelAndThreads(m - #ifndef _7ZIP_ST - , _numThreads - #endif - ); + COneMethodInfo oneMethodInfo; + if (!_methods.IsEmpty()) + oneMethodInfo = _methods[0]; + + SetGlobalLevelTo(oneMethodInfo); + + const bool numThreads_WasSpecifiedInMethod = (oneMethodInfo.Get_NumThreads() >= 0); + if (!numThreads_WasSpecifiedInMethod) { - FOR_VECTOR (j, m.Props) + // here we set the (NCoderPropID::kNumThreads) property in each method, only if there is no such property already + CMultiMethodProps::SetMethodThreadsTo_IfNotFinded(oneMethodInfo, numThreads); + } + + UInt64 cs = _numSolidBytes; + if (cs != XZ_PROPS__BLOCK_SIZE__AUTO) + oneMethodInfo.AddProp_BlockSize2(cs); + cs = oneMethodInfo.Get_Xz_BlockSize(); + + if (cs != XZ_PROPS__BLOCK_SIZE__AUTO && + cs != XZ_PROPS__BLOCK_SIZE__SOLID) + { + const UInt32 lzmaThreads = oneMethodInfo.Get_Lzma_NumThreads(); + const UInt32 numBlockThreads_Original = numThreads / lzmaThreads; + + if (numBlockThreads_Original > 1) { - const CProp &prop = m.Props[j]; - RINOK(NCompress::NLzma2::SetLzma2Prop(prop.Id, prop.Value, lzma2Props)); + UInt32 numBlockThreads = numBlockThreads_Original; + { + const UInt64 lzmaMemUsage = oneMethodInfo.Get_Lzma_MemUsage(false); + for (; numBlockThreads > 1; numBlockThreads--) + { + UInt64 size = numBlockThreads * (lzmaMemUsage + cs); + UInt32 numPackChunks = numBlockThreads + (numBlockThreads / 8) + 1; + if (cs < ((UInt32)1 << 26)) numPackChunks++; + if (cs < ((UInt32)1 << 24)) numPackChunks++; + if (cs < ((UInt32)1 << 22)) numPackChunks++; + size += numPackChunks * cs; + // printf("\nnumBlockThreads = %d, size = %d\n", (unsigned)(numBlockThreads), (unsigned)(size >> 20)); + if (size <= _memUsage_Compress) + break; + } + } + if (numBlockThreads == 0) + numBlockThreads = 1; + if (numBlockThreads != numBlockThreads_Original) + numThreads = numBlockThreads * lzmaThreads; } } } + xzProps.numTotalThreads = (int)numThreads; - #ifndef _7ZIP_ST - lzma2Props.numTotalThreads = _numThreads; - #endif + #endif // _7ZIP_ST - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; - lps->Init(updateCallback, true); - - CCompressProgressWrap progressWrap(progress); - CXzProps xzProps; - CXzFilterProps filter; - XzProps_Init(&xzProps); - XzFilterProps_Init(&filter); - xzProps.lzma2Props = &lzma2Props; - xzProps.filterProps = (_filterId != 0 ? &filter : NULL); - switch (_crcSize) + + xzProps.blockSize = _numSolidBytes; + if (_numSolidBytes == XZ_PROPS__BLOCK_SIZE__SOLID) { - case 0: xzProps.checkId = XZ_CHECK_NO; break; - case 4: xzProps.checkId = XZ_CHECK_CRC32; break; - case 8: xzProps.checkId = XZ_CHECK_CRC64; break; - case 32: xzProps.checkId = XZ_CHECK_SHA256; break; - default: return E_INVALIDARG; + xzProps.lzma2Props.blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; } - filter.id = _filterId; - if (_filterId == XZ_ID_Delta) + + RINOK(encoderSpec->SetCheckSize(_crcSize)); + { - bool deltaDefined = false; - FOR_VECTOR (j, _filterMethod.Props) + CXzFilterProps &filter = xzProps.filterProps; + + if (_filterId == XZ_ID_Delta) { - const CProp &prop = _filterMethod.Props[j]; - if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4) + bool deltaDefined = false; + FOR_VECTOR (j, _filterMethod.Props) { - UInt32 delta = (UInt32)prop.Value.ulVal; - if (delta < 1 || delta > 256) + const CProp &prop = _filterMethod.Props[j]; + if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4) + { + UInt32 delta = (UInt32)prop.Value.ulVal; + if (delta < 1 || delta > 256) + return E_INVALIDARG; + filter.delta = delta; + deltaDefined = true; + } + else return E_INVALIDARG; - filter.delta = delta; - deltaDefined = true; } + if (!deltaDefined) + return E_INVALIDARG; } - if (!deltaDefined) - return E_INVALIDARG; + filter.id = _filterId; } - SRes res = Xz_Encode(&seqOutStream.p, &seqInStream.p, &xzProps, &progressWrap.p); - if (res == SZ_OK) - return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); - return SResToHRESULT(res); + + FOR_VECTOR (i, _methods) + { + COneMethodInfo &m = _methods[i]; + + FOR_VECTOR (j, m.Props) + { + const CProp &prop = m.Props[j]; + RINOK(encoderSpec->SetCoderProp(prop.Id, prop.Value)); + } + } + + { + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + if (!fileInStream) + return S_FALSE; + { + CMyComPtr streamGetSize; + fileInStream.QueryInterface(IID_IStreamGetSize, &streamGetSize); + if (streamGetSize) + { + UInt64 size; + if (streamGetSize->GetSize(&size) == S_OK) + dataSize = size; + } + } + RINOK(updateCallback->SetTotal(dataSize)); + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + RINOK(encoderSpec->Code(fileInStream, outStream, NULL, NULL, progress)); + } + + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); } if (indexInArchive != 0) @@ -887,8 +1303,9 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (_stream) { - if (_phySize_Defined) - RINOK(updateCallback->SetTotal(_stat.PhySize)); + const CXzStatInfo *stat = GetStat(); + if (stat) + RINOK(updateCallback->SetTotal(stat->InSize)); RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); } @@ -901,16 +1318,76 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt COM_TRY_END } +#endif + + +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) +{ + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + #ifndef EXTRACT_ONLY + + if (name[0] == L's') + { + const wchar_t *s = name.Ptr(1); + if (*s == 0) + { + bool useStr = false; + bool isSolid; + switch (value.vt) + { + case VT_EMPTY: isSolid = true; break; + case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; + case VT_BSTR: + if (!StringToBool(value.bstrVal, isSolid)) + useStr = true; + break; + default: return E_INVALIDARG; + } + if (!useStr) + { + _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); + return S_OK; + } + } + return ParseSizeString(s, value, + 0, // percentsBase + _numSolidBytes) ? S_OK: E_INVALIDARG; + } + + return CMultiMethodProps::SetProperty(name, value); + + #else + + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; + } + + return E_INVALIDARG; + + #endif +} + + + STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN Init(); + for (UInt32 i = 0; i < numProps; i++) { RINOK(SetProperty(names[i], values[i])); } + #ifndef EXTRACT_ONLY + if (!_filterMethod.MethodName.IsEmpty()) { unsigned k; @@ -935,22 +1412,25 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR AString &methodName = _methods[0].MethodName; if (methodName.IsEmpty()) methodName = k_LZMA2_Name; - else if (!methodName.IsEqualTo_Ascii_NoCase(k_LZMA2_Name)) + else if ( + !methodName.IsEqualTo_Ascii_NoCase(k_LZMA2_Name) + && !methodName.IsEqualTo_Ascii_NoCase("xz")) return E_INVALIDARG; } + #endif + return S_OK; COM_TRY_END } -#endif REGISTER_ARC_IO( "xz", "xz txz", "* .tar", 0xC, - XZ_SIG, - 0, - NArcInfoFlags::kKeepName, - NULL) + XZ_SIG, 0 + , NArcInfoFlags::kKeepName + , 0 + , NULL) }} diff --git a/3rdparty/lzma/CPP/7zip/Archive/XzHandler.h b/3rdparty/lzma/CPP/7zip/Archive/XzHandler.h index 4a59e356a85..24e8eeb4a10 100644 --- a/3rdparty/lzma/CPP/7zip/Archive/XzHandler.h +++ b/3rdparty/lzma/CPP/7zip/Archive/XzHandler.h @@ -3,63 +3,9 @@ #ifndef __XZ_HANDLER_H #define __XZ_HANDLER_H -#include "../../../C/Xz.h" - -#include "../ICoder.h" - namespace NArchive { namespace NXz { -struct CXzUnpackerCPP -{ - Byte *InBuf; - Byte *OutBuf; - CXzUnpacker p; - - CXzUnpackerCPP(); - ~CXzUnpackerCPP(); -}; - -struct CStatInfo -{ - UInt64 InSize; - UInt64 OutSize; - UInt64 PhySize; - - UInt64 NumStreams; - UInt64 NumBlocks; - - bool UnpackSize_Defined; - - bool NumStreams_Defined; - bool NumBlocks_Defined; - - bool IsArc; - bool UnexpectedEnd; - bool DataAfterEnd; - bool Unsupported; - bool HeadersError; - bool DataError; - bool CrcError; - - CStatInfo() { Clear(); } - - void Clear(); -}; - -struct CDecoder: public CStatInfo -{ - CXzUnpackerCPP xzu; - SRes DecodeRes; // it's not HRESULT - - CDecoder(): DecodeRes(SZ_OK) {} - - /* Decode() can return ERROR code only if there is progress or stream error. - Decode() returns S_OK in case of xz decoding error, but DecodeRes and CStatInfo contain error information */ - HRESULT Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, ICompressProgressInfo *compressProgress); - Int32 Get_Extract_OperationResult() const; -}; - }} #endif diff --git a/3rdparty/lzma/CPP/7zip/Asm.mak b/3rdparty/lzma/CPP/7zip/Asm.mak index 4f77dddfad0..c4073e89090 100644 --- a/3rdparty/lzma/CPP/7zip/Asm.mak +++ b/3rdparty/lzma/CPP/7zip/Asm.mak @@ -2,7 +2,7 @@ !IF "$(CPU)" == "ARM" $(ASM_OBJS): ../../../../Asm/Arm/$(*B).asm $(COMPL_ASM) -!ELSEIF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" +!ELSEIF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" && "$(CPU)" != "ARM64" $(ASM_OBJS): ../../../../Asm/x86/$(*B).asm $(COMPL_ASM) !ENDIF diff --git a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/Alone.dsp b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/Alone.dsp index d220b198b37..217e6d684ac 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /FAc /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MT /W4 /WX /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAcs /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -69,7 +69,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -94,7 +94,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /Yu"StdAfx.h" /FD /c -# ADD CPP /nologo /Gr /MD /W4 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -121,7 +121,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /D "_MBCS" /Yu"StdAfx.h" /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -290,6 +290,14 @@ SOURCE=..\..\..\Common\DynamicBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\DynLimBuf.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\DynLimBuf.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\IntToString.cpp # End Source File # Begin Source File @@ -306,6 +314,10 @@ SOURCE=..\..\..\Common\ListFileUtils.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -342,6 +354,14 @@ SOURCE=..\..\..\Common\MyVector.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyWindows.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Common\MyWindows.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\NewHandler.cpp # End Source File # Begin Source File @@ -350,6 +370,10 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Sha256Reg.cpp # End Source File # Begin Source File @@ -406,6 +430,10 @@ SOURCE=..\..\..\Common\Wildcard.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\XzCrc64Init.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\XzCrc64Reg.cpp # End Source File # End Group @@ -478,6 +506,14 @@ SOURCE=..\..\..\Windows\FileName.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\FileSystem.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\FileSystem.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Handle.h # End Source File # Begin Source File @@ -506,6 +542,14 @@ SOURCE=..\..\..\Windows\PropVariantConv.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\Registry.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Registry.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.cpp # End Source File # Begin Source File @@ -522,6 +566,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # Begin Source File @@ -804,6 +856,22 @@ SOURCE=..\..\Compress\LzmaEncoder.h SOURCE=..\..\Compress\LzmaRegister.cpp # End Source File +# Begin Source File + +SOURCE=..\..\Compress\XzDecoder.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\XzDecoder.h +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\XzEncoder.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\XzEncoder.h +# End Source File # End Group # Begin Group "Archive" @@ -1205,34 +1273,6 @@ SOURCE=..\..\PropID.h # PROP Default_Filter "" # Begin Source File -SOURCE=..\..\..\..\C\Sha256.c - -!IF "$(CFG)" == "Alone - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "Alone - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" - -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sha256.h -# End Source File -# Begin Source File - SOURCE=..\..\..\..\C\Xz.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1429,6 +1469,10 @@ SOURCE=..\..\..\..\C\7zStream.c # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Aes.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1618,6 +1662,15 @@ SOURCE=..\..\..\..\C\Delta.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\IStream.h # End Source File # Begin Source File @@ -1640,6 +1693,30 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Compress\Lz\LzHash.h # End Source File # Begin Source File @@ -1676,6 +1753,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1769,16 +1874,73 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File -SOURCE=..\..\..\..\C\Threads.c +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + # SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + # End Source File # Begin Source File -SOURCE=..\..\..\..\C\Threads.h +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Sha256.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Sha256.h # End Source File # Begin Source File -SOURCE=..\..\..\..\C\Types.h +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Threads.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Threads.h # End Source File # End Group # Begin Group "Crypto" @@ -1806,6 +1968,10 @@ SOURCE=..\..\Crypto\MyAes.h # End Source File # Begin Source File +SOURCE=..\..\Crypto\MyAesReg.cpp +# End Source File +# Begin Source File + SOURCE=..\..\Crypto\RandGen.cpp # End Source File # Begin Source File diff --git a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile index 15c51f91987..f87684c717d 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile @@ -1,17 +1,17 @@ PROG = 7zr.exe -MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) -DPROG_VARIANT_R -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -!ENDIF +# USE_C_AES = 1 + +CFLAGS = $(CFLAGS) -DPROG_VARIANT_R COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ $O\CrcReg.obj \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\ListFileUtils.obj \ + $O\LzFindPrepare.obj \ $O\NewHandler.obj \ $O\StdInStream.obj \ $O\StdOutStream.obj \ @@ -22,6 +22,7 @@ COMMON_OBJS = \ $O\UTFConvert.obj \ $O\MyVector.obj \ $O\Wildcard.obj \ + $O\XzCrc64Init.obj \ $O\XzCrc64Reg.obj \ WIN_OBJS = \ @@ -32,11 +33,14 @@ WIN_OBJS = \ $O\FileIO.obj \ $O\FileLink.obj \ $O\FileName.obj \ + $O\FileSystem.obj \ $O\MemoryLock.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ + $O\Registry.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\SystemInfo.obj \ $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ @@ -109,6 +113,8 @@ COMPRESS_OBJS = \ $O\LzmaDecoder.obj \ $O\LzmaEncoder.obj \ $O\LzmaRegister.obj \ + $O\XzDecoder.obj \ + $O\XzEncoder.obj \ CRYPTO_OBJS = \ $O\7zAes.obj \ @@ -130,11 +136,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ - $O\Sha256.obj \ + $O\MtDec.obj \ $O\Sort.obj \ $O\Threads.obj \ $O\Xz.obj \ @@ -147,5 +154,8 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzFindOpt.mak" +!include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile.gcc b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile.gcc new file mode 100644 index 00000000000..f11fcde099b --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/makefile.gcc @@ -0,0 +1,278 @@ +PROG = 7zr + +CONSOLE_VARIANT_FLAGS=-DPROG_VARIANT_R + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +ifdef SystemDrive +IS_MINGW = 1 +else +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif +endif + +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef IS_MINGW +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/LzFindMt.o \ + $O/LzFindOpt.o \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ + $O/Threads.o \ + + + +endif + + + +LOCAL_FLAGS_SYS = + +ifdef IS_MINGW + +LOCAL_FLAGS_SYS = \ + -D_7ZIP_LARGE_PAGES \ + -DWIN_LONG_PATH \ + -DSUPPORT_DEVICE_FILE \ + +SYS_OBJS = \ + $O/FileSystem.o \ + $O/Registry.o \ + $O/MemoryLock.o \ + $O/DLL.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ + +# -D_LZMA_PROB32 + + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + $O/DynLimBuf.o \ + $O/ExtractCallbackConsole.o \ + $O/HashCon.o \ + $O/List.o \ + $O/Main.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UpdateCallbackConsole.o \ + $O/UserInputUtils.o \ + +UI_COMMON_OBJS = \ + $O/ArchiveCommandLine.o \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/Bench.o \ + $O/DefaultName.o \ + $O/EnumDirItems.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/HashCalc.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + $O/SetProperties.o \ + $O/SortUtils.o \ + $O/TempFiles.o \ + $O/Update.o \ + $O/UpdateAction.o \ + $O/UpdateCallback.o \ + $O/UpdatePair.o \ + $O/UpdateProduce.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/IntToString.o \ + $O/ListFileUtils.o \ + $O/LzFindPrepare.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/Sha256Prepare.o \ + $O/Sha256Reg.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + $O/XzCrc64Init.o \ + $O/XzCrc64Reg.o \ + +WIN_OBJS = \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileLink.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + $O/System.o \ + $O/SystemInfo.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/InOutTempBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/MethodId.o \ + $O/MethodProps.o \ + $O/OffsetStream.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + $O/UniqBlocks.o \ + +AR_OBJS = \ + $O/LzmaHandler.o \ + $O/SplitHandler.o \ + $O/XzHandler.o \ + +AR_COMMON_OBJS = \ + $O/CoderMixer2.o \ + $O/DummyOutStream.o \ + $O/HandlerOut.o \ + $O/InStreamWithCRC.o \ + $O/ItemNameUtils.o \ + $O/MultiStream.o \ + $O/OutStreamWithCRC.o \ + $O/ParseProperties.o \ + +7Z_OBJS = \ + $O/7zCompressionMode.o \ + $O/7zDecode.o \ + $O/7zEncode.o \ + $O/7zExtract.o \ + $O/7zFolderInStream.o \ + $O/7zHandler.o \ + $O/7zHandlerOut.o \ + $O/7zHeader.o \ + $O/7zIn.o \ + $O/7zOut.o \ + $O/7zProperties.o \ + $O/7zRegister.o \ + $O/7zSpecStream.o \ + $O/7zUpdate.o \ + +COMPRESS_OBJS = \ + $O/Bcj2Coder.o \ + $O/Bcj2Register.o \ + $O/BcjCoder.o \ + $O/BcjRegister.o \ + $O/BranchMisc.o \ + $O/BranchRegister.o \ + $O/ByteSwap.o \ + $O/CopyCoder.o \ + $O/CopyRegister.o \ + $O/DeltaFilter.o \ + $O/Lzma2Decoder.o \ + $O/Lzma2Encoder.o \ + $O/Lzma2Register.o \ + $O/LzmaDecoder.o \ + $O/LzmaEncoder.o \ + $O/LzmaRegister.o \ + $O/XzDecoder.o \ + $O/XzEncoder.o \ + +CRYPTO_OBJS = \ + $O/7zAes.o \ + $O/7zAesRegister.o \ + $O/MyAes.o \ + $O/MyAesReg.o \ + $O/RandGen.o \ + +C_OBJS = \ + $O/7zStream.o \ + $O/Alloc.o \ + $O/Bcj2.o \ + $O/Bcj2Enc.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/CpuArch.o \ + $O/Delta.o \ + $O/LzFind.o \ + $O/Lzma2Dec.o \ + $O/Lzma2DecMt.o \ + $O/Lzma2Enc.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/MtCoder.o \ + $O/MtDec.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/Sort.o \ + $O/Xz.o \ + $O/XzDec.o \ + $O/XzEnc.o \ + $O/XzIn.o \ + $O/XzCrc64.o \ + $O/XzCrc64Opt.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Aes.o \ + $O/AesOpt.o \ + + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(AR_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7Z_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/resource.rc b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/resource.rc index b3fb079a630..593785007a0 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/resource.rc +++ b/3rdparty/lzma/CPP/7zip/Bundles/Alone7z/resource.rc @@ -1,3 +1,7 @@ #include "../../../../C/7zVersion.rc" MY_VERSION_INFO_APP("7-Zip Reduced Standalone Console", "7zr") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "../../UI/Console/Console.manifest" +#endif diff --git a/3rdparty/lzma/CPP/7zip/Bundles/Format7zExtractR/makefile b/3rdparty/lzma/CPP/7zip/Bundles/Format7zExtractR/makefile index 724477e9493..756c8ae6922 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/Format7zExtractR/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/Format7zExtractR/makefile @@ -85,9 +85,12 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Bundles/Format7zR/makefile b/3rdparty/lzma/CPP/7zip/Bundles/Format7zR/makefile index 916dc568c38..c2237152615 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/Format7zR/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/Format7zR/makefile @@ -7,6 +7,7 @@ COMMON_OBJS = \ $O\CRC.obj \ $O\CrcReg.obj \ $O\IntToString.obj \ + $O\LzFindPrepare.obj \ $O\NewHandler.obj \ $O\MyString.obj \ $O\StringConvert.obj \ @@ -102,12 +103,16 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzFindOpt.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp index 178753e5373..71436f658af 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp +++ b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp @@ -2,7 +2,9 @@ #include "StdAfx.h" -#include +// #include + +#include "../../../../C/CpuArch.h" #if (defined(_WIN32) || defined(OS2) || defined(MSDOS)) && !defined(UNDER_CE) #include @@ -39,14 +41,16 @@ #include "../../UI/Console/BenchCon.h" #include "../../UI/Console/ConsoleClose.h" +bool g_LargePagesMode = false; + using namespace NCommandLineParser; static const unsigned kDictSizeLog = 24; -static const char *kCopyrightString = "\nLZMA " MY_VERSION_COPYRIGHT_DATE "\n\n"; +#define kCopyrightString "\nLZMA " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n" -static const char *kHelpString = - "Usage: LZMA [inputFile] [outputFile] [...]\n" +static const char * const kHelpString = + "Usage: lzma [inputFile] [outputFile] [...]\n" "\n" "\n" " e : Encode file\n" @@ -67,9 +71,9 @@ static const char *kHelpString = " -so : write data to stdout\n"; -static const char *kCantAllocate = "Can not allocate memory"; -static const char *kReadError = "Read error"; -static const char *kWriteError = "Write error"; +static const char * const kCantAllocate = "Cannot allocate memory"; +static const char * const kReadError = "Read error"; +static const char * const kWriteError = "Write error"; namespace NKey { @@ -95,28 +99,51 @@ enum Enum }; } +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL + +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_STRING SWFRM_1(NSwitchType::kString) + +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) + static const CSwitchForm kSwitchForms[] = { - { "?", NSwitchType::kSimple, false }, - { "H", NSwitchType::kSimple, false }, - { "MM", NSwitchType::kString, false, 1 }, - { "X", NSwitchType::kString, false, 1 }, - { "A", NSwitchType::kString, false, 1 }, - { "D", NSwitchType::kString, false, 1 }, - { "FB", NSwitchType::kString, false, 1 }, - { "MC", NSwitchType::kString, false, 1 }, - { "LC", NSwitchType::kString, false, 1 }, - { "LP", NSwitchType::kString, false, 1 }, - { "PB", NSwitchType::kString, false, 1 }, - { "MF", NSwitchType::kString, false, 1 }, - { "MT", NSwitchType::kString, false, 0 }, - { "EOS", NSwitchType::kSimple, false }, - { "SI", NSwitchType::kSimple, false }, - { "SO", NSwitchType::kSimple, false }, + { "?", SWFRM_SIMPLE }, + { "H", SWFRM_SIMPLE }, + { "MM", SWFRM_STRING_SINGL(1) }, + { "X", SWFRM_STRING_SINGL(1) }, + { "A", SWFRM_STRING_SINGL(1) }, + { "D", SWFRM_STRING_SINGL(1) }, + { "FB", SWFRM_STRING_SINGL(1) }, + { "MC", SWFRM_STRING_SINGL(1) }, + { "LC", SWFRM_STRING_SINGL(1) }, + { "LP", SWFRM_STRING_SINGL(1) }, + { "PB", SWFRM_STRING_SINGL(1) }, + { "MF", SWFRM_STRING_SINGL(1) }, + { "MT", SWFRM_STRING }, + { "EOS", SWFRM_SIMPLE }, + { "SI", SWFRM_SIMPLE }, + { "SO", SWFRM_SIMPLE }, { "F86", NSwitchType::kChar, false, 0, "+" } }; +static void Convert_UString_to_AString(const UString &s, AString &temp) +{ + int codePage = CP_OEMCP; + /* + int g_CodePage = -1; + int codePage = g_CodePage; + if (codePage == -1) + codePage = CP_OEMCP; + if (codePage == CP_UTF8) + ConvertUnicodeToUTF8(s, temp); + else + */ + UnicodeStringToMultiByte2(temp, s, (UINT)codePage); +} + static void PrintErr(const char *s) { fputs(s, stderr); @@ -135,10 +162,12 @@ static void PrintError(const char *s) PrintErr_LF(s); } -static void PrintError2(const char *s1, const wchar_t *s2) +static void PrintError2(const char *s1, const UString &s2) { PrintError(s1); - PrintErr_LF(GetOemString(s2)); + AString a; + Convert_UString_to_AString(s2, a); + PrintErr_LF(a); } static void PrintError_int(const char *s, int code) @@ -208,7 +237,7 @@ public: #define BACK_STR \ "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b" -static const char *kBackSpaces = +static const char * const kBackSpaces = BACK_STR " " BACK_STR; @@ -243,6 +272,7 @@ STDMETHODIMP CProgressPrint::SetRatioInfo(const UInt64 *inSize, const UInt64 *ou } +MY_ATTR_NORETURN static void IncorrectCommand() { throw "Incorrect command"; @@ -293,12 +323,14 @@ static int Error_HRESULT(const char *s, HRESULT res) return 1; } +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION PrintError("Unsupported Windows version"); return 1; +#endif static void AddProp(CObjectVector &props2, const char *name, const wchar_t *val) { CProperty &prop = props2.AddNew(); - prop.Name.SetFromAscii(name); + prop.Name = name; prop.Value = val; } @@ -322,10 +354,10 @@ static int main2(int numArgs, const char *args[]) for (int i = 1; i < numArgs; i++) commandStrings.Add(MultiByteToUnicodeString(args[i])); - CParser parser(ARRAY_SIZE(kSwitchForms)); + CParser parser; try { - if (!parser.ParseStrings(kSwitchForms, commandStrings)) + if (!parser.ParseStrings(kSwitchForms, ARRAY_SIZE(kSwitchForms), commandStrings)) { PrintError2(parser.ErrorMessage, parser.ErrorLine); return 1; @@ -376,7 +408,7 @@ static int main2(int numArgs, const char *args[]) AddProp(props2, "x", s); } - UString mf = L"BT4"; + UString mf ("BT4"); if (parser[NKey::kMatchFinder].ThereIs) mf = parser[NKey::kMatchFinder].PostStrings[0]; @@ -453,7 +485,7 @@ static int main2(int numArgs, const char *args[]) inStream = inStreamSpec; if (!inStreamSpec->Open(us2fs(inputName))) { - PrintError2("can not open input file", inputName); + PrintError2("Cannot open input file", inputName); return 1; } } @@ -473,7 +505,7 @@ static int main2(int numArgs, const char *args[]) outStream = outStreamSpec; if (!outStreamSpec->Create(us2fs(outputName), true)) { - PrintError2("can not open output file", outputName); + PrintError2("Cannot open output file", outputName); return 1; } } @@ -483,8 +515,8 @@ static int main2(int numArgs, const char *args[]) if (inStreamSpec) { - if (!inStreamSpec->File.GetLength(fileSize)) - throw "Can not get file length"; + if (!inStreamSpec->GetLength(fileSize)) + throw "Cannot get file length"; fileSizeDefined = true; if (!stdOutMode) Print_Size("Input size: ", fileSize); @@ -511,7 +543,7 @@ static int main2(int numArgs, const char *args[]) You can use xz format instead, if you want to use filters */ if (parser[NKey::kEOS].ThereIs || stdInMode) - throw "Can not use stdin in this mode"; + throw "Cannot use stdin in this mode"; size_t inSize = (size_t)fileSize; @@ -528,7 +560,7 @@ static int main2(int numArgs, const char *args[]) } if (ReadStream_FAIL(inStream, inBuffer, inSize) != S_OK) - throw "Can not read"; + throw "Cannot read"; Byte *outBuffer = NULL; size_t outSize; diff --git a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp index f8267a11f24..d7326efc0dc 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp +++ b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp @@ -140,6 +140,14 @@ SOURCE=..\..\..\Windows\PropVariant.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\Registry.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Registry.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.cpp # End Source File # Begin Source File @@ -156,6 +164,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SystemInfo.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\SystemInfo.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Thread.h # End Source File # End Group @@ -196,6 +212,10 @@ SOURCE=..\..\..\Common\IntToString.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\LzFindPrepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -300,6 +320,14 @@ SOURCE=..\..\Common\MethodProps.h # End Source File # Begin Source File +SOURCE=..\..\Common\StreamObjects.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\StreamObjects.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\StreamUtils.cpp # End Source File # Begin Source File @@ -425,6 +453,11 @@ SOURCE=..\..\..\..\C\LzFindMt.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\LzFindOpt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzHash.h # End Source File # Begin Source File diff --git a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile index 54919f73374..5e53327a0b0 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile @@ -1,6 +1,5 @@ PROG = lzma.exe MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) CURRENT_OBJS = \ $O\LzmaAlone.obj \ @@ -15,24 +14,27 @@ COMMON_OBJS = \ $O\CRC.obj \ $O\CrcReg.obj \ $O\IntToString.obj \ + $O\LzFindPrepare.obj \ $O\MyString.obj \ + $O\MyVector.obj \ $O\NewHandler.obj \ $O\StringConvert.obj \ $O\StringToInt.obj \ - $O\MyVector.obj WIN_OBJS = \ $O\FileIO.obj \ $O\PropVariant.obj \ - $O\System.obj + $O\Registry.obj \ + $O\System.obj \ + $O\SystemInfo.obj \ 7ZIP_COMMON_OBJS = \ - $O\CWrappers.obj \ $O\CreateCoder.obj \ + $O\CWrappers.obj \ $O\FileStreams.obj \ $O\FilterCoder.obj \ $O\MethodProps.obj \ - $O\OutBuffer.obj \ + $O\StreamObjects.obj \ $O\StreamUtils.obj \ UI_COMMON_OBJS = \ @@ -55,5 +57,7 @@ C_OBJS = \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzFindOpt.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile.gcc b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile.gcc index 97fe450841f..e45ebb68abf 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile.gcc +++ b/3rdparty/lzma/CPP/7zip/Bundles/LzmaCon/makefile.gcc @@ -1,195 +1,125 @@ PROG = lzma -CXX = g++ -O2 -# -Wall -Werror -Wno-delete-non-virtual-dtor -CXX_C = gcc -O2 -Wall -Werror -ifdef SystemDrive -IS_MINGW = 1 -endif +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 -ifdef IS_MINGW - -RM = del -CFLAGS = -c -LIB2 = -loleaut32 -luuid -LDFLAGS = -s +include ../../LzmaDec_gcc.mak -FILE_IO =FileIO -FILE_IO_2 =Windows/$(FILE_IO) +LOCAL_FLAGS_ST = +MT_OBJS = -MT_FILES = \ - LzFindMt.o \ - Threads.o \ +ifdef SystemDrive +IS_MINGW = 1 else - -RM = rm -f -CFLAGS = -c -D_7ZIP_ST - -FILE_IO =C_FileIO -FILE_IO_2 =Common/$(FILE_IO) - - +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif endif +ifdef ST_MODE -OBJS = \ - $(MT_FILES) \ - $(FILE_IO).o \ - LzmaAlone.o \ - Bench.o \ - BenchCon.o \ - ConsoleClose.o \ - LzmaDecoder.o \ - LzmaEncoder.o \ - LzmaRegister.o \ - CreateCoder.o \ - CWrappers.o \ - FileStreams.o \ - FilterCoder.o \ - MethodProps.o \ - StreamUtils.o \ - CommandLineParser.o \ - CRC.o \ - CrcReg.o \ - IntToString.o \ - MyString.o \ - MyVector.o \ - MyWindows.o \ - StringConvert.o \ - StringToInt.o \ - PropVariant.o \ - System.o \ - 7zCrc.o \ - 7zCrcOpt.o \ - Alloc.o \ - Bra86.o \ - CpuArch.o \ - LzFind.o \ - LzmaDec.o \ - LzmaEnc.o \ - Lzma86Dec.o \ - Lzma86Enc.o \ - - -all: $(PROG) - -$(PROG): $(OBJS) - $(CXX) -o $(PROG) $(LDFLAGS) $(OBJS) $(LIB2) - -LzmaAlone.o: LzmaAlone.cpp - $(CXX) $(CFLAGS) LzmaAlone.cpp - -Bench.o: ../../UI/Common/Bench.cpp - $(CXX) $(CFLAGS) ../../UI/Common/Bench.cpp - -BenchCon.o: ../../UI/Console/BenchCon.cpp - $(CXX) $(CFLAGS) ../../UI/Console/BenchCon.cpp - -ConsoleClose.o: ../../UI/Console/ConsoleClose.cpp - $(CXX) $(CFLAGS) ../../UI/Console/ConsoleClose.cpp - -LzmaDecoder.o: ../../Compress/LzmaDecoder.cpp - $(CXX) $(CFLAGS) ../../Compress/LzmaDecoder.cpp - -LzmaEncoder.o: ../../Compress/LzmaEncoder.cpp - $(CXX) $(CFLAGS) ../../Compress/LzmaEncoder.cpp - -LzmaRegister.o: ../../Compress/LzmaRegister.cpp - $(CXX) $(CFLAGS) ../../Compress/LzmaRegister.cpp - -CreateCoder.o: ../../Common/CreateCoder.cpp - $(CXX) $(CFLAGS) ../../Common/CreateCoder.cpp - -CWrappers.o: ../../Common/CWrappers.cpp - $(CXX) $(CFLAGS) ../../Common/CWrappers.cpp - -FileStreams.o: ../../Common/FileStreams.cpp - $(CXX) $(CFLAGS) ../../Common/FileStreams.cpp - -FilterCoder.o: ../../Common/FilterCoder.cpp - $(CXX) $(CFLAGS) ../../Common/FilterCoder.cpp - -MethodProps.o: ../../Common/MethodProps.cpp - $(CXX) $(CFLAGS) ../../Common/MethodProps.cpp - -StreamUtils.o: ../../Common/StreamUtils.cpp - $(CXX) $(CFLAGS) ../../Common/StreamUtils.cpp - -$(FILE_IO).o: ../../../$(FILE_IO_2).cpp - $(CXX) $(CFLAGS) ../../../$(FILE_IO_2).cpp - - -CommandLineParser.o: ../../../Common/CommandLineParser.cpp - $(CXX) $(CFLAGS) ../../../Common/CommandLineParser.cpp - -CRC.o: ../../../Common/CRC.cpp - $(CXX) $(CFLAGS) ../../../Common/CRC.cpp - -CrcReg.o: ../../../Common/CrcReg.cpp - $(CXX) $(CFLAGS) ../../../Common/CrcReg.cpp - -IntToString.o: ../../../Common/IntToString.cpp - $(CXX) $(CFLAGS) ../../../Common/IntToString.cpp - -MyString.o: ../../../Common/MyString.cpp - $(CXX) $(CFLAGS) ../../../Common/MyString.cpp - -MyVector.o: ../../../Common/MyVector.cpp - $(CXX) $(CFLAGS) ../../../Common/MyVector.cpp +LOCAL_FLAGS_ST = -D_7ZIP_ST -MyWindows.o: ../../../Common/MyWindows.cpp - $(CXX) $(CFLAGS) ../../../Common/MyWindows.cpp +else -StringConvert.o: ../../../Common/StringConvert.cpp - $(CXX) $(CFLAGS) ../../../Common/StringConvert.cpp +MT_OBJS = \ + $O/LzFindMt.o \ + $O/LzFindOpt.o \ + $O/Synchronization.o \ + $O/Threads.o \ -StringToInt.o: ../../../Common/StringToInt.cpp - $(CXX) $(CFLAGS) ../../../Common/StringToInt.cpp -PropVariant.o: ../../../Windows/PropVariant.cpp - $(CXX) $(CFLAGS) ../../../Windows/PropVariant.cpp -System.o: ../../../Windows/System.cpp - $(CXX) $(CFLAGS) ../../../Windows/System.cpp +endif -7zCrc.o: ../../../../C/7zCrc.c - $(CXX_C) $(CFLAGS) ../../../../C/7zCrc.c -7zCrcOpt.o: ../../../../C/7zCrcOpt.c - $(CXX_C) $(CFLAGS) ../../../../C/7zCrcOpt.c -Alloc.o: ../../../../C/Alloc.c - $(CXX_C) $(CFLAGS) ../../../../C/Alloc.c +LOCAL_FLAGS_SYS = -Bra86.o: ../../../../C/Bra86.c - $(CXX_C) $(CFLAGS) ../../../../C/Bra86.c +ifdef IS_MINGW -CpuArch.o: ../../../../C/CpuArch.c - $(CXX_C) $(CFLAGS) ../../../../C/CpuArch.c +SYS_OBJS = \ + $O/Registry.o \ + $O/resource.o \ -LzFind.o: ../../../../C/LzFind.c - $(CXX_C) $(CFLAGS) ../../../../C/LzFind.c +else -ifdef MT_FILES -LzFindMt.o: ../../../../C/LzFindMt.c - $(CXX_C) $(CFLAGS) ../../../../C/LzFindMt.c +SYS_OBJS = \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileName.o \ + $O/MyWindows.o \ + $O/TimeUtils.o \ -Threads.o: ../../../../C/Threads.c - $(CXX_C) $(CFLAGS) ../../../../C/Threads.c endif -LzmaDec.o: ../../../../C/LzmaDec.c - $(CXX_C) $(CFLAGS) ../../../../C/LzmaDec.c - -LzmaEnc.o: ../../../../C/LzmaEnc.c - $(CXX_C) $(CFLAGS) ../../../../C/LzmaEnc.c - -Lzma86Dec.o: ../../../../C/Lzma86Dec.c - $(CXX_C) $(CFLAGS) ../../../../C/Lzma86Dec.c +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/IntToString.o \ + $O/LzFindPrepare.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + +WIN_OBJS = \ + $O/FileIO.o \ + $O/PropVariant.o \ + $O/System.o \ + $O/SystemInfo.o \ + +COMPRESS_OBJS = \ + $O/LzmaDecoder.o \ + $O/LzmaEncoder.o \ + $O/LzmaRegister.o \ + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FileStreams.o \ + $O/FilterCoder.o \ + $O/MethodProps.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + +C_OBJS = \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Alloc.o \ + $O/Bra86.o \ + $O/CpuArch.o \ + $O/LzFind.o \ + $O/LzmaDec.o \ + $O/LzmaEnc.o \ + $O/Lzma86Dec.o \ + $O/Lzma86Enc.o \ -Lzma86Enc.o: ../../../../C/Lzma86Enc.c - $(CXX_C) $(CFLAGS) ../../../../C/Lzma86Enc.c - -clean: - -$(RM) $(PROG) $(OBJS) +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + $O/LzmaAlone.o \ + $O/Bench.o \ + +include ../../7zip_gcc.mak diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SFXCon.dsp b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SFXCon.dsp index b21b9e841e5..87e27c4f140 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SFXCon.dsp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SFXCon.dsp @@ -117,6 +117,10 @@ SOURCE=..\..\Archive\Common\CoderMixer2.h # End Source File # Begin Source File +SOURCE=..\..\Archive\Common\HandlerOut.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\ItemNameUtils.cpp # End Source File # Begin Source File @@ -293,6 +297,10 @@ SOURCE=..\..\Compress\Lzma2Decoder.cpp # End Source File # Begin Source File +SOURCE=..\..\Compress\Lzma2Decoder.h +# End Source File +# Begin Source File + SOURCE=..\..\Compress\Lzma2Register.cpp # End Source File # Begin Source File @@ -411,6 +419,14 @@ SOURCE=..\..\..\Windows\Synchronization.cpp SOURCE=..\..\..\Windows\Synchronization.h # End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File # End Group # Begin Group "Common" @@ -465,6 +481,10 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\StdInStream.cpp # End Source File # Begin Source File @@ -822,6 +842,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -831,6 +860,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -854,6 +892,11 @@ SOURCE=..\..\..\..\C\Sha256.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SfxCon.cpp b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SfxCon.cpp index d828922f314..cfa1ee7c6aa 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SfxCon.cpp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/SfxCon.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../../C/CpuArch.h" + #include "../../../Common/MyWindows.h" #include "../../../Common/MyInitGuid.h" @@ -11,8 +13,10 @@ #ifdef _WIN32 #include "../../../Windows/DLL.h" -#include "../../../Windows/FileDir.h" +#else +#include "../../../Common/StringConvert.h" #endif +#include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" #include "../../UI/Common/ExitCode.h" @@ -37,8 +41,8 @@ HINSTANCE g_hInstance = 0; int g_CodePage = -1; extern CStdOutStream *g_StdStream; -static const char *kCopyrightString = -"\n7-Zip SFX " MY_VERSION_COPYRIGHT_DATE "\n"; +static const char * const kCopyrightString = +"\n7-Zip SFX " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n"; static const int kNumSwitches = 6; @@ -65,7 +69,6 @@ enum EEnum } /* static const char kRecursedIDChar = 'R'; -static const wchar_t *kRecursedPostCharSet = L"0-"; namespace NRecursedPostCharIndex { enum EEnum @@ -81,14 +84,20 @@ static const char kImmediateNameID = '!'; static const char kSomeCludePostStringMinSize = 2; // at least <@|!>ame must be static const char kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!>ame must be */ + +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) + static const CSwitchForm kSwitchForms[kNumSwitches] = { - { "?", NSwitchType::kSimple }, - { "H", NSwitchType::kSimple }, - { "BD", NSwitchType::kSimple }, - { "Y", NSwitchType::kSimple }, - { "P", NSwitchType::kString, false, 1 }, - { "O", NSwitchType::kString, false, 1 }, + { "?", SWFRM_SIMPLE }, + { "H", SWFRM_SIMPLE }, + { "BD", SWFRM_SIMPLE }, + { "Y", SWFRM_SIMPLE }, + { "P", SWFRM_STRING_SINGL(1) }, + { "O", SWFRM_STRING_SINGL(1) }, }; static const int kNumCommandForms = 3; @@ -101,10 +110,9 @@ static const NRecursedType::EEnum kCommandRecursedDefault[kNumCommandForms] = // static const bool kTestExtractRecursedDefault = true; // static const bool kAddRecursedDefault = false; -static const wchar_t *kUniversalWildcard = L"*"; -static const int kCommandIndex = 0; +static const char * const kUniversalWildcard = "*"; -static const char *kHelpString = +static const char * const kHelpString = "\nUsage: 7zSFX [] [...] [...]\n" "\n" "\n" @@ -121,16 +129,16 @@ static const char *kHelpString = // --------------------------- // exception messages -static const char *kUserErrorMessage = "Incorrect command line"; // NExitCode::kUserError -// static const char *kIncorrectListFile = "Incorrect wildcard in listfile"; -static const char *kIncorrectWildcardInCommandLine = "Incorrect wildcard in command line"; +static const char * const kUserErrorMessage = "Incorrect command line"; // NExitCode::kUserError +// static const char * const kIncorrectListFile = "Incorrect wildcard in listfile"; +static const char * const kIncorrectWildcardInCommandLine = "Incorrect wildcard in command line"; // static const CSysString kFileIsNotArchiveMessageBefore = "File \""; // static const CSysString kFileIsNotArchiveMessageAfter = "\" is not archive"; -// static const char *kProcessArchiveMessage = " archive: "; +// static const char * const kProcessArchiveMessage = " archive: "; -static const char *kCantFindSFX = " cannot find sfx"; +static const char * const kCantFindSFX = " cannot find sfx"; namespace NCommandType { @@ -151,7 +159,7 @@ struct CArchiveCommand NRecursedType::EEnum DefaultRecursedType() const; }; -bool ParseArchiveCommand(const UString &commandString, CArchiveCommand &command) +static bool ParseArchiveCommand(const UString &commandString, CArchiveCommand &command) { UString s = commandString; s.MakeLower_Ascii(); @@ -171,17 +179,19 @@ NRecursedType::EEnum CArchiveCommand::DefaultRecursedType() const return kCommandRecursedDefault[CommandType]; } -void PrintHelp(void) +static void PrintHelp(void) { g_StdOut << kHelpString; } +MY_ATTR_NORETURN static void ShowMessageAndThrowException(const char *message, NExitCode::EEnum code) { g_StdOut << message << endl; throw code; } +MY_ATTR_NORETURN static void PrintHelpAndExit() // yyy { PrintHelp(); @@ -198,7 +208,7 @@ static bool AddNameToCensor(NWildcard::CCensor &wildcardCensor, if (!IsWildcardFilePathLegal(name)) return false; */ - bool isWildcard = DoesNameContainWildcard(name); + const bool isWildcard = DoesNameContainWildcard(name); bool recursed = false; switch (type) @@ -213,11 +223,14 @@ static bool AddNameToCensor(NWildcard::CCensor &wildcardCensor, recursed = false; break; } - wildcardCensor.AddPreItem(include, name, recursed, true); + + NWildcard::CCensorPathProps props; + props.Recursive = recursed; + wildcardCensor.AddPreItem(include, name, props); return true; } -void AddCommandLineWildcardToCensor(NWildcard::CCensor &wildcardCensor, +static void AddCommandLineWildcardToCensor(NWildcard::CCensor &wildcardCensor, const UString &name, bool include, NRecursedType::EEnum type) { if (!AddNameToCensor(wildcardCensor, name, include, type)) @@ -226,7 +239,7 @@ void AddCommandLineWildcardToCensor(NWildcard::CCensor &wildcardCensor, #ifndef _WIN32 -static void GetArguments(int numArgs, const char *args[], UStringVector &parts) +static void GetArguments(int numArgs, char *args[], UStringVector &parts) { parts.Clear(); for (int i = 0; i < numArgs; i++) @@ -237,9 +250,15 @@ static void GetArguments(int numArgs, const char *args[], UStringVector &parts) } #endif + int Main2( #ifndef _WIN32 - int numArgs, const char *args[] + int numArgs, char *args[] + #endif +); +int Main2( + #ifndef _WIN32 + int numArgs, char *args[] #endif ) { @@ -252,6 +271,10 @@ int Main2( SetFileApisToOEM(); #endif + #ifdef ENV_HAVE_LOCALE + MY_SetLocale(); + #endif + g_StdOut << kCopyrightString; UStringVector commandStrings; @@ -276,17 +299,23 @@ int Main2( #else - UString arcPath = commandStrings.Front(); + if (commandStrings.IsEmpty()) + return NExitCode::kFatalError; + + const FString arcPath = us2fs(commandStrings.Front()); #endif - commandStrings.Delete(0); + #ifndef UNDER_CE + if (commandStrings.Size() > 0) + commandStrings.Delete(0); + #endif - NCommandLineParser::CParser parser(kNumSwitches); + NCommandLineParser::CParser parser; try { - if (!parser.ParseStrings(kSwitchForms, commandStrings)) + if (!parser.ParseStrings(kSwitchForms, kNumSwitches, commandStrings)) { g_StdOut << "Command line error:" << endl << parser.ErrorMessage << endl @@ -331,7 +360,7 @@ int Main2( { if (nonSwitchStrings.Size() == curCommandIndex) - AddCommandLineWildcardToCensor(wildcardCensor, kUniversalWildcard, true, recursedType); + AddCommandLineWildcardToCensor(wildcardCensor, (UString)kUniversalWildcard, true, recursedType); for (; curCommandIndex < nonSwitchStrings.Size(); curCommandIndex++) { const UString &s = nonSwitchStrings[curCommandIndex]; @@ -352,7 +381,7 @@ int Main2( if (passwordEnabled) password = parser[NKey::kPassword].PostStrings[0]; - if (!NFind::DoesFileExist(arcPath)) + if (!NFind::DoesFileExist_FollowLink(arcPath)) throw kCantFindSFX; FString outputDir; diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile index 4c201c00a70..633bd775f6b 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile @@ -1,10 +1,11 @@ PROG = 7zCon.sfx MY_CONSOLE = 1 +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DEXTRACT_ONLY \ -DNO_READ_FROM_CODER \ -D_SFX \ - -D_CONSOLE \ CURRENT_OBJS = \ $O\SfxCon.obj \ @@ -28,8 +29,8 @@ COMMON_OBJS = \ $O\StdInStream.obj \ $O\StdOutStream.obj \ $O\StringConvert.obj \ - $O\Wildcard.obj \ $O\UTFConvert.obj \ + $O\Wildcard.obj \ WIN_OBJS = \ $O\DLL.obj \ @@ -41,6 +42,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\Synchronization.obj \ + $O\System.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ @@ -62,9 +64,9 @@ UI_COMMON_OBJS = \ $O\ArchiveExtractCallback.obj \ $O\ArchiveOpenCallback.obj \ $O\DefaultName.obj \ - $O\LoadCodecs.obj \ $O\Extract.obj \ $O\ExtractingFilePath.obj \ + $O\LoadCodecs.obj \ $O\OpenArchive.obj \ $O\PropIDUtils.obj \ @@ -77,7 +79,6 @@ AR_COMMON_OBJS = \ $O\MultiStream.obj \ $O\OutStreamWithCRC.obj \ - 7Z_OBJS = \ $O\7zDecode.obj \ $O\7zExtract.obj \ @@ -117,13 +118,16 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ - $O\Sha256.obj \ $O\Threads.obj \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile.gcc b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile.gcc new file mode 100644 index 00000000000..551b3e17dc3 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXCon/makefile.gcc @@ -0,0 +1,213 @@ +PROG = 7zCon + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + +ifdef SystemDrive +IS_MINGW = 1 +else +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif +endif + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef IS_MINGW +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ + $O/Threads.o \ + +endif + + + +LOCAL_FLAGS_SYS = + +ifdef IS_MINGW + +LOCAL_FLAGS_SYS = \ + +SYS_OBJS = \ + $O/DLL.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ + -DEXTRACT_ONLY \ + -DNO_READ_FROM_CODER \ + -D_SFX \ + + +CURRENT_OBJS = \ + $O/SfxCon.o \ + +CONSOLE_OBJS = \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/List.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UserInputUtils.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/IntToString.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/Sha256Prepare.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/StringConvert.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + +WIN_OBJS = \ + \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + \ + $O/System.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + \ + +UI_COMMON_OBJS = \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/DefaultName.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + +AR_OBJS = \ + $O/SplitHandler.o \ + +AR_COMMON_OBJS = \ + $O/CoderMixer2.o \ + $O/ItemNameUtils.o \ + $O/MultiStream.o \ + $O/OutStreamWithCRC.o \ + +7Z_OBJS = \ + $O/7zDecode.o \ + $O/7zExtract.o \ + $O/7zHandler.o \ + $O/7zIn.o \ + $O/7zRegister.o \ + +COMPRESS_OBJS = \ + $O/Bcj2Coder.o \ + $O/Bcj2Register.o \ + $O/BcjCoder.o \ + $O/BcjRegister.o \ + $O/BranchMisc.o \ + $O/BranchRegister.o \ + $O/CopyCoder.o \ + $O/CopyRegister.o \ + $O/DeltaFilter.o \ + $O/Lzma2Decoder.o \ + $O/Lzma2Register.o \ + $O/LzmaDecoder.o \ + $O/LzmaRegister.o \ + $O/PpmdDecoder.o \ + $O/PpmdRegister.o \ + +CRYPTO_OBJS = \ + $O/7zAes.o \ + $O/7zAesRegister.o \ + $O/MyAes.o \ + +C_OBJS = \ + $O/Alloc.o \ + $O/Bcj2.o \ + $O/Bra.o \ + $O/Bra86.o \ + $O/BraIA64.o \ + $O/CpuArch.o \ + $O/Delta.o \ + \ + $O/Lzma2Dec.o \ + $O/Lzma2DecMt.o \ + $O/LzmaDec.o \ + $O/MtDec.o \ + $O/Ppmd7.o \ + $O/Ppmd7Dec.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + $O/Aes.o \ + $O/AesOpt.o \ + +OBJS = \ + $(LZMA_DEC_OPT_OBJS) \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(SYS_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(AR_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7Z_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + $(CURRENT_OBJS) \ + +include ../../7zip_gcc.mak diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp index 793306303c5..8eaeabe7c3a 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp @@ -15,9 +15,9 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static LPCWSTR kCantDeleteFile = L"Can not delete output file"; -static LPCWSTR kCantOpenFile = L"Can not open output file"; -static LPCWSTR kUnsupportedMethod = L"Unsupported Method"; +static LPCSTR const kCantDeleteFile = "Cannot delete output file"; +static LPCSTR const kCantOpenFile = "Cannot open output file"; +static LPCSTR const kUnsupportedMethod = "Unsupported Method"; void CExtractCallbackImp::Init(IInArchive *archiveHandler, const FString &directoryPath, diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp index b1592993857..71b65c62001 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/ExtractEngine.cpp @@ -18,8 +18,8 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static LPCWSTR kCantFindArchive = L"Can not find archive file"; -static LPCWSTR kCantOpenArchive = L"Can not open the file as archive"; +static LPCSTR const kCantFindArchive = "Cannot find archive file"; +static LPCSTR const kCantOpenArchive = "Cannot open the file as archive"; struct CThreadExtracting { @@ -73,7 +73,7 @@ struct CThreadExtracting return; } - ExtractCallbackSpec->Init(ArchiveLink.GetArchive(), dirPath, L"Default", fi.MTime, 0); + ExtractCallbackSpec->Init(ArchiveLink.GetArchive(), dirPath, (UString)"Default", fi.MTime, 0); Result = ArchiveLink.GetArchive()->Extract(0, (UInt32)(Int32)-1 , BoolToInt(false), ExtractCallback); } diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp index 787c1bdefc5..754cc0e0ca3 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp @@ -449,6 +449,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -469,6 +477,14 @@ SOURCE=..\..\Common\CreateCoder.h # End Source File # Begin Source File +SOURCE=..\..\Common\CWrappers.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\CWrappers.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\FileStreams.cpp # End Source File # Begin Source File @@ -724,6 +740,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -733,6 +758,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp index aef2e1969c9..86b4f0fbb2c 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp @@ -33,7 +33,7 @@ using namespace NDir; HINSTANCE g_hInstance; -static CFSTR kTempDirPrefix = FTEXT("7zS"); +static CFSTR const kTempDirPrefix = FTEXT("7zS"); #define _SHELL_EXECUTE @@ -44,32 +44,32 @@ static bool ReadDataString(CFSTR fileName, LPCSTR startID, NIO::CInFile inFile; if (!inFile.Open(fileName)) return false; - const int kBufferSize = (1 << 12); + const size_t kBufferSize = (1 << 12); Byte buffer[kBufferSize]; - int signatureStartSize = MyStringLen(startID); - int signatureEndSize = MyStringLen(endID); + const unsigned signatureStartSize = MyStringLen(startID); + const unsigned signatureEndSize = MyStringLen(endID); - UInt32 numBytesPrev = 0; + size_t numBytesPrev = 0; bool writeMode = false; UInt64 posTotal = 0; for (;;) { if (posTotal > (1 << 20)) return (stringResult.IsEmpty()); - UInt32 numReadBytes = kBufferSize - numBytesPrev; - UInt32 processedSize; - if (!inFile.Read(buffer + numBytesPrev, numReadBytes, processedSize)) + const size_t numReadBytes = kBufferSize - numBytesPrev; + size_t processedSize; + if (!inFile.ReadFull(buffer + numBytesPrev, numReadBytes, processedSize)) return false; if (processedSize == 0) return true; - UInt32 numBytesInBuffer = numBytesPrev + processedSize; + const size_t numBytesInBuffer = numBytesPrev + processedSize; UInt32 pos = 0; for (;;) { if (writeMode) { - if (pos > numBytesInBuffer - signatureEndSize) + if (pos + signatureEndSize > numBytesInBuffer) break; if (memcmp(buffer + pos, endID, signatureEndSize) == 0) return true; @@ -81,7 +81,7 @@ static bool ReadDataString(CFSTR fileName, LPCSTR startID, } else { - if (pos > numBytesInBuffer - signatureStartSize) + if (pos + signatureStartSize > numBytesInBuffer) break; if (memcmp(buffer + pos, startID, signatureStartSize) == 0) { @@ -111,7 +111,9 @@ struct CInstallIDInit } g_CInstallIDInit; +#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION ShowErrorMessage(L"Unsupported Windows version"); return 1; +#endif static void ShowErrorMessageSpec(const UString &name) { @@ -169,7 +171,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, return 1; } - UString dirPrefix = L"." WSTRING_PATH_SEPARATOR; + UString dirPrefix ("." STRING_PATH_SEPARATOR); UString appLaunched; bool showProgress = true; if (!config.IsEmpty()) @@ -181,12 +183,12 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, ShowErrorMessage(L"Config failed"); return 1; } - UString friendlyName = GetTextConfigValue(pairs, L"Title"); - UString installPrompt = GetTextConfigValue(pairs, L"BeginPrompt"); - UString progress = GetTextConfigValue(pairs, L"Progress"); + UString friendlyName = GetTextConfigValue(pairs, "Title"); + UString installPrompt = GetTextConfigValue(pairs, "BeginPrompt"); + UString progress = GetTextConfigValue(pairs, "Progress"); if (progress.IsEqualTo_Ascii_NoCase("no")) showProgress = false; - int index = FindTextConfigItem(pairs, L"Directory"); + int index = FindTextConfigItem(pairs, "Directory"); if (index >= 0) dirPrefix = pairs[index].String; if (!installPrompt.IsEmpty() && !assumeYes) @@ -195,11 +197,11 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, MB_ICONQUESTION) != IDYES) return 0; } - appLaunched = GetTextConfigValue(pairs, L"RunProgram"); + appLaunched = GetTextConfigValue(pairs, "RunProgram"); #ifdef _SHELL_EXECUTE - executeFile = GetTextConfigValue(pairs, L"ExecuteFile"); - executeParameters = GetTextConfigValue(pairs, L"ExecuteParameters"); + executeFile = GetTextConfigValue(pairs, "ExecuteFile"); + executeParameters = GetTextConfigValue(pairs, "ExecuteParameters"); #endif } @@ -207,7 +209,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, if (!tempDir.Create(kTempDirPrefix)) { if (!assumeYes) - ShowErrorMessage(L"Can not create temp folder archive"); + ShowErrorMessage(L"Cannot create temp folder archive"); return 1; } @@ -217,7 +219,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, HRESULT result = codecs->Load(); if (result != S_OK) { - ShowErrorMessage(L"Can not load codecs"); + ShowErrorMessage(L"Cannot load codecs"); return 1; } } @@ -260,7 +262,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, #ifdef _SHELL_EXECUTE if (!executeFile.IsEmpty()) { - CSysString filePath = GetSystemString(executeFile); + CSysString filePath (GetSystemString(executeFile)); SHELLEXECUTEINFO execInfo; execInfo.cbSize = sizeof(execInfo); execInfo.fMask = SEE_MASK_NOCLOSEPROCESS @@ -278,7 +280,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, executeParameters += switches; } - CSysString parametersSys = GetSystemString(executeParameters); + CSysString parametersSys (GetSystemString(executeParameters)); if (parametersSys.IsEmpty()) execInfo.lpParameters = NULL; else @@ -292,7 +294,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, if (result <= 32) { if (!assumeYes) - ShowErrorMessage(L"Can not open file"); + ShowErrorMessage(L"Cannot open file"); return 1; } hProcess = execInfo.hProcess; @@ -303,10 +305,10 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, if (appLaunched.IsEmpty()) { appLaunched = L"setup.exe"; - if (!NFind::DoesFileExist(us2fs(appLaunched))) + if (!NFind::DoesFileExist_FollowLink(us2fs(appLaunched))) { if (!assumeYes) - ShowErrorMessage(L"Can not find setup.exe"); + ShowErrorMessage(L"Cannot find setup.exe"); return 1; } } @@ -337,7 +339,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, PROCESS_INFORMATION processInformation; - CSysString appLaunchedSys = GetSystemString(dirPrefix + appLaunched); + CSysString appLaunchedSys (GetSystemString(dirPrefix + appLaunched)); BOOL createResult = CreateProcess(NULL, (LPTSTR)(LPCTSTR)appLaunchedSys, NULL, NULL, FALSE, 0, NULL, NULL /*tempDir.GetPath() */, diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/makefile b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/makefile index 5849cf36907..9fddbc07827 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXSetup/makefile @@ -1,4 +1,6 @@ PROG = 7zS.sfx +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DNO_REGISTRY \ -DEXTRACT_ONLY \ @@ -33,6 +35,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\ResourceString.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -40,6 +43,7 @@ WIN_CTRL_OBJS = \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ + $O\CWrappers.obj \ $O\FileStreams.obj \ $O\InBuffer.obj \ $O\FilterCoder.obj \ @@ -102,9 +106,12 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index 301a6c5c756..8e1a70d22e1 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -537,6 +537,14 @@ SOURCE=..\..\..\Windows\Control\ListView.h # End Group # Begin Source File +SOURCE=..\..\..\Windows\Clipboard.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Clipboard.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\CommonDialog.cpp # End Source File # Begin Source File @@ -593,6 +601,14 @@ SOURCE=..\..\..\Windows\FileName.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\MemoryGlobal.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\MemoryGlobal.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\PropVariant.cpp # End Source File # Begin Source File @@ -633,6 +649,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -693,6 +717,10 @@ SOURCE=..\..\..\Common\NewHandler.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Sha256Prepare.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\StringConvert.cpp # End Source File # Begin Source File @@ -906,6 +934,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -915,6 +952,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -934,6 +980,11 @@ SOURCE=..\..\..\..\C\Sha256.c # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Sha256Opt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SfxWin.cpp b/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SfxWin.cpp index db7acfef7c8..23beade25ef 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SfxWin.cpp +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/SfxWin.cpp @@ -65,14 +65,14 @@ static DWORD GetDllVersion(LPCTSTR dllName) bool g_LVN_ITEMACTIVATE_Support = true; -static const wchar_t *kUnknownExceptionMessage = L"ERROR: Unknown Error!"; +static const wchar_t * const kUnknownExceptionMessage = L"ERROR: Unknown Error!"; -void ErrorMessageForHRESULT(HRESULT res) +static void ErrorMessageForHRESULT(HRESULT res) { ShowErrorMessage(HResultToMessage(res)); } -int APIENTRY WinMain2() +static int APIENTRY WinMain2() { // OleInitialize is required for ProgressBar in TaskBar. #ifndef UNDER_CE @@ -178,7 +178,7 @@ int APIENTRY WinMain2() v1.Add(fs2us(fullPath)); v2.Add(fs2us(fullPath)); NWildcard::CCensorNode wildcardCensor; - wildcardCensor.AddItem(true, L"*", true, true, true, true); + wildcardCensor.Add_Wildcard(); bool messageWasDisplayed = false; result = ExtractGUI(codecs, @@ -206,7 +206,9 @@ int APIENTRY WinMain2() return NExitCode::kFatalError; } +#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION ShowErrorMessage(L"Unsupported Windows version"); return NExitCode::kFatalError; +#endif int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, #ifdef UNDER_CE diff --git a/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/makefile b/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/makefile index dc48ae88a78..0ed36f8a857 100644 --- a/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/makefile +++ b/3rdparty/lzma/CPP/7zip/Bundles/SFXWin/makefile @@ -1,4 +1,6 @@ PROG = 7z.sfx +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DNO_REGISTRY \ -DEXTRACT_ONLY \ @@ -29,6 +31,7 @@ COMMON_OBJS = \ $O\Wildcard.obj \ WIN_OBJS = \ + $O\Clipboard.obj \ $O\CommonDialog.obj \ $O\DLL.obj \ $O\ErrorMsg.obj \ @@ -36,11 +39,13 @@ WIN_OBJS = \ $O\FileFind.obj \ $O\FileIO.obj \ $O\FileName.obj \ + $O\MemoryGlobal.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\ResourceString.obj \ $O\Shell.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -135,13 +140,16 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ - $O\Sha256.obj \ $O\Threads.obj \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" +!include "../../Sha256.mak" !include "../../7zip.mak" diff --git a/3rdparty/lzma/CPP/7zip/Common/CWrappers.cpp b/3rdparty/lzma/CPP/7zip/Common/CWrappers.cpp index a77b67e8664..ee4c36a2487 100644 --- a/3rdparty/lzma/CPP/7zip/Common/CWrappers.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/CWrappers.cpp @@ -1,4 +1,4 @@ -// CWrappers.h +// CWrappers.c #include "StdAfx.h" @@ -8,43 +8,74 @@ #include "StreamUtils.h" +SRes HRESULT_To_SRes(HRESULT res, SRes defaultRes) throw() +{ + switch (res) + { + case S_OK: return SZ_OK; + case E_OUTOFMEMORY: return SZ_ERROR_MEM; + case E_INVALIDARG: return SZ_ERROR_PARAM; + case E_ABORT: return SZ_ERROR_PROGRESS; + case S_FALSE: return SZ_ERROR_DATA; + case E_NOTIMPL: return SZ_ERROR_UNSUPPORTED; + } + return defaultRes; +} + + +HRESULT SResToHRESULT(SRes res) throw() +{ + switch (res) + { + case SZ_OK: return S_OK; + + case SZ_ERROR_DATA: + case SZ_ERROR_CRC: + case SZ_ERROR_INPUT_EOF: + return S_FALSE; + + case SZ_ERROR_MEM: return E_OUTOFMEMORY; + case SZ_ERROR_PARAM: return E_INVALIDARG; + case SZ_ERROR_PROGRESS: return E_ABORT; + case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; + // case SZ_ERROR_OUTPUT_EOF: + // case SZ_ERROR_READ: + // case SZ_ERROR_WRITE: + // case SZ_ERROR_THREAD: + // case SZ_ERROR_ARCHIVE: + // case SZ_ERROR_NO_ARCHIVE: + // return E_FAIL; + } + if (res < 0) + return res; + return E_FAIL; +} + + #define PROGRESS_UNKNOWN_VALUE ((UInt64)(Int64)-1) #define CONVERT_PR_VAL(x) (x == PROGRESS_UNKNOWN_VALUE ? NULL : &x) -static SRes CompressProgress(void *pp, UInt64 inSize, UInt64 outSize) throw() + +static SRes CompressProgress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) throw() { - CCompressProgressWrap *p = (CCompressProgressWrap *)pp; + CCompressProgressWrap *p = CONTAINER_FROM_VTBL(pp, CCompressProgressWrap, vt); p->Res = p->Progress->SetRatioInfo(CONVERT_PR_VAL(inSize), CONVERT_PR_VAL(outSize)); - return (SRes)p->Res; + return HRESULT_To_SRes(p->Res, SZ_ERROR_PROGRESS); } -CCompressProgressWrap::CCompressProgressWrap(ICompressProgressInfo *progress) throw() +void CCompressProgressWrap::Init(ICompressProgressInfo *progress) throw() { - p.Progress = CompressProgress; + vt.Progress = CompressProgress; Progress = progress; Res = SZ_OK; } static const UInt32 kStreamStepSize = (UInt32)1 << 31; -SRes HRESULT_To_SRes(HRESULT res, SRes defaultRes) +static SRes MyRead(const ISeqInStream *pp, void *data, size_t *size) throw() { - switch (res) - { - case S_OK: return SZ_OK; - case E_OUTOFMEMORY: return SZ_ERROR_MEM; - case E_INVALIDARG: return SZ_ERROR_PARAM; - case E_ABORT: return SZ_ERROR_PROGRESS; - case S_FALSE: return SZ_ERROR_DATA; - case E_NOTIMPL: return SZ_ERROR_UNSUPPORTED; - } - return defaultRes; -} - -static SRes MyRead(void *object, void *data, size_t *size) throw() -{ - CSeqInStreamWrap *p = (CSeqInStreamWrap *)object; + CSeqInStreamWrap *p = CONTAINER_FROM_VTBL(pp, CSeqInStreamWrap, vt); UInt32 curSize = ((*size < kStreamStepSize) ? (UInt32)*size : kStreamStepSize); p->Res = (p->Stream->Read(data, curSize, &curSize)); *size = curSize; @@ -54,9 +85,9 @@ static SRes MyRead(void *object, void *data, size_t *size) throw() return HRESULT_To_SRes(p->Res, SZ_ERROR_READ); } -static size_t MyWrite(void *object, const void *data, size_t size) throw() +static size_t MyWrite(const ISeqOutStream *pp, const void *data, size_t size) throw() { - CSeqOutStreamWrap *p = (CSeqOutStreamWrap *)object; + CSeqOutStreamWrap *p = CONTAINER_FROM_VTBL(pp, CSeqOutStreamWrap, vt); if (p->Stream) { p->Res = WriteStream(p->Stream, data, size); @@ -69,47 +100,36 @@ static size_t MyWrite(void *object, const void *data, size_t size) throw() return size; } -CSeqInStreamWrap::CSeqInStreamWrap(ISequentialInStream *stream) throw() + +void CSeqInStreamWrap::Init(ISequentialInStream *stream) throw() { - p.Read = MyRead; + vt.Read = MyRead; Stream = stream; Processed = 0; + Res = S_OK; } -CSeqOutStreamWrap::CSeqOutStreamWrap(ISequentialOutStream *stream) throw() +void CSeqOutStreamWrap::Init(ISequentialOutStream *stream) throw() { - p.Write = MyWrite; + vt.Write = MyWrite; Stream = stream; Res = SZ_OK; Processed = 0; } -HRESULT SResToHRESULT(SRes res) throw() -{ - switch (res) - { - case SZ_OK: return S_OK; - case SZ_ERROR_MEM: return E_OUTOFMEMORY; - case SZ_ERROR_PARAM: return E_INVALIDARG; - case SZ_ERROR_PROGRESS: return E_ABORT; - case SZ_ERROR_DATA: return S_FALSE; - case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; - } - return E_FAIL; -} -static SRes InStreamWrap_Read(void *pp, void *data, size_t *size) throw() +static SRes InStreamWrap_Read(const ISeekInStream *pp, void *data, size_t *size) throw() { - CSeekInStreamWrap *p = (CSeekInStreamWrap *)pp; + CSeekInStreamWrap *p = CONTAINER_FROM_VTBL(pp, CSeekInStreamWrap, vt); UInt32 curSize = ((*size < kStreamStepSize) ? (UInt32)*size : kStreamStepSize); p->Res = p->Stream->Read(data, curSize, &curSize); *size = curSize; return (p->Res == S_OK) ? SZ_OK : SZ_ERROR_READ; } -static SRes InStreamWrap_Seek(void *pp, Int64 *offset, ESzSeek origin) throw() +static SRes InStreamWrap_Seek(const ISeekInStream *pp, Int64 *offset, ESzSeek origin) throw() { - CSeekInStreamWrap *p = (CSeekInStreamWrap *)pp; + CSeekInStreamWrap *p = CONTAINER_FROM_VTBL(pp, CSeekInStreamWrap, vt); UInt32 moveMethod; switch (origin) { @@ -124,11 +144,11 @@ static SRes InStreamWrap_Seek(void *pp, Int64 *offset, ESzSeek origin) throw() return (p->Res == S_OK) ? SZ_OK : SZ_ERROR_READ; } -CSeekInStreamWrap::CSeekInStreamWrap(IInStream *stream) throw() +void CSeekInStreamWrap::Init(IInStream *stream) throw() { Stream = stream; - p.Read = InStreamWrap_Read; - p.Seek = InStreamWrap_Seek; + vt.Read = InStreamWrap_Read; + vt.Seek = InStreamWrap_Seek; Res = S_OK; } @@ -138,27 +158,27 @@ CSeekInStreamWrap::CSeekInStreamWrap(IInStream *stream) throw() void CByteInBufWrap::Free() throw() { ::MidFree(Buf); - Buf = 0; + Buf = NULL; } bool CByteInBufWrap::Alloc(UInt32 size) throw() { - if (Buf == 0 || size != Size) + if (!Buf || size != Size) { Free(); Lim = Cur = Buf = (Byte *)::MidAlloc((size_t)size); Size = size; } - return (Buf != 0); + return (Buf != NULL); } Byte CByteInBufWrap::ReadByteFromNewBlock() throw() { - if (Res == S_OK) + if (!Extra && Res == S_OK) { UInt32 avail; - Processed += (Cur - Buf); Res = Stream->Read(Buf, Size, &avail); + Processed += (size_t)(Cur - Buf); Cur = Buf; Lim = Buf + avail; if (avail != 0) @@ -168,55 +188,102 @@ Byte CByteInBufWrap::ReadByteFromNewBlock() throw() return 0; } -static Byte Wrap_ReadByte(void *pp) throw() +static Byte Wrap_ReadByte(const IByteIn *pp) throw() { - CByteInBufWrap *p = (CByteInBufWrap *)pp; + CByteInBufWrap *p = CONTAINER_FROM_VTBL_CLS(pp, CByteInBufWrap, vt); if (p->Cur != p->Lim) return *p->Cur++; return p->ReadByteFromNewBlock(); } -CByteInBufWrap::CByteInBufWrap(): Buf(0) +CByteInBufWrap::CByteInBufWrap(): Buf(NULL) { - p.Read = Wrap_ReadByte; + vt.Read = Wrap_ReadByte; } + +/* ---------- CByteOutBufWrap ---------- */ + +/* +void CLookToSequentialWrap::Free() throw() +{ + ::MidFree(BufBase); + BufBase = NULL; +} + +bool CLookToSequentialWrap::Alloc(UInt32 size) throw() +{ + if (!BufBase || size != Size) + { + Free(); + BufBase = (Byte *)::MidAlloc((size_t)size); + Size = size; + } + return (BufBase != NULL); +} +*/ + +/* +EXTERN_C_BEGIN + +void CLookToSequentialWrap_Look(ILookInSeqStream *pp) +{ + CLookToSequentialWrap *p = (CLookToSequentialWrap *)pp->Obj; + + if (p->Extra || p->Res != S_OK) + return; + { + UInt32 avail; + p->Res = p->Stream->Read(p->BufBase, p->Size, &avail); + p->Processed += avail; + pp->Buf = p->BufBase; + pp->Limit = pp->Buf + avail; + if (avail == 0) + p->Extra = true; + } +} + +EXTERN_C_END +*/ + + /* ---------- CByteOutBufWrap ---------- */ void CByteOutBufWrap::Free() throw() { ::MidFree(Buf); - Buf = 0; + Buf = NULL; } bool CByteOutBufWrap::Alloc(size_t size) throw() { - if (Buf == 0 || size != Size) + if (!Buf || size != Size) { Free(); Buf = (Byte *)::MidAlloc(size); Size = size; } - return (Buf != 0); + return (Buf != NULL); } HRESULT CByteOutBufWrap::Flush() throw() { if (Res == S_OK) { - size_t size = (Cur - Buf); + const size_t size = (size_t)(Cur - Buf); Res = WriteStream(Stream, Buf, size); if (Res == S_OK) Processed += size; - Cur = Buf; + // else throw 11; } + Cur = Buf; // reset pointer for later Wrap_WriteByte() return Res; } -static void Wrap_WriteByte(void *pp, Byte b) throw() +static void Wrap_WriteByte(const IByteOut *pp, Byte b) throw() { - CByteOutBufWrap *p = (CByteOutBufWrap *)pp; + CByteOutBufWrap *p = CONTAINER_FROM_VTBL_CLS(pp, CByteOutBufWrap, vt); Byte *dest = p->Cur; *dest = b; p->Cur = ++dest; @@ -224,7 +291,57 @@ static void Wrap_WriteByte(void *pp, Byte b) throw() p->Flush(); } -CByteOutBufWrap::CByteOutBufWrap() throw(): Buf(0) +CByteOutBufWrap::CByteOutBufWrap() throw(): Buf(NULL), Size(0) +{ + vt.Write = Wrap_WriteByte; +} + + +/* ---------- CLookOutWrap ---------- */ + +/* +void CLookOutWrap::Free() throw() +{ + ::MidFree(Buf); + Buf = NULL; +} + +bool CLookOutWrap::Alloc(size_t size) throw() +{ + if (!Buf || size != Size) + { + Free(); + Buf = (Byte *)::MidAlloc(size); + Size = size; + } + return (Buf != NULL); +} + +static size_t LookOutWrap_GetOutBuf(const ILookOutStream *pp, void **buf) throw() +{ + CLookOutWrap *p = CONTAINER_FROM_VTBL_CLS(pp, CLookOutWrap, vt); + *buf = p->Buf; + return p->Size; +} + +static size_t LookOutWrap_Write(const ILookOutStream *pp, size_t size) throw() +{ + CLookOutWrap *p = CONTAINER_FROM_VTBL_CLS(pp, CLookOutWrap, vt); + if (p->Res == S_OK && size != 0) + { + p->Res = WriteStream(p->Stream, p->Buf, size); + if (p->Res == S_OK) + { + p->Processed += size; + return size; + } + } + return 0; +} + +CLookOutWrap::CLookOutWrap() throw(): Buf(NULL), Size(0) { - p.Write = Wrap_WriteByte; + vt.GetOutBuf = LookOutWrap_GetOutBuf; + vt.Write = LookOutWrap_Write; } +*/ diff --git a/3rdparty/lzma/CPP/7zip/Common/CWrappers.h b/3rdparty/lzma/CPP/7zip/Common/CWrappers.h index acadc170edc..e7196a5c38d 100644 --- a/3rdparty/lzma/CPP/7zip/Common/CWrappers.h +++ b/3rdparty/lzma/CPP/7zip/Common/CWrappers.h @@ -6,49 +6,54 @@ #include "../ICoder.h" #include "../../Common/MyCom.h" +SRes HRESULT_To_SRes(HRESULT res, SRes defaultRes) throw(); +HRESULT SResToHRESULT(SRes res) throw(); + struct CCompressProgressWrap { - ICompressProgress p; + ICompressProgress vt; ICompressProgressInfo *Progress; HRESULT Res; - CCompressProgressWrap(ICompressProgressInfo *progress) throw(); + void Init(ICompressProgressInfo *progress) throw(); }; + struct CSeqInStreamWrap { - ISeqInStream p; + ISeqInStream vt; ISequentialInStream *Stream; HRESULT Res; UInt64 Processed; - CSeqInStreamWrap(ISequentialInStream *stream) throw(); + void Init(ISequentialInStream *stream) throw(); }; + struct CSeekInStreamWrap { - ISeekInStream p; + ISeekInStream vt; IInStream *Stream; HRESULT Res; - CSeekInStreamWrap(IInStream *stream) throw(); + void Init(IInStream *stream) throw(); }; + struct CSeqOutStreamWrap { - ISeqOutStream p; + ISeqOutStream vt; ISequentialOutStream *Stream; HRESULT Res; UInt64 Processed; - CSeqOutStreamWrap(ISequentialOutStream *stream) throw(); + void Init(ISequentialOutStream *stream) throw(); }; -HRESULT SResToHRESULT(SRes res) throw(); struct CByteInBufWrap { - IByteIn p; + IByteIn vt; const Byte *Cur; const Byte *Lim; Byte *Buf; @@ -69,7 +74,40 @@ struct CByteInBufWrap Extra = false; Res = S_OK; } - UInt64 GetProcessed() const { return Processed + (Cur - Buf); } + UInt64 GetProcessed() const { return Processed + (size_t)(Cur - Buf); } + Byte ReadByteFromNewBlock() throw(); + Byte ReadByte() + { + if (Cur != Lim) + return *Cur++; + return ReadByteFromNewBlock(); + } +}; + + +/* +struct CLookToSequentialWrap +{ + Byte *BufBase; + UInt32 Size; + ISequentialInStream *Stream; + UInt64 Processed; + bool Extra; + HRESULT Res; + + CLookToSequentialWrap(): BufBase(NULL) {} + ~CLookToSequentialWrap() { Free(); } + void Free() throw(); + bool Alloc(UInt32 size) throw(); + void Init() + { + // Lim = Cur = Buf; + Processed = 0; + Extra = false; + Res = S_OK; + } + // UInt64 GetProcessed() const { return Processed + (Cur - Buf); } + Byte ReadByteFromNewBlock() throw(); Byte ReadByte() { @@ -79,9 +117,16 @@ struct CByteInBufWrap } }; +EXTERN_C_BEGIN +// void CLookToSequentialWrap_Look(ILookInSeqStream *pp); +EXTERN_C_END +*/ + + + struct CByteOutBufWrap { - IByteOut p; + IByteOut vt; Byte *Cur; const Byte *Lim; Byte *Buf; @@ -101,7 +146,7 @@ struct CByteOutBufWrap Processed = 0; Res = S_OK; } - UInt64 GetProcessed() const { return Processed + (Cur - Buf); } + UInt64 GetProcessed() const { return Processed + (size_t)(Cur - Buf); } HRESULT Flush() throw(); void WriteByte(Byte b) { @@ -111,4 +156,27 @@ struct CByteOutBufWrap } }; + +/* +struct CLookOutWrap +{ + ILookOutStream vt; + Byte *Buf; + size_t Size; + ISequentialOutStream *Stream; + UInt64 Processed; + HRESULT Res; + + CLookOutWrap() throw(); + ~CLookOutWrap() { Free(); } + void Free() throw(); + bool Alloc(size_t size) throw(); + void Init() + { + Processed = 0; + Res = S_OK; + } +}; +*/ + #endif diff --git a/3rdparty/lzma/CPP/7zip/Common/CreateCoder.cpp b/3rdparty/lzma/CPP/7zip/Common/CreateCoder.cpp index 75074ad891f..872f17fca31 100644 --- a/3rdparty/lzma/CPP/7zip/Common/CreateCoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/CreateCoder.cpp @@ -11,19 +11,23 @@ #include "RegisterCodec.h" static const unsigned kNumCodecsMax = 64; +extern +unsigned g_NumCodecs; unsigned g_NumCodecs = 0; +extern +const CCodecInfo *g_Codecs[]; const CCodecInfo *g_Codecs[kNumCodecsMax]; // We use g_ExternalCodecs in other stages. -/* #ifdef EXTERNAL_CODECS +/* extern CExternalCodecs g_ExternalCodecs; #define CHECK_GLOBAL_CODECS \ if (!__externalCodecs || !__externalCodecs->IsSet()) __externalCodecs = &g_ExternalCodecs; -#endif */ - #define CHECK_GLOBAL_CODECS +#endif + void RegisterCodec(const CCodecInfo *codecInfo) throw() { @@ -32,7 +36,11 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw() } static const unsigned kNumHashersMax = 16; +extern +unsigned g_NumHashers; unsigned g_NumHashers = 0; +extern +const CHasherInfo *g_Hashers[]; const CHasherInfo *g_Hashers[kNumHashersMax]; void RegisterHasher(const CHasherInfo *hashInfo) throw() @@ -110,6 +118,7 @@ HRESULT CExternalCodecs::Load() } RINOK(ReadIsAssignedProp(GetCodecs, i, NMethodPropID::kEncoderIsAssigned, info.EncoderIsAssigned)); RINOK(ReadIsAssignedProp(GetCodecs, i, NMethodPropID::kDecoderIsAssigned, info.DecoderIsAssigned)); + RINOK(ReadIsAssignedProp(GetCodecs, i, NMethodPropID::kIsFilter, info.IsFilter)); Codecs.Add(info); } @@ -148,20 +157,23 @@ HRESULT CExternalCodecs::Load() #endif -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams) + bool encode, + CMethodId &methodId, + UInt32 &numStreams) { unsigned i; for (i = 0; i < g_NumCodecs; i++) { const CCodecInfo &codec = *g_Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.CreateEncoder : codec.CreateDecoder) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return (int)i; } } @@ -173,19 +185,51 @@ bool FindMethod( for (i = 0; i < __externalCodecs->Codecs.Size(); i++) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return (int)(g_NumCodecs + i); } } #endif - return false; + return -1; } + +static int FindMethod_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode) +{ + unsigned i; + for (i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &codec = *g_Codecs[i]; + if (codec.Id == methodId && (encode ? codec.CreateEncoder : codec.CreateDecoder)) + return (int)i; + } + + #ifdef EXTERNAL_CODECS + + CHECK_GLOBAL_CODECS + + if (__externalCodecs) + for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + { + const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; + if (codec.Id == methodId && (encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned)) + return (int)(g_NumCodecs + i); + } + + #endif + + return -1; +} + + bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, @@ -280,9 +324,11 @@ void GetHashMethods( #endif } -HRESULT CreateCoder( + + +HRESULT CreateCoder_Index( DECL_EXTERNAL_CODECS_LOC_VARS - CMethodId methodId, bool encode, + unsigned i, bool encode, CMyComPtr &filter, CCreatedCoder &cod) { @@ -290,11 +336,10 @@ HRESULT CreateCoder( cod.IsFilter = false; cod.NumStreams = 1; - unsigned i; - for (i = 0; i < g_NumCodecs; i++) + if (i < g_NumCodecs) { const CCodecInfo &codec = *g_Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -325,11 +370,12 @@ HRESULT CreateCoder( if (__externalCodecs) { + i -= g_NumCodecs; cod.IsExternal = true; - for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + if (i < __externalCodecs->Codecs.Size()) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -371,13 +417,50 @@ HRESULT CreateCoder( return S_OK; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod) +{ + CMyComPtr filter; + HRESULT res = CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + index, encode, + filter, cod); + + if (filter) + { + cod.IsFilter = true; + CFilterCoder *coderSpec = new CFilterCoder(encode); + cod.Coder = coderSpec; + coderSpec->Filter = filter; + } + + return res; +} + + +HRESULT CreateCoder_Id( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode, + CMyComPtr &filter, + CCreatedCoder &cod) +{ + int index = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS methodId, encode); + if (index < 0) + return S_OK; + return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS (unsigned)index, encode, filter, cod); +} + + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod) { CMyComPtr filter; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); @@ -393,13 +476,14 @@ HRESULT CreateCoder( return res; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &coder) { CCreatedCoder cod; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, cod); @@ -413,7 +497,7 @@ HRESULT CreateFilter( CMyComPtr &filter) { CCreatedCoder cod; - return CreateCoder( + return CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); diff --git a/3rdparty/lzma/CPP/7zip/Common/CreateCoder.h b/3rdparty/lzma/CPP/7zip/Common/CreateCoder.h index f06064b6a6f..24e6b66368f 100644 --- a/3rdparty/lzma/CPP/7zip/Common/CreateCoder.h +++ b/3rdparty/lzma/CPP/7zip/Common/CreateCoder.h @@ -35,8 +35,9 @@ struct CCodecInfoEx UInt32 NumStreams; bool EncoderIsAssigned; bool DecoderIsAssigned; + bool IsFilter; // it's unused - CCodecInfoEx(): EncoderIsAssigned(false), DecoderIsAssigned(false) {} + CCodecInfoEx(): EncoderIsAssigned(false), DecoderIsAssigned(false), IsFilter(false) {} }; struct CHasherInfoEx @@ -116,13 +117,12 @@ extern CExternalCodecs g_ExternalCodecs; #endif - - - -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams); + bool encode, + CMethodId &methodId, + UInt32 &numStreams); bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS @@ -152,18 +152,29 @@ struct CCreatedCoder }; -HRESULT CreateCoder( +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned codecIndex, bool encode, + CMyComPtr &filter, + CCreatedCoder &cod); + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod); + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &filter, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &coder); diff --git a/3rdparty/lzma/CPP/7zip/Common/FilePathAutoRename.cpp b/3rdparty/lzma/CPP/7zip/Common/FilePathAutoRename.cpp index d186e599333..1ebfd72bb93 100644 --- a/3rdparty/lzma/CPP/7zip/Common/FilePathAutoRename.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/FilePathAutoRename.cpp @@ -2,9 +2,6 @@ #include "StdAfx.h" -#include "../../Common/Defs.h" -#include "../../Common/IntToString.h" - #include "../../Windows/FileFind.h" #include "FilePathAutoRename.h" @@ -14,10 +11,8 @@ using namespace NWindows; static bool MakeAutoName(const FString &name, const FString &extension, UInt32 value, FString &path) { - char temp[16]; - ConvertUInt32ToString(value, temp); path = name; - path.AddAscii(temp); + path.Add_UInt32(value); path += extension; return NFile::NFind::DoesFileOrDirExist(path); } @@ -31,10 +26,10 @@ bool AutoRenamePath(FString &path) FString extension; if (dotPos > slashPos + 1) { - name.DeleteFrom(dotPos); - extension = path.Ptr(dotPos); + name.DeleteFrom((unsigned)dotPos); + extension = path.Ptr((unsigned)dotPos); } - name += FTEXT('_'); + name += '_'; FString temp; diff --git a/3rdparty/lzma/CPP/7zip/Common/FileStreams.cpp b/3rdparty/lzma/CPP/7zip/Common/FileStreams.cpp index f3a322fc9be..0349e907461 100644 --- a/3rdparty/lzma/CPP/7zip/Common/FileStreams.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/FileStreams.cpp @@ -2,42 +2,69 @@ #include "StdAfx.h" +// #include + #ifndef _WIN32 #include #include #include +#include +#include + +// for major()/minor(): +#if defined(__FreeBSD__) || defined(BSD) +#include +#else +#include #endif +#endif + +#include "../../Windows/FileFind.h" + #ifdef SUPPORT_DEVICE_FILE #include "../../../C/Alloc.h" #include "../../Common/Defs.h" #endif +#include "../PropID.h" + #include "FileStreams.h" -static inline HRESULT ConvertBoolToHRESULT(bool result) +static inline HRESULT GetLastError_HRESULT() { - #ifdef _WIN32 - if (result) - return S_OK; DWORD lastError = ::GetLastError(); if (lastError == 0) return E_FAIL; return HRESULT_FROM_WIN32(lastError); - #else - return result ? S_OK: E_FAIL; - #endif +} + +static inline HRESULT ConvertBoolToHRESULT(bool result) +{ + if (result) + return S_OK; + return GetLastError_HRESULT(); } +#ifdef SUPPORT_DEVICE_FILE static const UInt32 kClusterSize = 1 << 18; +#endif + CInFileStream::CInFileStream(): - #ifdef SUPPORT_DEVICE_FILE + #ifdef SUPPORT_DEVICE_FILE VirtPos(0), PhyPos(0), Buf(0), BufSize(0), - #endif + #endif + #ifndef _WIN32 + _uid(0), + _gid(0), + StoreOwnerId(false), + StoreOwnerName(false), + #endif + _info_WasLoaded(false), SupportHardLinks(false), Callback(NULL), CallbackRef(0) @@ -51,7 +78,7 @@ CInFileStream::~CInFileStream() #endif if (Callback) - Callback->InFileStream_On_Destroy(CallbackRef); + Callback->InFileStream_On_Destroy(this, CallbackRef); } STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) @@ -111,7 +138,7 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (alignedPos != PhyPos) { UInt64 realNewPosition; - bool result = File.Seek(alignedPos, FILE_BEGIN, realNewPosition); + bool result = File.Seek((Int64)alignedPos, FILE_BEGIN, realNewPosition); if (!result) return ConvertBoolToHRESULT(result); PhyPos = realNewPosition; @@ -140,7 +167,7 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (VirtPos != PhyPos) { UInt64 realNewPosition; - bool result = File.Seek(VirtPos, FILE_BEGIN, realNewPosition); + bool result = File.Seek((Int64)VirtPos, FILE_BEGIN, realNewPosition); if (!result) return ConvertBoolToHRESULT(result); PhyPos = VirtPos = realNewPosition; @@ -149,7 +176,7 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) #endif UInt32 realProcessedSize; - bool result = File.ReadPart(data, size, realProcessedSize); + const bool result = File.ReadPart(data, size, realProcessedSize); if (processedSize) *processedSize = realProcessedSize; @@ -161,33 +188,27 @@ STDMETHODIMP CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (result) return S_OK; + #else // USE_WIN_FILE + + if (processedSize) + *processedSize = 0; + const ssize_t res = File.read_part(data, (size_t)size); + if (res != -1) { - DWORD error = ::GetLastError(); + if (processedSize) + *processedSize = (UInt32)res; + return S_OK; + } + #endif // USE_WIN_FILE + { + const DWORD error = ::GetLastError(); if (Callback) return Callback->InFileStream_On_Error(CallbackRef, error); if (error == 0) return E_FAIL; - return HRESULT_FROM_WIN32(error); } - - #else - - if (processedSize) - *processedSize = 0; - ssize_t res = File.Read(data, (size_t)size); - if (res == -1) - { - if (Callback) - return Callback->InFileStream_On_Error(CallbackRef, E_FAIL); - return E_FAIL; - } - if (processedSize) - *processedSize = (UInt32)res; - return S_OK; - - #endif } #ifdef UNDER_CE @@ -228,7 +249,7 @@ STDMETHODIMP CStdInFileStream::Read(void *data, UInt32 size, UInt32 *processedSi } while (res < 0 && (errno == EINTR)); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); if (processedSize) *processedSize = (UInt32)res; return S_OK; @@ -257,15 +278,20 @@ STDMETHODIMP CInFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - VirtPos = offset; + VirtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } #endif - UInt64 realNewPosition; - bool result = File.Seek(offset, seekOrigin, realNewPosition); + UInt64 realNewPosition = 0; + const bool result = File.Seek(offset, seekOrigin, realNewPosition); + const HRESULT hres = ConvertBoolToHRESULT(result); + + /* 21.07: new File.Seek() in 21.07 already returns correct (realNewPosition) + in case of error. So we don't need additional code below */ + // if (!result) { realNewPosition = 0; File.GetPosition(realNewPosition); } #ifdef SUPPORT_DEVICE_FILE PhyPos = VirtPos = realNewPosition; @@ -273,13 +299,19 @@ STDMETHODIMP CInFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos if (newPosition) *newPosition = realNewPosition; - return ConvertBoolToHRESULT(result); + + return hres; #else - off_t res = File.Seek((off_t)offset, seekOrigin); + const off_t res = File.seek((off_t)offset, (int)seekOrigin); if (res == -1) - return E_FAIL; + { + const HRESULT hres = GetLastError_HRESULT(); + if (newPosition) + *newPosition = (UInt64)File.seekToCur(); + return hres; + } if (newPosition) *newPosition = (UInt64)res; return S_OK; @@ -296,8 +328,14 @@ STDMETHODIMP CInFileStream::GetSize(UInt64 *size) STDMETHODIMP CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aTime, FILETIME *mTime, UInt32 *attrib) { + if (!_info_WasLoaded) + RINOK(ReloadProps()); + const BY_HANDLE_FILE_INFORMATION &info = _info; + /* BY_HANDLE_FILE_INFORMATION info; - if (File.GetFileInformation(&info)) + if (!File.GetFileInformation(&info)) + return GetLastError_HRESULT(); + */ { if (size) *size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; if (cTime) *cTime = info.ftCreationTime; @@ -306,13 +344,18 @@ STDMETHODIMP CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aT if (attrib) *attrib = info.dwFileAttributes; return S_OK; } - return GetLastError(); } STDMETHODIMP CInFileStream::GetProps2(CStreamFileProps *props) { + if (!_info_WasLoaded) + RINOK(ReloadProps()); + const BY_HANDLE_FILE_INFORMATION &info = _info; + /* BY_HANDLE_FILE_INFORMATION info; - if (File.GetFileInformation(&info)) + if (!File.GetFileInformation(&info)) + return GetLastError_HRESULT(); + */ { props->Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; props->VolID = info.dwVolumeSerialNumber; @@ -325,11 +368,283 @@ STDMETHODIMP CInFileStream::GetProps2(CStreamFileProps *props) props->MTime = info.ftLastWriteTime; return S_OK; } - return GetLastError(); +} + +STDMETHODIMP CInFileStream::GetProperty(PROPID propID, PROPVARIANT *value) +{ + if (!_info_WasLoaded) + RINOK(ReloadProps()); + + if (!_info_WasLoaded) + return S_OK; + + NWindows::NCOM::CPropVariant prop; + + #ifdef SUPPORT_DEVICE_FILE + if (File.IsDeviceFile) + { + switch (propID) + { + case kpidSize: + if (File.SizeDefined) + prop = File.Size; + break; + // case kpidAttrib: prop = (UInt32)0; break; + case kpidPosixAttrib: + { + prop = (UInt32)NWindows::NFile::NFind::NAttributes:: + Get_PosixMode_From_WinAttrib(0); + /* GNU TAR by default can't extract file with MY_LIN_S_IFBLK attribute + so we don't use MY_LIN_S_IFBLK here */ + // prop = (UInt32)(MY_LIN_S_IFBLK | 0600); // for debug + break; + } + /* + case kpidDeviceMajor: + prop = (UInt32)8; // id for SCSI type device (sda) + break; + case kpidDeviceMinor: + prop = (UInt32)0; + break; + */ + } + } + else + #endif + { + switch (propID) + { + case kpidSize: + { + const UInt64 size = (((UInt64)_info.nFileSizeHigh) << 32) + _info.nFileSizeLow; + prop = size; + break; + } + case kpidAttrib: prop = (UInt32)_info.dwFileAttributes; break; + case kpidCTime: PropVariant_SetFrom_FiTime(prop, _info.ftCreationTime); break; + case kpidATime: PropVariant_SetFrom_FiTime(prop, _info.ftLastAccessTime); break; + case kpidMTime: PropVariant_SetFrom_FiTime(prop, _info.ftLastWriteTime); break; + case kpidPosixAttrib: + prop = (UInt32)NWindows::NFile::NFind::NAttributes:: + Get_PosixMode_From_WinAttrib(_info.dwFileAttributes); + // | (UInt32)(1 << 21); // for debug + break; + } + } + prop.Detach(value); + return S_OK; +} + + +STDMETHODIMP CInFileStream::ReloadProps() +{ + #ifdef SUPPORT_DEVICE_FILE + if (File.IsDeviceFile) + { + memset(&_info, 0, sizeof(_info)); + if (File.SizeDefined) + { + _info.nFileSizeHigh = (DWORD)(File.Size >> 32); + _info.nFileSizeLow = (DWORD)(File.Size); + } + _info.nNumberOfLinks = 1; + _info_WasLoaded = true; + return S_OK; + } + #endif + _info_WasLoaded = File.GetFileInformation(&_info); + if (!_info_WasLoaded) + return GetLastError_HRESULT(); + return S_OK; +} + + +#elif !defined(_WIN32) + +STDMETHODIMP CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aTime, FILETIME *mTime, UInt32 *attrib) +{ + if (!_info_WasLoaded) + RINOK(ReloadProps()); + const struct stat &st = _info; + /* + struct stat st; + if (File.my_fstat(&st) != 0) + return GetLastError_HRESULT(); + */ + + if (size) *size = (UInt64)st.st_size; + if (cTime) FiTime_To_FILETIME (ST_CTIME(st), *cTime); + if (aTime) FiTime_To_FILETIME (ST_ATIME(st), *aTime); + if (mTime) FiTime_To_FILETIME (ST_MTIME(st), *mTime); + if (attrib) *attrib = NWindows::NFile::NFind::Get_WinAttribPosix_From_PosixMode(st.st_mode); + + return S_OK; +} + +// #include + +STDMETHODIMP CInFileStream::GetProps2(CStreamFileProps *props) +{ + if (!_info_WasLoaded) + RINOK(ReloadProps()); + const struct stat &st = _info; + /* + struct stat st; + if (File.my_fstat(&st) != 0) + return GetLastError_HRESULT(); + */ + + props->Size = (UInt64)st.st_size; + /* + dev_t stat::st_dev: + GCC:Linux long unsigned int : __dev_t + Mac: int + */ + props->VolID = (UInt64)(Int64)st.st_dev; + props->FileID_Low = st.st_ino; + props->FileID_High = 0; + props->NumLinks = (UInt32)st.st_nlink; // we reduce to UInt32 from (nlink_t) that is (unsigned long) + props->Attrib = NWindows::NFile::NFind::Get_WinAttribPosix_From_PosixMode(st.st_mode); + + FiTime_To_FILETIME (ST_CTIME(st), props->CTime); + FiTime_To_FILETIME (ST_ATIME(st), props->ATime); + FiTime_To_FILETIME (ST_MTIME(st), props->MTime); + + /* + printf("\nGetProps2() NumLinks=%d = st_dev=%d st_ino = %d\n" + , (unsigned)(props->NumLinks) + , (unsigned)(st.st_dev) + , (unsigned)(st.st_ino) + ); + */ + + return S_OK; +} + +STDMETHODIMP CInFileStream::GetProperty(PROPID propID, PROPVARIANT *value) +{ + if (!_info_WasLoaded) + RINOK(ReloadProps()); + + if (!_info_WasLoaded) + return S_OK; + + const struct stat &st = _info; + + NWindows::NCOM::CPropVariant prop; + { + switch (propID) + { + case kpidSize: prop = (UInt64)st.st_size; break; + case kpidAttrib: + prop = (UInt32)NWindows::NFile::NFind::Get_WinAttribPosix_From_PosixMode(st.st_mode); + break; + case kpidCTime: PropVariant_SetFrom_FiTime(prop, ST_CTIME(st)); break; + case kpidATime: PropVariant_SetFrom_FiTime(prop, ST_ATIME(st)); break; + case kpidMTime: PropVariant_SetFrom_FiTime(prop, ST_MTIME(st)); break; + case kpidPosixAttrib: prop = (UInt32)st.st_mode; break; + + case kpidDeviceMajor: + { + // printf("\nst.st_rdev = %d\n", st.st_rdev); + if (S_ISCHR(st.st_mode) || + S_ISBLK(st.st_mode)) + prop = (UInt32)(major(st.st_rdev)); // + 1000); + // prop = (UInt32)12345678; // for debug + break; + } + + case kpidDeviceMinor: + if (S_ISCHR(st.st_mode) || + S_ISBLK(st.st_mode)) + prop = (UInt32)(minor(st.st_rdev)); // + 100); + // prop = (UInt32)(st.st_rdev); // for debug + // printf("\nst.st_rdev = %d\n", st.st_rdev); + // prop = (UInt32)123456789; // for debug + break; + + /* + case kpidDevice: + if (S_ISCHR(st.st_mode) || + S_ISBLK(st.st_mode)) + prop = (UInt64)(st.st_rdev); + break; + */ + + case kpidUserId: + { + if (StoreOwnerId) + prop = (UInt32)st.st_uid; + break; + } + case kpidGroupId: + { + if (StoreOwnerId) + prop = (UInt32)st.st_gid; + break; + } + case kpidUser: + { + if (StoreOwnerName) + { + const uid_t uid = st.st_uid; + { + if (!OwnerName.IsEmpty() && _uid == uid) + prop = OwnerName; + else + { + const passwd *pw = getpwuid(uid); + if (pw) + { + // we can use utf-8 here. + // prop = pw->pw_name; + } + } + } + } + break; + } + case kpidGroup: + { + if (StoreOwnerName) + { + const uid_t gid = st.st_gid; + { + if (!OwnerGroup.IsEmpty() && _gid == gid) + prop = OwnerGroup; + else + { + const group *gr = getgrgid(gid); + if (gr) + { + // we can use utf-8 here. + // prop = gr->gr_name; + } + } + } + } + break; + } + } + } + prop.Detach(value); + return S_OK; +} + + +STDMETHODIMP CInFileStream::ReloadProps() +{ + _info_WasLoaded = (File.my_fstat(&_info) == 0); + if (!_info_WasLoaded) + return GetLastError_HRESULT(); + return S_OK; } #endif + + + ////////////////////////// // COutFileStream @@ -343,7 +658,7 @@ STDMETHODIMP COutFileStream::Write(const void *data, UInt32 size, UInt32 *proces #ifdef USE_WIN_FILE UInt32 realProcessedSize; - bool result = File.Write(data, size, realProcessedSize); + const bool result = File.Write(data, size, realProcessedSize); ProcessedSize += realProcessedSize; if (processedSize) *processedSize = realProcessedSize; @@ -353,12 +668,13 @@ STDMETHODIMP COutFileStream::Write(const void *data, UInt32 size, UInt32 *proces if (processedSize) *processedSize = 0; - ssize_t res = File.Write(data, (size_t)size); - if (res == -1) - return E_FAIL; + size_t realProcessedSize; + const ssize_t res = File.write_full(data, (size_t)size, realProcessedSize); + ProcessedSize += realProcessedSize; if (processedSize) - *processedSize = (UInt32)res; - ProcessedSize += res; + *processedSize = (UInt32)realProcessedSize; + if (res == -1) + return GetLastError_HRESULT(); return S_OK; #endif @@ -371,17 +687,17 @@ STDMETHODIMP COutFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo #ifdef USE_WIN_FILE - UInt64 realNewPosition; - bool result = File.Seek(offset, seekOrigin, realNewPosition); + UInt64 realNewPosition = 0; + const bool result = File.Seek(offset, seekOrigin, realNewPosition); if (newPosition) *newPosition = realNewPosition; return ConvertBoolToHRESULT(result); #else - off_t res = File.Seek((off_t)offset, seekOrigin); + const off_t res = File.seek((off_t)offset, (int)seekOrigin); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); if (newPosition) *newPosition = (UInt64)res; return S_OK; @@ -391,21 +707,7 @@ STDMETHODIMP COutFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo STDMETHODIMP COutFileStream::SetSize(UInt64 newSize) { - #ifdef USE_WIN_FILE - - UInt64 currentPos; - if (!File.Seek(0, FILE_CURRENT, currentPos)) - return E_FAIL; - bool result = File.SetLength(newSize); - UInt64 currentPos2; - result = result && File.Seek(currentPos, currentPos2); - return result ? S_OK : E_FAIL; - - #else - - return E_FAIL; - - #endif + return ConvertBoolToHRESULT(File.SetLength_KeepPosition(newSize)); } HRESULT COutFileStream::GetSize(UInt64 *size) @@ -462,7 +764,7 @@ STDMETHODIMP CStdOutFileStream::Write(const void *data, UInt32 size, UInt32 *pro while (res < 0 && (errno == EINTR)); if (res == -1) - return E_FAIL; + return GetLastError_HRESULT(); _size += (size_t)res; if (processedSize) diff --git a/3rdparty/lzma/CPP/7zip/Common/FileStreams.h b/3rdparty/lzma/CPP/7zip/Common/FileStreams.h index ef2986fdd88..f2c19eea10a 100644 --- a/3rdparty/lzma/CPP/7zip/Common/FileStreams.h +++ b/3rdparty/lzma/CPP/7zip/Common/FileStreams.h @@ -7,42 +7,35 @@ #define USE_WIN_FILE #endif +#include "../../Common/MyCom.h" #include "../../Common/MyString.h" -#ifdef USE_WIN_FILE #include "../../Windows/FileIO.h" -#else -#include "../../Common/C_FileIO.h" -#endif - -#include "../../Common/MyCom.h" #include "../IStream.h" -#ifdef _WIN32 -typedef UINT_PTR My_UINT_PTR; -#else -typedef UINT My_UINT_PTR; -#endif +#include "UniqBlocks.h" + +class CInFileStream; struct IInFileStream_Callback { - virtual HRESULT InFileStream_On_Error(My_UINT_PTR val, DWORD error) = 0; - virtual void InFileStream_On_Destroy(My_UINT_PTR val) = 0; + virtual HRESULT InFileStream_On_Error(UINT_PTR val, DWORD error) = 0; + virtual void InFileStream_On_Destroy(CInFileStream *stream, UINT_PTR val) = 0; }; class CInFileStream: public IInStream, public IStreamGetSize, - #ifdef USE_WIN_FILE public IStreamGetProps, public IStreamGetProps2, - #endif + public IStreamGetProp, public CMyUnknownImp { + NWindows::NFile::NIO::CInFile File; public: + #ifdef USE_WIN_FILE - NWindows::NFile::NIO::CInFile File; #ifdef SUPPORT_DEVICE_FILE UInt64 VirtPos; @@ -52,35 +45,56 @@ public: UInt32 BufSize; #endif - #else - NC::NFile::NIO::CInFile File; #endif + #ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION _info; + #else + struct stat _info; + UInt32 _uid; + UInt32 _gid; + UString OwnerName; + UString OwnerGroup; + bool StoreOwnerId; + bool StoreOwnerName; + #endif + + bool _info_WasLoaded; bool SupportHardLinks; - IInFileStream_Callback *Callback; - My_UINT_PTR CallbackRef; + UINT_PTR CallbackRef; virtual ~CInFileStream(); CInFileStream(); + + void Set_PreserveATime(bool v) + { + File.PreserveATime = v; + } + + bool GetLength(UInt64 &length) const throw() + { + return File.GetLength(length); + } bool Open(CFSTR fileName) { + _info_WasLoaded = false; return File.Open(fileName); } bool OpenShared(CFSTR fileName, bool shareForWrite) { + _info_WasLoaded = false; return File.OpenShared(fileName, shareForWrite); } MY_QUERYINTERFACE_BEGIN2(IInStream) MY_QUERYINTERFACE_ENTRY(IStreamGetSize) - #ifdef USE_WIN_FILE MY_QUERYINTERFACE_ENTRY(IStreamGetProps) MY_QUERYINTERFACE_ENTRY(IStreamGetProps2) - #endif + MY_QUERYINTERFACE_ENTRY(IStreamGetProp) MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -88,10 +102,10 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); STDMETHOD(GetSize)(UInt64 *size); - #ifdef USE_WIN_FILE STDMETHOD(GetProps)(UInt64 *size, FILETIME *cTime, FILETIME *aTime, FILETIME *mTime, UInt32 *attrib); STDMETHOD(GetProps2)(CStreamFileProps *props); - #endif + STDMETHOD(GetProperty)(PROPID propID, PROPVARIANT *value); + STDMETHOD(ReloadProps)(); }; class CStdInFileStream: @@ -110,11 +124,8 @@ class COutFileStream: public CMyUnknownImp { public: - #ifdef USE_WIN_FILE NWindows::NFile::NIO::COutFile File; - #else - NC::NFile::NIO::COutFile File; - #endif + virtual ~COutFileStream() {} bool Create(CFSTR fileName, bool createAlways) { @@ -131,14 +142,11 @@ public: UInt64 ProcessedSize; - #ifdef USE_WIN_FILE - bool SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) + bool SetTime(const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) { return File.SetTime(cTime, aTime, mTime); } - bool SetMTime(const FILETIME *mTime) { return File.SetMTime(mTime); } - #endif - + bool SetMTime(const CFiTime *mTime) { return File.SetMTime(mTime); } MY_UNKNOWN_IMP1(IOutStream) @@ -146,6 +154,15 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); STDMETHOD(SetSize)(UInt64 newSize); + bool SeekToBegin_bool() + { + #ifdef USE_WIN_FILE + return File.SeekToBegin(); + #else + return File.seekToBegin() == 0; + #endif + } + HRESULT GetSize(UInt64 *size); }; diff --git a/3rdparty/lzma/CPP/7zip/Common/FilterCoder.cpp b/3rdparty/lzma/CPP/7zip/Common/FilterCoder.cpp index 4f3ae4e7dcf..fb99f610e23 100644 --- a/3rdparty/lzma/CPP/7zip/Common/FilterCoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/FilterCoder.cpp @@ -7,6 +7,23 @@ #include "FilterCoder.h" #include "StreamUtils.h" +#ifdef _WIN32 + #define alignedMidBuffer_Alloc g_MidAlloc +#else + #define alignedMidBuffer_Alloc g_AlignedAlloc +#endif + +CAlignedMidBuffer::~CAlignedMidBuffer() +{ + ISzAlloc_Free(&alignedMidBuffer_Alloc, _buf); +} + +void CAlignedMidBuffer::AllocAligned(size_t size) +{ + ISzAlloc_Free(&alignedMidBuffer_Alloc, _buf); + _buf = (Byte *)ISzAlloc_Alloc(&alignedMidBuffer_Alloc, size); +} + /* AES filters need 16-bytes alignment for HARDWARE-AES instructions. So we call IFilter::Filter(, size), where (size != 16 * N) only for last data block. @@ -36,7 +53,7 @@ HRESULT CFilterCoder::Alloc() size = kMinSize; if (!_buf || _bufSize != size) { - AllocAlignedMask(size, 16 - 1); + AllocAligned(size); if (!_buf) return E_OUTOFMEMORY; _bufSize = size; @@ -69,47 +86,47 @@ STDMETHODIMP CFilterCoder::Code(ISequentialInStream *inStream, ISequentialOutStr { RINOK(Init_and_Alloc()); + UInt64 prev = 0; UInt64 nowPos64 = 0; bool inputFinished = false; UInt32 pos = 0; while (!outSize || nowPos64 < *outSize) { - UInt32 endPos = pos; - if (!inputFinished) { size_t processedSize = _bufSize - pos; RINOK(ReadStream(inStream, _buf + pos, &processedSize)); - endPos = pos + (UInt32)processedSize; - inputFinished = (endPos != _bufSize); + pos += (UInt32)processedSize; + inputFinished = (pos != _bufSize); } - pos = Filter->Filter(_buf, endPos); + if (pos == 0) + return S_OK; + + UInt32 filtered = Filter->Filter(_buf, pos); - if (pos > endPos) + if (filtered > pos) { // AES - if (!inputFinished || pos > _bufSize) + if (!inputFinished || filtered > _bufSize) return E_FAIL; if (!_encodeMode) return S_FALSE; + Byte *buf = _buf; do - _buf[endPos] = 0; - while (++endPos != pos); + buf[pos] = 0; + while (++pos != filtered); - if (pos != Filter->Filter(_buf, pos)) + if (filtered != Filter->Filter(buf, filtered)) return E_FAIL; } - if (endPos == 0) - return S_OK; - - UInt32 size = (pos != 0 ? pos : endPos); + UInt32 size = (filtered != 0 ? filtered : pos); if (outSize) { - UInt64 remSize = *outSize - nowPos64; + const UInt64 remSize = *outSize - nowPos64; if (size > remSize) size = (UInt32)remSize; } @@ -117,16 +134,17 @@ STDMETHODIMP CFilterCoder::Code(ISequentialInStream *inStream, ISequentialOutStr RINOK(WriteStream(outStream, _buf, size)); nowPos64 += size; - if (pos == 0) + if (filtered == 0) return S_OK; + pos -= filtered; + for (UInt32 i = 0; i < pos; i++) + _buf[i] = _buf[filtered++]; - if (progress) + if (progress && (nowPos64 - prev) >= (1 << 22)) + { + prev = nowPos64; RINOK(progress->SetRatioInfo(&nowPos64, &nowPos64)); - - UInt32 i = 0; - while (pos < endPos) - _buf[i++] = _buf[pos++]; - pos = i; + } } return S_OK; diff --git a/3rdparty/lzma/CPP/7zip/Common/FilterCoder.h b/3rdparty/lzma/CPP/7zip/Common/FilterCoder.h index 224ff2ca459..6668fdb31f8 100644 --- a/3rdparty/lzma/CPP/7zip/Common/FilterCoder.h +++ b/3rdparty/lzma/CPP/7zip/Common/FilterCoder.h @@ -19,41 +19,11 @@ struct CAlignedMidBuffer { - #ifdef _WIN32 - Byte *_buf; CAlignedMidBuffer(): _buf(NULL) {} - ~CAlignedMidBuffer() { ::MidFree(_buf); } - - void AllocAlignedMask(size_t size, size_t) - { - ::MidFree(_buf); - _buf = (Byte *)::MidAlloc(size); - } - - #else - - Byte *_bufBase; - Byte *_buf; - - CAlignedMidBuffer(): _bufBase(NULL), _buf(NULL) {} - ~CAlignedMidBuffer() { ::MidFree(_bufBase); } - - void AllocAlignedMask(size_t size, size_t alignMask) - { - ::MidFree(_bufBase); - _buf = NULL; - _bufBase = (Byte *)::MidAlloc(size + alignMask); - - if (_bufBase) - { - // _buf = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - _buf = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - } - } - - #endif + ~CAlignedMidBuffer(); + void AllocAligned(size_t size); }; class CFilterCoder: @@ -153,6 +123,15 @@ public: ~C_OutStream_Releaser() { if (FilterCoder) FilterCoder->ReleaseOutStream(); } }; + class C_Filter_Releaser + { + public: + CFilterCoder *FilterCoder; + C_Filter_Releaser(): FilterCoder(NULL) {} + ~C_Filter_Releaser() { if (FilterCoder) FilterCoder->Filter.Release(); } + }; + + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetOutStreamSize) diff --git a/3rdparty/lzma/CPP/7zip/Common/InBuffer.cpp b/3rdparty/lzma/CPP/7zip/Common/InBuffer.cpp index 133d95b3885..fe6d1490257 100644 --- a/3rdparty/lzma/CPP/7zip/Common/InBuffer.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/InBuffer.cpp @@ -56,7 +56,7 @@ bool CInBufferBase::ReadBlock() #endif if (_wasFinished) return false; - _processedSize += (_buf - _bufBase); + _processedSize += (size_t)(_buf - _bufBase); _buf = _bufBase; _bufLim = _bufBase; UInt32 processed; @@ -77,7 +77,8 @@ bool CInBufferBase::ReadByte_FromNewBlock(Byte &b) { if (!ReadBlock()) { - NumExtraBytes++; + // 22.00: we don't increment (NumExtraBytes) here + // NumExtraBytes++; b = 0xFF; return false; } @@ -97,6 +98,33 @@ Byte CInBufferBase::ReadByte_FromNewBlock() size_t CInBufferBase::ReadBytes(Byte *buf, size_t size) { + size_t num = 0; + for (;;) + { + const size_t rem = (size_t)(_bufLim - _buf); + if (size <= rem) + { + if (size != 0) + { + memcpy(buf, _buf, size); + _buf += size; + num += size; + } + return num; + } + if (rem != 0) + { + memcpy(buf, _buf, rem); + _buf += rem; + buf += rem; + num += rem; + size -= rem; + } + if (!ReadBlock()) + return num; + } + + /* if ((size_t)(_bufLim - _buf) >= size) { const Byte *src = _buf; @@ -113,6 +141,7 @@ size_t CInBufferBase::ReadBytes(Byte *buf, size_t size) buf[i] = *_buf++; } return size; + */ } size_t CInBufferBase::Skip(size_t size) @@ -120,7 +149,7 @@ size_t CInBufferBase::Skip(size_t size) size_t processed = 0; for (;;) { - size_t rem = (_bufLim - _buf); + const size_t rem = (size_t)(_bufLim - _buf); if (rem >= size) { _buf += size; diff --git a/3rdparty/lzma/CPP/7zip/Common/InBuffer.h b/3rdparty/lzma/CPP/7zip/Common/InBuffer.h index 4b8662bb9d5..fa063949f5a 100644 --- a/3rdparty/lzma/CPP/7zip/Common/InBuffer.h +++ b/3rdparty/lzma/CPP/7zip/Common/InBuffer.h @@ -38,8 +38,16 @@ public: CInBufferBase() throw(); - UInt64 GetStreamSize() const { return _processedSize + (_buf - _bufBase); } - UInt64 GetProcessedSize() const { return _processedSize + NumExtraBytes + (_buf - _bufBase); } + // the size of portion of data in real stream that was already read from this object + // it doesn't include unused data in buffer + // it doesn't include virtual Extra bytes after the end of real stream data + UInt64 GetStreamSize() const { return _processedSize + (size_t)(_buf - _bufBase); } + + // the size of virtual data that was read from this object + // it doesn't include unused data in buffers + // it includes any virtual Extra bytes after the end of real data + UInt64 GetProcessedSize() const { return _processedSize + NumExtraBytes + (size_t)(_buf - _bufBase); } + bool WasFinished() const { return _wasFinished; } void SetStream(ISequentialInStream *stream) { _stream = stream; } @@ -59,7 +67,8 @@ public: } void Init() throw(); - + + MY_FORCE_INLINE bool ReadByte(Byte &b) { if (_buf >= _bufLim) @@ -67,7 +76,17 @@ public: b = *_buf++; return true; } + + MY_FORCE_INLINE + bool ReadByte_FromBuf(Byte &b) + { + if (_buf >= _bufLim) + return false; + b = *_buf++; + return true; + } + MY_FORCE_INLINE Byte ReadByte() { if (_buf >= _bufLim) diff --git a/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.cpp b/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.cpp index 85d6a1aa921..cae6b8037a7 100644 --- a/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.cpp @@ -2,64 +2,72 @@ #include "StdAfx.h" -#include "../../../C/7zCrc.h" - -#include "../../Common/Defs.h" - #include "InOutTempBuffer.h" #include "StreamUtils.h" +#ifdef USE_InOutTempBuffer_FILE + +#include "../../../C/7zCrc.h" + using namespace NWindows; using namespace NFile; using namespace NDir; static const size_t kTempBufSize = (1 << 20); -static CFSTR kTempFilePrefixString = FTEXT("7zt"); +#define kTempFilePrefixString FTEXT("7zt") +CInOutTempBuffer::~CInOutTempBuffer() +{ + delete []_buf; +} +#endif -CInOutTempBuffer::CInOutTempBuffer(): _buf(NULL) { } +CInOutTempBuffer::CInOutTempBuffer() + #ifdef USE_InOutTempBuffer_FILE + : _buf(NULL) + #endif +{ } void CInOutTempBuffer::Create() { + #ifdef USE_InOutTempBuffer_FILE if (!_buf) _buf = new Byte[kTempBufSize]; -} - -CInOutTempBuffer::~CInOutTempBuffer() -{ - delete []_buf; + #endif } void CInOutTempBuffer::InitWriting() { + #ifdef USE_InOutTempBuffer_FILE _bufPos = 0; + _crc = CRC_INIT_VAL; _tempFileCreated = false; + #endif _size = 0; - _crc = CRC_INIT_VAL; } -bool CInOutTempBuffer::WriteToFile(const void *data, UInt32 size) + +#ifdef USE_InOutTempBuffer_FILE + +static inline HRESULT Get_HRESULT_LastError() { - if (size == 0) - return true; - if (!_tempFileCreated) - { - if (!_tempFile.CreateRandomInTempFolder(kTempFilePrefixString, &_outFile)) - return false; - _tempFileCreated = true; - } - UInt32 processed; - if (!_outFile.Write(data, size, processed)) - return false; - _crc = CrcUpdate(_crc, data, processed); - _size += processed; - return (processed == size); + #ifdef _WIN32 + DWORD lastError = ::GetLastError(); + if (lastError != 0) + return HRESULT_FROM_WIN32(lastError); + #endif + return E_FAIL; } -bool CInOutTempBuffer::Write(const void *data, UInt32 size) +#endif + + +HRESULT CInOutTempBuffer::Write_HRESULT(const void *data, UInt32 size) { + #ifdef USE_InOutTempBuffer_FILE + if (size == 0) - return true; + return S_OK; size_t cur = kTempBufSize - _bufPos; if (cur != 0) { @@ -72,11 +80,42 @@ bool CInOutTempBuffer::Write(const void *data, UInt32 size) size -= (UInt32)cur; data = ((const Byte *)data) + cur; } - return WriteToFile(data, size); + + if (size == 0) + return S_OK; + + if (!_tempFileCreated) + { + if (!_tempFile.CreateRandomInTempFolder(kTempFilePrefixString, &_outFile)) + return Get_HRESULT_LastError(); + _tempFileCreated = true; + } + UInt32 processed; + if (!_outFile.Write(data, size, processed)) + return Get_HRESULT_LastError(); + _crc = CrcUpdate(_crc, data, processed); + _size += processed; + return (processed == size) ? S_OK : E_FAIL; + + #else + + const size_t newSize = _size + size; + if (newSize < _size) + return E_OUTOFMEMORY; + if (!_dynBuffer.EnsureCapacity(newSize)) + return E_OUTOFMEMORY; + memcpy(((Byte *)_dynBuffer) + _size, data, size); + _size = newSize; + return S_OK; + + #endif } + HRESULT CInOutTempBuffer::WriteToStream(ISequentialOutStream *stream) { + #ifdef USE_InOutTempBuffer_FILE + if (!_outFile.Close()) return E_FAIL; @@ -107,8 +146,13 @@ HRESULT CInOutTempBuffer::WriteToStream(ISequentialOutStream *stream) size += processed; } } - return (_crc == crc && size == _size) ? S_OK : E_FAIL; + + #else + + return WriteStream(stream, (const Byte *)_dynBuffer, _size); + + #endif } /* diff --git a/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.h b/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.h index 204a105f1de..755935eadb0 100644 --- a/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.h +++ b/3rdparty/lzma/CPP/7zip/Common/InOutTempBuffer.h @@ -3,30 +3,48 @@ #ifndef __IN_OUT_TEMP_BUFFER_H #define __IN_OUT_TEMP_BUFFER_H -#include "../../Common/MyCom.h" +#ifdef _WIN32 +// #define USE_InOutTempBuffer_FILE +#endif + +#ifdef USE_InOutTempBuffer_FILE #include "../../Windows/FileDir.h" +#else +#include "StreamObjects.h" +#endif #include "../IStream.h" class CInOutTempBuffer { + #ifdef USE_InOutTempBuffer_FILE + NWindows::NFile::NDir::CTempFile _tempFile; NWindows::NFile::NIO::COutFile _outFile; + bool _tempFileCreated; Byte *_buf; size_t _bufPos; UInt64 _size; UInt32 _crc; - bool _tempFileCreated; - bool WriteToFile(const void *data, UInt32 size); + #else + + CByteDynBuffer _dynBuffer; + size_t _size; + + #endif + + CLASS_NO_COPY(CInOutTempBuffer); public: CInOutTempBuffer(); - ~CInOutTempBuffer(); void Create(); - void InitWriting(); - bool Write(const void *data, UInt32 size); + #ifdef USE_InOutTempBuffer_FILE + ~CInOutTempBuffer(); + #endif + void InitWriting(); + HRESULT Write_HRESULT(const void *data, UInt32 size); HRESULT WriteToStream(ISequentialOutStream *stream); UInt64 GetDataSize() const { return _size; } }; diff --git a/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.cpp b/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.cpp index 8e032561662..980c795d20f 100644 --- a/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.cpp @@ -67,7 +67,7 @@ STDMETHODIMP CLimitedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *new } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -115,7 +115,7 @@ STDMETHODIMP CClusterInStream::Read(void *data, UInt32 size, UInt32 *processedSi _curRem = blockSize - offsetInBlock; - for (int i = 1; i < 64 && (virtBlock + i) < (UInt32)Vector.Size() && phyBlock + i == Vector[virtBlock + i]; i++) + for (unsigned i = 1; i < 64 && (virtBlock + i) < (UInt32)Vector.Size() && phyBlock + i == Vector[virtBlock + i]; i++) _curRem += (UInt32)1 << BlockSizeLog; } @@ -143,9 +143,9 @@ STDMETHODIMP CClusterInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *new return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; if (_virtPos != (UInt64)offset) _curRem = 0; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -154,44 +154,70 @@ STDMETHODIMP CExtentsStream::Read(void *data, UInt32 size, UInt32 *processedSize { if (processedSize) *processedSize = 0; - if (_virtPos >= Extents.Back().Virt) + const UInt64 virt = _virtPos; + if (virt >= Extents.Back().Virt) return S_OK; if (size == 0) return S_OK; - unsigned left = 0, right = Extents.Size() - 1; - for (;;) + unsigned left = _prevExtentIndex; + if (virt < Extents[left].Virt || + virt >= Extents[left + 1].Virt) { - unsigned mid = (left + right) / 2; - if (mid == left) - break; - if (_virtPos < Extents[mid].Virt) - right = mid; - else - left = mid; + left = 0; + unsigned right = Extents.Size() - 1; + for (;;) + { + const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + if (mid == left) + break; + if (virt < Extents[mid].Virt) + right = mid; + else + left = mid; + } + _prevExtentIndex = left; } - const CSeekExtent &extent = Extents[left]; - UInt64 phyPos = extent.Phy + (_virtPos - extent.Virt); - if (_needStartSeek || _phyPos != phyPos) { - _needStartSeek = false; - _phyPos = phyPos; - RINOK(SeekToPhys()); + const UInt64 rem = Extents[left + 1].Virt - virt; + if (size > rem) + size = (UInt32)rem; } - UInt64 rem = Extents[left + 1].Virt - _virtPos; - if (size > rem) - size = (UInt32)rem; + const CSeekExtent &extent = Extents[left]; - HRESULT res = Stream->Read(data, size, &size); - _phyPos += size; + if (extent.Is_ZeroFill()) + { + memset(data, 0, size); + _virtPos += size; + if (processedSize) + *processedSize = size; + return S_OK; + } + + { + const UInt64 phy = extent.Phy + (virt - extent.Virt); + if (_phyPos != phy) + { + _phyPos = (UInt64)0 - 1; // we don't trust seek_pos in case of error + RINOK(Stream->Seek((Int64)phy, STREAM_SEEK_SET, NULL)); + _phyPos = phy; + } + } + + const HRESULT res = Stream->Read(data, size, &size); _virtPos += size; + if (res == S_OK) + _phyPos += size; + else + _phyPos = (UInt64)0 - 1; // we don't trust seek_pos in case of error if (processedSize) *processedSize = size; return res; } + STDMETHODIMP CExtentsStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) { switch (seekOrigin) @@ -203,7 +229,7 @@ STDMETHODIMP CExtentsStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -268,10 +294,10 @@ STDMETHODIMP CTailInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPos } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; - return Stream->Seek(Offset + _virtPos, STREAM_SEEK_SET, NULL); + return Stream->Seek((Int64)(Offset + _virtPos), STREAM_SEEK_SET, NULL); } STDMETHODIMP CLimitedCachedInStream::Read(void *data, UInt32 size, UInt32 *processedSize) @@ -325,7 +351,7 @@ STDMETHODIMP CLimitedCachedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt6 } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; return S_OK; @@ -354,10 +380,10 @@ STDMETHODIMP CTailOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPo } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _virtPos = offset; + _virtPos = (UInt64)offset; if (newPosition) *newPosition = _virtPos; - return Stream->Seek(Offset + _virtPos, STREAM_SEEK_SET, NULL); + return Stream->Seek((Int64)(Offset + _virtPos), STREAM_SEEK_SET, NULL); } STDMETHODIMP CTailOutStream::SetSize(UInt64 newSize) diff --git a/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.h b/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.h index fb1ac3cd6bd..50c7cd853b5 100644 --- a/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.h +++ b/3rdparty/lzma/CPP/7zip/Common/LimitedStreams.h @@ -44,7 +44,7 @@ class CLimitedInStream: UInt64 _size; UInt64 _startOffset; - HRESULT SeekToPhys() { return _stream->Seek(_physPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return _stream->Seek((Int64)_physPos, STREAM_SEEK_SET, NULL); } public: void SetStream(IInStream *stream) { _stream = stream; } HRESULT InitAndSeek(UInt64 startOffset, UInt64 size) @@ -80,7 +80,7 @@ public: CRecordVector Vector; UInt64 StartOffset; - HRESULT SeekToPhys() { return Stream->Seek(_physPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return Stream->Seek((Int64)_physPos, STREAM_SEEK_SET, NULL); } HRESULT InitAndSeek() { @@ -101,21 +101,26 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); }; + + +const UInt64 k_SeekExtent_Phy_Type_ZeroFill = (UInt64)(Int64)-1; + struct CSeekExtent { - UInt64 Phy; UInt64 Virt; + UInt64 Phy; + + void SetAs_ZeroFill() { Phy = k_SeekExtent_Phy_Type_ZeroFill; } + bool Is_ZeroFill() const { return Phy == k_SeekExtent_Phy_Type_ZeroFill; } }; class CExtentsStream: public IInStream, public CMyUnknownImp { - UInt64 _phyPos; UInt64 _virtPos; - bool _needStartSeek; - - HRESULT SeekToPhys() { return Stream->Seek(_phyPos, STREAM_SEEK_SET, NULL); } + UInt64 _phyPos; + unsigned _prevExtentIndex; public: CMyComPtr Stream; @@ -129,11 +134,13 @@ public: void Init() { _virtPos = 0; - _phyPos = 0; - _needStartSeek = true; + _phyPos = (UInt64)0 - 1; // we need Seek() for Stream + _prevExtentIndex = 0; } }; + + class CLimitedSequentialOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -177,7 +184,7 @@ public: STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); - HRESULT SeekToStart() { return Stream->Seek(Offset, STREAM_SEEK_SET, NULL); } + HRESULT SeekToStart() { return Stream->Seek((Int64)Offset, STREAM_SEEK_SET, NULL); } }; class CLimitedCachedInStream: @@ -195,7 +202,7 @@ class CLimitedCachedInStream: size_t _cachePhyPos; - HRESULT SeekToPhys() { return _stream->Seek(_physPos, STREAM_SEEK_SET, NULL); } + HRESULT SeekToPhys() { return _stream->Seek((Int64)_physPos, STREAM_SEEK_SET, NULL); } public: CByteBuffer Buffer; diff --git a/3rdparty/lzma/CPP/7zip/Common/MethodProps.cpp b/3rdparty/lzma/CPP/7zip/Common/MethodProps.cpp index 09f7b29cc24..0e8ff1c61ca 100644 --- a/3rdparty/lzma/CPP/7zip/Common/MethodProps.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/MethodProps.cpp @@ -8,9 +8,39 @@ using namespace NWindows; -bool StringToBool(const UString &s, bool &res) +UInt64 Calc_From_Val_Percents(UInt64 val, UInt64 percents) { - if (s.IsEmpty() || (s[0] == '+' && s[1] == 0) || StringsAreEqualNoCase_Ascii(s, "ON")) + // if (percents == 0) return 0; + const UInt64 q = percents / 100; + const UInt32 r = (UInt32)(percents % 100); + UInt64 res = 0; + + if (q != 0) + { + if (val > (UInt64)(Int64)-1 / q) + return (UInt64)(Int64)-1; + res = val * q; + } + + if (r != 0) + { + UInt64 v2; + if (val <= (UInt64)(Int64)-1 / r) + v2 = val * r / 100; + else + v2 = val / 100 * r; + res += v2; + if (res < v2) + return (UInt64)(Int64)-1; + } + + return res; +} + + +bool StringToBool(const wchar_t *s, bool &res) +{ + if (s[0] == 0 || (s[0] == '+' && s[1] == 0) || StringsAreEqualNoCase_Ascii(s, "ON")) { res = true; return true; @@ -42,10 +72,18 @@ unsigned ParseStringToUInt32(const UString &srcString, UInt32 &number) return (unsigned)(end - start); } +static unsigned ParseStringToUInt64(const UString &srcString, UInt64 &number) +{ + const wchar_t *start = srcString; + const wchar_t *end; + number = ConvertStringToUInt64(start, &end); + return (unsigned)(end - start); +} + HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 &resValue) { // =VT_UI4 - // =VT_EMPTY + // =VT_EMPTY : it doesn't change (resValue), and returns S_OK // {stringUInt32}=VT_EMPTY if (prop.vt == VT_UI4) @@ -66,66 +104,151 @@ HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 & return S_OK; } -HRESULT ParseMtProp(const UString &name, const PROPVARIANT &prop, UInt32 defaultNumThreads, UInt32 &numThreads) + + +HRESULT ParseMtProp2(const UString &name, const PROPVARIANT &prop, UInt32 &numThreads, bool &force) { + force = false; + UString s; if (name.IsEmpty()) { - switch (prop.vt) + if (prop.vt == VT_UI4) { - case VT_UI4: - numThreads = prop.ulVal; - break; - default: + numThreads = prop.ulVal; + force = true; + return S_OK; + } + bool val; + HRESULT res = PROPVARIANT_to_bool(prop, val); + if (res == S_OK) + { + if (!val) { - bool val; - RINOK(PROPVARIANT_to_bool(prop, val)); - numThreads = (val ? defaultNumThreads : 1); - break; + numThreads = 1; + force = true; } + // force = true; for debug + // "(VT_BOOL = VARIANT_TRUE)" set "force = false" and doesn't change numThreads + return S_OK; } - return S_OK; + if (prop.vt != VT_BSTR) + return res; + s.SetFromBstr(prop.bstrVal); + if (s.IsEmpty()) + return E_INVALIDARG; } - if (prop.vt != VT_EMPTY) + else + { + if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + s = name; + } + + s.MakeLower_Ascii(); + const wchar_t *start = s; + UInt32 v = numThreads; + + /* we force up, if threads number specified + only `d` will force it down */ + bool force_loc = true; + for (;;) + { + const wchar_t c = *start; + if (!c) + break; + if (c == 'd') + { + force_loc = false; // force down + start++; + continue; + } + if (c == 'u') + { + force_loc = true; // force up + start++; + continue; + } + bool isPercent = false; + if (c == 'p') + { + isPercent = true; + start++; + } + const wchar_t *end; + v = ConvertStringToUInt32(start, &end); + if (end == start) + return E_INVALIDARG; + if (isPercent) + v = numThreads * v / 100; + start = end; + } + + numThreads = v; + force = force_loc; + return S_OK; +} + + + +static HRESULT SetLogSizeProp(UInt64 number, NCOM::CPropVariant &destProp) +{ + if (number >= 64) return E_INVALIDARG; - return ParsePropToUInt32(name, prop, numThreads); + UInt32 val32; + if (number < 32) + val32 = (UInt32)1 << (unsigned)number; + /* + else if (number == 32 && reduce_4GB_to_32bits) + val32 = (UInt32)(Int32)-1; + */ + else + { + destProp = (UInt64)((UInt64)1 << (unsigned)number); + return S_OK; + } + destProp = (UInt32)val32; + return S_OK; } static HRESULT StringToDictSize(const UString &s, NCOM::CPropVariant &destProp) { + /* if (reduce_4GB_to_32bits) we can reduce (4 GiB) property to (4 GiB - 1). + to fit the value to UInt32 for clients that do not support 64-bit values */ + const wchar_t *end; - UInt32 number = ConvertStringToUInt32(s, &end); - unsigned numDigits = (unsigned)(end - s); + const UInt64 number = ConvertStringToUInt64(s, &end); + const unsigned numDigits = (unsigned)(end - s.Ptr()); if (numDigits == 0 || s.Len() > numDigits + 1) return E_INVALIDARG; if (s.Len() == numDigits) - { - if (number >= 64) - return E_INVALIDARG; - if (number < 32) - destProp = (UInt32)((UInt32)1 << (unsigned)number); - else - destProp = (UInt64)((UInt64)1 << (unsigned)number); - return S_OK; - } + return SetLogSizeProp(number, destProp); unsigned numBits; switch (MyCharLower_Ascii(s[numDigits])) { - case 'b': destProp = number; return S_OK; + case 'b': numBits = 0; break; case 'k': numBits = 10; break; case 'm': numBits = 20; break; case 'g': numBits = 30; break; default: return E_INVALIDARG; } - if (number < ((UInt32)1 << (32 - numBits))) - destProp = (UInt32)(number << numBits); + const UInt64 range4g = ((UInt64)1 << (32 - numBits)); + if (number < range4g) + destProp = (UInt32)((UInt32)number << numBits); + /* + else if (number == range4g && reduce_4GB_to_32bits) + destProp = (UInt32)(Int32)-1; + */ + else if (numBits == 0) + destProp = (UInt64)number; + else if (number >= ((UInt64)1 << (64 - numBits))) + return E_INVALIDARG; else destProp = (UInt64)((UInt64)number << numBits); - return S_OK; } @@ -133,28 +256,32 @@ static HRESULT StringToDictSize(const UString &s, NCOM::CPropVariant &destProp) static HRESULT PROPVARIANT_to_DictSize(const PROPVARIANT &prop, NCOM::CPropVariant &destProp) { if (prop.vt == VT_UI4) + return SetLogSizeProp(prop.ulVal, destProp); + + if (prop.vt == VT_BSTR) { - UInt32 v = prop.ulVal; - if (v >= 64) - return E_INVALIDARG; - if (v < 32) - destProp = (UInt32)((UInt32)1 << (unsigned)v); - else - destProp = (UInt64)((UInt64)1 << (unsigned)v); - return S_OK; + UString s; + s = prop.bstrVal; + return StringToDictSize(s, destProp); } - if (prop.vt == VT_BSTR) - return StringToDictSize(prop.bstrVal, destProp); return E_INVALIDARG; } -void CProps::AddProp32(PROPID propid, UInt32 level) +void CProps::AddProp32(PROPID propid, UInt32 val) { CProp &prop = Props.AddNew(); prop.IsOptional = true; prop.Id = propid; - prop.Value = (UInt32)level; + prop.Value = (UInt32)val; +} + +void CProps::AddPropBool(PROPID propid, bool val) +{ + CProp &prop = Props.AddNew(); + prop.IsOptional = true; + prop.Id = propid; + prop.Value = val; } class CCoderProps @@ -164,10 +291,12 @@ class CCoderProps unsigned _numProps; unsigned _numPropsMax; public: - CCoderProps(unsigned numPropsMax) + CCoderProps(unsigned numPropsMax): + _propIDs(NULL), + _props(NULL), + _numProps(0), + _numPropsMax(numPropsMax) { - _numPropsMax = numPropsMax; - _numProps = 0; _propIDs = new PROPID[numPropsMax]; _props = new NCOM::CPropVariant[numPropsMax]; } @@ -194,7 +323,15 @@ void CCoderProps::AddProp(const CProp &prop) HRESULT CProps::SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce) const { - CCoderProps coderProps(Props.Size() + (dataSizeReduce ? 1 : 0)); + return SetCoderProps_DSReduce_Aff(scp, dataSizeReduce, NULL); +} + +HRESULT CProps::SetCoderProps_DSReduce_Aff( + ICompressSetCoderProperties *scp, + const UInt64 *dataSizeReduce, + const UInt64 *affinity) const +{ + CCoderProps coderProps(Props.Size() + (dataSizeReduce ? 1 : 0) + (affinity ? 1 : 0) ); FOR_VECTOR (i, Props) coderProps.AddProp(Props[i]); if (dataSizeReduce) @@ -204,27 +341,34 @@ HRESULT CProps::SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *da prop.Value = *dataSizeReduce; coderProps.AddProp(prop); } + if (affinity) + { + CProp prop; + prop.Id = NCoderPropID::kAffinity; + prop.Value = *affinity; + coderProps.AddProp(prop); + } return coderProps.SetProps(scp); } int CMethodProps::FindProp(PROPID id) const { - for (int i = Props.Size() - 1; i >= 0; i--) - if (Props[i].Id == id) - return i; + for (unsigned i = Props.Size(); i != 0;) + if (Props[--i].Id == id) + return (int)i; return -1; } -int CMethodProps::GetLevel() const +unsigned CMethodProps::GetLevel() const { int i = FindProp(NCoderPropID::kLevel); if (i < 0) return 5; - if (Props[i].Value.vt != VT_UI4) + if (Props[(unsigned)i].Value.vt != VT_UI4) return 9; - UInt32 level = Props[i].Value.ulVal; - return level > 9 ? 9 : (int)level; + UInt32 level = Props[(unsigned)i].Value.ulVal; + return level > 9 ? 9 : (unsigned)level; } struct CNameToPropID @@ -233,6 +377,9 @@ struct CNameToPropID const char *Name; }; + +// the following are related to NCoderPropID::EEnum values + static const CNameToPropID g_NameToPropID[] = { { VT_UI4, "" }, @@ -251,14 +398,19 @@ static const CNameToPropID g_NameToPropID[] = { VT_UI4, "mt" }, { VT_BOOL, "eos" }, { VT_UI4, "x" }, - { VT_UI4, "reduceSize" } + { VT_UI8, "reduce" }, + { VT_UI8, "expect" }, + { VT_UI4, "b" }, + { VT_UI4, "check" }, + { VT_BSTR, "filter" }, + { VT_UI8, "memuse" } }; static int FindPropIdExact(const UString &name) { for (unsigned i = 0; i < ARRAY_SIZE(g_NameToPropID); i++) if (StringsAreEqualNoCase_Ascii(name, g_NameToPropID[i].Name)) - return i; + return (int)i; return -1; } @@ -269,6 +421,13 @@ static bool ConvertProperty(const PROPVARIANT &srcProp, VARTYPE varType, NCOM::C destProp = srcProp; return true; } + + if (varType == VT_UI8 && srcProp.vt == VT_UI4) + { + destProp = (UInt64)srcProp.ulVal; + return true; + } + if (varType == VT_BOOL) { bool res; @@ -311,8 +470,8 @@ static void SplitParam(const UString ¶m, UString &name, UString &value) int eqPos = param.Find(L'='); if (eqPos >= 0) { - name.SetFrom(param, eqPos); - value = param.Ptr(eqPos + 1); + name.SetFrom(param, (unsigned)eqPos); + value = param.Ptr((unsigned)(eqPos + 1)); return; } unsigned i; @@ -333,7 +492,8 @@ static bool IsLogSizeProp(PROPID propid) case NCoderPropID::kDictionarySize: case NCoderPropID::kUsedMemorySize: case NCoderPropID::kBlockSize: - case NCoderPropID::kReduceSize: + case NCoderPropID::kBlockSize2: + // case NCoderPropID::kReduceSize: return true; } return false; @@ -346,7 +506,7 @@ HRESULT CMethodProps::SetParam(const UString &name, const UString &value) return E_INVALIDARG; const CNameToPropID &nameToPropID = g_NameToPropID[(unsigned)index]; CProp prop; - prop.Id = index; + prop.Id = (unsigned)index; if (IsLogSizeProp(prop.Id)) { @@ -366,9 +526,22 @@ HRESULT CMethodProps::SetParam(const UString &name, const UString &value) } else if (!value.IsEmpty()) { - UInt32 number; - if (ParseStringToUInt32(value, number) == value.Len()) - propValue = number; + if (nameToPropID.VarType == VT_UI4) + { + UInt32 number; + if (ParseStringToUInt32(value, number) == value.Len()) + propValue = number; + else + propValue = value; + } + else if (nameToPropID.VarType == VT_UI8) + { + UInt64 number; + if (ParseStringToUInt64(value, number) == value.Len()) + propValue = number; + else + propValue = value; + } else propValue = value; } @@ -414,7 +587,7 @@ HRESULT CMethodProps::ParseParamsFromPROPVARIANT(const UString &realName, const return E_INVALIDARG; const CNameToPropID &nameToPropID = g_NameToPropID[(unsigned)index]; CProp prop; - prop.Id = index; + prop.Id = (unsigned)index; if (IsLogSizeProp(prop.Id)) { @@ -429,6 +602,59 @@ HRESULT CMethodProps::ParseParamsFromPROPVARIANT(const UString &realName, const return S_OK; } + +static UInt64 GetMemoryUsage_LZMA(UInt32 dict, bool isBt, UInt32 numThreads) +{ + UInt32 hs = dict - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + if (hs >= (1 << 24)) + hs >>= 1; + hs |= (1 << 16) - 1; + // if (numHashBytes >= 5) + if (!isBt) + hs |= (256 << 10) - 1; + hs++; + UInt64 size1 = (UInt64)hs * 4; + size1 += (UInt64)dict * 4; + if (isBt) + size1 += (UInt64)dict * 4; + size1 += (2 << 20); + + if (numThreads > 1 && isBt) + size1 += (2 << 20) + (4 << 20); + return size1; +} + +static const UInt32 kLzmaMaxDictSize = (UInt32)15 << 28; + +UInt64 CMethodProps::Get_Lzma_MemUsage(bool addSlidingWindowSize) const +{ + const UInt64 dicSize = Get_Lzma_DicSize(); + const bool isBt = Get_Lzma_MatchFinder_IsBt(); + const UInt32 dict32 = (dicSize >= kLzmaMaxDictSize ? kLzmaMaxDictSize : (UInt32)dicSize); + const UInt32 numThreads = Get_Lzma_NumThreads(); + UInt64 size = GetMemoryUsage_LZMA(dict32, isBt, numThreads); + + if (addSlidingWindowSize) + { + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)(1 << 16); + UInt64 blockSize = (UInt64)dict32 + (1 << 16) + + (numThreads > 1 ? (1 << 20) : 0); + blockSize += (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)); + if (blockSize >= kBlockSizeMax) + blockSize = kBlockSizeMax; + size += blockSize; + } + return size; +} + + + + HRESULT COneMethodInfo::ParseMethodFromString(const UString &s) { MethodName.Empty(); @@ -436,14 +662,14 @@ HRESULT COneMethodInfo::ParseMethodFromString(const UString &s) { UString temp = s; if (splitPos >= 0) - temp.DeleteFrom(splitPos); + temp.DeleteFrom((unsigned)splitPos); if (!temp.IsAscii()) return E_INVALIDARG; MethodName.SetFromWStr_if_Ascii(temp); } if (splitPos < 0) return S_OK; - PropsString = s.Ptr(splitPos + 1); + PropsString = s.Ptr((unsigned)(splitPos + 1)); return ParseParamsFromString(PropsString); } @@ -454,5 +680,7 @@ HRESULT COneMethodInfo::ParseMethodFromPROPVARIANT(const UString &realName, cons // -m{N}=method if (value.vt != VT_BSTR) return E_INVALIDARG; - return ParseMethodFromString(value.bstrVal); + UString s; + s = value.bstrVal; + return ParseMethodFromString(s); } diff --git a/3rdparty/lzma/CPP/7zip/Common/MethodProps.h b/3rdparty/lzma/CPP/7zip/Common/MethodProps.h index 765e425df6d..5b5c96a457f 100644 --- a/3rdparty/lzma/CPP/7zip/Common/MethodProps.h +++ b/3rdparty/lzma/CPP/7zip/Common/MethodProps.h @@ -4,17 +4,46 @@ #define __7Z_METHOD_PROPS_H #include "../../Common/MyString.h" +#include "../../Common/Defs.h" + +#include "../../Windows/Defs.h" #include "../../Windows/PropVariant.h" #include "../ICoder.h" -bool StringToBool(const UString &s, bool &res); +// UInt64 GetMemoryUsage_LZMA(UInt32 dict, bool isBt, UInt32 numThreads); + +inline UInt64 Calc_From_Val_Percents_Less100(UInt64 val, UInt64 percents) +{ + if (percents == 0) + return 0; + if (val <= (UInt64)(Int64)-1 / percents) + return val * percents / 100; + return val / 100 * percents; +} + +UInt64 Calc_From_Val_Percents(UInt64 val, UInt64 percents); + +bool StringToBool(const wchar_t *s, bool &res); HRESULT PROPVARIANT_to_bool(const PROPVARIANT &prop, bool &dest); unsigned ParseStringToUInt32(const UString &srcString, UInt32 &number); + +/* +if (name.IsEmpty() && prop.vt == VT_EMPTY), it doesn't change (resValue) and returns S_OK. + So you must set (resValue) for default value before calling */ HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 &resValue); -HRESULT ParseMtProp(const UString &name, const PROPVARIANT &prop, UInt32 defaultNumThreads, UInt32 &numThreads); +/* input: (numThreads = the_number_of_processors) */ +HRESULT ParseMtProp2(const UString &name, const PROPVARIANT &prop, UInt32 &numThreads, bool &force); + +inline HRESULT ParseMtProp(const UString &name, const PROPVARIANT &prop, UInt32 numCPUs, UInt32 &numThreads) +{ + bool forced = false; + numThreads = numCPUs; + return ParseMtProp2(name, prop, numThreads, forced); +} + struct CProp { @@ -38,7 +67,9 @@ struct CProps return false; } - void AddProp32(PROPID propid, UInt32 level); + void AddProp32(PROPID propid, UInt32 val); + + void AddPropBool(PROPID propid, bool val); void AddProp_Ascii(PROPID propid, const char *s) { @@ -48,33 +79,45 @@ struct CProps prop.Value = s; } - HRESULT SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce) const; + HRESULT SetCoderProps(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce = NULL) const; + HRESULT SetCoderProps_DSReduce_Aff(ICompressSetCoderProperties *scp, const UInt64 *dataSizeReduce, const UInt64 *affinity) const; }; class CMethodProps: public CProps { HRESULT SetParam(const UString &name, const UString &value); public: - int GetLevel() const; + unsigned GetLevel() const; int Get_NumThreads() const { - int i = FindProp(NCoderPropID::kNumThreads); + const int i = FindProp(NCoderPropID::kNumThreads); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return (int)Props[i].Value.ulVal; + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + return (int)val.ulVal; + } return -1; } - bool Get_DicSize(UInt32 &res) const + bool Get_DicSize(UInt64 &res) const { res = 0; - int i = FindProp(NCoderPropID::kDictionarySize); + const int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + { + res = val.ulVal; + return true; + } + if (val.vt == VT_UI8) { - res = Props[i].Value.ulVal; + res = val.uhVal.QuadPart; return true; } + } return false; } @@ -84,19 +127,50 @@ public: { int i = FindProp(NCoderPropID::kAlgorithm); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return Props[i].Value.ulVal; + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + return val.ulVal; + } return GetLevel() >= 5 ? 1 : 0; } - UInt32 Get_Lzma_DicSize() const + UInt64 Get_Lzma_DicSize() const + { + UInt64 v; + if (Get_DicSize(v)) + return v; + const unsigned level = GetLevel(); + const UInt32 dictSize = + ( level <= 3 ? ((UInt32)1 << (level * 2 + 16)) : + ( level <= 6 ? ((UInt32)1 << (level + 19)) : + ( level <= 7 ? ((UInt32)1 << 25) : ((UInt32)1 << 26) + ))); + return dictSize; + } + + bool Get_Lzma_MatchFinder_IsBt() const { - int i = FindProp(NCoderPropID::kDictionarySize); + const int i = FindProp(NCoderPropID::kMatchFinder); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return Props[i].Value.ulVal; - int level = GetLevel(); - return level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26)); + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_BSTR) + return ((val.bstrVal[0] | 0x20) != 'h'); // check for "hc" + } + return GetLevel() >= 5; + } + + bool Get_Lzma_Eos() const + { + const int i = FindProp(NCoderPropID::kEndMarker); + if (i >= 0) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_BOOL) + return VARIANT_BOOLToBool(val.boolVal); + } + return false; } bool Are_Lzma_Model_Props_Defined() const @@ -107,18 +181,68 @@ public: return false; } - UInt32 Get_Lzma_NumThreads(bool &fixedNumber) const + UInt32 Get_Lzma_NumThreads() const { - fixedNumber = false; + if (Get_Lzma_Algo() == 0) + return 1; int numThreads = Get_NumThreads(); if (numThreads >= 0) - { - fixedNumber = true; return numThreads < 2 ? 1 : 2; + return 2; + } + + UInt64 Get_Lzma_MemUsage(bool addSlidingWindowSize) const; + + /* returns -1, if numThreads is unknown */ + int Get_Xz_NumThreads(UInt32 &lzmaThreads) const + { + lzmaThreads = 1; + int numThreads = Get_NumThreads(); + if (numThreads >= 0 && numThreads <= 1) + return 1; + if (Get_Lzma_Algo() != 0) + lzmaThreads = 2; + return numThreads; + } + + UInt64 GetProp_BlockSize(PROPID id) const + { + const int i = FindProp(id); + if (i >= 0) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) { return val.ulVal; } + if (val.vt == VT_UI8) { return val.uhVal.QuadPart; } } - return Get_Lzma_Algo() == 0 ? 1 : 2; + return 0; } + UInt64 Get_Xz_BlockSize() const + { + { + UInt64 blockSize1 = GetProp_BlockSize(NCoderPropID::kBlockSize); + UInt64 blockSize2 = GetProp_BlockSize(NCoderPropID::kBlockSize2); + UInt64 minSize = MyMin(blockSize1, blockSize2); + if (minSize != 0) + return minSize; + UInt64 maxSize = MyMax(blockSize1, blockSize2); + if (maxSize != 0) + return maxSize; + } + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + const UInt64 dictSize = Get_Lzma_DicSize(); + /* lzma2 code uses fake 4 GiB to calculate ChunkSize. So we do same */ + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + return blockSize; + } + + UInt32 Get_BZip2_NumThreads(bool &fixedNumber) const { fixedNumber = false; @@ -127,37 +251,47 @@ public: { fixedNumber = true; if (numThreads < 1) return 1; - if (numThreads > 64) return 64; - return numThreads; + const unsigned kNumBZip2ThreadsMax = 64; + if ((unsigned)numThreads > kNumBZip2ThreadsMax) return kNumBZip2ThreadsMax; + return (unsigned)numThreads; } return 1; } UInt32 Get_BZip2_BlockSize() const { - int i = FindProp(NCoderPropID::kDictionarySize); + const int i = FindProp(NCoderPropID::kDictionarySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) { - UInt32 blockSize = Props[i].Value.ulVal; + UInt32 blockSize = val.ulVal; const UInt32 kDicSizeMin = 100000; const UInt32 kDicSizeMax = 900000; if (blockSize < kDicSizeMin) blockSize = kDicSizeMin; if (blockSize > kDicSizeMax) blockSize = kDicSizeMax; return blockSize; } - int level = GetLevel(); + } + const unsigned level = GetLevel(); return 100000 * (level >= 5 ? 9 : (level >= 1 ? level * 2 - 1: 1)); } - UInt32 Get_Ppmd_MemSize() const + UInt64 Get_Ppmd_MemSize() const { - int i = FindProp(NCoderPropID::kUsedMemorySize); + const int i = FindProp(NCoderPropID::kUsedMemorySize); if (i >= 0) - if (Props[i].Value.vt == VT_UI4) - return Props[i].Value.ulVal; - int level = GetLevel(); - return level >= 9 ? (192 << 20) : ((UInt32)1 << (level + 19)); + { + const NWindows::NCOM::CPropVariant &val = Props[(unsigned)i].Value; + if (val.vt == VT_UI4) + return val.ulVal; + if (val.vt == VT_UI8) + return val.uhVal.QuadPart; + } + const unsigned level = GetLevel(); + const UInt32 mem = (UInt32)1 << (level + 19); + return mem; } void AddProp_Level(UInt32 level) @@ -170,6 +304,23 @@ public: AddProp32(NCoderPropID::kNumThreads, numThreads); } + void AddProp_EndMarker_if_NotFound(bool eos) + { + if (FindProp(NCoderPropID::kEndMarker) < 0) + AddPropBool(NCoderPropID::kEndMarker, eos); + } + + void AddProp_BlockSize2(UInt64 blockSize2) + { + if (FindProp(NCoderPropID::kBlockSize2) < 0) + { + CProp &prop = Props.AddNew(); + prop.IsOptional = true; + prop.Id = NCoderPropID::kBlockSize2; + prop.Value = blockSize2; + } + } + HRESULT ParseParamsFromString(const UString &srcString); HRESULT ParseParamsFromPROPVARIANT(const UString &realName, const PROPVARIANT &value); }; diff --git a/3rdparty/lzma/CPP/7zip/Common/OffsetStream.cpp b/3rdparty/lzma/CPP/7zip/Common/OffsetStream.cpp index 368d39b6514..b16124c2c71 100644 --- a/3rdparty/lzma/CPP/7zip/Common/OffsetStream.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/OffsetStream.cpp @@ -10,7 +10,7 @@ HRESULT COffsetOutStream::Init(IOutStream *stream, UInt64 offset) { _offset = offset; _stream = stream; - return _stream->Seek(offset, STREAM_SEEK_SET, NULL); + return _stream->Seek((Int64)offset, STREAM_SEEK_SET, NULL); } STDMETHODIMP COffsetOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) @@ -20,13 +20,13 @@ STDMETHODIMP COffsetOutStream::Write(const void *data, UInt32 size, UInt32 *proc STDMETHODIMP COffsetOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) { - UInt64 absoluteNewPosition; if (seekOrigin == STREAM_SEEK_SET) { if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; offset += _offset; } + UInt64 absoluteNewPosition = 0; // =0 for gcc-10 HRESULT result = _stream->Seek(offset, seekOrigin, &absoluteNewPosition); if (newPosition) *newPosition = absoluteNewPosition - _offset; diff --git a/3rdparty/lzma/CPP/7zip/Common/OutBuffer.h b/3rdparty/lzma/CPP/7zip/Common/OutBuffer.h index 3bdfb87c592..d7ca9f6a229 100644 --- a/3rdparty/lzma/CPP/7zip/Common/OutBuffer.h +++ b/3rdparty/lzma/CPP/7zip/Common/OutBuffer.h @@ -47,8 +47,11 @@ public: void WriteByte(Byte b) { - _buf[_pos++] = b; - if (_pos == _limitPos) + UInt32 pos = _pos; + _buf[pos] = b; + pos++; + _pos = pos; + if (pos == _limitPos) FlushWithCheck(); } void WriteBytes(const void *data, size_t size) diff --git a/3rdparty/lzma/CPP/7zip/Common/PropId.cpp b/3rdparty/lzma/CPP/7zip/Common/PropId.cpp index 11d20d55753..0e643e85365 100644 --- a/3rdparty/lzma/CPP/7zip/Common/PropId.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/PropId.cpp @@ -104,5 +104,12 @@ const Byte k7z_PROPID_To_VARTYPE[kpid_NUM_DEFINED] = VT_UI8, VT_BOOL, VT_BSTR, - VT_BSTR + VT_BSTR, + VT_BSTR, + VT_BOOL, + VT_FILETIME, // kpidChangeTime + VT_UI4, + VT_UI4, + VT_UI4, + VT_UI4 // kpidDeviceMinor }; diff --git a/3rdparty/lzma/CPP/7zip/Common/RegisterArc.h b/3rdparty/lzma/CPP/7zip/Common/RegisterArc.h index 3421ba1b164..a0384fad6ea 100644 --- a/3rdparty/lzma/CPP/7zip/Common/RegisterArc.h +++ b/3rdparty/lzma/CPP/7zip/Common/RegisterArc.h @@ -7,7 +7,7 @@ struct CArcInfo { - UInt16 Flags; + UInt32 Flags; Byte Id; Byte SignatureSize; UInt16 SignatureOffset; @@ -17,6 +17,8 @@ struct CArcInfo const char *Ext; const char *AddExt; + UInt32 TimeFlags; + Func_CreateInArchive CreateInArchive; Func_CreateOutArchive CreateOutArchive; Func_IsArc IsArc; @@ -39,22 +41,22 @@ void RegisterArc(const CArcInfo *arcInfo) throw(); #define IMP_CreateArcOut static IOutArchive *CreateArcOut() { return new CHandler(); } #endif -#define REGISTER_ARC_V(n, e, ae, id, sigSize, sig, offs, flags, crIn, crOut, isArc) \ - static const CArcInfo g_ArcInfo = { flags, id, sigSize, offs, sig, n, e, ae, crIn, crOut, isArc } ; \ +#define REGISTER_ARC_V(n, e, ae, id, sigSize, sig, offs, flags, tf, crIn, crOut, isArc) \ + static const CArcInfo g_ArcInfo = { flags, id, sigSize, offs, sig, n, e, ae, tf, crIn, crOut, isArc } ; \ -#define REGISTER_ARC_R(n, e, ae, id, sigSize, sig, offs, flags, crIn, crOut, isArc) \ - REGISTER_ARC_V(n, e, ae, id, sigSize, sig, offs, flags, crIn, crOut, isArc) \ +#define REGISTER_ARC_R(n, e, ae, id, sigSize, sig, offs, flags, tf, crIn, crOut, isArc) \ + REGISTER_ARC_V (n, e, ae, id, sigSize, sig, offs, flags, tf, crIn, crOut, isArc) \ struct CRegisterArc { CRegisterArc() { RegisterArc(&g_ArcInfo); }}; \ static CRegisterArc g_RegisterArc; #define REGISTER_ARC_I_CLS(cls, n, e, ae, id, sig, offs, flags, isArc) \ IMP_CreateArcIn_2(cls) \ - REGISTER_ARC_R(n, e, ae, id, ARRAY_SIZE(sig), sig, offs, flags, CreateArc, NULL, isArc) + REGISTER_ARC_R(n, e, ae, id, ARRAY_SIZE(sig), sig, offs, flags, 0, CreateArc, NULL, isArc) #define REGISTER_ARC_I_CLS_NO_SIG(cls, n, e, ae, id, offs, flags, isArc) \ IMP_CreateArcIn_2(cls) \ - REGISTER_ARC_R(n, e, ae, id, 0, NULL, offs, flags, CreateArc, NULL, isArc) + REGISTER_ARC_R(n, e, ae, id, 0, NULL, offs, flags, 0, CreateArc, NULL, isArc) #define REGISTER_ARC_I(n, e, ae, id, sig, offs, flags, isArc) \ REGISTER_ARC_I_CLS(CHandler(), n, e, ae, id, sig, offs, flags, isArc) @@ -63,15 +65,15 @@ void RegisterArc(const CArcInfo *arcInfo) throw(); REGISTER_ARC_I_CLS_NO_SIG(CHandler(), n, e, ae, id, offs, flags, isArc) -#define REGISTER_ARC_IO(n, e, ae, id, sig, offs, flags, isArc) \ +#define REGISTER_ARC_IO(n, e, ae, id, sig, offs, flags, tf, isArc) \ IMP_CreateArcIn \ IMP_CreateArcOut \ - REGISTER_ARC_R(n, e, ae, id, ARRAY_SIZE(sig), sig, offs, flags, CreateArc, CreateArcOut, isArc) + REGISTER_ARC_R(n, e, ae, id, ARRAY_SIZE(sig), sig, offs, flags, tf, CreateArc, CreateArcOut, isArc) -#define REGISTER_ARC_IO_DECREMENT_SIG(n, e, ae, id, sig, offs, flags, isArc) \ +#define REGISTER_ARC_IO_DECREMENT_SIG(n, e, ae, id, sig, offs, flags, tf, isArc) \ IMP_CreateArcIn \ IMP_CreateArcOut \ - REGISTER_ARC_V(n, e, ae, id, ARRAY_SIZE(sig), sig, offs, flags, CreateArc, CreateArcOut, isArc) \ + REGISTER_ARC_V(n, e, ae, id, ARRAY_SIZE(sig), sig, offs, flags, tf, CreateArc, CreateArcOut, isArc) \ struct CRegisterArcDecSig { CRegisterArcDecSig() { sig[0]--; RegisterArc(&g_ArcInfo); }}; \ static CRegisterArcDecSig g_RegisterArc; diff --git a/3rdparty/lzma/CPP/7zip/Common/RegisterCodec.h b/3rdparty/lzma/CPP/7zip/Common/RegisterCodec.h index 7ddb7604b9a..a942da7a129 100644 --- a/3rdparty/lzma/CPP/7zip/Common/RegisterCodec.h +++ b/3rdparty/lzma/CPP/7zip/Common/RegisterCodec.h @@ -26,11 +26,11 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw(); #define REGISTER_CODEC_CREATE(name, cls) REGISTER_CODEC_CREATE_2(name, cls, ICompressCoder) #define REGISTER_CODEC_NAME(x) CRegisterCodec ## x -#define REGISTER_CODEC_VAR static const CCodecInfo g_CodecInfo = +#define REGISTER_CODEC_VAR(x) static const CCodecInfo g_CodecInfo_ ## x = #define REGISTER_CODEC(x) struct REGISTER_CODEC_NAME(x) { \ - REGISTER_CODEC_NAME(x)() { RegisterCodec(&g_CodecInfo); }}; \ - static REGISTER_CODEC_NAME(x) g_RegisterCodec; + REGISTER_CODEC_NAME(x)() { RegisterCodec(&g_CodecInfo_ ## x); }}; \ + static REGISTER_CODEC_NAME(x) g_RegisterCodec_ ## x; #define REGISTER_CODECS_NAME(x) CRegisterCodecs ## x @@ -43,7 +43,7 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw(); #define REGISTER_CODEC_2(x, crDec, crEnc, id, name) \ - REGISTER_CODEC_VAR \ + REGISTER_CODEC_VAR(x) \ { crDec, crEnc, id, name, 1, false }; \ REGISTER_CODEC(x) @@ -67,19 +67,19 @@ void RegisterCodec(const CCodecInfo *codecInfo) throw(); { crDec, crEnc, id, name, 1, true } #define REGISTER_FILTER(x, crDec, crEnc, id, name) \ - REGISTER_CODEC_VAR \ + REGISTER_CODEC_VAR(x) \ REGISTER_FILTER_ITEM(crDec, crEnc, id, name); \ REGISTER_CODEC(x) #ifdef EXTRACT_ONLY #define REGISTER_FILTER_E(x, clsDec, clsEnc, id, name) \ - REGISTER_FILTER_CREATE(CreateDec, clsDec) \ - REGISTER_FILTER(x, CreateDec, NULL, id, name) + REGISTER_FILTER_CREATE(x ## _CreateDec, clsDec) \ + REGISTER_FILTER(x, x ## _CreateDec, NULL, id, name) #else #define REGISTER_FILTER_E(x, clsDec, clsEnc, id, name) \ - REGISTER_FILTER_CREATE(CreateDec, clsDec) \ - REGISTER_FILTER_CREATE(CreateEnc, clsEnc) \ - REGISTER_FILTER(x, CreateDec, CreateEnc, id, name) + REGISTER_FILTER_CREATE(x ## _CreateDec, clsDec) \ + REGISTER_FILTER_CREATE(x ## _CreateEnc, clsEnc) \ + REGISTER_FILTER(x, x ## _CreateDec, x ## _CreateEnc, id, name) #endif diff --git a/3rdparty/lzma/CPP/7zip/Common/StreamBinder.cpp b/3rdparty/lzma/CPP/7zip/Common/StreamBinder.cpp index 435440c6649..6b6e0e5844e 100644 --- a/3rdparty/lzma/CPP/7zip/Common/StreamBinder.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/StreamBinder.cpp @@ -14,7 +14,7 @@ class CBinderInStream: public: MY_UNKNOWN_IMP1(ISequentialInStream) STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); - ~CBinderInStream() { _binder->CloseRead(); } + ~CBinderInStream() { _binder->CloseRead_CallOnce(); } CBinderInStream(CStreamBinder *binder): _binder(binder) {} }; @@ -37,19 +37,24 @@ STDMETHODIMP CBinderOutStream::Write(const void *data, UInt32 size, UInt32 *proc { return _binder->Write(data, size, processedSize); } - -WRes CStreamBinder::CreateEvents() +static HRESULT Event__Create_or_Reset(NWindows::NSynchronization::CAutoResetEvent &event) { - RINOK(_canWrite_Event.Create()); - RINOK(_canRead_Event.Create()); - return _readingWasClosed_Event.Create(); + WRes wres; + if (event.IsCreated()) + wres = event.Reset(); + else + wres = event.Create(); + return HRESULT_FROM_WIN32(wres); } -void CStreamBinder::ReInit() +HRESULT CStreamBinder::Create_ReInit() { - _canWrite_Event.Reset(); - _canRead_Event.Reset(); - _readingWasClosed_Event.Reset(); + RINOK(Event__Create_or_Reset(_canRead_Event)); + // RINOK(Event__Create_or_Reset(_canWrite_Event)); + + // _canWrite_Semaphore.Close(); + // we need at least 3 items of maxCount: 1 for normal unlock in Read(), 2 items for unlock in CloseRead_CallOnce() + _canWrite_Semaphore.OptCreateInit(0, 3); // _readingWasClosed = false; _readingWasClosed2 = false; @@ -59,27 +64,14 @@ void CStreamBinder::ReInit() _buf = NULL; ProcessedSize = 0; // WritingWasCut = false; + return S_OK; } -void CStreamBinder::CreateStreams(ISequentialInStream **inStream, ISequentialOutStream **outStream) +void CStreamBinder::CreateStreams2(CMyComPtr &inStream, CMyComPtr &outStream) { - // _readingWasClosed = false; - _readingWasClosed2 = false; - - _waitWrite = true; - _bufSize = 0; - _buf = NULL; - ProcessedSize = 0; - // WritingWasCut = false; - - CBinderInStream *inStreamSpec = new CBinderInStream(this); - CMyComPtr inStreamLoc(inStreamSpec); - *inStream = inStreamLoc.Detach(); - - CBinderOutStream *outStreamSpec = new CBinderOutStream(this); - CMyComPtr outStreamLoc(outStreamSpec); - *outStream = outStreamLoc.Detach(); + inStream = new CBinderInStream(this); + outStream = new CBinderOutStream(this); } // (_canRead_Event && _bufSize == 0) means that stream is finished. @@ -92,7 +84,9 @@ HRESULT CStreamBinder::Read(void *data, UInt32 size, UInt32 *processedSize) { if (_waitWrite) { - RINOK(_canRead_Event.Lock()); + WRes wres = _canRead_Event.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); _waitWrite = false; } if (size > _bufSize) @@ -105,17 +99,25 @@ HRESULT CStreamBinder::Read(void *data, UInt32 size, UInt32 *processedSize) if (processedSize) *processedSize = size; _bufSize -= size; + + /* + if (_bufSize == 0), then we have read whole buffer + we have two ways here: + - if we check (_bufSize == 0) here, we unlock Write only after full data Reading - it reduces the number of syncs + - if we don't check (_bufSize == 0) here, we unlock Write after partial data Reading + */ if (_bufSize == 0) { _waitWrite = true; - _canRead_Event.Reset(); - _canWrite_Event.Set(); + // _canWrite_Event.Set(); + _canWrite_Semaphore.Release(); } } } return S_OK; } + HRESULT CStreamBinder::Write(const void *data, UInt32 size, UInt32 *processedSize) { if (processedSize) @@ -135,20 +137,20 @@ HRESULT CStreamBinder::Write(const void *data, UInt32 size, UInt32 *processedSiz _readingWasClosed2 = true; */ - HANDLE events[2] = { _canWrite_Event, _readingWasClosed_Event }; - DWORD waitResult = ::WaitForMultipleObjects(2, events, FALSE, INFINITE); - if (waitResult >= WAIT_OBJECT_0 + 2) - return E_FAIL; + _canWrite_Semaphore.Lock(); + // _bufSize : is remain size that was not read size -= _bufSize; + + // size : is size of data that was read if (size != 0) { + // if some data was read, then we report that size and return if (processedSize) *processedSize = size; return S_OK; } - // if (waitResult == WAIT_OBJECT_0 + 1) - _readingWasClosed2 = true; + _readingWasClosed2 = true; } // WritingWasCut = true; diff --git a/3rdparty/lzma/CPP/7zip/Common/StreamBinder.h b/3rdparty/lzma/CPP/7zip/Common/StreamBinder.h index 12088a94586..16c872fb57c 100644 --- a/3rdparty/lzma/CPP/7zip/Common/StreamBinder.h +++ b/3rdparty/lzma/CPP/7zip/Common/StreamBinder.h @@ -8,45 +8,63 @@ #include "../IStream.h" /* -We don't use probably UNSAFE version: -reader thread: +We can use one from two code versions here: with Event or with Semaphore to unlock Writer thread +The difference for cases where Reading must be closed before Writing closing + +1) Event Version: _canWrite_Event + We call _canWrite_Event.Set() without waiting _canRead_Event in CloseRead() function. + The writer thread can get (_readingWasClosed) status in one from two iterations. + It's ambiguity of processing flow. But probably it's SAFE to use, if Event functions provide memory barriers. + reader thread: _canWrite_Event.Set(); - _readingWasClosed = true + _readingWasClosed = true; _canWrite_Event.Set(); -writer thread: + writer thread: _canWrite_Event.Wait() if (_readingWasClosed) -Can second call of _canWrite_Event.Set() be executed without memory barrier, if event is already set? + +2) Semaphore Version: _canWrite_Semaphore + writer thread always will detect closing of reading in latest iteration after all data processing iterations */ class CStreamBinder { - NWindows::NSynchronization::CAutoResetEvent _canWrite_Event; - NWindows::NSynchronization::CManualResetEvent _canRead_Event; - NWindows::NSynchronization::CManualResetEvent _readingWasClosed_Event; + NWindows::NSynchronization::CAutoResetEvent _canRead_Event; + // NWindows::NSynchronization::CAutoResetEvent _canWrite_Event; + NWindows::NSynchronization::CSemaphore _canWrite_Semaphore; - // bool _readingWasClosed; - bool _readingWasClosed2; + // bool _readingWasClosed; // set it in reader thread and check it in write thread + bool _readingWasClosed2; // use it in writer thread // bool WritingWasCut; - bool _waitWrite; + bool _waitWrite; // use it in reader thread UInt32 _bufSize; const void *_buf; public: - UInt64 ProcessedSize; + UInt64 ProcessedSize; // the size that was read by reader thread - WRes CreateEvents(); - void CreateStreams(ISequentialInStream **inStream, ISequentialOutStream **outStream); + void CreateStreams2(CMyComPtr &inStream, CMyComPtr &outStream); - void ReInit(); + HRESULT Create_ReInit(); HRESULT Read(void *data, UInt32 size, UInt32 *processedSize); HRESULT Write(const void *data, UInt32 size, UInt32 *processedSize); - void CloseRead() + void CloseRead_CallOnce() { - _readingWasClosed_Event.Set(); - // _readingWasClosed = true; - // _canWrite_Event.Set(); + // call it only once: for example, in destructor + + /* + _readingWasClosed = true; + _canWrite_Event.Set(); + */ + + /* + We must relase Semaphore only once !!! + we must release at least 2 items of Semaphore: + one item to unlock partial Write(), if Read() have read some items + then additional item to stop writing (_bufSize will be 0) + */ + _canWrite_Semaphore.Release(2); } void CloseWrite() diff --git a/3rdparty/lzma/CPP/7zip/Common/StreamObjects.cpp b/3rdparty/lzma/CPP/7zip/Common/StreamObjects.cpp index 8136716d66f..2d941df6d52 100644 --- a/3rdparty/lzma/CPP/7zip/Common/StreamObjects.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/StreamObjects.cpp @@ -37,9 +37,9 @@ STDMETHODIMP CBufferInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newP } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -72,9 +72,9 @@ STDMETHODIMP CBufInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosi } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } @@ -108,14 +108,10 @@ bool CByteDynBuffer::EnsureCapacity(size_t cap) throw() { if (cap <= _capacity) return true; - size_t delta; - if (_capacity > 64) - delta = _capacity / 4; - else if (_capacity > 8) - delta = 16; - else - delta = 4; - cap = MyMax(_capacity + delta, cap); + size_t delta = _capacity / 4; + size_t cap2 = _capacity + delta; + if (cap < cap2) + cap = cap2; Byte *buf = (Byte *)realloc(_buf, cap); if (!buf) return false; @@ -185,9 +181,9 @@ static const UInt64 kEmptyTag = (UInt64)(Int64)-1; void CCachedInStream::Free() throw() { MyFree(_tags); - _tags = 0; + _tags = NULL; MidFree(_data); - _data = 0; + _data = NULL; } bool CCachedInStream::Alloc(unsigned blockSizeLog, unsigned numBlocksLog) throw() @@ -196,19 +192,19 @@ bool CCachedInStream::Alloc(unsigned blockSizeLog, unsigned numBlocksLog) throw( if (sizeLog >= sizeof(size_t) * 8) return false; size_t dataSize = (size_t)1 << sizeLog; - if (_data == 0 || dataSize != _dataSize) + if (!_data || dataSize != _dataSize) { MidFree(_data); _data = (Byte *)MidAlloc(dataSize); - if (_data == 0) + if (!_data) return false; _dataSize = dataSize; } - if (_tags == 0 || numBlocksLog != _numBlocksLog) + if (!_tags || numBlocksLog != _numBlocksLog) { MyFree(_tags); _tags = (UInt64 *)MyAlloc(sizeof(UInt64) << numBlocksLog); - if (_tags == 0) + if (!_tags) return false; _numBlocksLog = numBlocksLog; } @@ -242,21 +238,32 @@ STDMETHODIMP CCachedInStream::Read(void *data, UInt32 size, UInt32 *processedSiz while (size != 0) { - UInt64 cacheTag = _pos >> _blockSizeLog; - size_t cacheIndex = (size_t)cacheTag & (((size_t)1 << _numBlocksLog) - 1); + const UInt64 cacheTag = _pos >> _blockSizeLog; + const size_t cacheIndex = (size_t)cacheTag & (((size_t)1 << _numBlocksLog) - 1); Byte *p = _data + (cacheIndex << _blockSizeLog); + if (_tags[cacheIndex] != cacheTag) { + _tags[cacheIndex] = kEmptyTag; UInt64 remInBlock = _size - (cacheTag << _blockSizeLog); size_t blockSize = (size_t)1 << _blockSizeLog; if (blockSize > remInBlock) blockSize = (size_t)remInBlock; + RINOK(ReadBlock(cacheTag, p, blockSize)); + _tags[cacheIndex] = cacheTag; } - size_t offset = (size_t)_pos & (((size_t)1 << _blockSizeLog) - 1); - UInt32 cur = (UInt32)MyMin(((size_t)1 << _blockSizeLog) - offset, (size_t)size); + + const size_t kBlockSize = (size_t)1 << _blockSizeLog; + const size_t offset = (size_t)_pos & (kBlockSize - 1); + UInt32 cur = size; + const size_t rem = kBlockSize - offset; + if (cur > rem) + cur = (UInt32)rem; + memcpy(data, p + offset, cur); + if (processedSize) *processedSize += cur; data = (void *)((const Byte *)data + cur); @@ -266,6 +273,7 @@ STDMETHODIMP CCachedInStream::Read(void *data, UInt32 size, UInt32 *processedSiz return S_OK; } + STDMETHODIMP CCachedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition) { @@ -278,8 +286,8 @@ STDMETHODIMP CCachedInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newP } if (offset < 0) return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; - _pos = offset; + _pos = (UInt64)offset; if (newPosition) - *newPosition = offset; + *newPosition = (UInt64)offset; return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/Common/StreamObjects.h b/3rdparty/lzma/CPP/7zip/Common/StreamObjects.h index e20e9bd80de..a8fb229c8fd 100644 --- a/3rdparty/lzma/CPP/7zip/Common/StreamObjects.h +++ b/3rdparty/lzma/CPP/7zip/Common/StreamObjects.h @@ -24,6 +24,7 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); }; + struct CReferenceBuf: public IUnknown, public CMyUnknownImp @@ -32,6 +33,7 @@ struct CReferenceBuf: MY_UNKNOWN_IMP }; + class CBufInStream: public IInStream, public CMyUnknownImp @@ -41,7 +43,7 @@ class CBufInStream: size_t _size; CMyComPtr _ref; public: - void Init(const Byte *data, size_t size, IUnknown *ref = 0) + void Init(const Byte *data, size_t size, IUnknown *ref = NULL) { _data = data; _size = size; @@ -55,17 +57,20 @@ public: STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition); }; + void Create_BufInStream_WithReference(const void *data, size_t size, IUnknown *ref, ISequentialInStream **stream); void Create_BufInStream_WithNewBuffer(const void *data, size_t size, ISequentialInStream **stream); inline void Create_BufInStream_WithNewBuffer(const CByteBuffer &buf, ISequentialInStream **stream) { Create_BufInStream_WithNewBuffer(buf, buf.Size(), stream); } + class CByteDynBuffer { size_t _capacity; Byte *_buf; + CLASS_NO_COPY(CByteDynBuffer); public: - CByteDynBuffer(): _capacity(0), _buf(0) {}; + CByteDynBuffer(): _capacity(0), _buf(NULL) {}; // there is no copy constructor. So don't copy this object. ~CByteDynBuffer() { Free(); } void Free() throw(); @@ -75,6 +80,7 @@ public: bool EnsureCapacity(size_t capacity) throw(); }; + class CDynBufSeqOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -94,6 +100,7 @@ public: STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; + class CBufPtrSeqOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -114,6 +121,7 @@ public: STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; + class CSequentialOutStreamSizeCount: public ISequentialOutStream, public CMyUnknownImp @@ -129,6 +137,7 @@ public: STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; + class CCachedInStream: public IInStream, public CMyUnknownImp @@ -143,7 +152,7 @@ class CCachedInStream: protected: virtual HRESULT ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize) = 0; public: - CCachedInStream(): _tags(0), _data(0) {} + CCachedInStream(): _tags(NULL), _data(NULL) {} virtual ~CCachedInStream() { Free(); } // the destructor must be virtual (release calls it) !!! void Free() throw(); bool Alloc(unsigned blockSizeLog, unsigned numBlocksLog) throw(); diff --git a/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.cpp b/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.cpp index 8f754e177e8..32dc27626ed 100644 --- a/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.cpp @@ -11,10 +11,10 @@ unsigned CUniqBlocks::AddUniq(const Byte *data, size_t size) unsigned left = 0, right = Sorted.Size(); while (left != right) { - unsigned mid = (left + right) / 2; - unsigned index = Sorted[mid]; + const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned index = Sorted[mid]; const CByteBuffer &buf = Bufs[index]; - size_t sizeMid = buf.Size(); + const size_t sizeMid = buf.Size(); if (size < sizeMid) right = mid; else if (size > sizeMid) @@ -23,7 +23,7 @@ unsigned CUniqBlocks::AddUniq(const Byte *data, size_t size) { if (size == 0) return index; - int cmp = memcmp(data, buf, size); + const int cmp = memcmp(data, buf, size); if (cmp == 0) return index; if (cmp < 0) diff --git a/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.h b/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.h index a376024e3df..8479d68cb48 100644 --- a/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.h +++ b/3rdparty/lzma/CPP/7zip/Common/UniqBlocks.h @@ -3,9 +3,24 @@ #ifndef __UNIQ_BLOCKS_H #define __UNIQ_BLOCKS_H -#include "../../Common/MyTypes.h" #include "../../Common/MyBuffer.h" -#include "../../Common/MyVector.h" +#include "../../Common/MyString.h" + +struct C_UInt32_UString_Map +{ + CRecordVector Numbers; + UStringVector Strings; + + void Add_UInt32(const UInt32 n) + { + Numbers.AddToUniqueSorted(n); + } + int Find(const UInt32 n) + { + return Numbers.FindInSorted(n); + } +}; + struct CUniqBlocks { @@ -19,7 +34,7 @@ struct CUniqBlocks bool IsOnlyEmpty() const { - return (Bufs.Size() == 0 || Bufs.Size() == 1 && Bufs[0].Size() == 0); + return (Bufs.Size() == 0 || (Bufs.Size() == 1 && Bufs[0].Size() == 0)); } }; diff --git a/3rdparty/lzma/CPP/7zip/Common/VirtThread.cpp b/3rdparty/lzma/CPP/7zip/Common/VirtThread.cpp index 77e3c1acfd8..bf24bb1cc58 100644 --- a/3rdparty/lzma/CPP/7zip/Common/VirtThread.cpp +++ b/3rdparty/lzma/CPP/7zip/Common/VirtThread.cpp @@ -19,20 +19,20 @@ static THREAD_FUNC_DECL CoderThread(void *p) WRes CVirtThread::Create() { - RINOK(StartEvent.CreateIfNotCreated()); - RINOK(FinishedEvent.CreateIfNotCreated()); - StartEvent.Reset(); - FinishedEvent.Reset(); + RINOK_WRes(StartEvent.CreateIfNotCreated_Reset()); + RINOK_WRes(FinishedEvent.CreateIfNotCreated_Reset()); + // StartEvent.Reset(); + // FinishedEvent.Reset(); Exit = false; if (Thread.IsCreated()) return S_OK; return Thread.Create(CoderThread, this); } -void CVirtThread::Start() +WRes CVirtThread::Start() { Exit = false; - StartEvent.Set(); + return StartEvent.Set(); } void CVirtThread::WaitThreadFinish() @@ -42,7 +42,6 @@ void CVirtThread::WaitThreadFinish() StartEvent.Set(); if (Thread.IsCreated()) { - Thread.Wait(); - Thread.Close(); + Thread.Wait_Close(); } } diff --git a/3rdparty/lzma/CPP/7zip/Common/VirtThread.h b/3rdparty/lzma/CPP/7zip/Common/VirtThread.h index ebee158ca5b..b4d8a5a99c7 100644 --- a/3rdparty/lzma/CPP/7zip/Common/VirtThread.h +++ b/3rdparty/lzma/CPP/7zip/Common/VirtThread.h @@ -16,9 +16,9 @@ struct CVirtThread ~CVirtThread() { WaitThreadFinish(); } void WaitThreadFinish(); // call it in destructor of child class ! WRes Create(); - void Start(); + WRes Start(); virtual void Execute() = 0; - void WaitExecuteFinish() { FinishedEvent.Lock(); } + WRes WaitExecuteFinish() { return FinishedEvent.Lock(); } }; #endif diff --git a/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.cpp b/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.cpp index 96150c5f8b2..561fd08b732 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.cpp @@ -234,14 +234,14 @@ HRESULT CEncoder::CodeReal(ISequentialInStream * const *inStreams, const UInt64 Bcj2Enc_Encode(&enc); - currentInPos = totalStreamRead - numBytes_in_ReadBuf + (enc.src - _bufs[BCJ2_NUM_STREAMS]) - enc.tempPos; + currentInPos = totalStreamRead - numBytes_in_ReadBuf + (size_t)(enc.src - _bufs[BCJ2_NUM_STREAMS]) - enc.tempPos; if (Bcj2Enc_IsFinished(&enc)) break; if (enc.state < BCJ2_NUM_STREAMS) { - size_t curSize = enc.bufs[enc.state] - _bufs[enc.state]; + const size_t curSize = (size_t)(enc.bufs[enc.state] - _bufs[enc.state]); // printf("Write stream = %2d %6d\n", enc.state, curSize); RINOK(WriteStream(outStreams[enc.state], _bufs[enc.state], curSize)); if (enc.state == BCJ2_STREAM_RC) @@ -286,7 +286,7 @@ HRESULT CEncoder::CodeReal(ISequentialInStream * const *inStreams, const UInt64 if (progress && currentInPos - prevProgress >= (1 << 20)) { - UInt64 outSize2 = currentInPos + outSizeRc + enc.bufs[BCJ2_STREAM_RC] - enc.bufs[BCJ2_STREAM_RC]; + const UInt64 outSize2 = currentInPos + outSizeRc + (size_t)(enc.bufs[BCJ2_STREAM_RC] - enc.bufs[BCJ2_STREAM_RC]); prevProgress = currentInPos; // printf("progress %8d, %8d\n", (int)inSize2, (int)outSize2); RINOK(progress->SetRatioInfo(¤tInPos, &outSize2)); @@ -295,7 +295,7 @@ HRESULT CEncoder::CodeReal(ISequentialInStream * const *inStreams, const UInt64 for (int i = 0; i < BCJ2_NUM_STREAMS; i++) { - RINOK(WriteStream(outStreams[i], _bufs[i], enc.bufs[i] - _bufs[i])); + RINOK(WriteStream(outStreams[i], _bufs[i], (size_t)(enc.bufs[i] - _bufs[i]))); } // if (currentInPos != subStreamStartPos + subStreamSize) return E_FAIL; @@ -440,7 +440,7 @@ HRESULT CDecoder::Code(ISequentialInStream * const *inStreams, const UInt64 * co } else // if (dec.state <= BCJ2_STATE_ORIG) { - size_t curSize = dec.dest - _bufs[BCJ2_NUM_STREAMS]; + const size_t curSize = (size_t)(dec.dest - _bufs[BCJ2_NUM_STREAMS]); if (curSize != 0) { outSizeProcessed += curSize; @@ -463,17 +463,17 @@ HRESULT CDecoder::Code(ISequentialInStream * const *inStreams, const UInt64 * co if (progress) { - UInt64 outSize2 = outSizeProcessed + (dec.dest - _bufs[BCJ2_NUM_STREAMS]); + const UInt64 outSize2 = outSizeProcessed + (size_t)(dec.dest - _bufs[BCJ2_NUM_STREAMS]); if (outSize2 - prevProgress >= (1 << 22)) { - UInt64 inSize2 = outSize2 + _inStreamsProcessed[BCJ2_STREAM_RC] - (dec.lims[BCJ2_STREAM_RC] - dec.bufs[BCJ2_STREAM_RC]); + const UInt64 inSize2 = outSize2 + _inStreamsProcessed[BCJ2_STREAM_RC] - (size_t)(dec.lims[BCJ2_STREAM_RC] - dec.bufs[BCJ2_STREAM_RC]); RINOK(progress->SetRatioInfo(&inSize2, &outSize2)); prevProgress = outSize2; } } } - size_t curSize = dec.dest - _bufs[BCJ2_NUM_STREAMS]; + const size_t curSize = (size_t)(dec.dest - _bufs[BCJ2_NUM_STREAMS]); if (curSize != 0) { outSizeProcessed += curSize; @@ -498,7 +498,7 @@ HRESULT CDecoder::Code(ISequentialInStream * const *inStreams, const UInt64 * co { for (int i = 0; i < BCJ2_NUM_STREAMS; i++) { - size_t rem = dec.lims[i] - dec.bufs[i] + _extraReadSizes[i]; + const size_t rem = (size_t)(dec.lims[i] - dec.bufs[i]) + _extraReadSizes[i]; /* if (rem != 0) return S_FALSE; @@ -655,4 +655,12 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) return res; } + +STDMETHODIMP CDecoder::GetInStreamProcessedSize2(UInt32 streamIndex, UInt64 *value) +{ + const size_t rem = (size_t)(dec.lims[streamIndex] - dec.bufs[streamIndex]) + _extraReadSizes[streamIndex]; + *value = _inStreamsProcessed[streamIndex] - rem; + return S_OK; +} + }} diff --git a/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.h b/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.h index 381b9f54a57..ca6a1e4ed9b 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/Bcj2Coder.h @@ -62,6 +62,7 @@ public: class CDecoder: public ICompressCoder2, public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize2, public ICompressSetInStream2, public ISequentialInStream, public ICompressSetOutStreamSize, @@ -84,9 +85,10 @@ class CDecoder: // HRESULT ReadSpec(); public: - MY_UNKNOWN_IMP6( + MY_UNKNOWN_IMP7( ICompressCoder2, ICompressSetFinishMode, + ICompressGetInStreamProcessedSize2, ICompressSetInStream2, ISequentialInStream, ICompressSetOutStreamSize, @@ -98,6 +100,7 @@ public: ICompressProgressInfo *progress); STDMETHOD(SetFinishMode)(UInt32 finishMode); + STDMETHOD(GetInStreamProcessedSize2)(UInt32 streamIndex, UInt64 *value); STDMETHOD(SetInStream2)(UInt32 streamIndex, ISequentialInStream *inStream); STDMETHOD(ReleaseInStream2)(UInt32 streamIndex); diff --git a/3rdparty/lzma/CPP/7zip/Compress/Bcj2Register.cpp b/3rdparty/lzma/CPP/7zip/Compress/Bcj2Register.cpp index 7a48f91cfac..2868f1c9dcd 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Bcj2Register.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/Bcj2Register.cpp @@ -16,7 +16,7 @@ REGISTER_CODEC_CREATE_2(CreateCodecOut, CEncoder(), ICompressCoder2) #define CreateCodecOut NULL #endif -REGISTER_CODEC_VAR +REGISTER_CODEC_VAR(BCJ2) { CreateCodec, CreateCodecOut, 0x303011B, "BCJ2", 4, false }; REGISTER_CODEC(BCJ2) diff --git a/3rdparty/lzma/CPP/7zip/Compress/CodecExports.cpp b/3rdparty/lzma/CPP/7zip/Compress/CodecExports.cpp index 99085040d07..5bb6ff8d21d 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/CodecExports.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/CodecExports.cpp @@ -82,6 +82,14 @@ static HRESULT FindCodecClassId(const GUID *clsid, bool isCoder2, bool isFilter, return S_OK; } +/* +#ifdef __GNUC__ +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wduplicated-branches" +#endif +#endif +*/ + static HRESULT CreateCoderMain(unsigned index, bool encode, void **coder) { COM_TRY_BEGIN @@ -97,12 +105,15 @@ static HRESULT CreateCoderMain(unsigned index, bool encode, void **coder) if (c) { IUnknown *unk; + unk = (IUnknown *)c; + /* if (codec.IsFilter) unk = (IUnknown *)(ICompressFilter *)c; else if (codec.NumStreams != 1) unk = (IUnknown *)(ICompressCoder2 *)c; else unk = (IUnknown *)(ICompressCoder *)c; + */ unk->AddRef(); *coder = c; } @@ -136,16 +147,22 @@ static HRESULT CreateCoder2(bool encode, UInt32 index, const GUID *iid, void **o return CreateCoderMain(index, encode, outObject); } + +STDAPI CreateDecoder(UInt32 index, const GUID *iid, void **outObject); STDAPI CreateDecoder(UInt32 index, const GUID *iid, void **outObject) { return CreateCoder2(false, index, iid, outObject); } + +STDAPI CreateEncoder(UInt32 index, const GUID *iid, void **outObject); STDAPI CreateEncoder(UInt32 index, const GUID *iid, void **outObject) { return CreateCoder2(true, index, iid, outObject); } + +STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject); STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject) { *outObject = NULL; @@ -175,6 +192,8 @@ STDAPI CreateCoder(const GUID *clsid, const GUID *iid, void **outObject) return CreateCoderMain(codecIndex, encode, outObject); } + +STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value); STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) { ::VariantClear((VARIANTARG *)value); @@ -211,15 +230,12 @@ STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) value->ulVal = (ULONG)codec.NumStreams; } break; - /* case NMethodPropID::kIsFilter: - // if (codec.IsFilter) { value->vt = VT_BOOL; value->boolVal = BoolToVARIANT_BOOL(codec.IsFilter); } break; - */ /* case NMethodPropID::kDecoderFlags: { @@ -238,6 +254,8 @@ STDAPI GetMethodProperty(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) return S_OK; } + +STDAPI GetNumberOfMethods(UINT32 *numCodecs); STDAPI GetNumberOfMethods(UINT32 *numCodecs) { *numCodecs = g_NumCodecs; @@ -270,6 +288,7 @@ static HRESULT CreateHasher2(UInt32 index, IHasher **hasher) COM_TRY_END } +STDAPI CreateHasher(const GUID *clsid, IHasher **outObject); STDAPI CreateHasher(const GUID *clsid, IHasher **outObject) { COM_TRY_BEGIN @@ -281,6 +300,7 @@ STDAPI CreateHasher(const GUID *clsid, IHasher **outObject) COM_TRY_END } +STDAPI GetHasherProp(UInt32 codecIndex, PROPID propID, PROPVARIANT *value); STDAPI GetHasherProp(UInt32 codecIndex, PROPID propID, PROPVARIANT *value) { ::VariantClear((VARIANTARG *)value); @@ -318,6 +338,7 @@ public: STDMETHOD(CreateHasher)(UInt32 index, IHasher **hasher); }; +STDAPI GetHashers(IHashers **hashers); STDAPI GetHashers(IHashers **hashers) { COM_TRY_BEGIN diff --git a/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.cpp index ac6ab2e718a..a49bba8a519 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.cpp @@ -15,6 +15,11 @@ CCopyCoder::~CCopyCoder() ::MidFree(_buf); } +STDMETHODIMP CCopyCoder::SetFinishMode(UInt32 /* finishMode */) +{ + return S_OK; +} + STDMETHODIMP CCopyCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 *outSize, @@ -32,12 +37,34 @@ STDMETHODIMP CCopyCoder::Code(ISequentialInStream *inStream, for (;;) { UInt32 size = kBufSize; - if (outSize && size > *outSize - TotalSize) - size = (UInt32)(*outSize - TotalSize); - if (size == 0) - return S_OK; + if (outSize) + { + const UInt64 rem = *outSize - TotalSize; + if (size > rem) + { + size = (UInt32)rem; + if (size == 0) + return S_OK; + } + } - HRESULT readRes = inStream->Read(_buf, size, &size); + HRESULT readRes; + { + UInt32 pos = 0; + do + { + const UInt32 curSize = size - pos; + UInt32 processed = 0; + readRes = inStream->Read(_buf + pos, curSize, &processed); + if (processed > curSize) + return E_FAIL; // internal code failure + pos += processed; + if (readRes != S_OK || processed == 0) + break; + } + while (pos < kBufSize); + size = pos; + } if (size == 0) return readRes; @@ -47,12 +74,15 @@ STDMETHODIMP CCopyCoder::Code(ISequentialInStream *inStream, UInt32 pos = 0; do { - UInt32 curSize = size - pos; - HRESULT res = outStream->Write(_buf + pos, curSize, &curSize); - pos += curSize; - TotalSize += curSize; + const UInt32 curSize = size - pos; + UInt32 processed = 0; + const HRESULT res = outStream->Write(_buf + pos, curSize, &processed); + if (processed > curSize) + return E_FAIL; // internal code failure + pos += processed; + TotalSize += processed; RINOK(res); - if (curSize == 0) + if (processed == 0) return E_FAIL; } while (pos < size); @@ -62,7 +92,10 @@ STDMETHODIMP CCopyCoder::Code(ISequentialInStream *inStream, RINOK(readRes); - if (progress) + if (size != kBufSize) + return S_OK; + + if (progress && (TotalSize & (((UInt32)1 << 22) - 1)) == 0) { RINOK(progress->SetRatioInfo(&TotalSize, &TotalSize)); } diff --git a/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.h b/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.h index a21c0988a91..a9d0b6db269 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/CopyCoder.h @@ -13,6 +13,7 @@ class CCopyCoder: public ICompressCoder, public ICompressSetInStream, public ISequentialInStream, + public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, public CMyUnknownImp { @@ -24,10 +25,11 @@ public: CCopyCoder(): _buf(0), TotalSize(0) {}; ~CCopyCoder(); - MY_UNKNOWN_IMP4( + MY_UNKNOWN_IMP5( ICompressCoder, ICompressSetInStream, ISequentialInStream, + ICompressSetFinishMode, ICompressGetInStreamProcessedSize) STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, @@ -35,6 +37,7 @@ public: STDMETHOD(SetInStream)(ISequentialInStream *inStream); STDMETHOD(ReleaseInStream)(); STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); + STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); }; diff --git a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.cpp index 1a378bbe3a1..653fe2de088 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.cpp @@ -2,263 +2,262 @@ #include "StdAfx.h" +// #include + #include "../../../C/Alloc.h" +// #include "../../../C/CpuTicks.h" #include "../Common/StreamUtils.h" #include "Lzma2Decoder.h" -static HRESULT SResToHRESULT(SRes res) -{ - switch (res) - { - case SZ_OK: return S_OK; - case SZ_ERROR_MEM: return E_OUTOFMEMORY; - case SZ_ERROR_PARAM: return E_INVALIDARG; - // case SZ_ERROR_PROGRESS: return E_ABORT; - case SZ_ERROR_DATA: return S_FALSE; - } - return E_FAIL; -} - namespace NCompress { namespace NLzma2 { CDecoder::CDecoder(): - _inBuf(NULL), - _inBufSize(0), - _inBufSizeNew(1 << 20), - _outStepSize(1 << 22), - _outSizeDefined(false), - _finishMode(false) -{ - Lzma2Dec_Construct(&_state); -} - -STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSizeNew = size; return S_OK; } -STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outStepSize = size; return S_OK; } + _dec(NULL) + , _inProcessed(0) + , _prop(0xFF) + , _finishMode(false) + , _inBufSize(1 << 20) + , _outStep(1 << 20) + #ifndef _7ZIP_ST + , _tryMt(1) + , _numThreads(1) + , _memUsage((UInt64)(sizeof(size_t)) << 28) + #endif +{} CDecoder::~CDecoder() { - Lzma2Dec_Free(&_state, &g_Alloc); - MidFree(_inBuf); + if (_dec) + Lzma2DecMt_Destroy(_dec); } +STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSize = size; return S_OK; } +STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outStep = size; return S_OK; } + STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { if (size != 1) return E_NOTIMPL; - - RINOK(SResToHRESULT(Lzma2Dec_Allocate(&_state, prop[0], &g_Alloc))); - if (!_inBuf || _inBufSize != _inBufSizeNew) - { - MidFree(_inBuf); - _inBufSize = 0; - _inBuf = (Byte *)MidAlloc(_inBufSizeNew); - if (!_inBuf) - return E_OUTOFMEMORY; - _inBufSize = _inBufSizeNew; - } - + if (prop[0] > 40) + return E_NOTIMPL; + _prop = prop[0]; return S_OK; } -STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) { *value = _inSizeProcessed; return S_OK; } -STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) { _inStream = inStream; return S_OK; } -STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); return S_OK; } -STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) +STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) { - _outSizeDefined = (outSize != NULL); - _outSize = 0; - if (_outSizeDefined) - _outSize = *outSize; - - Lzma2Dec_Init(&_state); - - _inPos = _inSize = 0; - _inSizeProcessed = _outSizeProcessed = 0; + _finishMode = (finishMode != 0); return S_OK; } -STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) + + +#ifndef _7ZIP_ST + +static UInt64 Get_ExpectedBlockSize_From_Dict(UInt32 dictSize) { - _finishMode = (finishMode != 0); - return S_OK; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + return blockSize; } -STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, - ISequentialOutStream *outStream, const UInt64 *inSize, - const UInt64 *outSize, ICompressProgressInfo *progress) +#define LZMA2_DIC_SIZE_FROM_PROP_FULL(p) ((p) == 40 ? 0xFFFFFFFF : (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))) + +#endif + +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + +STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (!_inBuf) - return S_FALSE; - SetOutStreamSize(outSize); - - UInt32 step = _outStepSize; - const UInt32 kOutStepSize_Min = 1 << 12; - if (step < kOutStepSize_Min) - step = kOutStepSize_Min; - - SizeT wrPos = _state.decoder.dicPos; + _inProcessed = 0; + + if (!_dec) + { + _dec = Lzma2DecMt_Create( + // &g_AlignedAlloc, + &g_Alloc, + &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } - SizeT next = (_state.decoder.dicBufSize - _state.decoder.dicPos < step) ? - _state.decoder.dicBufSize : - _state.decoder.dicPos + step; + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); - HRESULT hres = S_OK; + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; - for (;;) + #ifndef _7ZIP_ST { - if (_inPos == _inSize) - { - _inPos = _inSize = 0; - hres = inStream->Read(_inBuf, _inBufSize, &_inSize); - if (hres != S_OK) - break; - } + props.numThreads = 1; + UInt32 numThreads = _numThreads; - SizeT dicPos = _state.decoder.dicPos; - SizeT curSize = next - dicPos; - - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) + if (_tryMt && numThreads >= 1) { - const UInt64 rem = _outSize - _outSizeProcessed; - if (curSize >= rem) + UInt64 useLimit = _memUsage; + UInt32 dictSize = LZMA2_DIC_SIZE_FROM_PROP_FULL(_prop); + UInt64 expectedBlockSize64 = Get_ExpectedBlockSize_From_Dict(dictSize); + size_t expectedBlockSize = (size_t)expectedBlockSize64; + size_t inBlockMax = expectedBlockSize + expectedBlockSize / 16; + if (expectedBlockSize == expectedBlockSize64 && inBlockMax >= expectedBlockSize) { - curSize = (SizeT)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; + props.outBlockMax = expectedBlockSize; + props.inBlockMax = inBlockMax; + const size_t kOverheadSize = props.inBufSize_MT + (1 << 16); + UInt64 okThreads = useLimit / (props.outBlockMax + props.inBlockMax + kOverheadSize); + if (numThreads > okThreads) + numThreads = (UInt32)okThreads; + if (numThreads == 0) + numThreads = 1; + props.numThreads = numThreads; } } + } + #endif - SizeT inSizeProcessed = _inSize - _inPos; - ELzmaStatus status; - SRes res = Lzma2Dec_DecodeToDic(&_state, dicPos + curSize, _inBuf + _inPos, &inSizeProcessed, finishMode, &status); + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; - _inPos += (UInt32)inSizeProcessed; - _inSizeProcessed += inSizeProcessed; - SizeT outSizeProcessed = _state.decoder.dicPos - dicPos; - _outSizeProcessed += outSizeProcessed; + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); - bool finished = (inSizeProcessed == 0 && outSizeProcessed == 0 - || status == LZMA_STATUS_FINISHED_WITH_MARK); - bool outFinished = (_outSizeDefined && _outSizeProcessed >= _outSize); + SRes res; - if (res != 0 - || _state.decoder.dicPos >= next - || finished - || outFinished) - { - HRESULT res2 = WriteStream(outStream, _state.decoder.dic + wrPos, _state.decoder.dicPos - wrPos); + UInt64 inProcessed = 0; + int isMT = False; - if (_state.decoder.dicPos == _state.decoder.dicBufSize) - _state.decoder.dicPos = 0; - - wrPos = _state.decoder.dicPos; + #ifndef _7ZIP_ST + isMT = _tryMt; + #endif - next = (_state.decoder.dicBufSize - _state.decoder.dicPos < step) ? - _state.decoder.dicBufSize : - _state.decoder.dicPos + step; + // UInt64 cpuTicks = GetCpuTicks(); - if (res != 0) - return S_FALSE; - RINOK(res2); + res = Lzma2DecMt_Decode(_dec, _prop, &props, + &outWrap.vt, outSize, _finishMode, + &inWrap.vt, + &inProcessed, + &isMT, + progress ? &progressWrap.vt : NULL); - if (finished) - { - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - { - if (_finishMode && inSize && *inSize != _inSizeProcessed) - return S_FALSE; - if (finishMode == LZMA_FINISH_END && !outFinished) - return S_FALSE; - return S_OK; - } - return (finishMode == LZMA_FINISH_END) ? S_FALSE : S_OK; - } + /* + cpuTicks = GetCpuTicks() - cpuTicks; + printf("\n ticks = %10I64u\n", cpuTicks / 1000000); + */ - if (outFinished && finishMode == LZMA_FINISH_ANY) - return S_OK; - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&_inSizeProcessed, &_outSizeProcessed)); - } + + #ifndef _7ZIP_ST + /* we reset _tryMt, only if p->props.numThreads was changed */ + if (props.numThreads > 1) + _tryMt = isMT; + #endif + + _inProcessed = inProcessed; + + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) + + if (res == SZ_OK && _finishMode) + { + if (inSize && *inSize != inProcessed) + res = SZ_ERROR_DATA; + if (outSize && *outSize != outWrap.Processed) + res = SZ_ERROR_DATA; } - HRESULT res2 = WriteStream(outStream, _state.decoder.dic + wrPos, _state.decoder.dicPos - wrPos); - if (hres != S_OK) - return hres; - return res2; + return SResToHRESULT(res); } + +STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = _inProcessed; + return S_OK; +} + + +#ifndef _7ZIP_ST + +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; + return S_OK; +} + +STDMETHODIMP CDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + + #ifndef NO_READ_FROM_CODER -STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) { - UInt32 totalProcessed = 0; + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; + + _inProcessed = 0; + + if (!_dec) + { + _dec = Lzma2DecMt_Create(&g_AlignedAlloc, &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + + _inWrap.Init(_inStream); + SRes res = Lzma2DecMt_Init(_dec, _prop, &props, outSize, _finishMode, &_inWrap.vt); + + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; +} + + +STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) { _inStream = inStream; return S_OK; } +STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); return S_OK; } + + +STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) +{ if (processedSize) *processedSize = 0; - for (;;) - { - if (_inPos == _inSize) - { - _inPos = _inSize = 0; - RINOK(_inStream->Read(_inBuf, _inBufSize, &_inSize)); - } - { - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) - { - const UInt64 rem = _outSize - _outSizeProcessed; - if (rem <= size) - { - size = (UInt32)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; - } - } + size_t size2 = size; + UInt64 inProcessed = 0; - SizeT outProcessed = size; - SizeT inProcessed = _inSize - _inPos; - - ELzmaStatus status; - SRes res = Lzma2Dec_DecodeToBuf(&_state, (Byte *)data, &outProcessed, - _inBuf + _inPos, &inProcessed, finishMode, &status); - - _inPos += (UInt32)inProcessed; - _inSizeProcessed += inProcessed; - _outSizeProcessed += outProcessed; - size -= (UInt32)outProcessed; - data = (Byte *)data + outProcessed; - - totalProcessed += (UInt32)outProcessed; - if (processedSize) - *processedSize = totalProcessed; - - if (res != SZ_OK) - { - if (totalProcessed != 0) - return S_OK; - return SResToHRESULT(res); - } - - if (inProcessed == 0 && outProcessed == 0) - return S_OK; - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - return S_OK; - if (outProcessed != 0) - { - if (finishMode != LZMA_FINISH_END || _outSize != _outSizeProcessed) - return S_OK; - } - } - } + SRes res = Lzma2DecMt_Read(_dec, (Byte *)data, &size2, &inProcessed); + + _inProcessed += inProcessed; + if (processedSize) + *processedSize = (UInt32)size2; + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; } #endif diff --git a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.h b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.h index a87912fb596..e14148848f6 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Decoder.h @@ -3,11 +3,9 @@ #ifndef __LZMA2_DECODER_H #define __LZMA2_DECODER_H -#include "../../../C/Lzma2Dec.h" +#include "../../../C/Lzma2DecMt.h" -#include "../../Common/MyCom.h" - -#include "../ICoder.h" +#include "../Common/CWrappers.h" namespace NCompress { namespace NLzma2 { @@ -18,69 +16,79 @@ class CDecoder: public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, public ICompressSetBufSize, + #ifndef NO_READ_FROM_CODER public ICompressSetInStream, public ICompressSetOutStreamSize, public ISequentialInStream, #endif - public CMyUnknownImp -{ - CMyComPtr _inStream; - Byte *_inBuf; - UInt32 _inPos; - UInt32 _inSize; - bool _finishMode; - bool _outSizeDefined; - UInt64 _outSize; + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif - UInt64 _inSizeProcessed; - UInt64 _outSizeProcessed; - + public CMyUnknownImp +{ + CLzma2DecMtHandle _dec; + UInt64 _inProcessed; + Byte _prop; + int _finishMode; UInt32 _inBufSize; - UInt32 _inBufSizeNew; - UInt32 _outStepSize; + UInt32 _outStep; - CLzma2Dec _state; public: - MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) MY_QUERYINTERFACE_ENTRY(ICompressSetBufSize) + #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) MY_QUERYINTERFACE_ENTRY(ICompressSetOutStreamSize) MY_QUERYINTERFACE_ENTRY(ISequentialInStream) #endif + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END MY_ADDREF_RELEASE STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); - STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); - STDMETHOD(SetFinishMode)(UInt32 finishMode); - STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); - STDMETHOD(SetInBufSize)(UInt32 streamIndex, UInt32 size); STDMETHOD(SetOutBufSize)(UInt32 streamIndex, UInt32 size); - STDMETHOD(SetInStream)(ISequentialInStream *inStream); - STDMETHOD(ReleaseInStream)(); - - STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); + #ifndef _7ZIP_ST +private: + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; +public: + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif #ifndef NO_READ_FROM_CODER +private: + CMyComPtr _inStream; + CSeqInStreamWrap _inWrap; +public: + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); + STDMETHOD(SetInStream)(ISequentialInStream *inStream); + STDMETHOD(ReleaseInStream)(); STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); #endif CDecoder(); virtual ~CDecoder(); - }; }} diff --git a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.cpp index 06a11f10eae..6a5d67976e6 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.cpp @@ -21,18 +21,20 @@ namespace NLzma2 { CEncoder::CEncoder() { - _encoder = 0; - _encoder = Lzma2Enc_Create(&g_Alloc, &g_BigAlloc); - if (_encoder == 0) + _encoder = NULL; + _encoder = Lzma2Enc_Create(&g_AlignedAlloc, &g_BigAlloc); + if (!_encoder) throw 1; } CEncoder::~CEncoder() { - if (_encoder != 0) + if (_encoder) Lzma2Enc_Destroy(_encoder); } + +HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props); HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props) { switch (propID) @@ -42,24 +44,23 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm if (prop.vt == VT_UI4) lzma2Props.blockSize = prop.ulVal; else if (prop.vt == VT_UI8) - { - size_t v = (size_t)prop.uhVal.QuadPart; - if (v != prop.uhVal.QuadPart) - return E_INVALIDARG; - lzma2Props.blockSize = v; - } + lzma2Props.blockSize = prop.uhVal.QuadPart; else return E_INVALIDARG; break; } case NCoderPropID::kNumThreads: - if (prop.vt != VT_UI4) return E_INVALIDARG; lzma2Props.numTotalThreads = (int)(prop.ulVal); break; + if (prop.vt != VT_UI4) + return E_INVALIDARG; + lzma2Props.numTotalThreads = (int)(prop.ulVal); + break; default: RINOK(NLzma::SetLzmaProp(propID, prop, lzma2Props.lzmaProps)); } return S_OK; } + STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) { @@ -73,26 +74,52 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, return SResToHRESULT(Lzma2Enc_SetProps(_encoder, &lzma2Props)); } + +STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID == NCoderPropID::kExpectedDataSize) + if (prop.vt == VT_UI8) + Lzma2Enc_SetDataSize(_encoder, prop.uhVal.QuadPart); + } + return S_OK; +} + + STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) { Byte prop = Lzma2Enc_WriteProperties(_encoder); return WriteStream(outStream, &prop, 1); } + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) { - CSeqInStreamWrap inWrap(inStream); - CSeqOutStreamWrap outWrap(outStream); - CCompressProgressWrap progressWrap(progress); - - SRes res = Lzma2Enc_Encode(_encoder, &outWrap.p, &inWrap.p, progress ? &progressWrap.p : NULL); - if (res == SZ_ERROR_READ && inWrap.Res != S_OK) - return inWrap.Res; - if (res == SZ_ERROR_WRITE && outWrap.Res != S_OK) - return outWrap.Res; - if (res == SZ_ERROR_PROGRESS && progressWrap.Res != S_OK) - return progressWrap.Res; + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + SRes res = Lzma2Enc_Encode2(_encoder, + &outWrap.vt, NULL, NULL, + &inWrap.vt, NULL, 0, + progress ? &progressWrap.vt : NULL); + + RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + return SResToHRESULT(res); } diff --git a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.h b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.h index 8ff6e838351..f681ad16ff3 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/Lzma2Encoder.h @@ -16,16 +16,22 @@ class CEncoder: public ICompressCoder, public ICompressSetCoderProperties, public ICompressWriteCoderProperties, + public ICompressSetCoderPropertiesOpt, public CMyUnknownImp { CLzma2EncHandle _encoder; public: - MY_UNKNOWN_IMP3(ICompressCoder, ICompressSetCoderProperties, ICompressWriteCoderProperties) + MY_UNKNOWN_IMP4( + ICompressCoder, + ICompressSetCoderProperties, + ICompressWriteCoderProperties, + ICompressSetCoderPropertiesOpt) STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); CEncoder(); virtual ~CEncoder(); diff --git a/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.cpp index 9002678ae08..a25d36d144a 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.cpp @@ -24,243 +24,321 @@ static HRESULT SResToHRESULT(SRes res) namespace NCompress { namespace NLzma { -CDecoder::CDecoder(): _inBuf(0), _propsWereSet(false), _outSizeDefined(false), - _inBufSize(1 << 20), - _outBufSize(1 << 22), +CDecoder::CDecoder(): + _inBuf(NULL), + _lzmaStatus(LZMA_STATUS_NOT_SPECIFIED), FinishStream(false), - NeedMoreInput(false) + _propsWereSet(false), + _outSizeDefined(false), + _outStep(1 << 20), + _inBufSize(0), + _inBufSizeNew(1 << 20) { - _inSizeProcessed = 0; - _inPos = _inSize = 0; + _inProcessed = 0; + _inPos = _inLim = 0; + + /* + AlignOffsetAlloc_CreateVTable(&_alloc); + _alloc.numAlignBits = 7; + _alloc.offset = 0; + */ LzmaDec_Construct(&_state); } CDecoder::~CDecoder() { - LzmaDec_Free(&_state, &g_Alloc); + LzmaDec_Free(&_state, &g_AlignedAlloc); // &_alloc.vt MyFree(_inBuf); } -STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSize = size; return S_OK; } -STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outBufSize = size; return S_OK; } +STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSizeNew = size; return S_OK; } +STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outStep = size; return S_OK; } HRESULT CDecoder::CreateInputBuffer() { - if (_inBuf == 0 || _inBufSize != _inBufSizeAllocated) + if (!_inBuf || _inBufSizeNew != _inBufSize) { MyFree(_inBuf); - _inBuf = (Byte *)MyAlloc(_inBufSize); - if (_inBuf == 0) + _inBufSize = 0; + _inBuf = (Byte *)MyAlloc(_inBufSizeNew); + if (!_inBuf) return E_OUTOFMEMORY; - _inBufSizeAllocated = _inBufSize; + _inBufSize = _inBufSizeNew; } return S_OK; } + STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { - RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_Alloc))); + RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_AlignedAlloc))) // &_alloc.vt _propsWereSet = true; return CreateInputBuffer(); } + void CDecoder::SetOutStreamSizeResume(const UInt64 *outSize) { _outSizeDefined = (outSize != NULL); + _outSize = 0; if (_outSizeDefined) _outSize = *outSize; - _outSizeProcessed = 0; - _wrPos = 0; + _outProcessed = 0; + _lzmaStatus = LZMA_STATUS_NOT_SPECIFIED; + LzmaDec_Init(&_state); } + STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) { - _inSizeProcessed = 0; - _inPos = _inSize = 0; - NeedMoreInput = false; + _inProcessed = 0; + _inPos = _inLim = 0; SetOutStreamSizeResume(outSize); return S_OK; } + STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) { FinishStream = (finishMode != 0); return S_OK; } + +STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = _inProcessed; + return S_OK; +} + + HRESULT CDecoder::CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress) { - if (_inBuf == 0 || !_propsWereSet) + if (!_inBuf || !_propsWereSet) return S_FALSE; + + const UInt64 startInProgress = _inProcessed; + SizeT wrPos = _state.dicPos; + HRESULT readRes = S_OK; - UInt64 startInProgress = _inSizeProcessed; - - SizeT next = (_state.dicBufSize - _state.dicPos < _outBufSize) ? _state.dicBufSize : (_state.dicPos + _outBufSize); for (;;) { - if (_inPos == _inSize) + if (_inPos == _inLim && readRes == S_OK) { - _inPos = _inSize = 0; - RINOK(inStream->Read(_inBuf, _inBufSizeAllocated, &_inSize)); + _inPos = _inLim = 0; + readRes = inStream->Read(_inBuf, _inBufSize, &_inLim); + } + + const SizeT dicPos = _state.dicPos; + SizeT size; + { + SizeT next = _state.dicBufSize; + if (next - wrPos > _outStep) + next = wrPos + _outStep; + size = next - dicPos; } - SizeT dicPos = _state.dicPos; - SizeT curSize = next - dicPos; - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; if (_outSizeDefined) { - const UInt64 rem = _outSize - _outSizeProcessed; - if (rem <= curSize) + const UInt64 rem = _outSize - _outProcessed; + if (size >= rem) { - curSize = (SizeT)rem; + size = (SizeT)rem; if (FinishStream) finishMode = LZMA_FINISH_END; } } - SizeT inSizeProcessed = _inSize - _inPos; + SizeT inProcessed = _inLim - _inPos; ELzmaStatus status; - SRes res = LzmaDec_DecodeToDic(&_state, dicPos + curSize, _inBuf + _inPos, &inSizeProcessed, finishMode, &status); - _inPos += (UInt32)inSizeProcessed; - _inSizeProcessed += inSizeProcessed; - SizeT outSizeProcessed = _state.dicPos - dicPos; - _outSizeProcessed += outSizeProcessed; + SRes res = LzmaDec_DecodeToDic(&_state, dicPos + size, _inBuf + _inPos, &inProcessed, finishMode, &status); + + _lzmaStatus = status; + _inPos += (UInt32)inProcessed; + _inProcessed += inProcessed; + const SizeT outProcessed = _state.dicPos - dicPos; + _outProcessed += outProcessed; + + // we check for LZMA_STATUS_NEEDS_MORE_INPUT to allow RangeCoder initialization, if (_outSizeDefined && _outSize == 0) + bool outFinished = (_outSizeDefined && _outProcessed >= _outSize); - bool finished = (inSizeProcessed == 0 && outSizeProcessed == 0); - bool stopDecoding = (_outSizeDefined && _outSizeProcessed >= _outSize); + bool needStop = (res != 0 + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (outFinished && status != LZMA_STATUS_NEEDS_MORE_INPUT)); - if (res != 0 || _state.dicPos == next || finished || stopDecoding) + if (needStop || outProcessed >= size) { - HRESULT res2 = WriteStream(outStream, _state.dic + _wrPos, _state.dicPos - _wrPos); + HRESULT res2 = WriteStream(outStream, _state.dic + wrPos, _state.dicPos - wrPos); - _wrPos = _state.dicPos; if (_state.dicPos == _state.dicBufSize) - { _state.dicPos = 0; - _wrPos = 0; - } - next = (_state.dicBufSize - _state.dicPos < _outBufSize) ? _state.dicBufSize : (_state.dicPos + _outBufSize); - - if (res != 0) - return S_FALSE; + wrPos = _state.dicPos; + RINOK(res2); - if (stopDecoding) + + if (needStop) { - if (status == LZMA_STATUS_NEEDS_MORE_INPUT) - NeedMoreInput = true; - if (FinishStream && - status != LZMA_STATUS_FINISHED_WITH_MARK && - status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) + if (res != 0) + { + // return SResToHRESULT(res); return S_FALSE; - return S_OK; - } - if (finished) - { - if (status == LZMA_STATUS_NEEDS_MORE_INPUT) - NeedMoreInput = true; - return (status == LZMA_STATUS_FINISHED_WITH_MARK ? S_OK : S_FALSE); + } + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (FinishStream) + if (_outSizeDefined && _outSize != _outProcessed) + return S_FALSE; + return readRes; + } + + if (outFinished && status != LZMA_STATUS_NEEDS_MORE_INPUT) + if (!FinishStream || status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) + return readRes; + + return S_FALSE; } } + if (progress) { - UInt64 inSize = _inSizeProcessed - startInProgress; - RINOK(progress->SetRatioInfo(&inSize, &_outSizeProcessed)); + const UInt64 inSize = _inProcessed - startInProgress; + RINOK(progress->SetRatioInfo(&inSize, &_outProcessed)); } } } + STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (_inBuf == 0) + if (!_inBuf) return E_INVALIDARG; SetOutStreamSize(outSize); - return CodeSpec(inStream, outStream, progress); + HRESULT res = CodeSpec(inStream, outStream, progress); + if (res == S_OK) + if (FinishStream && inSize && *inSize != _inProcessed) + res = S_FALSE; + return res; } + #ifndef NO_READ_FROM_CODER STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) { _inStream = inStream; return S_OK; } STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); return S_OK; } + STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) { if (processedSize) *processedSize = 0; - do + + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + if (_outSizeDefined) { - if (_inPos == _inSize) + const UInt64 rem = _outSize - _outProcessed; + if (size >= rem) { - _inPos = _inSize = 0; - RINOK(_inStream->Read(_inBuf, _inBufSizeAllocated, &_inSize)); + size = (UInt32)rem; + if (FinishStream) + finishMode = LZMA_FINISH_END; } + } + + HRESULT readRes = S_OK; + + for (;;) + { + if (_inPos == _inLim && readRes == S_OK) { - SizeT inProcessed = _inSize - _inPos; + _inPos = _inLim = 0; + readRes = _inStream->Read(_inBuf, _inBufSize, &_inLim); + } - if (_outSizeDefined) - { - const UInt64 rem = _outSize - _outSizeProcessed; - if (rem < size) - size = (UInt32)rem; - } + SizeT inProcessed = _inLim - _inPos; + SizeT outProcessed = size; + ELzmaStatus status; + + SRes res = LzmaDec_DecodeToBuf(&_state, (Byte *)data, &outProcessed, + _inBuf + _inPos, &inProcessed, finishMode, &status); + + _lzmaStatus = status; + _inPos += (UInt32)inProcessed; + _inProcessed += inProcessed; + _outProcessed += outProcessed; + size -= (UInt32)outProcessed; + data = (Byte *)data + outProcessed; + if (processedSize) + *processedSize += (UInt32)outProcessed; + + if (res != 0) + return S_FALSE; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; - SizeT outProcessed = size; - ELzmaStatus status; - SRes res = LzmaDec_DecodeToBuf(&_state, (Byte *)data, &outProcessed, - _inBuf + _inPos, &inProcessed, LZMA_FINISH_ANY, &status); - _inPos += (UInt32)inProcessed; - _inSizeProcessed += inProcessed; - _outSizeProcessed += outProcessed; - size -= (UInt32)outProcessed; - data = (Byte *)data + outProcessed; - if (processedSize) - *processedSize += (UInt32)outProcessed; - RINOK(SResToHRESULT(res)); - if (inProcessed == 0 && outProcessed == 0) - return S_OK; + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (FinishStream + && _outSizeDefined && _outProcessed >= _outSize + && status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) + return S_FALSE; + return readRes; } + */ + + if (inProcessed == 0 && outProcessed == 0) + return readRes; } - while (size != 0); - return S_OK; } + HRESULT CDecoder::CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress) { SetOutStreamSizeResume(outSize); return CodeSpec(_inStream, outStream, progress); } + HRESULT CDecoder::ReadFromInputStream(void *data, UInt32 size, UInt32 *processedSize) { RINOK(CreateInputBuffer()); + if (processedSize) *processedSize = 0; - while (size > 0) + + HRESULT readRes = S_OK; + + while (size != 0) { - if (_inPos == _inSize) + if (_inPos == _inLim) { - _inPos = _inSize = 0; - RINOK(_inStream->Read(_inBuf, _inBufSizeAllocated, &_inSize)); - if (_inSize == 0) + _inPos = _inLim = 0; + if (readRes == S_OK) + readRes = _inStream->Read(_inBuf, _inBufSize, &_inLim); + if (_inLim == 0) break; } - { - UInt32 curSize = _inSize - _inPos; - if (curSize > size) - curSize = size; - memcpy(data, _inBuf + _inPos, curSize); - _inPos += curSize; - _inSizeProcessed += curSize; - size -= curSize; - data = (Byte *)data + curSize; - if (processedSize) - *processedSize += curSize; - } + + UInt32 cur = _inLim - _inPos; + if (cur > size) + cur = size; + memcpy(data, _inBuf + _inPos, cur); + _inPos += cur; + _inProcessed += cur; + size -= cur; + data = (Byte *)data + cur; + if (processedSize) + *processedSize += cur; } - return S_OK; + + return readRes; } #endif diff --git a/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.h b/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.h index f1f839a4f1b..37dec0258f5 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/LzmaDecoder.h @@ -3,6 +3,7 @@ #ifndef __LZMA_DECODER_H #define __LZMA_DECODER_H +// #include "../../../C/Alloc.h" #include "../../../C/LzmaDec.h" #include "../../Common/MyCom.h" @@ -15,6 +16,7 @@ class CDecoder: public ICompressCoder, public ICompressSetDecoderProperties2, public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, public ICompressSetBufSize, #ifndef NO_READ_FROM_CODER public ICompressSetInStream, @@ -23,21 +25,29 @@ class CDecoder: #endif public CMyUnknownImp { - CMyComPtr _inStream; Byte *_inBuf; UInt32 _inPos; - UInt32 _inSize; - CLzmaDec _state; + UInt32 _inLim; + + ELzmaStatus _lzmaStatus; + +public: + bool FinishStream; // set it before decoding, if you need to decode full LZMA stream + +private: bool _propsWereSet; bool _outSizeDefined; UInt64 _outSize; - UInt64 _inSizeProcessed; - UInt64 _outSizeProcessed; + UInt64 _inProcessed; + UInt64 _outProcessed; - UInt32 _inBufSizeAllocated; + UInt32 _outStep; UInt32 _inBufSize; - UInt32 _outBufSize; - SizeT _wrPos; + UInt32 _inBufSizeNew; + + // CAlignOffsetAlloc _alloc; + + CLzmaDec _state; HRESULT CreateInputBuffer(); HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); @@ -47,6 +57,7 @@ public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) MY_QUERYINTERFACE_ENTRY(ICompressSetBufSize) #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) @@ -60,11 +71,16 @@ public: const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); STDMETHOD(SetFinishMode)(UInt32 finishMode); + STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInBufSize)(UInt32 streamIndex, UInt32 size); STDMETHOD(SetOutBufSize)(UInt32 streamIndex, UInt32 size); #ifndef NO_READ_FROM_CODER + +private: + CMyComPtr _inStream; +public: STDMETHOD(SetInStream)(ISequentialInStream *inStream); STDMETHOD(ReleaseInStream)(); @@ -72,18 +88,24 @@ public: HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress); HRESULT ReadFromInputStream(void *data, UInt32 size, UInt32 *processedSize); - UInt64 GetInputProcessedSize() const { return _inSizeProcessed; } - + #endif - bool FinishStream; // set it before decoding, if you need to decode full LZMA stream - - bool NeedMoreInput; // it's set by decoder, if it needs more input data to decode stream + UInt64 GetInputProcessedSize() const { return _inProcessed; } CDecoder(); virtual ~CDecoder(); - UInt64 GetOutputProcessedSize() const { return _outSizeProcessed; } + UInt64 GetOutputProcessedSize() const { return _outProcessed; } + + bool NeedsMoreInput() const { return _lzmaStatus == LZMA_STATUS_NEEDS_MORE_INPUT; } + + bool CheckFinishStatus(bool withEndMark) const + { + return _lzmaStatus == (withEndMark ? + LZMA_STATUS_FINISHED_WITH_MARK : + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK); + } }; }} diff --git a/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.cpp index 0a7e294d4f9..dabd77a0fe2 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.cpp @@ -9,13 +9,28 @@ #include "LzmaEncoder.h" +// #define LOG_LZMA_THREADS + +#ifdef LOG_LZMA_THREADS + +#include + +#include "../../Common/IntToString.h" +#include "../../Windows/TimeUtils.h" + +EXTERN_C_BEGIN +void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]); +EXTERN_C_END + +#endif + namespace NCompress { namespace NLzma { CEncoder::CEncoder() { _encoder = NULL; - _encoder = LzmaEnc_Create(&g_Alloc); + _encoder = LzmaEnc_Create(&g_AlignedAlloc); if (!_encoder) throw 1; } @@ -23,49 +38,51 @@ CEncoder::CEncoder() CEncoder::~CEncoder() { if (_encoder) - LzmaEnc_Destroy(_encoder, &g_Alloc, &g_BigAlloc); + LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc); } -static inline wchar_t GetUpperChar(wchar_t c) +static inline wchar_t GetLowCharFast(wchar_t c) { - if (c >= 'a' && c <= 'z') - c -= 0x20; - return c; + return c |= 0x20; } static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes) { - wchar_t c = GetUpperChar(*s++); - if (c == L'H') + wchar_t c = GetLowCharFast(*s++); + if (c == 'h') { - if (GetUpperChar(*s++) != L'C') + if (GetLowCharFast(*s++) != 'c') return 0; - int numHashBytesLoc = (int)(*s++ - L'0'); - if (numHashBytesLoc < 4 || numHashBytesLoc > 4) + int num = (int)(*s++ - L'0'); + if (num < 4 || num > 5) return 0; if (*s != 0) return 0; *btMode = 0; - *numHashBytes = numHashBytesLoc; + *numHashBytes = num; return 1; } - if (c != L'B') + if (c != 'b') return 0; - if (GetUpperChar(*s++) != L'T') - return 0; - int numHashBytesLoc = (int)(*s++ - L'0'); - if (numHashBytesLoc < 2 || numHashBytesLoc > 4) - return 0; - if (*s != 0) - return 0; - *btMode = 1; - *numHashBytes = numHashBytesLoc; - return 1; + { + if (GetLowCharFast(*s++) != 't') + return 0; + int num = (int)(*s++ - L'0'); + if (num < 2 || num > 5) + return 0; + if (*s != 0) + return 0; + *btMode = 1; + *numHashBytes = num; + return 1; + } } -#define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break; +#define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break; +#define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break; +HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep); HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) { if (propID == NCoderPropID::kMatchFinder) @@ -74,25 +91,61 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep) return E_INVALIDARG; return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG; } + + if (propID == NCoderPropID::kAffinity) + { + if (prop.vt == VT_UI8) + ep.affinity = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + return S_OK; + } + if (propID > NCoderPropID::kReduceSize) return S_OK; + if (propID == NCoderPropID::kReduceSize) { if (prop.vt == VT_UI8) ep.reduceSize = prop.uhVal.QuadPart; + else + return E_INVALIDARG; return S_OK; } + + if (propID == NCoderPropID::kDictionarySize) + { + if (prop.vt == VT_UI8) + { + // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB) + const UInt64 v = prop.uhVal.QuadPart; + if (v > ((UInt64)1 << 32)) + return E_INVALIDARG; + UInt32 dict; + if (v == ((UInt64)1 << 32)) + dict = (UInt32)(Int32)-1; + else + dict = (UInt32)v; + ep.dictSize = dict; + return S_OK; + } + } + if (prop.vt != VT_UI4) return E_INVALIDARG; UInt32 v = prop.ulVal; switch (propID) { - case NCoderPropID::kDefaultProp: if (v > 31) return E_INVALIDARG; ep.dictSize = (UInt32)1 << (unsigned)v; break; + case NCoderPropID::kDefaultProp: + if (v > 32) + return E_INVALIDARG; + ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v; + break; SET_PROP_32(kLevel, level) SET_PROP_32(kNumFastBytes, fb) - SET_PROP_32(kMatchFinderCycles, mc) + SET_PROP_32U(kMatchFinderCycles, mc) SET_PROP_32(kAlgorithm, algo) - SET_PROP_32(kDictionarySize, dictSize) + SET_PROP_32U(kDictionarySize, dictSize) SET_PROP_32(kPosStateBits, pb) SET_PROP_32(kLitPosBits, lp) SET_PROP_32(kLitContextBits, lc) @@ -115,7 +168,10 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, switch (propID) { case NCoderPropID::kEndMarker: - if (prop.vt != VT_BOOL) return E_INVALIDARG; props.writeEndMark = (prop.boolVal != VARIANT_FALSE); break; + if (prop.vt != VT_BOOL) + return E_INVALIDARG; + props.writeEndMark = (prop.boolVal != VARIANT_FALSE); + break; default: RINOK(SetLzmaProp(propID, prop, props)); } @@ -123,6 +179,22 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props)); } + +STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID == NCoderPropID::kExpectedDataSize) + if (prop.vt == VT_UI8) + LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart); + } + return S_OK; +} + + STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) { Byte props[LZMA_PROPS_SIZE]; @@ -131,21 +203,144 @@ STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream) return WriteStream(outStream, props, size); } + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + + + +#ifdef LOG_LZMA_THREADS + +static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } + +static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') +{ + char temp[64]; + char *p = temp + 32; + ConvertUInt64ToString(val, p); + unsigned len = (unsigned)strlen(p); + for (; len < numDigits; len++) + *--p = c; + printf("%s", p); +} + +static void PrintTime(const char *s, UInt64 val, UInt64 total) +{ + printf(" %s :", s); + const UInt32 kFreq = 10000000; + UInt64 sec = val / kFreq; + PrintNum(sec, 6); + printf(" ."); + UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000); + PrintNum(ms, 3, '0'); + + while (val > ((UInt64)1 << 56)) + { + val >>= 1; + total >>= 1; + } + + UInt64 percent = 0; + if (total != 0) + percent = val * 100 / total; + printf(" ="); + PrintNum(percent, 4); + printf("%%"); +} + + +struct CBaseStat +{ + UInt64 kernelTime, userTime; + + BOOL Get(HANDLE thread, const CBaseStat *prevStat) + { + FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT; + BOOL res = GetThreadTimes(thread + , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT); + if (res) + { + kernelTime = GetTime64(kernelTimeFT); + userTime = GetTime64(userTimeFT); + if (prevStat) + { + kernelTime -= prevStat->kernelTime; + userTime -= prevStat->userTime; + } + } + return res; + } +}; + + +static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat) +{ + CBaseStat newStat; + if (!newStat.Get(thread, prevStat)) + return; + + PrintTime("K", newStat.kernelTime, totalTime); + + const UInt64 processTime = newStat.kernelTime + newStat.userTime; + + PrintTime("U", newStat.userTime, totalTime); + PrintTime("S", processTime, totalTime); + printf("\n"); + // PrintTime("G ", totalTime, totalTime); +} + +#endif + + + STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) { - CSeqInStreamWrap inWrap(inStream); - CSeqOutStreamWrap outWrap(outStream); - CCompressProgressWrap progressWrap(progress); + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + #ifdef LOG_LZMA_THREADS + + FILETIME startTimeFT; + NWindows::NTime::GetCurUtcFileTime(startTimeFT); + UInt64 totalTime = GetTime64(startTimeFT); + CBaseStat oldStat; + if (!oldStat.Get(GetCurrentThread(), NULL)) + return E_FAIL; + + #endif + + + SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, + progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc); - SRes res = LzmaEnc_Encode(_encoder, &outWrap.p, &inWrap.p, progress ? &progressWrap.p : NULL, &g_Alloc, &g_BigAlloc); _inputProcessed = inWrap.Processed; - if (res == SZ_ERROR_READ && inWrap.Res != S_OK) - return inWrap.Res; - if (res == SZ_ERROR_WRITE && outWrap.Res != S_OK) - return outWrap.Res; - if (res == SZ_ERROR_PROGRESS && progressWrap.Res != S_OK) - return progressWrap.Res; + + RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + + + #ifdef LOG_LZMA_THREADS + + NWindows::NTime::GetCurUtcFileTime(startTimeFT); + totalTime = GetTime64(startTimeFT) - totalTime; + HANDLE lz_threads[2]; + LzmaEnc_GetLzThreads(_encoder, lz_threads); + printf("\n"); + printf("Main: "); PrintStat(GetCurrentThread(), totalTime, &oldStat); + printf("Hash: "); PrintStat(lz_threads[0], totalTime, NULL); + printf("BinT: "); PrintStat(lz_threads[1], totalTime, NULL); + // PrintTime("Total: ", totalTime, totalTime); + printf("\n"); + + #endif + return SResToHRESULT(res); } diff --git a/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.h b/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.h index f919ac2cc49..7d706ad7025 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/LzmaEncoder.h @@ -16,21 +16,29 @@ class CEncoder: public ICompressCoder, public ICompressSetCoderProperties, public ICompressWriteCoderProperties, + public ICompressSetCoderPropertiesOpt, public CMyUnknownImp { CLzmaEncHandle _encoder; UInt64 _inputProcessed; public: - MY_UNKNOWN_IMP3(ICompressCoder, ICompressSetCoderProperties, ICompressWriteCoderProperties) + MY_UNKNOWN_IMP4( + ICompressCoder, + ICompressSetCoderProperties, + ICompressWriteCoderProperties, + ICompressSetCoderPropertiesOpt) STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); CEncoder(); virtual ~CEncoder(); + UInt64 GetInputProcessedSize() const { return _inputProcessed; } + bool IsWriteEndMark() const { return LzmaEnc_IsWriteEndMark(_encoder) != 0; } }; }} diff --git a/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.cpp index c868730ea00..7f54ec3bcec 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.cpp @@ -1,5 +1,5 @@ // PpmdDecoder.cpp -// 2009-03-11 : Igor Pavlov : Public domain +// 2020-07-03 : Igor Pavlov : Public domain #include "StdAfx.h" @@ -13,13 +13,13 @@ namespace NCompress { namespace NPpmd { -static const UInt32 kBufSize = (1 << 20); +static const UInt32 kBufSize = (1 << 16); enum { kStatus_NeedInit, kStatus_Normal, - kStatus_Finished, + kStatus_Finished_With_Mark, kStatus_Error }; @@ -35,7 +35,8 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *props, UInt32 size) return E_INVALIDARG; _order = props[0]; UInt32 memSize = GetUi32(props + 1); - if (_order < PPMD7_MIN_ORDER || + if ( + // _order < PPMD7_MIN_ORDER || _order > PPMD7_MAX_ORDER || memSize < PPMD7_MIN_MEM_SIZE || memSize > PPMD7_MAX_MEM_SIZE) @@ -47,23 +48,36 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *props, UInt32 size) return S_OK; } +#define _rangeDec _ppmd.rc.dec + +#define CHECK_EXTRA_ERROR \ + if (_inStream.Extra) { \ + _status = kStatus_Error; \ + return (_res = (_inStream.Res != SZ_OK ? _inStream.Res: S_FALSE)); } + + HRESULT CDecoder::CodeSpec(Byte *memStream, UInt32 size) { + if (_res != S_OK) + return _res; + switch (_status) { - case kStatus_Finished: return S_OK; + case kStatus_Finished_With_Mark: return S_OK; case kStatus_Error: return S_FALSE; case kStatus_NeedInit: _inStream.Init(); if (!Ppmd7z_RangeDec_Init(&_rangeDec)) { _status = kStatus_Error; - return S_FALSE; + return (_res = S_FALSE); } + CHECK_EXTRA_ERROR _status = kStatus_Normal; Ppmd7_Init(&_ppmd, _order); break; } + if (_outSizeDefined) { const UInt64 rem = _outSize - _processedSize; @@ -71,29 +85,54 @@ HRESULT CDecoder::CodeSpec(Byte *memStream, UInt32 size) size = (UInt32)rem; } - UInt32 i; int sym = 0; - for (i = 0; i != size; i++) { - sym = Ppmd7_DecodeSymbol(&_ppmd, &_rangeDec.p); - if (_inStream.Extra || sym < 0) - break; - memStream[i] = (Byte)sym; + Byte *buf = memStream; + const Byte *lim = buf + size; + for (; buf != lim; buf++) + { + sym = Ppmd7z_DecodeSymbol(&_ppmd); + if (_inStream.Extra || sym < 0) + break; + *buf = (Byte)sym; + } + /* + buf = Ppmd7z_DecodeSymbols(&_ppmd, buf, lim); + sym = _ppmd.LastSymbol; + */ + _processedSize += (size_t)(buf - memStream); + } + + CHECK_EXTRA_ERROR + + if (sym >= 0) + { + if (!FinishStream + || !_outSizeDefined + || _outSize != _processedSize + || _rangeDec.Code == 0) + return S_OK; + /* + // We can decode additional End Marker here: + sym = Ppmd7z_DecodeSymbol(&_ppmd); + CHECK_EXTRA_ERROR + */ } - _processedSize += i; - if (_inStream.Extra) + if (sym != PPMD7_SYM_END || _rangeDec.Code != 0) { _status = kStatus_Error; - return _inStream.Res; + return (_res = S_FALSE); } - if (sym < 0) - _status = (sym < -1) ? kStatus_Error : kStatus_Finished; + + _status = kStatus_Finished_With_Mark; return S_OK; } + + STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { if (!_outBuf) { @@ -112,18 +151,23 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream size_t processed = (size_t)(_processedSize - startPos); RINOK(WriteStream(outStream, _outBuf, processed)); RINOK(res); - if (_status == kStatus_Finished) + if (_status == kStatus_Finished_With_Mark) break; if (progress) { - UInt64 inSize = _inStream.GetProcessed(); - RINOK(progress->SetRatioInfo(&inSize, &_processedSize)); + const UInt64 inProcessed = _inStream.GetProcessed(); + RINOK(progress->SetRatioInfo(&inProcessed, &_processedSize)); } } while (!_outSizeDefined || _processedSize < _outSize); + + if (FinishStream && inSize && *inSize != _inStream.GetProcessed()) + return S_FALSE; + return S_OK; } + STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) { _outSizeDefined = (outSize != NULL); @@ -131,6 +175,19 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) _outSize = *outSize; _processedSize = 0; _status = kStatus_NeedInit; + _res = SZ_OK; + return S_OK; +} + +STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) +{ + FinishStream = (finishMode != 0); + return S_OK; +} + +STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = _inStream.GetProcessed(); return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.h b/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.h index 8ebcd700c7d..f1a8ee2118c 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/PpmdDecoder.h @@ -1,5 +1,5 @@ // PpmdDecoder.h -// 2009-03-11 : Igor Pavlov : Public domain +// 2020-07-03 : Igor Pavlov : Public domain #ifndef __COMPRESS_PPMD_DECODER_H #define __COMPRESS_PPMD_DECODER_H @@ -8,16 +8,18 @@ #include "../../Common/MyCom.h" -#include "../Common/CWrappers.h" - #include "../ICoder.h" +#include "../Common/CWrappers.h" + namespace NCompress { namespace NPpmd { class CDecoder : public ICompressCoder, public ICompressSetDecoderProperties2, + public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, #ifndef NO_READ_FROM_CODER public ICompressSetInStream, public ICompressSetOutStreamSize, @@ -26,12 +28,13 @@ class CDecoder : public CMyUnknownImp { Byte *_outBuf; - CPpmd7z_RangeDec _rangeDec; CByteInBufWrap _inStream; CPpmd7 _ppmd; Byte _order; + bool FinishStream; bool _outSizeDefined; + HRESULT _res; int _status; UInt64 _outSize; UInt64 _processedSize; @@ -42,19 +45,27 @@ public: #ifndef NO_READ_FROM_CODER CMyComPtr InSeqStream; - MY_UNKNOWN_IMP4( - ICompressSetDecoderProperties2, - ICompressSetInStream, - ICompressSetOutStreamSize, - ISequentialInStream) - #else - MY_UNKNOWN_IMP1( - ICompressSetDecoderProperties2) #endif + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) + MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + #ifndef NO_READ_FROM_CODER + MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) + MY_QUERYINTERFACE_ENTRY(ICompressSetOutStreamSize) + MY_QUERYINTERFACE_ENTRY(ISequentialInStream) + #endif + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE + + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); + STDMETHOD(SetFinishMode)(UInt32 finishMode); + STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); #ifndef NO_READ_FROM_CODER @@ -63,11 +74,13 @@ public: STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); #endif - CDecoder(): _outBuf(NULL), _outSizeDefined(false) + CDecoder(): + _outBuf(NULL), + FinishStream(false), + _outSizeDefined(false) { - Ppmd7z_RangeDec_CreateVTable(&_rangeDec); - _rangeDec.Stream = &_inStream.p; Ppmd7_Construct(&_ppmd); + _ppmd.rc.dec.Stream = &_inStream.vt; } ~CDecoder(); diff --git a/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.cpp index 00ea9668b00..d41d2acad2b 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.cpp +++ b/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.cpp @@ -3,7 +3,6 @@ #include "StdAfx.h" #include "../../../C/Alloc.h" -#include "../../../C/CpuArch.h" #include "../Common/StreamUtils.h" @@ -21,7 +20,7 @@ void CEncProps::Normalize(int level) if (level < 0) level = 5; if (level > 9) level = 9; if (MemSize == (UInt32)(Int32)-1) - MemSize = level >= 9 ? ((UInt32)192 << 20) : ((UInt32)1 << (level + 19)); + MemSize = (UInt32)1 << (level + 19); const unsigned kMult = 16; if (MemSize / kMult > ReduceSize) { @@ -43,8 +42,8 @@ CEncoder::CEncoder(): _inBuf(NULL) { _props.Normalize(-1); - _rangeEnc.Stream = &_outStream.p; Ppmd7_Construct(&_ppmd); + _ppmd.rc.enc.Stream = &_outStream.vt; } CEncoder::~CEncoder() @@ -60,7 +59,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA for (UInt32 i = 0; i < numProps; i++) { const PROPVARIANT &prop = coderProps[i]; - PROPID propID = propIDs[i]; + const PROPID propID = propIDs[i]; if (propID > NCoderPropID::kReduceSize) continue; if (propID == NCoderPropID::kReduceSize) @@ -69,16 +68,50 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA props.ReduceSize = (UInt32)prop.uhVal.QuadPart; continue; } + + if (propID == NCoderPropID::kUsedMemorySize) + { + // here we have selected (4 GiB - 1 KiB) as replacement for (4 GiB) MEM_SIZE. + const UInt32 kPpmd_Default_4g = (UInt32)0 - ((UInt32)1 << 10); + UInt32 v; + if (prop.vt == VT_UI8) + { + // 21.03 : we support 64-bit values (for 4 GiB value) + const UInt64 v64 = prop.uhVal.QuadPart; + if (v64 > ((UInt64)1 << 32)) + return E_INVALIDARG; + if (v64 == ((UInt64)1 << 32)) + v = kPpmd_Default_4g; + else + v = (UInt32)v64; + } + else if (prop.vt == VT_UI4) + v = (UInt32)prop.ulVal; + else + return E_INVALIDARG; + if (v > PPMD7_MAX_MEM_SIZE) + v = kPpmd_Default_4g; + + /* here we restrict MEM_SIZE for Encoder. + It's for better performance of encoding and decoding. + The Decoder still supports more MEM_SIZE values. */ + if (v < ((UInt32)1 << 16) || (v & 3) != 0) + return E_INVALIDARG; + // if (v < PPMD7_MIN_MEM_SIZE) return E_INVALIDARG; // (1 << 11) + /* + Supported MEM_SIZE range : + [ (1 << 11) , 0xFFFFFFFF - 12 * 3 ] - current 7-Zip's Ppmd7 constants + [ 1824 , 0xFFFFFFFF ] - real limits of Ppmd7 code + */ + props.MemSize = v; + continue; + } + if (prop.vt != VT_UI4) return E_INVALIDARG; UInt32 v = (UInt32)prop.ulVal; switch (propID) { - case NCoderPropID::kUsedMemorySize: - if (v < (1 << 16) || v > PPMD7_MAX_MEM_SIZE || (v & 3) != 0) - return E_INVALIDARG; - props.MemSize = v; - break; case NCoderPropID::kOrder: if (v < 2 || v > 32) return E_INVALIDARG; @@ -120,8 +153,8 @@ HRESULT CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outS _outStream.Stream = outStream; _outStream.Init(); - Ppmd7z_RangeEnc_Init(&_rangeEnc); - Ppmd7_Init(&_ppmd, _props.Order); + Ppmd7z_Init_RangeEnc(&_ppmd); + Ppmd7_Init(&_ppmd, (unsigned)_props.Order); UInt64 processed = 0; for (;;) @@ -131,19 +164,27 @@ HRESULT CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outS if (size == 0) { // We don't write EndMark in PPMD-7z. - // Ppmd7_EncodeSymbol(&_ppmd, &_rangeEnc, -1); - Ppmd7z_RangeEnc_FlushData(&_rangeEnc); + // Ppmd7z_EncodeSymbol(&_ppmd, -1); + Ppmd7z_Flush_RangeEnc(&_ppmd); return _outStream.Flush(); } - for (UInt32 i = 0; i < size; i++) + const Byte *buf = _inBuf; + const Byte *lim = buf + size; + /* + for (; buf < lim; buf++) { - Ppmd7_EncodeSymbol(&_ppmd, &_rangeEnc, _inBuf[i]); + Ppmd7z_EncodeSymbol(&_ppmd, *buf); RINOK(_outStream.Res); } + */ + + Ppmd7z_EncodeSymbols(&_ppmd, buf, lim); + RINOK(_outStream.Res); + processed += size; if (progress) { - UInt64 outSize = _outStream.GetProcessed(); + const UInt64 outSize = _outStream.GetProcessed(); RINOK(progress->SetRatioInfo(&processed, &outSize)); } } diff --git a/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.h b/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.h index 671a5353e7a..0104663f562 100644 --- a/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.h +++ b/3rdparty/lzma/CPP/7zip/Compress/PpmdEncoder.h @@ -37,7 +37,6 @@ class CEncoder : { Byte *_inBuf; CByteOutBufWrap _outStream; - CPpmd7z_RangeEnc _rangeEnc; CPpmd7 _ppmd; CEncProps _props; public: diff --git a/3rdparty/lzma/CPP/7zip/Compress/XzDecoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/XzDecoder.cpp new file mode 100644 index 00000000000..0371173c36f --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Compress/XzDecoder.cpp @@ -0,0 +1,150 @@ +// XzDecoder.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" + +#include "../Common/CWrappers.h" + +#include "XzDecoder.h" + +namespace NCompress { +namespace NXz { + +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + +static HRESULT SResToHRESULT_Code(SRes res) throw() +{ + if (res < 0) + return res; + switch (res) + { + case SZ_OK: return S_OK; + case SZ_ERROR_MEM: return E_OUTOFMEMORY; + case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; + } + return S_FALSE; +} + + +HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, + const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress) +{ + MainDecodeSRes = SZ_OK; + MainDecodeSRes_wasUsed = false; + XzStatInfo_Clear(&Stat); + + if (!xz) + { + xz = XzDecMt_Create(&g_Alloc, &g_MidAlloc); + if (!xz) + return E_OUTOFMEMORY; + } + + CXzDecMtProps props; + XzDecMtProps_Init(&props); + + int isMT = False; + + #ifndef _7ZIP_ST + { + props.numThreads = 1; + UInt32 numThreads = _numThreads; + + if (_tryMt && numThreads > 1) + { + size_t memUsage = (size_t)_memUsage; + if (memUsage != _memUsage) + memUsage = (size_t)0 - 1; + props.memUseMax = memUsage; + isMT = (numThreads > 1); + } + + props.numThreads = numThreads; + } + #endif + + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(seqInStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + SRes res = XzDecMt_Decode(xz, + &props, + outSizeLimit, finishStream, + &outWrap.vt, + &inWrap.vt, + &Stat, + &isMT, + progress ? &progressWrap.vt : NULL); + + MainDecodeSRes = res; + + #ifndef _7ZIP_ST + // _tryMt = isMT; + #endif + + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) + + // return E_OUTOFMEMORY; // for debug check + + MainDecodeSRes_wasUsed = true; + + if (res == SZ_OK && finishStream) + { + /* + if (inSize && *inSize != Stat.PhySize) + res = SZ_ERROR_DATA; + */ + if (outSizeLimit && *outSizeLimit != outWrap.Processed) + res = SZ_ERROR_DATA; + } + + return SResToHRESULT_Code(res); +} + + +HRESULT CComDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + return Decode(inStream, outStream, outSize, _finishStream, progress); +} + +STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) +{ + _finishStream = (finishMode != 0); + return S_OK; +} + +STDMETHODIMP CComDecoder::GetInStreamProcessedSize(UInt64 *value) +{ + *value = Stat.InSize; + return S_OK; +} + +#ifndef _7ZIP_ST + +STDMETHODIMP CComDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; + return S_OK; +} + +STDMETHODIMP CComDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + +}} diff --git a/3rdparty/lzma/CPP/7zip/Compress/XzDecoder.h b/3rdparty/lzma/CPP/7zip/Compress/XzDecoder.h new file mode 100644 index 00000000000..7ad81f49aba --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Compress/XzDecoder.h @@ -0,0 +1,93 @@ +// XzDecoder.h + +#ifndef __XZ_DECODER_H +#define __XZ_DECODER_H + +#include "../../../C/Xz.h" + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +namespace NCompress { +namespace NXz { + +struct CDecoder +{ + CXzDecMtHandle xz; + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; + + SRes MainDecodeSRes; // it's not HRESULT + bool MainDecodeSRes_wasUsed; + CXzStatInfo Stat; + + CDecoder(): + xz(NULL), + _tryMt(True), + _numThreads(1), + _memUsage((UInt64)(sizeof(size_t)) << 28), + MainDecodeSRes(SZ_OK), + MainDecodeSRes_wasUsed(false) + {} + + ~CDecoder() + { + if (xz) + XzDecMt_Destroy(xz); + } + + /* Decode() can return S_OK, if there is data after good xz streams, and that data is not new xz stream. + check also (Stat.DataAfterEnd) flag */ + + HRESULT Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, + const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *compressProgress); +}; + + +class CComDecoder: + public ICompressCoder, + public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + + public CMyUnknownImp, + public CDecoder +{ + bool _finishStream; + +public: + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + + MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE + + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD(SetFinishMode)(UInt32 finishMode); + STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + + #ifndef _7ZIP_ST + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + + CComDecoder(): _finishStream(false) {} +}; + +}} + +#endif diff --git a/3rdparty/lzma/CPP/7zip/Compress/XzEncoder.cpp b/3rdparty/lzma/CPP/7zip/Compress/XzEncoder.cpp new file mode 100644 index 00000000000..d6f42caf2d3 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Compress/XzEncoder.cpp @@ -0,0 +1,245 @@ +// XzEncoder.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" + +#include "../../Common/MyString.h" +#include "../../Common/StringToInt.h" + +#include "../Common/CWrappers.h" +#include "../Common/StreamUtils.h" + +#include "XzEncoder.h" + +namespace NCompress { + +namespace NLzma2 { + +HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props); + +} + +namespace NXz { + +void CEncoder::InitCoderProps() +{ + XzProps_Init(&xzProps); +} + +CEncoder::CEncoder() +{ + XzProps_Init(&xzProps); + _encoder = NULL; + _encoder = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!_encoder) + throw 1; +} + +CEncoder::~CEncoder() +{ + if (_encoder) + XzEnc_Destroy(_encoder); +} + + +struct CMethodNamePair +{ + UInt32 Id; + const char *Name; +}; + +static const CMethodNamePair g_NamePairs[] = +{ + { XZ_ID_Delta, "Delta" }, + { XZ_ID_X86, "BCJ" }, + { XZ_ID_PPC, "PPC" }, + { XZ_ID_IA64, "IA64" }, + { XZ_ID_ARM, "ARM" }, + { XZ_ID_ARMT, "ARMT" }, + { XZ_ID_SPARC, "SPARC" } + // { XZ_ID_LZMA2, "LZMA2" } +}; + +static int FilterIdFromName(const wchar_t *name) +{ + for (unsigned i = 0; i < ARRAY_SIZE(g_NamePairs); i++) + { + const CMethodNamePair &pair = g_NamePairs[i]; + if (StringsAreEqualNoCase_Ascii(name, pair.Name)) + return (int)pair.Id; + } + return -1; +} + + +HRESULT CEncoder::SetCheckSize(UInt32 checkSizeInBytes) +{ + unsigned id; + switch (checkSizeInBytes) + { + case 0: id = XZ_CHECK_NO; break; + case 4: id = XZ_CHECK_CRC32; break; + case 8: id = XZ_CHECK_CRC64; break; + case 32: id = XZ_CHECK_SHA256; break; + default: return E_INVALIDARG; + } + xzProps.checkId = id; + return S_OK; +} + + +HRESULT CEncoder::SetCoderProp(PROPID propID, const PROPVARIANT &prop) +{ + if (propID == NCoderPropID::kNumThreads) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + xzProps.numTotalThreads = (int)(prop.ulVal); + return S_OK; + } + + if (propID == NCoderPropID::kCheckSize) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + return SetCheckSize(prop.ulVal); + } + + if (propID == NCoderPropID::kBlockSize2) + { + if (prop.vt == VT_UI4) + xzProps.blockSize = prop.ulVal; + else if (prop.vt == VT_UI8) + xzProps.blockSize = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + return S_OK; + } + + if (propID == NCoderPropID::kReduceSize) + { + if (prop.vt == VT_UI8) + xzProps.reduceSize = prop.uhVal.QuadPart; + else + return E_INVALIDARG; + return S_OK; + } + + if (propID == NCoderPropID::kFilter) + { + if (prop.vt == VT_UI4) + { + UInt32 id32 = prop.ulVal; + if (id32 == XZ_ID_Delta) + return E_INVALIDARG; + xzProps.filterProps.id = prop.ulVal; + } + else + { + if (prop.vt != VT_BSTR) + return E_INVALIDARG; + + const wchar_t *name = prop.bstrVal; + const wchar_t *end; + + UInt32 id32 = ConvertStringToUInt32(name, &end); + + if (end != name) + name = end; + else + { + if (IsString1PrefixedByString2_NoCase_Ascii(name, "Delta")) + { + name += 5; // strlen("Delta"); + id32 = XZ_ID_Delta; + } + else + { + int filterId = FilterIdFromName(prop.bstrVal); + if (filterId < 0 /* || filterId == XZ_ID_LZMA2 */) + return E_INVALIDARG; + id32 = (unsigned)filterId; + } + } + + if (id32 == XZ_ID_Delta) + { + wchar_t c = *name; + if (c != '-' && c != ':') + return E_INVALIDARG; + name++; + UInt32 delta = ConvertStringToUInt32(name, &end); + if (end == name || *end != 0 || delta == 0 || delta > 256) + return E_INVALIDARG; + xzProps.filterProps.delta = delta; + } + + xzProps.filterProps.id = id32; + } + + return S_OK; + } + + return NLzma2::SetLzma2Prop(propID, prop, xzProps.lzma2Props); +} + + +STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + XzProps_Init(&xzProps); + + for (UInt32 i = 0; i < numProps; i++) + { + RINOK(SetCoderProp(propIDs[i], coderProps[i])); + } + + return S_OK; + // return SResToHRESULT(XzEnc_SetProps(_encoder, &xzProps)); +} + + +STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + PROPID propID = propIDs[i]; + if (propID == NCoderPropID::kExpectedDataSize) + if (prop.vt == VT_UI8) + XzEnc_SetDataSize(_encoder, prop.uhVal.QuadPart); + } + return S_OK; +} + + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + +STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) +{ + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + SRes res = XzEnc_SetProps(_encoder, &xzProps); + if (res == SZ_OK) + res = XzEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL); + + // SRes res = Xz_Encode(&outWrap.vt, &inWrap.vt, &xzProps, progress ? &progressWrap.vt : NULL); + + RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + + return SResToHRESULT(res); +} + +}} diff --git a/3rdparty/lzma/CPP/7zip/Compress/XzEncoder.h b/3rdparty/lzma/CPP/7zip/Compress/XzEncoder.h new file mode 100644 index 00000000000..ea5190ee6ea --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Compress/XzEncoder.h @@ -0,0 +1,46 @@ +// XzEncoder.h + +#ifndef __XZ_ENCODER_H +#define __XZ_ENCODER_H + +#include "../../../C/XzEnc.h" + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +namespace NCompress { +namespace NXz { + + +class CEncoder: + public ICompressCoder, + public ICompressSetCoderProperties, + public ICompressSetCoderPropertiesOpt, + public CMyUnknownImp +{ + CXzEncHandle _encoder; +public: + CXzProps xzProps; + + MY_UNKNOWN_IMP3( + ICompressCoder, + ICompressSetCoderProperties, + ICompressSetCoderPropertiesOpt) + + void InitCoderProps(); + HRESULT SetCheckSize(UInt32 checkSizeInBytes); + HRESULT SetCoderProp(PROPID propID, const PROPVARIANT &prop); + + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + + CEncoder(); + virtual ~CEncoder(); +}; + +}} + +#endif diff --git a/3rdparty/lzma/CPP/7zip/Crc.mak b/3rdparty/lzma/CPP/7zip/Crc.mak index f2b0874cb45..815142db856 100644 --- a/3rdparty/lzma/CPP/7zip/Crc.mak +++ b/3rdparty/lzma/CPP/7zip/Crc.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\7zCrc.obj -!IF "$(CPU)" == "IA64" || "$(CPU)" == "MIPS" || "$(CPU)" == "ARM" +!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/3rdparty/lzma/CPP/7zip/Crc64.mak b/3rdparty/lzma/CPP/7zip/Crc64.mak index 57301c22345..d58a483261e 100644 --- a/3rdparty/lzma/CPP/7zip/Crc64.mak +++ b/3rdparty/lzma/CPP/7zip/Crc64.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\XzCrc64.obj -!IF "$(CPU)" == "IA64" || "$(CPU)" == "MIPS" || "$(CPU)" == "ARM" +!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/3rdparty/lzma/CPP/7zip/Crypto/7zAes.cpp b/3rdparty/lzma/CPP/7zip/Crypto/7zAes.cpp index d33b562a3f6..7f34e910847 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/7zAes.cpp +++ b/3rdparty/lzma/CPP/7zip/Crypto/7zAes.cpp @@ -2,9 +2,11 @@ #include "StdAfx.h" +#include "../../../C/CpuArch.h" #include "../../../C/Sha256.h" #include "../../Common/ComTry.h" +#include "../../Common/MyBuffer2.h" #ifndef _7ZIP_ST #include "../../Windows/Synchronization.h" @@ -48,31 +50,65 @@ void CKeyInfo::CalcKey() } else { - size_t bufSize = 8 + SaltSize + Password.Size(); - CObjArray buf(bufSize); + const unsigned kUnrPow = 6; + const UInt32 numUnroll = (UInt32)1 << (NumCyclesPower <= kUnrPow ? (unsigned)NumCyclesPower : kUnrPow); + + const size_t bufSize = 8 + SaltSize + Password.Size(); + const size_t unrollSize = bufSize * numUnroll; + + // MY_ALIGN (16) + // CSha256 sha; + CAlignedBuffer sha(sizeof(CSha256) + unrollSize + bufSize * 2); + Byte *buf = sha + sizeof(CSha256); + memcpy(buf, Salt, SaltSize); memcpy(buf + SaltSize, Password, Password.Size()); + memset(buf + bufSize - 8, 0, 8); - CSha256 sha; - Sha256_Init(&sha); - - Byte *ctr = buf + SaltSize + Password.Size(); - - for (unsigned i = 0; i < 8; i++) - ctr[i] = 0; + Sha256_Init((CSha256 *)(void *)(Byte *)sha); + { + { + Byte *dest = buf; + for (UInt32 i = 1; i < numUnroll; i++) + { + dest += bufSize; + memcpy(dest, buf, bufSize); + } + } + + const UInt32 numRounds = (UInt32)1 << NumCyclesPower; + UInt32 r = 0; + do + { + Byte *dest = buf + bufSize - 8; + UInt32 i = r; + r += numUnroll; + do + { + SetUi32(dest, i); i++; dest += bufSize; + // SetUi32(dest, i); i++; dest += bufSize; + } + while (i < r); + Sha256_Update((CSha256 *)(void *)(Byte *)sha, buf, unrollSize); + } + while (r < numRounds); + } + /* UInt64 numRounds = (UInt64)1 << NumCyclesPower; do { - Sha256_Update(&sha, buf, bufSize); + Sha256_Update((CSha256 *)(Byte *)sha, buf, bufSize); for (unsigned i = 0; i < 8; i++) if (++(ctr[i]) != 0) break; } while (--numRounds != 0); + */ - Sha256_Final(&sha, Key); + Sha256_Final((CSha256 *)(void *)(Byte *)sha, Key); + memset(sha, 0, sha.Size()); } } @@ -164,8 +200,8 @@ STDMETHODIMP CEncoder::ResetInitVector() { for (unsigned i = 0; i < sizeof(_iv); i++) _iv[i] = 0; - _ivSize = 8; - g_RandomGenerator.Generate(_iv, _ivSize); + _ivSize = 16; + MY_RAND_GEN(_iv, _ivSize); return S_OK; } @@ -250,6 +286,7 @@ STDMETHODIMP CBaseCoder::CryptoSetPassword(const Byte *data, UInt32 size) { COM_TRY_BEGIN + _key.Password.Wipe(); _key.Password.CopyFrom(data, (size_t)size); return S_OK; diff --git a/3rdparty/lzma/CPP/7zip/Crypto/7zAes.h b/3rdparty/lzma/CPP/7zip/Crypto/7zAes.h index 84e07ac9613..a67ac0b61b3 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/7zAes.h +++ b/3rdparty/lzma/CPP/7zip/Crypto/7zAes.h @@ -37,6 +37,17 @@ public: for (unsigned i = 0; i < sizeof(Salt); i++) Salt[i] = 0; } + + void Wipe() + { + Password.Wipe(); + NumCyclesPower = 0; + SaltSize = 0; + MY_memset_0_ARRAY(Salt); + MY_memset_0_ARRAY(Key); + } + + ~CKeyInfo() { Wipe(); } }; class CKeyInfoCache diff --git a/3rdparty/lzma/CPP/7zip/Crypto/7zAesRegister.cpp b/3rdparty/lzma/CPP/7zip/Crypto/7zAesRegister.cpp index f9d59699b7c..35605f4352d 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/7zAesRegister.cpp +++ b/3rdparty/lzma/CPP/7zip/Crypto/7zAesRegister.cpp @@ -9,9 +9,9 @@ namespace NCrypto { namespace N7z { -REGISTER_FILTER_E(7zAES, - CDecoder(), - CEncoder(), +REGISTER_FILTER_E(_7zAES, + CDecoder, + CEncoder, 0x6F10701, "7zAES") }} diff --git a/3rdparty/lzma/CPP/7zip/Crypto/MyAes.cpp b/3rdparty/lzma/CPP/7zip/Crypto/MyAes.cpp index 52eaab7ae2c..5cb7f46359b 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/MyAes.cpp +++ b/3rdparty/lzma/CPP/7zip/Crypto/MyAes.cpp @@ -10,91 +10,179 @@ namespace NCrypto { static struct CAesTabInit { CAesTabInit() { AesGenTables();} } g_AesTabInit; -CAesCbcCoder::CAesCbcCoder(bool encodeMode, unsigned keySize): +CAesCoder::CAesCoder(bool encodeMode, unsigned keySize, bool ctrMode): _keySize(keySize), _keyIsSet(false), - _encodeMode(encodeMode) + _encodeMode(encodeMode), + _ctrMode(ctrMode), + _aes(AES_NUM_IVMRK_WORDS * 4 + AES_BLOCK_SIZE * 2) { - _offset = ((0 - (unsigned)(ptrdiff_t)_aes) & 0xF) / sizeof(UInt32); + // _offset = ((0 - (unsigned)(ptrdiff_t)_aes) & 0xF) / sizeof(UInt32); memset(_iv, 0, AES_BLOCK_SIZE); + /* + // we can use the following code to test 32-bit overflow case for AES-CTR + for (unsigned i = 0; i < 16; i++) _iv[i] = (Byte)(i + 1); + _iv[0] = 0xFE; _iv[1] = _iv[2] = _iv[3] = 0xFF; + */ SetFunctions(0); } -STDMETHODIMP CAesCbcCoder::Init() +STDMETHODIMP CAesCoder::Init() { - AesCbc_Init(_aes + _offset, _iv); - return _keyIsSet ? S_OK : E_FAIL; + AesCbc_Init(Aes(), _iv); + return _keyIsSet ? S_OK : E_NOTIMPL; // E_FAIL } -STDMETHODIMP_(UInt32) CAesCbcCoder::Filter(Byte *data, UInt32 size) +STDMETHODIMP_(UInt32) CAesCoder::Filter(Byte *data, UInt32 size) { if (!_keyIsSet) return 0; if (size == 0) return 0; if (size < AES_BLOCK_SIZE) + { + #ifndef _SFX + if (_ctrMode) + { + // use that code only for last block !!! + Byte *ctr = (Byte *)(Aes() + AES_NUM_IVMRK_WORDS); + memset(ctr, 0, AES_BLOCK_SIZE); + memcpy(ctr, data, size); + _codeFunc(Aes(), ctr, 1); + memcpy(data, ctr, size); + return size; + } + #endif return AES_BLOCK_SIZE; + } size >>= 4; - _codeFunc(_aes + _offset, data, size); + _codeFunc(Aes(), data, size); return size << 4; } -STDMETHODIMP CAesCbcCoder::SetKey(const Byte *data, UInt32 size) +STDMETHODIMP CAesCoder::SetKey(const Byte *data, UInt32 size) { if ((size & 0x7) != 0 || size < 16 || size > 32) return E_INVALIDARG; if (_keySize != 0 && size != _keySize) return E_INVALIDARG; - AES_SET_KEY_FUNC setKeyFunc = _encodeMode ? Aes_SetKey_Enc : Aes_SetKey_Dec; - setKeyFunc(_aes + _offset + 4, data, size); + AES_SET_KEY_FUNC setKeyFunc = (_ctrMode | _encodeMode) ? Aes_SetKey_Enc : Aes_SetKey_Dec; + setKeyFunc(Aes() + 4, data, size); _keyIsSet = true; return S_OK; } -STDMETHODIMP CAesCbcCoder::SetInitVector(const Byte *data, UInt32 size) +STDMETHODIMP CAesCoder::SetInitVector(const Byte *data, UInt32 size) { if (size != AES_BLOCK_SIZE) return E_INVALIDARG; memcpy(_iv, data, size); - CAesCbcCoder::Init(); // don't call virtual function here !!! + CAesCoder::Init(); // don't call virtual function here !!! return S_OK; } -EXTERN_C_BEGIN +#ifndef _SFX -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); +#ifdef MY_CPU_X86_OR_AMD64 + #define USE_HW_AES +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + #if defined(__clang__) + #if (__clang_major__ >= 8) // fix that check + #define USE_HW_AES + #endif + #elif defined(__GNUC__) + #if (__GNUC__ >= 6) // fix that check + #define USE_HW_AES + #endif + #elif defined(_MSC_VER) + #if _MSC_VER >= 1910 + #define USE_HW_AES + #endif + #endif +#endif -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); +#endif -EXTERN_C_END -bool CAesCbcCoder::SetFunctions(UInt32 algo) +bool CAesCoder::SetFunctions(UInt32 + #ifndef _SFX + algo + #endif + ) { - _codeFunc = _encodeMode ? - g_AesCbc_Encode : - g_AesCbc_Decode; + _codeFunc = g_AesCbc_Decode; + + #ifdef _SFX + + return true; + + #else + + if (_ctrMode) + _codeFunc = g_AesCtr_Code; + else if (_encodeMode) + _codeFunc = g_AesCbc_Encode; + + if (algo < 1) + return true; + if (algo == 1) { - _codeFunc = _encodeMode ? - AesCbc_Encode: - AesCbc_Decode; + _codeFunc = AesCbc_Decode; + + #ifndef _SFX + if (_ctrMode) + _codeFunc = AesCtr_Code; + else if (_encodeMode) + _codeFunc = AesCbc_Encode; + #endif + return true; } - if (algo == 2) + + #ifdef USE_HW_AES + // if (CPU_IsSupported_AES()) { - #ifdef MY_CPU_X86_OR_AMD64 - if (g_AesCbc_Encode != AesCbc_Encode_Intel) + if (algo == 2) + if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW) + { + _codeFunc = AesCbc_Decode_HW; + #ifndef _SFX + if (_ctrMode) + _codeFunc = AesCtr_Code_HW; + else if (_encodeMode) + _codeFunc = AesCbc_Encode_HW; + #endif + return true; + } + + #if defined(MY_CPU_X86_OR_AMD64) + if (algo == 3) + if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW_256) + { + _codeFunc = AesCbc_Decode_HW_256; + #ifndef _SFX + if (_ctrMode) + _codeFunc = AesCtr_Code_HW_256; + else if (_encodeMode) + _codeFunc = AesCbc_Encode_HW; + #endif + return true; + } #endif - return false; } - return true; + #endif + + return false; + + #endif } -STDMETHODIMP CAesCbcCoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) + +#ifndef _SFX + +STDMETHODIMP CAesCoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) { + UInt32 algo = 0; for (UInt32 i = 0; i < numProps; i++) { const PROPVARIANT &prop = coderProps[i]; @@ -102,11 +190,16 @@ STDMETHODIMP CAesCbcCoder::SetCoderProperties(const PROPID *propIDs, const PROPV { if (prop.vt != VT_UI4) return E_INVALIDARG; - if (!SetFunctions(prop.ulVal)) + if (prop.ulVal > 3) return E_NOTIMPL; + algo = prop.ulVal; } } + if (!SetFunctions(algo)) + return E_NOTIMPL; return S_OK; } +#endif + } diff --git a/3rdparty/lzma/CPP/7zip/Crypto/MyAes.h b/3rdparty/lzma/CPP/7zip/Crypto/MyAes.h index 84c21de30e7..a60042be33e 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/MyAes.h +++ b/3rdparty/lzma/CPP/7zip/Crypto/MyAes.h @@ -5,49 +5,73 @@ #include "../../../C/Aes.h" +#include "../../Common/MyBuffer2.h" #include "../../Common/MyCom.h" #include "../ICoder.h" namespace NCrypto { -class CAesCbcCoder: +class CAesCoder: public ICompressFilter, public ICryptoProperties, + #ifndef _SFX public ICompressSetCoderProperties, + #endif public CMyUnknownImp { AES_CODE_FUNC _codeFunc; - unsigned _offset; + // unsigned _offset; unsigned _keySize; bool _keyIsSet; bool _encodeMode; - UInt32 _aes[AES_NUM_IVMRK_WORDS + 3]; + bool _ctrMode; + + // UInt32 _aes[AES_NUM_IVMRK_WORDS + 3]; + CAlignedBuffer _aes; + Byte _iv[AES_BLOCK_SIZE]; + // UInt32 *Aes() { return _aes + _offset; } + UInt32 *Aes() { return (UInt32 *)(void *)(Byte *)_aes; } + bool SetFunctions(UInt32 algo); public: - CAesCbcCoder(bool encodeMode, unsigned keySize); + CAesCoder(bool encodeMode, unsigned keySize, bool ctrMode); - MY_UNKNOWN_IMP3(ICompressFilter, ICryptoProperties, ICompressSetCoderProperties) + virtual ~CAesCoder() {}; // we need virtual destructor for derived classes + + MY_QUERYINTERFACE_BEGIN2(ICompressFilter) + MY_QUERYINTERFACE_ENTRY(ICryptoProperties) + #ifndef _SFX + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderProperties) + #endif + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE INTERFACE_ICompressFilter(;) + + void SetKeySize(unsigned size) { _keySize = size; } STDMETHOD(SetKey)(const Byte *data, UInt32 size); STDMETHOD(SetInitVector)(const Byte *data, UInt32 size); + #ifndef _SFX STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + #endif }; -struct CAesCbcEncoder: public CAesCbcCoder +#ifndef _SFX +struct CAesCbcEncoder: public CAesCoder { - CAesCbcEncoder(unsigned keySize = 0): CAesCbcCoder(true, keySize) {} + CAesCbcEncoder(unsigned keySize = 0): CAesCoder(true, keySize, false) {} }; +#endif -struct CAesCbcDecoder: public CAesCbcCoder +struct CAesCbcDecoder: public CAesCoder { - CAesCbcDecoder(unsigned keySize = 0): CAesCbcCoder(false, keySize) {} + CAesCbcDecoder(unsigned keySize = 0): CAesCoder(false, keySize, false) {} }; } diff --git a/3rdparty/lzma/CPP/7zip/Crypto/MyAesReg.cpp b/3rdparty/lzma/CPP/7zip/Crypto/MyAesReg.cpp index 28006835549..86bd25474fb 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/MyAesReg.cpp +++ b/3rdparty/lzma/CPP/7zip/Crypto/MyAesReg.cpp @@ -8,9 +8,23 @@ namespace NCrypto { -REGISTER_FILTER_E(AES256CBC, - CAesCbcDecoder(32), - CAesCbcEncoder(32), - 0x6F00181, "AES256CBC") +#ifndef _SFX + +#define REGISTER_AES_2(name, nameString, keySize, isCtr) \ + REGISTER_FILTER_E(name, \ + CAesCoder(false, keySize, isCtr), \ + CAesCoder(true , keySize, isCtr), \ + 0x6F00100 | ((keySize - 16) * 8) | (isCtr ? 4 : 1), \ + nameString) \ + +#define REGISTER_AES(name, nameString, isCtr) \ + /* REGISTER_AES_2(AES128 ## name, "AES128" nameString, 16, isCtr) */ \ + /* REGISTER_AES_2(AES192 ## name, "AES192" nameString, 24, isCtr) */ \ + REGISTER_AES_2(AES256 ## name, "AES256" nameString, 32, isCtr) \ + +REGISTER_AES(CBC, "CBC", false) +// REGISTER_AES(CTR, "CTR", true) + +#endif } diff --git a/3rdparty/lzma/CPP/7zip/Crypto/RandGen.cpp b/3rdparty/lzma/CPP/7zip/Crypto/RandGen.cpp index 408f73f1686..65633fd8f3b 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/RandGen.cpp +++ b/3rdparty/lzma/CPP/7zip/Crypto/RandGen.cpp @@ -2,14 +2,44 @@ #include "StdAfx.h" +#include "RandGen.h" + +#ifndef USE_STATIC_SYSTEM_RAND + #ifndef _7ZIP_ST #include "../../Windows/Synchronization.h" #endif -#include "RandGen.h" -#ifndef _WIN32 +#ifdef _WIN32 + +#ifdef _WIN64 +#define USE_STATIC_RtlGenRandom +#endif + +#ifdef USE_STATIC_RtlGenRandom + +// #include + +EXTERN_C_BEGIN +#ifndef RtlGenRandom + #define RtlGenRandom SystemFunction036 + BOOLEAN WINAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); +#endif +EXTERN_C_END + +#else +EXTERN_C_BEGIN +typedef BOOLEAN (WINAPI * Func_RtlGenRandom)(PVOID RandomBuffer, ULONG RandomBufferLength); +EXTERN_C_END +#endif + + +#else #include +#include +#include +#include #define USE_POSIX_TIME #define USE_POSIX_TIME2 #endif @@ -21,38 +51,115 @@ #endif #endif -// This is not very good random number generator. -// Please use it only for salt. -// First generated data block depends from timer and processID. +// The seed and first generated data block depend from processID, +// theadID, timer and system random generator, if available. // Other generated data blocks depend from previous state -// Maybe it's possible to restore original timer value from generated value. #define HASH_UPD(x) Sha256_Update(&hash, (const Byte *)&x, sizeof(x)); void CRandomGenerator::Init() { + MY_ALIGN (16) CSha256 hash; Sha256_Init(&hash); + unsigned numIterations = 1000; + + { + #ifndef UNDER_CE + const unsigned kNumIterations_Small = 100; + const unsigned kBufSize = 32; + MY_ALIGN (16) + Byte buf[kBufSize]; + #endif + #ifdef _WIN32 + DWORD w = ::GetCurrentProcessId(); HASH_UPD(w); w = ::GetCurrentThreadId(); HASH_UPD(w); + + #ifdef UNDER_CE + /* + if (CeGenRandom(kBufSize, buf)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } + */ + #elif defined(USE_STATIC_RtlGenRandom) + if (RtlGenRandom(buf, kBufSize)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } #else + { + HMODULE hModule = ::LoadLibrary(TEXT("Advapi32.dll")); + if (hModule) + { + // SystemFunction036() is real name of RtlGenRandom() function + Func_RtlGenRandom my_RtlGenRandom = (Func_RtlGenRandom)(void *)GetProcAddress(hModule, "SystemFunction036"); + if (my_RtlGenRandom) + { + if (my_RtlGenRandom(buf, kBufSize)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } + } + ::FreeLibrary(hModule); + } + } + #endif + + #else + pid_t pid = getpid(); HASH_UPD(pid); pid = getppid(); HASH_UPD(pid); + + { + int f = open("/dev/urandom", O_RDONLY); + unsigned numBytes = kBufSize; + if (f >= 0) + { + do + { + ssize_t n = read(f, buf, numBytes); + if (n <= 0) + break; + Sha256_Update(&hash, buf, (size_t)n); + numBytes -= (unsigned)n; + } + while (numBytes); + close(f); + if (numBytes == 0) + numIterations = kNumIterations_Small; + } + } + /* + { + int n = getrandom(buf, kBufSize, 0); + if (n > 0) + { + Sha256_Update(&hash, buf, n); + if (n == kBufSize) + numIterations = kNumIterations_Small; + } + } + */ + + #endif + } + + #ifdef _DEBUG + numIterations = 2; #endif - for (unsigned i = 0; i < - #ifdef _DEBUG - 2; - #else - 1000; - #endif - i++) + do { #ifdef _WIN32 LARGE_INTEGER v; @@ -85,6 +192,8 @@ void CRandomGenerator::Init() Sha256_Update(&hash, _buff, SHA256_DIGEST_SIZE); } } + while (--numIterations); + Sha256_Final(&hash, _buff); _needInit = false; } @@ -104,6 +213,7 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) Init(); while (size != 0) { + MY_ALIGN (16) CSha256 hash; Sha256_Init(&hash); @@ -114,6 +224,7 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) UInt32 salt = 0xF672ABD1; HASH_UPD(salt); Sha256_Update(&hash, _buff, SHA256_DIGEST_SIZE); + MY_ALIGN (16) Byte buff[SHA256_DIGEST_SIZE]; Sha256_Final(&hash, buff); for (unsigned i = 0; i < SHA256_DIGEST_SIZE && size != 0; i++, size--) @@ -122,3 +233,5 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) } CRandomGenerator g_RandomGenerator; + +#endif diff --git a/3rdparty/lzma/CPP/7zip/Crypto/RandGen.h b/3rdparty/lzma/CPP/7zip/Crypto/RandGen.h index cfdcd60d012..5122ec4b447 100644 --- a/3rdparty/lzma/CPP/7zip/Crypto/RandGen.h +++ b/3rdparty/lzma/CPP/7zip/Crypto/RandGen.h @@ -5,6 +5,21 @@ #include "../../../C/Sha256.h" +#ifdef _WIN64 +// #define USE_STATIC_SYSTEM_RAND +#endif + +#ifdef USE_STATIC_SYSTEM_RAND + +#ifdef _WIN32 +#include +#define MY_RAND_GEN(data, size) RtlGenRandom(data, size) +#else +#define MY_RAND_GEN(data, size) getrandom(data, size, 0) +#endif + +#else + class CRandomGenerator { Byte _buff[SHA256_DIGEST_SIZE]; @@ -18,4 +33,8 @@ public: extern CRandomGenerator g_RandomGenerator; +#define MY_RAND_GEN(data, size) g_RandomGenerator.Generate(data, size) + +#endif + #endif diff --git a/3rdparty/lzma/CPP/7zip/GuiCommon.rc b/3rdparty/lzma/CPP/7zip/GuiCommon.rc index 6b26ddd85c9..b67409b922f 100644 --- a/3rdparty/lzma/CPP/7zip/GuiCommon.rc +++ b/3rdparty/lzma/CPP/7zip/GuiCommon.rc @@ -71,6 +71,9 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US DEFPUSHBUTTON "OK", IDOK, bx2, by, bxs, bys \ PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys +#define MY_BUTTON__CLOSE \ + DEFPUSHBUTTON "&Close", IDCLOSE, bx1, by, bxs, bys + #define MY_COMBO CBS_DROPDOWNLIST | WS_VSCROLL | WS_TABSTOP #define MY_COMBO_SORTED MY_COMBO | CBS_SORT diff --git a/3rdparty/lzma/CPP/7zip/Guid.txt b/3rdparty/lzma/CPP/7zip/Guid.txt index 7edab6ead87..8fcdd849d4c 100644 --- a/3rdparty/lzma/CPP/7zip/Guid.txt +++ b/3rdparty/lzma/CPP/7zip/Guid.txt @@ -19,9 +19,11 @@ 0F IOutFolderArchive 10 IFolderArchiveUpdateCallback2 11 IFolderScanProgress + 12 IFolderSetZoneIdMode 20 IFileExtractCallback.h::IGetProp - 30 IFileExtractCallback.h::IFolderExtractToStreamCallback + 30 IFileExtractCallback.h::IFolderExtractToStreamCallback (old) + 31 IFileExtractCallback.h::IFolderExtractToStreamCallback (new 21.04) 03 IStream.h @@ -33,6 +35,7 @@ 07 IOutStreamFinish 08 IStreamGetProps 09 IStreamGetProps2 + 0A IStreamGetProp 04 ICoder.h @@ -40,6 +43,7 @@ 04 ICompressProgressInfo 05 ICompressCoder 18 ICompressCoder2 + 1F ICompressSetCoderPropertiesOpt 20 ICompressSetCoderProperties 21 ICompressSetDecoderProperties // 22 ICompressSetDecoderProperties2 @@ -47,6 +51,9 @@ 24 ICompressGetInStreamProcessedSize 25 ICompressSetCoderMt 26 ICompressSetFinishMode + 27 ICompressGetInStreamProcessedSize2 + 28 ICompressSetMemLimit + 29 ICompressReadUnusedFromInBuf 30 ICompressGetSubStreamSize 31 ICompressSetInStream @@ -100,6 +107,9 @@ 80 IArchiveUpdateCallback 82 IArchiveUpdateCallback2 83 IArchiveUpdateCallbackFile + 84 IArchiveGetDiskProperty + 85 IArchiveUpdateCallbackArcProp (Reserved) + A0 IOutArchive @@ -163,6 +173,13 @@ Handler GUIDs: 0C xz 0D ppmd + C0 AVB + C1 LP + C2 Sparse + C3 APFS + C4 Vhdx + C5 Base64 + C6 COFF C7 Ext C8 VMDK C9 VDI diff --git a/3rdparty/lzma/CPP/7zip/ICoder.h b/3rdparty/lzma/CPP/7zip/ICoder.h index 75db9399379..9177f5e5094 100644 --- a/3rdparty/lzma/CPP/7zip/ICoder.h +++ b/3rdparty/lzma/CPP/7zip/ICoder.h @@ -43,6 +43,7 @@ CODER_INTERFACE(ICompressCoder2, 0x18) S_OK : OK S_FALSE : data error (for decoders) E_OUTOFMEMORY : memory allocation error + E_NOTIMPL : unsupported encoding method (for decoders) another error code : some error. For example, it can be error code received from inStream or outStream function. Parameters: @@ -104,25 +105,42 @@ namespace NCoderPropID enum EEnum { kDefaultProp = 0, - kDictionarySize, - kUsedMemorySize, - kOrder, - kBlockSize, - kPosStateBits, - kLitContextBits, - kLitPosBits, - kNumFastBytes, - kMatchFinder, - kMatchFinderCycles, - kNumPasses, - kAlgorithm, - kNumThreads, - kEndMarker, - kLevel, - kReduceSize // estimated size of data that will be compressed. Encoder can use this value to reduce dictionary size. + kDictionarySize, // VT_UI4 + kUsedMemorySize, // VT_UI4 + kOrder, // VT_UI4 + kBlockSize, // VT_UI4 or VT_UI8 + kPosStateBits, // VT_UI4 + kLitContextBits, // VT_UI4 + kLitPosBits, // VT_UI4 + kNumFastBytes, // VT_UI4 + kMatchFinder, // VT_BSTR + kMatchFinderCycles, // VT_UI4 + kNumPasses, // VT_UI4 + kAlgorithm, // VT_UI4 + kNumThreads, // VT_UI4 + kEndMarker, // VT_BOOL + kLevel, // VT_UI4 + kReduceSize, // VT_UI8 : it's estimated size of largest data stream that will be compressed + // encoder can use this value to reduce dictionary size and allocate data buffers + + kExpectedDataSize, // VT_UI8 : for ICompressSetCoderPropertiesOpt : + // it's estimated size of current data stream + // real data size can differ from that size + // encoder can use this value to optimize encoder initialization + + kBlockSize2, // VT_UI4 or VT_UI8 + kCheckSize, // VT_UI4 : size of digest in bytes + kFilter, // VT_BSTR + kMemUse, // VT_UI8 + kAffinity // VT_UI8 }; } +CODER_INTERFACE(ICompressSetCoderPropertiesOpt, 0x1F) +{ + STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) PURE; +}; + CODER_INTERFACE(ICompressSetCoderProperties, 0x20) { STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) PURE; @@ -170,6 +188,33 @@ CODER_INTERFACE(ICompressSetFinishMode, 0x26) 1 : full decoding. The stream must be finished at the end of decoding. */ }; +CODER_INTERFACE(ICompressGetInStreamProcessedSize2, 0x27) +{ + STDMETHOD(GetInStreamProcessedSize2)(UInt32 streamIndex, UInt64 *value) PURE; +}; + +CODER_INTERFACE(ICompressSetMemLimit, 0x28) +{ + STDMETHOD(SetMemLimit)(UInt64 memUsage) PURE; +}; + + +/* + ICompressReadUnusedFromInBuf is supported by ICoder object + call ReadUnusedFromInBuf() after ICoder::Code(inStream, ...). + ICoder::Code(inStream, ...) decodes data, and the ICoder object is allowed + to read from inStream to internal buffers more data than minimal data required for decoding. + So we can call ReadUnusedFromInBuf() from same ICoder object to read unused input + data from the internal buffer. + in ReadUnusedFromInBuf(): the Coder is not allowed to use (ISequentialInStream *inStream) object, that was sent to ICoder::Code(). +*/ + +CODER_INTERFACE(ICompressReadUnusedFromInBuf, 0x29) +{ + STDMETHOD(ReadUnusedFromInBuf)(void *data, UInt32 size, UInt32 *processedSize) PURE; +}; + + CODER_INTERFACE(ICompressGetSubStreamSize, 0x30) { @@ -256,8 +301,24 @@ CODER_INTERFACE(ICompressSetInStreamSize2, 0x39) /* ICompressFilter - Filter() converts as most as possible bytes + Filter() converts as most as possible bytes required for fast processing. + Some filters have (smallest_fast_block). + For example, (smallest_fast_block == 16) for AES CBC/CTR filters. + If data stream is not finished, caller must call Filter() for larger block: + where (size >= smallest_fast_block). + if (size >= smallest_fast_block) + { + The filter can leave some bytes at the end of data without conversion: + if there are data alignment reasons or speed reasons. + The caller must read additional data from stream and call Filter() again. + } + If data stream was finished, caller can call Filter() for (size < smallest_fast_block) + + data : must be aligned for at least 16 bytes for some filters (AES) + returns: (outSize): + if (outSize == 0) : Filter have not converted anything. + So the caller can stop processing, if data stream was finished. if (outSize <= size) : Filter have converted outSize bytes if (outSize > size) : Filter have not converted anything. and it needs at least outSize bytes to convert one block @@ -333,7 +394,8 @@ namespace NMethodPropID kDescription, kDecoderIsAssigned, kEncoderIsAssigned, - kDigestSize + kDigestSize, + kIsFilter }; } diff --git a/3rdparty/lzma/CPP/7zip/IPassword.h b/3rdparty/lzma/CPP/7zip/IPassword.h index 7ea45537e65..4ccc9ac742b 100644 --- a/3rdparty/lzma/CPP/7zip/IPassword.h +++ b/3rdparty/lzma/CPP/7zip/IPassword.h @@ -10,11 +10,41 @@ #define PASSWORD_INTERFACE(i, x) DECL_INTERFACE(i, 5, x) +/* +How to use output parameter (BSTR *password): + +in: The caller is required to set BSTR value as NULL (no string). + The callee (in 7-Zip code) ignores the input value stored in BSTR variable, + +out: The callee rewrites BSTR variable (*password) with new allocated string pointer. + The caller must free BSTR string with function SysFreeString(); +*/ + PASSWORD_INTERFACE(ICryptoGetTextPassword, 0x10) { STDMETHOD(CryptoGetTextPassword)(BSTR *password) PURE; }; + +/* +CryptoGetTextPassword2() +in: + The caller is required to set BSTR value as NULL (no string). + The caller is not required to set (*passwordIsDefined) value. + +out: + Return code: != S_OK : error code + Return code: S_OK : success + + if (*passwordIsDefined == 1), the variable (*password) contains password string + + if (*passwordIsDefined == 0), the password is not defined, + but the callee still could set (*password) to some allocated string, for example, as empty string. + + The caller must free BSTR string with function SysFreeString() +*/ + + PASSWORD_INTERFACE(ICryptoGetTextPassword2, 0x11) { STDMETHOD(CryptoGetTextPassword2)(Int32 *passwordIsDefined, BSTR *password) PURE; diff --git a/3rdparty/lzma/CPP/7zip/IStream.h b/3rdparty/lzma/CPP/7zip/IStream.h index 9a0bcbf3d72..2793a1e9b17 100644 --- a/3rdparty/lzma/CPP/7zip/IStream.h +++ b/3rdparty/lzma/CPP/7zip/IStream.h @@ -62,12 +62,21 @@ STREAM_INTERFACE(ISequentialOutStream, 0x02) */ }; +#ifdef _WIN32 + #ifdef __HRESULT_FROM_WIN32 #define HRESULT_WIN32_ERROR_NEGATIVE_SEEK __HRESULT_FROM_WIN32(ERROR_NEGATIVE_SEEK) #else #define HRESULT_WIN32_ERROR_NEGATIVE_SEEK HRESULT_FROM_WIN32(ERROR_NEGATIVE_SEEK) #endif +#else + +#define HRESULT_WIN32_ERROR_NEGATIVE_SEEK MY__E_ERROR_NEGATIVE_SEEK + +#endif + + /* Seek() Function If you seek before the beginning of the stream, Seek() function returns error code: Recommended error code is __HRESULT_FROM_WIN32(ERROR_NEGATIVE_SEEK). @@ -124,4 +133,11 @@ STREAM_INTERFACE(IStreamGetProps2, 0x09) STDMETHOD(GetProps2)(CStreamFileProps *props) PURE; }; + +STREAM_INTERFACE(IStreamGetProp, 0x0a) +{ + STDMETHOD(GetProperty)(PROPID propID, PROPVARIANT *value) PURE; + STDMETHOD(ReloadProps)() PURE; +}; + #endif diff --git a/3rdparty/lzma/CPP/7zip/LzFindOpt.mak b/3rdparty/lzma/CPP/7zip/LzFindOpt.mak new file mode 100644 index 00000000000..169e10f0b1e --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/LzFindOpt.mak @@ -0,0 +1,7 @@ +!IF defined(USE_C_LZFINDOPT) || "$(PLATFORM)" != "x64" +C_OBJS = $(C_OBJS) \ + $O\LzFindOpt.obj +!ELSE +ASM_OBJS = $(ASM_OBJS) \ + $O\LzFindOpt.obj +!ENDIF diff --git a/3rdparty/lzma/CPP/7zip/LzmaDec.mak b/3rdparty/lzma/CPP/7zip/LzmaDec.mak new file mode 100644 index 00000000000..bed4700a4cd --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/LzmaDec.mak @@ -0,0 +1,7 @@ +!IF "$(PLATFORM)" == "x64" +!IFNDEF NO_ASM +CFLAGS_C_SPEC = -D_LZMA_DEC_OPT +ASM_OBJS = $(ASM_OBJS) \ + $O\LzmaDecOpt.obj +!ENDIF +!ENDIF diff --git a/3rdparty/lzma/CPP/7zip/LzmaDec_gcc.mak b/3rdparty/lzma/CPP/7zip/LzmaDec_gcc.mak new file mode 100644 index 00000000000..51924f50565 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/LzmaDec_gcc.mak @@ -0,0 +1,14 @@ +ifdef USE_ASM +ifdef IS_X64 +USE_LZMA_DEC_ASM=1 +endif +ifdef IS_ARM64 +USE_LZMA_DEC_ASM=1 +endif +endif + +ifdef USE_LZMA_DEC_ASM + +LZMA_DEC_OPT_OBJS= $O/LzmaDecOpt.o + +endif diff --git a/3rdparty/lzma/CPP/7zip/PropID.h b/3rdparty/lzma/CPP/7zip/PropID.h index 1822f402b9f..2da636fa362 100644 --- a/3rdparty/lzma/CPP/7zip/PropID.h +++ b/3rdparty/lzma/CPP/7zip/PropID.h @@ -103,6 +103,13 @@ enum kpidReadOnly, kpidOutName, kpidCopyLink, + kpidArcFileName, + kpidIsHash, + kpidChangeTime, + kpidUserId, + kpidGroupId, + kpidDeviceMajor, + kpidDeviceMinor, kpid_NUM_DEFINED, @@ -124,4 +131,46 @@ const UInt32 kpv_ErrorFlags_DataError = 1 << 9; const UInt32 kpv_ErrorFlags_CrcError = 1 << 10; // const UInt32 kpv_ErrorFlags_Unsupported = 1 << 11; +/* +linux ctime : + file metadata was last changed. + changing the file modification time + counts as a metadata change, so will also have the side effect of updating the ctime. + +PROPVARIANT for timestamps in 7-Zip: +{ + vt = VT_FILETIME + wReserved1: set precision level + 0 : base value (backward compatibility value) + only filetime is used (7 digits precision). + wReserved2 and wReserved3 can contain random data + 1 : Unix (1 sec) + 2 : DOS (2 sec) + 3 : High Precision (1 ns) + 16 - 3 : (reserved) = 1 day + 16 - 2 : (reserved) = 1 hour + 16 - 1 : (reserved) = 1 minute + 16 + 0 : 1 sec (0 digits after point) + 16 + (1,2,3,4,5,6,7,8,9) : set subsecond precision level : + (number of decimal digits after point) + 16 + 9 : 1 ns (9 digits after point) + wReserved2 = ns % 100 : if (8 or 9 digits pecision) + = 0 : if not (8 or 9 digits pecision) + wReserved3 = 0; + filetime +} + +NOTE: TAR-PAX archives created by GNU TAR don't keep + whole information about original level of precision, + and timestamp are stored in reduced form, where tail zero + digits after point are removed. + So 7-Zip can return different precision levels for different items for such TAR archives. +*/ + +/* +TimePrec returned by IOutArchive::GetFileTimeType() +is used only for updating, when we compare MTime timestamp +from archive with timestamp from directory. +*/ + #endif diff --git a/3rdparty/lzma/CPP/7zip/Sha256.mak b/3rdparty/lzma/CPP/7zip/Sha256.mak new file mode 100644 index 00000000000..0bdbcb60283 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/Sha256.mak @@ -0,0 +1,13 @@ +COMMON_OBJS = $(COMMON_OBJS) \ + $O\Sha256Prepare.obj + +C_OBJS = $(C_OBJS) \ + $O\Sha256.obj + +!IF defined(USE_C_SHA) || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +C_OBJS = $(C_OBJS) \ + $O\Sha256Opt.obj +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" +ASM_OBJS = $(ASM_OBJS) \ + $O\Sha256Opt.obj +!ENDIF diff --git a/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.cpp b/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.cpp index 55650cfa0fe..a55fa22d699 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.cpp @@ -28,42 +28,74 @@ #include "../../../../C/7zVersion.h" #ifdef _WIN32 +extern +HINSTANCE g_hInstance; HINSTANCE g_hInstance = 0; #endif -// Tou can find the list of all GUIDs in Guid.txt file. -// use another CLSIDs, if you want to support other formats (zip, rar, ...). -// {23170F69-40C1-278A-1000-000110070000} +// You can find full list of all GUIDs supported by 7-Zip in Guid.txt file. +// 7z format GUID: {23170F69-40C1-278A-1000-000110070000} -DEFINE_GUID(CLSID_CFormat7z, - 0x23170F69, 0x40C1, 0x278A, 0x10, 0x00, 0x00, 0x01, 0x10, 0x07, 0x00, 0x00); -DEFINE_GUID(CLSID_CFormatXz, - 0x23170F69, 0x40C1, 0x278A, 0x10, 0x00, 0x00, 0x01, 0x10, 0x0C, 0x00, 0x00); +#define DEFINE_GUID_ARC(name, id) DEFINE_GUID(name, \ + 0x23170F69, 0x40C1, 0x278A, 0x10, 0x00, 0x00, 0x01, 0x10, id, 0x00, 0x00); -#define CLSID_Format CLSID_CFormat7z -// #define CLSID_Format CLSID_CFormatXz +enum +{ + kId_Zip = 1, + kId_BZip2 = 2, + kId_7z = 7, + kId_Xz = 0xC, + kId_Tar = 0xEE, + kId_GZip = 0xEF +}; + +// use another id, if you want to support other formats (zip, Xz, ...). +// DEFINE_GUID_ARC (CLSID_Format, kId_Zip) +// DEFINE_GUID_ARC (CLSID_Format, kId_BZip2) +// DEFINE_GUID_ARC (CLSID_Format, kId_Xz) +// DEFINE_GUID_ARC (CLSID_Format, kId_Tar) +// DEFINE_GUID_ARC (CLSID_Format, kId_GZip) +DEFINE_GUID_ARC (CLSID_Format, kId_7z) using namespace NWindows; using namespace NFile; using namespace NDir; +#ifdef _WIN32 #define kDllName "7z.dll" +#else +#define kDllName "7z.so" +#endif -static const char *kCopyrightString = "\n7-Zip " MY_VERSION -" (" kDllName " client) " -MY_COPYRIGHT " " MY_DATE "\n"; +static const char * const kCopyrightString = + "\n" + "7-Zip" + " (" kDllName " client)" + " " MY_VERSION + " : " MY_COPYRIGHT_DATE + "\n"; -static const char *kHelpString = -"Usage: Client7z.exe [a | l | x ] archive.7z [fileName ...]\n" +static const char * const kHelpString = +"Usage: 7zcl.exe [a | l | x] archive.7z [fileName ...]\n" "Examples:\n" -" Client7z.exe a archive.7z f1.txt f2.txt : compress two files to archive.7z\n" -" Client7z.exe l archive.7z : List contents of archive.7z\n" -" Client7z.exe x archive.7z : eXtract files from archive.7z\n"; +" 7zcl.exe a archive.7z f1.txt f2.txt : compress two files to archive.7z\n" +" 7zcl.exe l archive.7z : List contents of archive.7z\n" +" 7zcl.exe x archive.7z : eXtract files from archive.7z\n"; -static AString FStringToConsoleString(const FString &s) +static void Convert_UString_to_AString(const UString &s, AString &temp) { - return GetOemString(fs2us(s)); + int codePage = CP_OEMCP; + /* + int g_CodePage = -1; + int codePage = g_CodePage; + if (codePage == -1) + codePage = CP_OEMCP; + if (codePage == CP_UTF8) + ConvertUnicodeToUTF8(s, temp); + else + */ + UnicodeStringToMultiByte2(temp, s, (UINT)codePage); } static FString CmdStringToFString(const char *s) @@ -71,42 +103,54 @@ static FString CmdStringToFString(const char *s) return us2fs(GetUnicodeString(s)); } -static void PrintString(const UString &s) +static void Print(const char *s) +{ + fputs(s, stdout); +} + +static void Print(const AString &s) +{ + Print(s.Ptr()); +} + +static void Print(const UString &s) { - printf("%s", (LPCSTR)GetOemString(s)); + AString as; + Convert_UString_to_AString(s, as); + Print(as); } -static void PrintString(const AString &s) +static void Print(const wchar_t *s) { - printf("%s", (LPCSTR)s); + Print(UString(s)); } static void PrintNewLine() { - PrintString("\n"); + Print("\n"); } -static void PrintStringLn(const AString &s) +static void PrintStringLn(const char *s) { - PrintString(s); + Print(s); PrintNewLine(); } -static void PrintError(const char *message, const FString &name) +static void PrintError(const char *message) { - printf("Error: %s", (LPCSTR)message); + Print("Error: "); PrintNewLine(); - PrintString(FStringToConsoleString(name)); + Print(message); PrintNewLine(); } -static void PrintError(const AString &s) +static void PrintError(const char *message, const FString &name) { - PrintNewLine(); - PrintString(s); - PrintNewLine(); + PrintError(message); + Print(name); } + static HRESULT IsArchiveItemProp(IInArchive *archive, UInt32 index, PROPID propID, bool &result) { NCOM::CPropVariant prop; @@ -126,7 +170,7 @@ static HRESULT IsArchiveItemFolder(IInArchive *archive, UInt32 index, bool &resu } -static const wchar_t *kEmptyFileAlias = L"[Content]"; +static const wchar_t * const kEmptyFileAlias = L"[Content]"; ////////////////////////////////////////////////////////////// @@ -176,21 +220,106 @@ STDMETHODIMP CArchiveOpenCallback::CryptoGetTextPassword(BSTR *password) } + +static const char * const kIncorrectCommand = "incorrect command"; + ////////////////////////////////////////////////////////////// // Archive Extracting callback class -static const char *kTestingString = "Testing "; -static const char *kExtractingString = "Extracting "; -static const char *kSkippingString = "Skipping "; +static const char * const kTestingString = "Testing "; +static const char * const kExtractingString = "Extracting "; +static const char * const kSkippingString = "Skipping "; +static const char * const kReadingString = "Reading "; + +static const char * const kUnsupportedMethod = "Unsupported Method"; +static const char * const kCRCFailed = "CRC Failed"; +static const char * const kDataError = "Data Error"; +static const char * const kUnavailableData = "Unavailable data"; +static const char * const kUnexpectedEnd = "Unexpected end of data"; +static const char * const kDataAfterEnd = "There are some data after the end of the payload data"; +static const char * const kIsNotArc = "Is not archive"; +static const char * const kHeadersError = "Headers Error"; + + +struct CArcTime +{ + FILETIME FT; + UInt16 Prec; + Byte Ns100; + bool Def; + + CArcTime() + { + Clear(); + } + + void Clear() + { + FT.dwHighDateTime = FT.dwLowDateTime = 0; + Prec = 0; + Ns100 = 0; + Def = false; + } + + bool IsZero() const + { + return FT.dwLowDateTime == 0 && FT.dwHighDateTime == 0 && Ns100 == 0; + } + + int GetNumDigits() const + { + if (Prec == k_PropVar_TimePrec_Unix || + Prec == k_PropVar_TimePrec_DOS) + return 0; + if (Prec == k_PropVar_TimePrec_HighPrec) + return 9; + if (Prec == k_PropVar_TimePrec_0) + return 7; + int digits = (int)Prec - (int)k_PropVar_TimePrec_Base; + if (digits < 0) + digits = 0; + return digits; + } + + void Write_To_FiTime(CFiTime &dest) const + { + #ifdef _WIN32 + dest = FT; + #else + if (FILETIME_To_timespec(FT, dest)) + if ((Prec == k_PropVar_TimePrec_Base + 8 || + Prec == k_PropVar_TimePrec_Base + 9) + && Ns100 != 0) + { + dest.tv_nsec += Ns100; + } + #endif + } + + void Set_From_Prop(const PROPVARIANT &prop) + { + FT = prop.filetime; + unsigned prec = 0; + unsigned ns100 = 0; + const unsigned prec_Temp = prop.wReserved1; + if (prec_Temp != 0 + && prec_Temp <= k_PropVar_TimePrec_1ns + && prop.wReserved3 == 0) + { + const unsigned ns100_Temp = prop.wReserved2; + if (ns100_Temp < 100) + { + ns100 = ns100_Temp; + prec = prec_Temp; + } + } + Prec = (UInt16)prec; + Ns100 = (Byte)ns100; + Def = true; + } +}; + -static const char *kUnsupportedMethod = "Unsupported Method"; -static const char *kCRCFailed = "CRC Failed"; -static const char *kDataError = "Data Error"; -static const char *kUnavailableData = "Unavailable data"; -static const char *kUnexpectedEnd = "Unexpected end of data"; -static const char *kDataAfterEnd = "There are some data after the end of the payload data"; -static const char *kIsNotArc = "Is not archive"; -static const char *kHeadersError = "Headers Error"; class CArchiveExtractCallback: public IArchiveExtractCallback, @@ -220,11 +349,10 @@ private: bool _extractMode; struct CProcessedFileInfo { - FILETIME MTime; + CArcTime MTime; UInt32 Attrib; bool isDir; - bool AttribDefined; - bool MTimeDefined; + bool Attrib_Defined; } _processedFileInfo; COutFileStream *_outFileStreamSpec; @@ -291,32 +419,31 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, if (prop.vt == VT_EMPTY) { _processedFileInfo.Attrib = 0; - _processedFileInfo.AttribDefined = false; + _processedFileInfo.Attrib_Defined = false; } else { if (prop.vt != VT_UI4) return E_FAIL; _processedFileInfo.Attrib = prop.ulVal; - _processedFileInfo.AttribDefined = true; + _processedFileInfo.Attrib_Defined = true; } } RINOK(IsArchiveItemFolder(_archiveHandler, index, _processedFileInfo.isDir)); { + _processedFileInfo.MTime.Clear(); // Get Modified Time NCOM::CPropVariant prop; RINOK(_archiveHandler->GetProperty(index, kpidMTime, &prop)); - _processedFileInfo.MTimeDefined = false; switch (prop.vt) { case VT_EMPTY: // _processedFileInfo.MTime = _utcMTimeDefault; break; case VT_FILETIME: - _processedFileInfo.MTime = prop.filetime; - _processedFileInfo.MTimeDefined = true; + _processedFileInfo.MTime.Set_From_Prop(prop); break; default: return E_FAIL; @@ -353,7 +480,7 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, { if (!DeleteFileAlways(fullProcessedPath)) { - PrintError("Can not delete output file", fullProcessedPath); + PrintError("Cannot delete output file", fullProcessedPath); return E_ABORT; } } @@ -362,7 +489,7 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, CMyComPtr outStreamLoc(_outFileStreamSpec); if (!_outFileStreamSpec->Open(fullProcessedPath, CREATE_ALWAYS)) { - PrintError("Can not open output file", fullProcessedPath); + PrintError("Cannot open output file", fullProcessedPath); return E_ABORT; } _outFileStream = outStreamLoc; @@ -380,11 +507,14 @@ STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) }; switch (askExtractMode) { - case NArchive::NExtract::NAskMode::kExtract: PrintString(kExtractingString); break; - case NArchive::NExtract::NAskMode::kTest: PrintString(kTestingString); break; - case NArchive::NExtract::NAskMode::kSkip: PrintString(kSkippingString); break; + case NArchive::NExtract::NAskMode::kExtract: Print(kExtractingString); break; + case NArchive::NExtract::NAskMode::kTest: Print(kTestingString); break; + case NArchive::NExtract::NAskMode::kSkip: Print(kSkippingString); break; + case NArchive::NExtract::NAskMode::kReadExternal: Print(kReadingString); break; + default: + Print("??? "); break; }; - PrintString(_filePath); + Print(_filePath); return S_OK; } @@ -397,7 +527,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 operationResult) default: { NumErrors++; - PrintString(" : "); + Print(" : "); const char *s = NULL; switch (operationResult) { @@ -428,28 +558,32 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 operationResult) } if (s) { - PrintString("Error : "); - PrintString(s); + Print("Error : "); + Print(s); } else { char temp[16]; ConvertUInt32ToString(operationResult, temp); - PrintString("Error #"); - PrintString(temp); + Print("Error #"); + Print(temp); } } } if (_outFileStream) { - if (_processedFileInfo.MTimeDefined) - _outFileStreamSpec->SetMTime(&_processedFileInfo.MTime); + if (_processedFileInfo.MTime.Def) + { + CFiTime ft; + _processedFileInfo.MTime.Write_To_FiTime(ft); + _outFileStreamSpec->SetMTime(&ft); + } RINOK(_outFileStreamSpec->Close()); } _outFileStream.Release(); - if (_extractMode && _processedFileInfo.AttribDefined) - SetFileAttrib(_diskFilePath, _processedFileInfo.Attrib); + if (_extractMode && _processedFileInfo.Attrib_Defined) + SetFileAttrib_PosixHighDetect(_diskFilePath, _processedFileInfo.Attrib); PrintNewLine(); return S_OK; } @@ -473,17 +607,14 @@ STDMETHODIMP CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password) ////////////////////////////////////////////////////////////// // Archive Creating callback class -struct CDirItem +struct CDirItem: public NWindows::NFile::NFind::CFileInfoBase { - UInt64 Size; - FILETIME CTime; - FILETIME ATime; - FILETIME MTime; - UString Name; - FString FullPath; - UInt32 Attrib; + UString Path_For_Handler; + FString FullPath; // for filesystem - bool isDir() const { return (Attrib & FILE_ATTRIBUTE_DIRECTORY) != 0 ; } + CDirItem(const NWindows::NFile::NFind::CFileInfo &fi): + CFileInfoBase(fi) + {} }; class CArchiveUpdateCallback: @@ -526,7 +657,11 @@ public: FStringVector FailedFiles; CRecordVector FailedCodes; - CArchiveUpdateCallback(): PasswordIsDefined(false), AskPassword(false), DirItems(0) {}; + CArchiveUpdateCallback(): + DirItems(NULL), + PasswordIsDefined(false), + AskPassword(false) + {} ~CArchiveUpdateCallback() { Finilize(); } HRESULT Finilize(); @@ -574,16 +709,17 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR } { - const CDirItem &dirItem = (*DirItems)[index]; + const CDirItem &di = (*DirItems)[index]; switch (propID) { - case kpidPath: prop = dirItem.Name; break; - case kpidIsDir: prop = dirItem.isDir(); break; - case kpidSize: prop = dirItem.Size; break; - case kpidAttrib: prop = dirItem.Attrib; break; - case kpidCTime: prop = dirItem.CTime; break; - case kpidATime: prop = dirItem.ATime; break; - case kpidMTime: prop = dirItem.MTime; break; + case kpidPath: prop = di.Path_For_Handler; break; + case kpidIsDir: prop = di.IsDir(); break; + case kpidSize: prop = di.Size; break; + case kpidCTime: PropVariant_SetFrom_FiTime(prop, di.CTime); break; + case kpidATime: PropVariant_SetFrom_FiTime(prop, di.ATime); break; + case kpidMTime: PropVariant_SetFrom_FiTime(prop, di.MTime); break; + case kpidAttrib: prop = (UInt32)di.GetWinAttrib(); break; + case kpidPosixAttrib: prop = (UInt32)di.GetPosixAttrib(); break; } } prop.Detach(value); @@ -602,10 +738,10 @@ HRESULT CArchiveUpdateCallback::Finilize() static void GetStream2(const wchar_t *name) { - PrintString("Compressing "); + Print("Compressing "); if (name[0] == 0) name = kEmptyFileAlias; - PrintString(name); + Print(name); } STDMETHODIMP CArchiveUpdateCallback::GetStream(UInt32 index, ISequentialInStream **inStream) @@ -613,9 +749,9 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream(UInt32 index, ISequentialInStream RINOK(Finilize()); const CDirItem &dirItem = (*DirItems)[index]; - GetStream2(dirItem.Name); + GetStream2(dirItem.Path_For_Handler); - if (dirItem.isDir()) + if (dirItem.IsDir()) return S_OK; { @@ -631,7 +767,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream(UInt32 index, ISequentialInStream { PrintNewLine(); PrintError("WARNING: can't open file"); - // PrintString(NError::MyFormatMessageW(systemError)); + // Print(NError::MyFormatMessageW(systemError)); return S_FALSE; } // return sysError; @@ -665,7 +801,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetVolumeStream(UInt32 index, ISequentialOu while (res.Len() < 2) res.InsertAtFront(L'0'); UString fileName = VolName; - fileName += L'.'; + fileName += '.'; fileName += res; fileName += VolExt; COutFileStream *streamSpec = new COutFileStream; @@ -696,62 +832,115 @@ STDMETHODIMP CArchiveUpdateCallback::CryptoGetTextPassword2(Int32 *passwordIsDef // Main function +#if defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION PrintError("Unsupported Windows version"); return 1; +#endif int MY_CDECL main(int numArgs, const char *args[]) { NT_CHECK + #ifdef ENV_HAVE_LOCALE + MY_SetLocale(); + #endif + PrintStringLn(kCopyrightString); - if (numArgs < 3) + if (numArgs < 2) { PrintStringLn(kHelpString); - return 1; + return 0; } - + + FString dllPrefix; + + #ifdef _WIN32 + dllPrefix = NDLL::GetModuleDirPrefix(); + #else + { + AString s (args[0]); + int sep = s.ReverseFind_PathSepar(); + s.DeleteFrom(sep + 1); + dllPrefix = s; + } + #endif + NDLL::CLibrary lib; - if (!lib.Load(NDLL::GetModuleDirPrefix() + FTEXT(kDllName))) + if (!lib.Load(dllPrefix + FTEXT(kDllName))) { - PrintError("Can not load 7-zip library"); + PrintError("Cannot load 7-zip library"); return 1; } Func_CreateObject createObjectFunc = (Func_CreateObject)lib.GetProc("CreateObject"); if (!createObjectFunc) { - PrintError("Can not get CreateObject"); + PrintError("Cannot get CreateObject"); return 1; } - char c; + char c = 0; + UString password; + bool passwordIsDefined = false; + CObjectVector params; + + for (int curCmd = 1; curCmd < numArgs; curCmd++) { - AString command = args[1]; - if (command.Len() != 1) + AString a(args[curCmd]); + + if (!a.IsEmpty()) + { + if (a[0] == '-') + { + if (!passwordIsDefined && a[1] == 'p') + { + password = GetUnicodeString(a.Ptr(2)); + passwordIsDefined = true; + continue; + } + } + else + { + if (c) + { + params.Add(CmdStringToFString(a)); + continue; + } + if (a.Len() == 1) + { + c = (char)MyCharLower_Ascii(a[0]); + continue; + } + } + } { - PrintError("incorrect command"); + PrintError(kIncorrectCommand); return 1; } - c = (char)MyCharLower_Ascii(command[0]); } - FString archiveName = CmdStringToFString(args[2]); + if (!c || params.Size() < 1) + { + PrintError(kIncorrectCommand); + return 1; + } + + const FString &archiveName = params[0]; if (c == 'a') { // create archive command - if (numArgs < 4) + if (params.Size() < 2) { - PrintStringLn(kHelpString); + PrintError(kIncorrectCommand); return 1; } CObjectVector dirItems; { - int i; - for (i = 3; i < numArgs; i++) + unsigned i; + for (i = 1; i < params.Size(); i++) { - CDirItem di; - FString name = CmdStringToFString(args[i]); + const FString &name = params[i]; NFind::CFileInfo fi; if (!fi.Find(name)) @@ -759,13 +948,10 @@ int MY_CDECL main(int numArgs, const char *args[]) PrintError("Can't find file", name); return 1; } + + CDirItem di(fi); - di.Attrib = fi.Attrib; - di.Size = fi.Size; - di.CTime = fi.CTime; - di.ATime = fi.ATime; - di.MTime = fi.MTime; - di.Name = fs2us(name); + di.Path_For_Handler = fs2us(name); di.FullPath = name; dirItems.Add(di); } @@ -782,26 +968,28 @@ int MY_CDECL main(int numArgs, const char *args[]) CMyComPtr outArchive; if (createObjectFunc(&CLSID_Format, &IID_IOutArchive, (void **)&outArchive) != S_OK) { - PrintError("Can not get class object"); + PrintError("Cannot get class object"); return 1; } CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; CMyComPtr updateCallback(updateCallbackSpec); updateCallbackSpec->Init(&dirItems); - // updateCallbackSpec->PasswordIsDefined = true; - // updateCallbackSpec->Password = L"1"; + updateCallbackSpec->PasswordIsDefined = passwordIsDefined; + updateCallbackSpec->Password = password; /* { const wchar_t *names[] = { + L"m", L"s", L"x" }; const unsigned kNumProps = ARRAY_SIZE(names); NCOM::CPropVariant values[kNumProps] = { + L"lzma", false, // solid mode OFF (UInt32)9 // compression level = 9 - ultra }; @@ -812,7 +1000,11 @@ int MY_CDECL main(int numArgs, const char *args[]) PrintError("ISetProperties unsupported"); return 1; } - RINOK(setProperties->SetProperties(names, values, kNumProps)); + if (setProperties->SetProperties(names, values, kNumProps) != S_OK) + { + PrintError("SetProperties() error"); + return 1; + } } */ @@ -837,9 +1029,9 @@ int MY_CDECL main(int numArgs, const char *args[]) } else { - if (numArgs != 3) + if (params.Size() != 1) { - PrintStringLn(kHelpString); + PrintError(kIncorrectCommand); return 1; } @@ -851,14 +1043,14 @@ int MY_CDECL main(int numArgs, const char *args[]) listCommand = false; else { - PrintError("incorrect command"); + PrintError(kIncorrectCommand); return 1; } CMyComPtr archive; if (createObjectFunc(&CLSID_Format, &IID_IInArchive, (void **)&archive) != S_OK) { - PrintError("Can not get class object"); + PrintError("Cannot get class object"); return 1; } @@ -867,21 +1059,20 @@ int MY_CDECL main(int numArgs, const char *args[]) if (!fileSpec->Open(archiveName)) { - PrintError("Can not open archive file", archiveName); + PrintError("Cannot open archive file", archiveName); return 1; } { CArchiveOpenCallback *openCallbackSpec = new CArchiveOpenCallback; CMyComPtr openCallback(openCallbackSpec); - openCallbackSpec->PasswordIsDefined = false; - // openCallbackSpec->PasswordIsDefined = true; - // openCallbackSpec->Password = L"1"; + openCallbackSpec->PasswordIsDefined = passwordIsDefined; + openCallbackSpec->Password = password; const UInt64 scanSize = 1 << 23; if (archive->Open(file, &scanSize, openCallback) != S_OK) { - PrintError("Can not open file as archive", archiveName); + PrintError("Cannot open file as archive", archiveName); return 1; } } @@ -899,17 +1090,17 @@ int MY_CDECL main(int numArgs, const char *args[]) archive->GetProperty(i, kpidSize, &prop); char s[32]; ConvertPropVariantToShortString(prop, s); - PrintString(s); - PrintString(" "); + Print(s); + Print(" "); } { // Get name of file NCOM::CPropVariant prop; archive->GetProperty(i, kpidPath, &prop); if (prop.vt == VT_BSTR) - PrintString(prop.bstrVal); + Print(prop.bstrVal); else if (prop.vt != VT_EMPTY) - PrintString("ERROR!"); + Print("ERROR!"); } PrintNewLine(); } @@ -919,10 +1110,9 @@ int MY_CDECL main(int numArgs, const char *args[]) // Extract command CArchiveExtractCallback *extractCallbackSpec = new CArchiveExtractCallback; CMyComPtr extractCallback(extractCallbackSpec); - extractCallbackSpec->Init(archive, FTEXT("")); // second parameter is output folder path - extractCallbackSpec->PasswordIsDefined = false; - // extractCallbackSpec->PasswordIsDefined = true; - // extractCallbackSpec->Password = L"1"; + extractCallbackSpec->Init(archive, FString()); // second parameter is output folder path + extractCallbackSpec->PasswordIsDefined = passwordIsDefined; + extractCallbackSpec->Password = password; /* const wchar_t *names[] = @@ -939,7 +1129,13 @@ int MY_CDECL main(int numArgs, const char *args[]) CMyComPtr setProperties; archive->QueryInterface(IID_ISetProperties, (void **)&setProperties); if (setProperties) - setProperties->SetProperties(names, values, kNumProps); + { + if (setProperties->SetProperties(names, values, kNumProps) != S_OK) + { + PrintError("SetProperties() error"); + return 1; + } + } */ HRESULT result = archive->Extract(NULL, (UInt32)(Int32)(-1), false, extractCallback); diff --git a/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.dsp b/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.dsp index 8ad7fb133dd..b412c8e9a7d 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.dsp +++ b/3rdparty/lzma/CPP/7zip/UI/Client7z/Client7z.dsp @@ -66,7 +66,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /Yu"stdafx.h" /FD /GZ /c -# ADD CPP /nologo /W4 /WX /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /Yu"stdafx.h" /FD /GZ /c +# ADD CPP /nologo /MDd /W4 /WX /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /Yu"stdafx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe diff --git a/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile b/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile index 99a6d494778..988701eff6a 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile +++ b/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile @@ -8,9 +8,9 @@ COMMON_OBJS = \ $O\IntToString.obj \ $O\NewHandler.obj \ $O\MyString.obj \ + $O\MyVector.obj \ $O\StringConvert.obj \ $O\StringToInt.obj \ - $O\MyVector.obj \ $O\Wildcard.obj \ WIN_OBJS = \ diff --git a/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile.gcc b/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile.gcc new file mode 100644 index 00000000000..3f97205d2cf --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/UI/Client7z/makefile.gcc @@ -0,0 +1,69 @@ +PROG = 7zcl +IS_NOT_STANDALONE = 1 + +# IS_X64 = 1 + + + +ifdef SystemDrive +IS_MINGW = 1 +else +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif +endif + + +ifdef IS_MINGW + +SYS_OBJS = \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + $O/TimeUtils.o \ + +endif + + +LOCAL_FLAGS = \ + + +CURRENT_OBJS = \ + $O/Client7z.o \ + +COMMON_OBJS = \ + $O/IntToString.o \ + $O/MyString.o \ + $O/MyVector.o \ + $O/NewHandler.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + $O/Wildcard.o \ + +WIN_OBJS = \ + $O/DLL.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + +7ZIP_COMMON_OBJS = \ + $O/FileStreams.o \ + + +OBJS = \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(SYS_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(CURRENT_OBJS) \ + + +include ../../7zip_gcc.mak diff --git a/3rdparty/lzma/CPP/7zip/UI/Client7z/resource.rc b/3rdparty/lzma/CPP/7zip/UI/Client7z/resource.rc index a09bb0441ca..462df6fa752 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Client7z/resource.rc +++ b/3rdparty/lzma/CPP/7zip/UI/Client7z/resource.rc @@ -1,3 +1,3 @@ #include "../../MyVersionInfo.rc" -MY_VERSION_INFO_APP("7-Zip client", "7zcl") +MY_VERSION_INFO_APP("7-Zip client" , "7zcl") diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index 832ec66d9c5..d88c8516e01 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -8,27 +8,53 @@ #ifndef UNDER_CE #include #endif +#else +// for isatty() +#include #endif + #include +#ifdef _7ZIP_LARGE_PAGES +#include "../../../../C/Alloc.h" +#endif + +#include "../../../Common/IntToString.h" #include "../../../Common/ListFileUtils.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" +#include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" +#include "../../../Windows/System.h" #ifdef _WIN32 #include "../../../Windows/FileMapping.h" +#include "../../../Windows/MemoryLock.h" #include "../../../Windows/Synchronization.h" #endif #include "ArchiveCommandLine.h" #include "EnumDirItems.h" -#include "SortUtils.h" #include "Update.h" #include "UpdateAction.h" extern bool g_CaseSensitive; +extern bool g_PathTrailReplaceMode; + +#ifdef _7ZIP_LARGE_PAGES +extern +bool g_LargePagesMode; +bool g_LargePagesMode = false; +#endif + +/* +#ifdef ENV_HAVE_LSTAT +EXTERN_C_BEGIN +extern int global_use_lstat; +EXTERN_C_END +#endif +*/ #ifdef UNDER_CE @@ -36,13 +62,18 @@ extern bool g_CaseSensitive; #else -#if _MSC_VER >= 1400 -#define MY_isatty_fileno(x) _isatty(_fileno(x)) -#else -#define MY_isatty_fileno(x) isatty(fileno(x)) -#endif - -#define MY_IS_TERMINAL(x) (MY_isatty_fileno(x) != 0); +// #define MY_isatty_fileno(x) (isatty(fileno(x))) +// #define MY_IS_TERMINAL(x) (MY_isatty_fileno(x) != 0); +static inline bool MY_IS_TERMINAL(FILE *x) +{ + return ( + #if defined(_MSC_VER) && (_MSC_VER >= 1400) + _isatty(_fileno(x)) + #else + isatty(fileno(x)) + #endif + != 0); +} #endif @@ -59,17 +90,6 @@ static bool StringToUInt32(const wchar_t *s, UInt32 &v) return *end == 0; } -CArcCmdLineException::CArcCmdLineException(const char *a, const wchar_t *u) -{ - (*this) += MultiByteToUnicodeString(a); - if (u) - { - this->Add_LF(); - (*this) += u; - } -} - -int g_CodePage = -1; namespace NKey { enum Enum @@ -113,6 +133,8 @@ enum Enum kSfx, kEmail, kHash, + // kHashGenFile, + kHashDir, kStdIn, kStdOut, @@ -121,22 +143,34 @@ enum Enum kListfileCharSet, kConsoleCharSet, kTechMode, + kListFields, + kPreserveATime, kShareForWrite, + kStopAfterOpenError, kCaseSensitive, kArcNameMode, + kUseSlashMark, kDisableWildcardParsing, kElimDup, kFullPathMode, kHardLinks, + kSymLinks_AllowDangerous, kSymLinks, kNtSecurity, + + kStoreOwnerId, + kStoreOwnerName, + + kZoneFile, kAltStreams, kReplaceColonForAltStream, kWriteToAltStreamIfColon, + kNameTrailReplace, + kDeleteAfterCompressing, kSetArcMTime @@ -149,13 +183,13 @@ enum Enum static const wchar_t kRecursedIDChar = 'r'; -static const char *kRecursedPostCharSet = "0-"; +static const char * const kRecursedPostCharSet = "0-"; -static const char *k_ArcNameMode_PostCharSet = "sea"; +static const char * const k_ArcNameMode_PostCharSet = "sea"; -static const char *k_Stream_PostCharSet = "012"; +static const char * const k_Stream_PostCharSet = "012"; -static inline const EArcNameMode ParseArcNameMode(int postCharIndex) +static inline EArcNameMode ParseArcNameMode(int postCharIndex) { switch (postCharIndex) { @@ -173,14 +207,17 @@ namespace NRecursedPostCharIndex { }; } -static const char kImmediateNameID = '!'; -static const char kMapNameID = '#'; -static const char kFileListID = '@'; +// static const char +#define kImmediateNameID '!' +#ifdef _WIN32 +#define kMapNameID '#' +#endif +#define kFileListID '@' -static const char kSomeCludePostStringMinSize = 2; // at least <@|!>ame must be -static const char kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!>ame must be +static const Byte kSomeCludePostStringMinSize = 2; // at least <@|!>ame must be +static const Byte kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!>ame must be -static const char *kOverwritePostCharSet = "asut"; +static const char * const kOverwritePostCharSet = "asut"; static const NExtract::NOverwriteMode::EEnum k_OverwriteModes[] = { @@ -190,90 +227,116 @@ static const NExtract::NOverwriteMode::EEnum k_OverwriteModes[] = NExtract::NOverwriteMode::kRenameExisting }; + + +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL + +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_MINUS SWFRM_1(NSwitchType::kMinus) +#define SWFRM_STRING SWFRM_1(NSwitchType::kString) + +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) +#define SWFRM_STRING_MULT(mi) SWFRM_3(NSwitchType::kString, true, mi) + + static const CSwitchForm kSwitchForms[] = { - { "?" }, - { "h" }, - { "-help" }, + { "?", SWFRM_SIMPLE }, + { "h", SWFRM_SIMPLE }, + { "-help", SWFRM_SIMPLE }, - { "ba" }, - { "bd" }, - { "bt" }, - { "bb", NSwitchType::kString, false, 0 }, + { "ba", SWFRM_SIMPLE }, + { "bd", SWFRM_SIMPLE }, + { "bt", SWFRM_SIMPLE }, + { "bb", SWFRM_STRING_SINGL(0) }, { "bso", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, { "bse", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, { "bsp", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, - { "y" }, + { "y", SWFRM_SIMPLE }, - { "ad" }, + { "ad", SWFRM_SIMPLE }, { "ao", NSwitchType::kChar, false, 1, kOverwritePostCharSet}, - { "t", NSwitchType::kString, false, 1 }, - { "stx", NSwitchType::kString, true, 1 }, + { "t", SWFRM_STRING_SINGL(1) }, + { "stx", SWFRM_STRING_MULT(1) }, - { "m", NSwitchType::kString, true, 1 }, - { "o", NSwitchType::kString, false, 1 }, - { "w", NSwitchType::kString }, - - { "i", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "x", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "ai", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "ax", NSwitchType::kString, true, kSomeCludePostStringMinSize}, - { "an" }, + { "m", SWFRM_STRING_MULT(1) }, + { "o", SWFRM_STRING_SINGL(1) }, + { "w", SWFRM_STRING }, + + { "i", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "x", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "ai", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "ax", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "an", SWFRM_SIMPLE }, - { "u", NSwitchType::kString, true, 1}, - { "v", NSwitchType::kString, true, 1}, + { "u", SWFRM_STRING_MULT(1) }, + { "v", SWFRM_STRING_MULT(1) }, { "r", NSwitchType::kChar, false, 0, kRecursedPostCharSet }, - { "stm", NSwitchType::kString }, - { "sfx", NSwitchType::kString }, - { "seml", NSwitchType::kString, false, 0}, - { "scrc", NSwitchType::kString, true, 0 }, + { "stm", SWFRM_STRING }, + { "sfx", SWFRM_STRING }, + { "seml", SWFRM_STRING_SINGL(0) }, + { "scrc", SWFRM_STRING_MULT(0) }, + // { "scrf", SWFRM_STRING_SINGL(1) }, + { "shd", SWFRM_STRING_SINGL(1) }, - { "si", NSwitchType::kString }, - { "so" }, - - { "slp", NSwitchType::kMinus }, - { "scs", NSwitchType::kString }, - { "scc", NSwitchType::kString }, - { "slt" }, - - { "ssw" }, - { "ssc", NSwitchType::kMinus }, + { "si", SWFRM_STRING }, + { "so", SWFRM_SIMPLE }, + + { "slp", SWFRM_STRING }, + { "scs", SWFRM_STRING }, + { "scc", SWFRM_STRING }, + { "slt", SWFRM_SIMPLE }, + { "slf", SWFRM_STRING_SINGL(1) }, + + { "ssp", SWFRM_SIMPLE }, + { "ssw", SWFRM_SIMPLE }, + { "sse", SWFRM_SIMPLE }, + { "ssc", SWFRM_MINUS }, { "sa", NSwitchType::kChar, false, 1, k_ArcNameMode_PostCharSet }, - { "spd" }, - { "spe", NSwitchType::kMinus }, - { "spf", NSwitchType::kString, false, 0 }, + { "spm", SWFRM_STRING_SINGL(0) }, + { "spd", SWFRM_SIMPLE }, + { "spe", SWFRM_MINUS }, + { "spf", SWFRM_STRING_SINGL(0) }, - { "snh", NSwitchType::kMinus }, - { "snl", NSwitchType::kMinus }, - { "sni" }, - { "sns", NSwitchType::kMinus }, - { "snr" }, - { "snc" }, + { "snh", SWFRM_MINUS }, + { "snld", SWFRM_MINUS }, + { "snl", SWFRM_MINUS }, + { "sni", SWFRM_SIMPLE }, + + { "snoi", SWFRM_MINUS }, + { "snon", SWFRM_MINUS }, - { "sdel" }, - { "stl" } + { "snz", SWFRM_STRING_SINGL(0) }, + { "sns", SWFRM_MINUS }, + { "snr", SWFRM_SIMPLE }, + { "snc", SWFRM_SIMPLE }, + + { "snt", SWFRM_MINUS }, + + { "sdel", SWFRM_SIMPLE }, + { "stl", SWFRM_SIMPLE } #ifndef _NO_CRYPTO - , { "p", NSwitchType::kString } + , { "p", SWFRM_STRING } #endif }; -static const wchar_t *kUniversalWildcard = L"*"; +static const char * const kUniversalWildcard = "*"; static const unsigned kMinNonSwitchWords = 1; static const unsigned kCommandIndex = 0; -// static const char *kUserErrorMessage = "Incorrect command line"; -static const char *kCannotFindListFile = "Cannot find listfile"; -static const char *kIncorrectListFile = "Incorrect item in listfile.\nCheck charset encoding and -scs switch."; -static const char *kTerminalOutError = "I won't write compressed data to a terminal"; -static const char *kSameTerminalError = "I won't write data and program's messages to same stream"; -static const char *kEmptyFilePath = "Empty file path"; -static const char *kCannotFindArchive = "Cannot find archive"; +// static const char * const kUserErrorMessage = "Incorrect command line"; +// static const char * const kCannotFindListFile = "Cannot find listfile"; +static const char * const kIncorrectListFile = "Incorrect item in listfile.\nCheck charset encoding and -scs switch."; +static const char * const kTerminalOutError = "I won't write compressed data to a terminal"; +static const char * const kSameTerminalError = "I won't write data and program's messages to same stream"; +static const char * const kEmptyFilePath = "Empty file path"; bool CArcCommand::IsFromExtractGroup() const { @@ -283,8 +346,9 @@ bool CArcCommand::IsFromExtractGroup() const case NCommandType::kExtract: case NCommandType::kExtractFull: return true; + default: + return false; } - return false; } NExtract::NPathMode::EEnum CArcCommand::GetPathMode() const @@ -294,8 +358,9 @@ NExtract::NPathMode::EEnum CArcCommand::GetPathMode() const case NCommandType::kTest: case NCommandType::kExtractFull: return NExtract::NPathMode::kFullPaths; + default: + return NExtract::NPathMode::kNoPaths; } - return NExtract::NPathMode::kNoPaths; } bool CArcCommand::IsFromUpdateGroup() const @@ -307,8 +372,9 @@ bool CArcCommand::IsFromUpdateGroup() const case NCommandType::kDelete: case NCommandType::kRename: return true; + default: + return false; } - return false; } static NRecursedType::EEnum GetRecursedTypeFromIndex(int index) @@ -328,7 +394,7 @@ static const char *g_Commands = "audtexlbih"; static bool ParseArchiveCommand(const UString &commandString, CArcCommand &command) { - UString s = commandString; + UString s (commandString); s.MakeLower_Ascii(); if (s.Len() == 1) { @@ -351,12 +417,28 @@ static bool ParseArchiveCommand(const UString &commandString, CArcCommand &comma // ------------------------------------------------------------------ // filenames functions +struct CNameOption +{ + bool Include; + bool WildcardMatching; + Byte MarkMode; + NRecursedType::EEnum RecursedType; + + CNameOption(): + Include(true), + WildcardMatching(true), + MarkMode(NWildcard::kMark_FileOrDir), + RecursedType(NRecursedType::kNonRecursed) + {} +}; + + static void AddNameToCensor(NWildcard::CCensor &censor, - const UString &name, bool include, NRecursedType::EEnum type, bool wildcardMatching) + const CNameOption &nop, const UString &name) { bool recursed = false; - switch (type) + switch (nop.RecursedType) { case NRecursedType::kWildcardOnlyRecursed: recursed = DoesNameContainWildcard(name); @@ -364,8 +446,15 @@ static void AddNameToCensor(NWildcard::CCensor &censor, case NRecursedType::kRecursed: recursed = true; break; + default: + break; } - censor.AddPreItem(include, name, recursed, wildcardMatching); + + NWildcard::CCensorPathProps props; + props.Recursive = recursed; + props.WildcardMatching = nop.WildcardMatching; + props.MarkMode = nop.MarkMode; + censor.AddPreItem(nop.Include, name, props); } static void AddRenamePair(CObjectVector *renamePairs, @@ -386,9 +475,9 @@ static void AddRenamePair(CObjectVector *renamePairs, val += pair.NewName; val.Add_LF(); if (type == NRecursedType::kRecursed) - val.AddAscii("-r"); + val += "-r"; else if (type == NRecursedType::kWildcardOnlyRecursed) - val.AddAscii("-r0"); + val += "-r0"; throw CArcCmdLineException("Unsupported rename command:", val); } } @@ -396,13 +485,26 @@ static void AddRenamePair(CObjectVector *renamePairs, static void AddToCensorFromListFile( CObjectVector *renamePairs, NWildcard::CCensor &censor, - LPCWSTR fileName, bool include, NRecursedType::EEnum type, bool wildcardMatching, Int32 codePage) + const CNameOption &nop, LPCWSTR fileName, UInt32 codePage) { UStringVector names; - if (!NFind::DoesFileExist(us2fs(fileName))) + /* + if (!NFind::DoesFileExist_FollowLink(us2fs(fileName))) throw CArcCmdLineException(kCannotFindListFile, fileName); - if (!ReadNamesFromListFile(us2fs(fileName), names, codePage)) + */ + DWORD lastError = 0; + if (!ReadNamesFromListFile2(us2fs(fileName), names, codePage, lastError)) + { + if (lastError != 0) + { + UString m; + m = "The file operation error for listfile"; + m.Add_LF(); + m += NError::MyFormatMessage(lastError); + throw CArcCmdLineException(m, fileName); + } throw CArcCmdLineException(kIncorrectListFile, fileName); + } if (renamePairs) { if ((names.Size() & 1) != 0) @@ -410,55 +512,73 @@ static void AddToCensorFromListFile( for (unsigned i = 0; i < names.Size(); i += 2) { // change type !!!! - AddRenamePair(renamePairs, names[i], names[i + 1], type, wildcardMatching); + AddRenamePair(renamePairs, names[i], names[i + 1], nop.RecursedType, nop.WildcardMatching); } } else FOR_VECTOR (i, names) - AddNameToCensor(censor, names[i], include, type, wildcardMatching); + AddNameToCensor(censor, nop, names[i]); } static void AddToCensorFromNonSwitchesStrings( CObjectVector *renamePairs, unsigned startIndex, NWildcard::CCensor &censor, - const UStringVector &nonSwitchStrings, NRecursedType::EEnum type, - bool wildcardMatching, - bool thereAreSwitchIncludes, Int32 codePage) + const UStringVector &nonSwitchStrings, + int stopSwitchIndex, + const CNameOption &nop, + bool thereAreSwitchIncludes, UInt32 codePage) { + // another default if ((renamePairs || nonSwitchStrings.Size() == startIndex) && !thereAreSwitchIncludes) - AddNameToCensor(censor, kUniversalWildcard, true, type, - true // wildcardMatching - ); + { + /* for rename command: -i switch sets the mask for archive item reading. + if (thereAreSwitchIncludes), { we don't use UniversalWildcard. } + also for non-rename command: we set UniversalWildcard, only if there are no nonSwitches. */ + // we use default fileds in (CNameOption) for UniversalWildcard. + CNameOption nop2; + // recursive mode is not important for UniversalWildcard (*) + // nop2.RecursedType = nop.RecursedType; // we don't need it + /* + nop2.RecursedType = NRecursedType::kNonRecursed; + nop2.Include = true; + nop2.WildcardMatching = true; + nop2.MarkMode = NWildcard::kMark_FileOrDir; + */ + AddNameToCensor(censor, nop2, UString(kUniversalWildcard)); + } int oldIndex = -1; + if (stopSwitchIndex < 0) + stopSwitchIndex = (int)nonSwitchStrings.Size(); + for (unsigned i = startIndex; i < nonSwitchStrings.Size(); i++) { const UString &s = nonSwitchStrings[i]; if (s.IsEmpty()) throw CArcCmdLineException(kEmptyFilePath); - if (s[0] == kFileListID) - AddToCensorFromListFile(renamePairs, censor, s.Ptr(1), true, type, wildcardMatching, codePage); + if (i < (unsigned)stopSwitchIndex && s[0] == kFileListID) + AddToCensorFromListFile(renamePairs, censor, nop, s.Ptr(1), codePage); else if (renamePairs) { if (oldIndex == -1) - oldIndex = i; + oldIndex = (int)i; else { // NRecursedType::EEnum type is used for global wildcard (-i! switches) - AddRenamePair(renamePairs, nonSwitchStrings[oldIndex], s, NRecursedType::kNonRecursed, wildcardMatching); + AddRenamePair(renamePairs, nonSwitchStrings[(unsigned)oldIndex], s, NRecursedType::kNonRecursed, nop.WildcardMatching); // AddRenamePair(renamePairs, nonSwitchStrings[oldIndex], s, type); oldIndex = -1; } } else - AddNameToCensor(censor, s, true, type, wildcardMatching); + AddNameToCensor(censor, nop, s); } if (oldIndex != -1) { - throw CArcCmdLineException("There is no second file name for rename pair:", nonSwitchStrings[oldIndex]); + throw CArcCmdLineException("There is no second file name for rename pair:", nonSwitchStrings[(unsigned)oldIndex]); } } @@ -477,35 +597,34 @@ struct CEventSetEnd } }; -const char *k_IncorrectMapCommand = "Incorrect Map command"; +static const char * const k_IncorrectMapCommand = "Incorrect Map command"; static const char *ParseMapWithPaths( NWildcard::CCensor &censor, - const UString &s2, bool include, - NRecursedType::EEnum commonRecursedType, - bool wildcardMatching) + const UString &s2, + const CNameOption &nop) { - UString s = s2; + UString s (s2); int pos = s.Find(L':'); if (pos < 0) return k_IncorrectMapCommand; - int pos2 = s.Find(L':', pos + 1); + int pos2 = s.Find(L':', (unsigned)(pos + 1)); if (pos2 < 0) return k_IncorrectMapCommand; - CEventSetEnd eventSetEnd((const wchar_t *)s + ((unsigned)pos2 + 1)); - s.DeleteFrom(pos2); + CEventSetEnd eventSetEnd((const wchar_t *)s + (unsigned)(pos2 + 1)); + s.DeleteFrom((unsigned)pos2); UInt32 size; - if (!StringToUInt32(s.Ptr(pos + 1), size) + if (!StringToUInt32(s.Ptr((unsigned)(pos + 1)), size) || size < sizeof(wchar_t) || size > ((UInt32)1 << 31) || size % sizeof(wchar_t) != 0) return "Unsupported Map data size"; - s.DeleteFrom(pos); + s.DeleteFrom((unsigned)pos); CFileMapping map; if (map.Open(FILE_MAP_READ, GetSystemString(s)) != 0) - return "Can not open mapping"; + return "Cannot open mapping"; LPVOID data = map.Map(FILE_MAP_READ, 0, size); if (!data) return "MapViewOfFile error"; @@ -522,7 +641,7 @@ static const char *ParseMapWithPaths( if (c == 0) { // MessageBoxW(0, name, L"7-Zip", 0); - AddNameToCensor(censor, name, include, commonRecursedType, wildcardMatching); + AddNameToCensor(censor, nop, name); name.Empty(); } else @@ -538,17 +657,15 @@ static const char *ParseMapWithPaths( static void AddSwitchWildcardsToCensor( NWildcard::CCensor &censor, - const UStringVector &strings, bool include, - NRecursedType::EEnum commonRecursedType, - bool wildcardMatching, - Int32 codePage) + const UStringVector &strings, + const CNameOption &nop, + UInt32 codePage) { const char *errorMessage = NULL; unsigned i; for (i = 0; i < strings.Size(); i++) { const UString &name = strings[i]; - NRecursedType::EEnum recursedType; unsigned pos = 0; if (name.Len() < kSomeCludePostStringMinSize) @@ -556,37 +673,118 @@ static void AddSwitchWildcardsToCensor( errorMessage = "Too short switch"; break; } - - if (::MyCharLower_Ascii(name[pos]) == kRecursedIDChar) + + if (!nop.Include) { - pos++; - wchar_t c = name[pos]; - int index = -1; - if (c <= 0x7F) - index = FindCharPosInString(kRecursedPostCharSet, (char)c); - recursedType = GetRecursedTypeFromIndex(index); - if (index >= 0) + if (name.IsEqualTo_Ascii_NoCase("td")) + { + censor.ExcludeDirItems = true; + continue; + } + if (name.IsEqualTo_Ascii_NoCase("tf")) + { + censor.ExcludeFileItems = true; + continue; + } + } + + CNameOption nop2 = nop; + + bool type_WasUsed = false; + bool recursed_WasUsed = false; + bool matching_WasUsed = false; + bool error = false; + + for (;;) + { + wchar_t c = ::MyCharLower_Ascii(name[pos]); + if (c == kRecursedIDChar) + { + if (recursed_WasUsed) + { + error = true; + break; + } + recursed_WasUsed = true; + pos++; + c = name[pos]; + int index = -1; + if (c <= 0x7F) + index = FindCharPosInString(kRecursedPostCharSet, (char)c); + nop2.RecursedType = GetRecursedTypeFromIndex(index); + if (index >= 0) + { + pos++; + continue; + } + } + + if (c == 'w') + { + if (matching_WasUsed) + { + error = true; + break; + } + matching_WasUsed = true; + nop2.WildcardMatching = true; pos++; + if (name[pos] == '-') + { + nop2.WildcardMatching = false; + pos++; + } + } + else if (c == 'm') + { + if (type_WasUsed) + { + error = true; + break; + } + type_WasUsed = true; + pos++; + nop2.MarkMode = NWildcard::kMark_StrictFile; + c = name[pos]; + if (c == '-') + { + nop2.MarkMode = NWildcard::kMark_FileOrDir; + pos++; + } + else if (c == '2') + { + nop2.MarkMode = NWildcard::kMark_StrictFile_IfWildcard; + pos++; + } + } + else + break; } - else - recursedType = commonRecursedType; + if (error) + { + errorMessage = "inorrect switch"; + break; + } + if (name.Len() < pos + kSomeCludeAfterRecursedPostStringMinSize) { errorMessage = "Too short switch"; break; } - UString tail = name.Ptr(pos + 1); - - if (name[pos] == kImmediateNameID) - AddNameToCensor(censor, tail, include, recursedType, wildcardMatching); - else if (name[pos] == kFileListID) - AddToCensorFromListFile(NULL, censor, tail, include, recursedType, wildcardMatching, codePage); + const UString tail = name.Ptr(pos + 1); + + const wchar_t c = name[pos]; + + if (c == kImmediateNameID) + AddNameToCensor(censor, nop2, tail); + else if (c == kFileListID) + AddToCensorFromListFile(NULL, censor, nop2, tail, codePage); #ifdef _WIN32 - else if (name[pos] == kMapNameID) + else if (c == kMapNameID) { - errorMessage = ParseMapWithPaths(censor, tail, include, recursedType, wildcardMatching); + errorMessage = ParseMapWithPaths(censor, tail, nop2); if (errorMessage) break; } @@ -597,88 +795,11 @@ static void AddSwitchWildcardsToCensor( break; } } + if (i != strings.Size()) throw CArcCmdLineException(errorMessage, strings[i]); } -#ifdef _WIN32 - -// This code converts all short file names to long file names. - -static void ConvertToLongName(const UString &prefix, UString &name) -{ - if (name.IsEmpty() || DoesNameContainWildcard(name)) - return; - NFind::CFileInfo fi; - const FString path = us2fs(prefix + name); - #ifndef UNDER_CE - if (NFile::NName::IsDevicePath(path)) - return; - #endif - if (fi.Find(path)) - name = fs2us(fi.Name); -} - -static void ConvertToLongNames(const UString &prefix, CObjectVector &items) -{ - FOR_VECTOR (i, items) - { - NWildcard::CItem &item = items[i]; - if (item.Recursive || item.PathParts.Size() != 1) - continue; - if (prefix.IsEmpty() && item.IsDriveItem()) - continue; - ConvertToLongName(prefix, item.PathParts.Front()); - } -} - -static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) -{ - ConvertToLongNames(prefix, node.IncludeItems); - ConvertToLongNames(prefix, node.ExcludeItems); - unsigned i; - for (i = 0; i < node.SubNodes.Size(); i++) - { - UString &name = node.SubNodes[i].Name; - if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) - continue; - ConvertToLongName(prefix, name); - } - // mix folders with same name - for (i = 0; i < node.SubNodes.Size(); i++) - { - NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; - for (unsigned j = i + 1; j < node.SubNodes.Size();) - { - const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; - if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) - { - nextNode1.IncludeItems += nextNode2.IncludeItems; - nextNode1.ExcludeItems += nextNode2.ExcludeItems; - node.SubNodes.Delete(j); - } - else - j++; - } - } - for (i = 0; i < node.SubNodes.Size(); i++) - { - NWildcard::CCensorNode &nextNode = node.SubNodes[i]; - ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); - } -} - -void ConvertToLongNames(NWildcard::CCensor &censor) -{ - FOR_VECTOR (i, censor.Pairs) - { - NWildcard::CPair &pair = censor.Pairs[i]; - ConvertToLongNames(pair.Prefix, pair.Head); - } -} - -#endif - /* static NUpdateArchive::NPairAction::EEnum GetUpdatePairActionType(int i) { @@ -693,11 +814,11 @@ static NUpdateArchive::NPairAction::EEnum GetUpdatePairActionType(int i) } */ -static const wchar_t *kUpdatePairStateIDSet = L"pqrxyzw"; +static const char * const kUpdatePairStateIDSet = "pqrxyzw"; static const int kUpdatePairStateNotSupportedActions[] = {2, 2, 1, -1, -1, -1, -1}; static const unsigned kNumUpdatePairActions = 4; -static const char *kUpdateIgnoreItselfPostStringID = "-"; +static const char * const kUpdateIgnoreItselfPostStringID = "-"; static const wchar_t kUpdateNewArchivePostCharID = '!'; @@ -707,8 +828,8 @@ static bool ParseUpdateCommandString2(const UString &command, for (unsigned i = 0; i < command.Len();) { wchar_t c = MyCharLower_Ascii(command[i]); - int statePos = FindCharPosInString(kUpdatePairStateIDSet, c); - if (statePos < 0) + int statePos = FindCharPosInString(kUpdatePairStateIDSet, (char)c); + if (c > 0x7F || statePos < 0) { postString = command.Ptr(i); return true; @@ -717,9 +838,9 @@ static bool ParseUpdateCommandString2(const UString &command, if (i >= command.Len()) return false; c = command[i]; - if (c < '0' || c >= '0' + kNumUpdatePairActions) + if (c < '0' || c >= (wchar_t)('0' + kNumUpdatePairActions)) return false; - unsigned actionPos = c - '0'; + unsigned actionPos = (unsigned)(c - '0'); actionSet.StateActions[(unsigned)statePos] = (NUpdateArchive::NPairAction::EEnum)(actionPos); if (kUpdatePairStateNotSupportedActions[(unsigned)statePos] == (int)actionPos) return false; @@ -841,26 +962,37 @@ static void SetMethodOptions(const CParser &parser, CObjectVector &pr int index = prop.Name.Find(L'='); if (index >= 0) { - prop.Value = prop.Name.Ptr(index + 1); - prop.Name.DeleteFrom(index); + prop.Value = prop.Name.Ptr((unsigned)(index + 1)); + prop.Name.DeleteFrom((unsigned)index); } properties.Add(prop); } } } -CArcCmdLineParser::CArcCmdLineParser(): parser(ARRAY_SIZE(kSwitchForms)) {} static inline void SetStreamMode(const CSwitchResult &sw, unsigned &res) { if (sw.ThereIs) - res = sw.PostCharIndex; + res = (unsigned)sw.PostCharIndex; } + +#if defined(_WIN32) && !defined(UNDER_CE) +static void PrintHex(UString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} +#endif + + void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, CArcCmdLineOptions &options) { - if (!parser.ParseStrings(kSwitchForms, commandStrings)) + Parse1Log.Empty(); + if (!parser.ParseStrings(kSwitchForms, ARRAY_SIZE(kSwitchForms), commandStrings)) throw CArcCmdLineException(parser.ErrorMessage, parser.ErrorLine); options.IsInTerminal = MY_IS_TERMINAL(stdin); @@ -872,6 +1004,11 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, options.StdInMode = parser[NKey::kStdIn].ThereIs; options.StdOutMode = parser[NKey::kStdOut].ThereIs; options.EnableHeaders = !parser[NKey::kDisableHeaders].ThereIs; + if (parser[NKey::kListFields].ThereIs) + { + const UString &s = parser[NKey::kListFields].PostStrings[0]; + options.ListFields = GetAnsiString(s); + } options.TechMode = parser[NKey::kTechMode].ThereIs; options.ShowTime = parser[NKey::kShowTime].ThereIs; @@ -903,14 +1040,52 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, if (parser[NKey::kCaseSensitive].ThereIs) { + options.CaseSensitive = g_CaseSensitive = !parser[NKey::kCaseSensitive].WithMinus; - options.CaseSensitiveChange = true; - options.CaseSensitive = g_CaseSensitive; + options.CaseSensitive_Change = true; } - options.LargePages = false; + + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_SymLink(); + #endif + + // options.LargePages = false; + if (parser[NKey::kLargePages].ThereIs) - options.LargePages = !parser[NKey::kLargePages].WithMinus; + { + unsigned slp = 0; + const UString &s = parser[NKey::kLargePages].PostStrings[0]; + if (s.IsEmpty()) + slp = 1; + else if (s != L"-") + { + if (!StringToUInt32(s, slp)) + throw CArcCmdLineException("Unsupported switch postfix for -slp", s); + } + + #ifdef _7ZIP_LARGE_PAGES + if (slp > + #if defined(_WIN32) && !defined(UNDER_CE) + (unsigned)NSecurity::Get_LargePages_RiskLevel() + #else + 0 + #endif + ) + { + #ifdef _WIN32 // change it ! + SetLargePageSize(); + #endif + // note: this process also can inherit that Privilege from parent process + g_LargePagesMode = + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_LockMemory(); + #else + true; + #endif + } + #endif + } #ifndef UNDER_CE @@ -920,32 +1095,83 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, const UString &s = parser[NKey::kAffinity].PostStrings[0]; if (!s.IsEmpty()) { - UInt32 v = 0; AString a; a.SetFromWStr_if_Ascii(s); - if (!a.IsEmpty()) + Parse1Log += "Set process affinity mask: "; + + #ifdef _WIN32 + + UInt64 v = 0; { const char *end; - v = ConvertHexStringToUInt32(a, &end); + v = ConvertHexStringToUInt64(a, &end); if (*end != 0) a.Empty(); } if (a.IsEmpty()) throw CArcCmdLineException("Unsupported switch postfix -stm", s); + + { + #ifndef _WIN64 + if (v >= ((UInt64)1 << 32)) + throw CArcCmdLineException("unsupported value -stm", s); + #endif + { + PrintHex(Parse1Log, v); + if (!SetProcessAffinityMask(GetCurrentProcess(), (DWORD_PTR)v)) + { + DWORD lastError = GetLastError(); + Parse1Log += " : ERROR : "; + Parse1Log += NError::MyFormatMessage(lastError); + } + } + } - #ifdef _WIN32 - SetProcessAffinityMask(GetCurrentProcess(), v); - #endif + #else // _WIN32 + + { + Parse1Log += a; + NSystem::CProcessAffinity aff; + aff.CpuZero(); + for (unsigned i = 0; i < a.Len(); i++) + { + char c = a[i]; + unsigned v; + if (c >= '0' && c <= '9') v = (unsigned)(c - '0'); + else if (c >= 'A' && c <= 'F') v = 10 + (unsigned)(c - 'A'); + else if (c >= 'a' && c <= 'f') v = 10 + (unsigned)(c - 'a'); + else + throw CArcCmdLineException("Unsupported switch postfix -stm", s); + for (unsigned k = 0; k < 4; k++) + { + const unsigned cpu = (a.Len() - 1 - i) * 4 + k; + if (v & ((unsigned)1 << k)) + aff.CpuSet(cpu); + } + } + + if (!aff.SetProcAffinity()) + { + DWORD lastError = GetLastError(); + Parse1Log += " : ERROR : "; + Parse1Log += NError::MyFormatMessage(lastError); + } + } + #endif // _WIN32 + + Parse1Log.Add_LF(); } } #endif } + + struct CCodePagePair { const char *Name; - Int32 CodePage; + UInt32 CodePage; }; static const unsigned kNumByteOnlyCodePages = 3; @@ -965,7 +1191,7 @@ static Int32 FindCharset(const NCommandLineParser::CParser &parser, unsigned key if (!parser[keyIndex].ThereIs) return defaultVal; - UString name = parser[keyIndex].PostStrings.Back(); + UString name (parser[keyIndex].PostStrings.Back()); UInt32 v; if (StringToUInt32(name, v)) if (v < ((UInt32)1 << 16)) @@ -978,68 +1204,10 @@ static Int32 FindCharset(const NCommandLineParser::CParser &parser, unsigned key throw CArcCmdLineException("Unsupported charset:", name); const CCodePagePair &pair = g_CodePagePairs[i]; if (name.IsEqualTo(pair.Name)) - return pair.CodePage; + return (Int32)pair.CodePage; } } -HRESULT EnumerateDirItemsAndSort( - NWildcard::CCensor &censor, - NWildcard::ECensorPathMode censorPathMode, - const UString &addPathPrefix, - UStringVector &sortedPaths, - UStringVector &sortedFullPaths, - CDirItemsStat &st, - IDirItemsCallback *callback) -{ - FStringVector paths; - - { - CDirItems dirItems; - dirItems.Callback = callback; - { - HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); - st = dirItems.Stat; - RINOK(res); - } - - FOR_VECTOR (i, dirItems.Items) - { - const CDirItem &dirItem = dirItems.Items[i]; - if (!dirItem.IsDir()) - paths.Add(dirItems.GetPhyPath(i)); - } - } - - if (paths.Size() == 0) - throw CArcCmdLineException(kCannotFindArchive); - - UStringVector fullPaths; - - unsigned i; - - for (i = 0; i < paths.Size(); i++) - { - FString fullPath; - NFile::NDir::MyGetFullPathName(paths[i], fullPath); - fullPaths.Add(fs2us(fullPath)); - } - - CUIntVector indices; - SortFileNames(fullPaths, indices); - sortedPaths.ClearAndReserve(indices.Size()); - sortedFullPaths.ClearAndReserve(indices.Size()); - - for (i = 0; i < indices.Size(); i++) - { - unsigned index = indices[i]; - sortedPaths.AddInReserved(fs2us(paths[index])); - sortedFullPaths.AddInReserved(fullPaths[index]); - if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) - throw CArcCmdLineException("Duplicate archive path:", sortedFullPaths[i]); - } - - return S_OK; -} static void SetBoolPair(NCommandLineParser::CParser &parser, unsigned switchID, CBoolPair &bp) { @@ -1051,7 +1219,7 @@ static void SetBoolPair(NCommandLineParser::CParser &parser, unsigned switchID, void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) { const UStringVector &nonSwitchStrings = parser.NonSwitchStrings; - unsigned numNonSwitchStrings = nonSwitchStrings.Size(); + const unsigned numNonSwitchStrings = nonSwitchStrings.Size(); if (numNonSwitchStrings < kMinNonSwitchWords) throw CArcCmdLineException("The command must be specified"); @@ -1060,6 +1228,27 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kHash].ThereIs) options.HashMethods = parser[NKey::kHash].PostStrings; + + /* + if (parser[NKey::kHashGenFile].ThereIs) + { + const UString &s = parser[NKey::kHashGenFile].PostStrings[0]; + for (unsigned i = 0 ; i < s.Len();) + { + const wchar_t c = s[i++]; + if (!options.HashOptions.ParseFlagCharOption(c, true)) + { + if (c != '=') + throw CArcCmdLineException("Unsupported hash mode switch:", s); + options.HashOptions.HashFilePath = s.Ptr(i); + break; + } + } + } + */ + + if (parser[NKey::kHashDir].ThereIs) + options.ExtractOptions.HashDir = parser[NKey::kHashDir].PostStrings[0]; if (parser[NKey::kElimDup].ThereIs) { @@ -1082,31 +1271,51 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) } } - NRecursedType::EEnum recursedType; + if (parser[NKey::kNameTrailReplace].ThereIs) + g_PathTrailReplaceMode = !parser[NKey::kNameTrailReplace].WithMinus; + + CNameOption nop; + if (parser[NKey::kRecursed].ThereIs) - recursedType = GetRecursedTypeFromIndex(parser[NKey::kRecursed].PostCharIndex); - else - recursedType = NRecursedType::kNonRecursed; + nop.RecursedType = GetRecursedTypeFromIndex(parser[NKey::kRecursed].PostCharIndex); - bool wildcardMatching = true; if (parser[NKey::kDisableWildcardParsing].ThereIs) - wildcardMatching = false; + nop.WildcardMatching = false; + + if (parser[NKey::kUseSlashMark].ThereIs) + { + const UString &s = parser[NKey::kUseSlashMark].PostStrings[0]; + if (s.IsEmpty()) + nop.MarkMode = NWildcard::kMark_StrictFile; + else if (s.IsEqualTo_Ascii_NoCase("-")) + nop.MarkMode = NWildcard::kMark_FileOrDir; + else if (s.IsEqualTo_Ascii_NoCase("2")) + nop.MarkMode = NWildcard::kMark_StrictFile_IfWildcard; + else + throw CArcCmdLineException("Unsupported -spm:", s); + } - g_CodePage = FindCharset(parser, NKey::kConsoleCharSet, true, -1); - Int32 codePage = FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); + + options.ConsoleCodePage = FindCharset(parser, NKey::kConsoleCharSet, true, -1); + + UInt32 codePage = (UInt32)FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); bool thereAreSwitchIncludes = false; - + if (parser[NKey::kInclude].ThereIs) { thereAreSwitchIncludes = true; + nop.Include = true; AddSwitchWildcardsToCensor(options.Censor, - parser[NKey::kInclude].PostStrings, true, recursedType, wildcardMatching, codePage); + parser[NKey::kInclude].PostStrings, nop, codePage); } if (parser[NKey::kExclude].ThereIs) + { + nop.Include = false; AddSwitchWildcardsToCensor(options.Censor, - parser[NKey::kExclude].PostStrings, false, recursedType, wildcardMatching, codePage); + parser[NKey::kExclude].PostStrings, nop, codePage); + } unsigned curCommandIndex = kCommandIndex + 1; bool thereIsArchiveName = !parser[NKey::kNoArName].ThereIs && @@ -1114,9 +1323,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) options.Command.CommandType != NCommandType::kInfo && options.Command.CommandType != NCommandType::kHash; - bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); - bool isExtractOrList = isExtractGroupCommand || options.Command.CommandType == NCommandType::kList; - bool isRename = options.Command.CommandType == NCommandType::kRename; + const bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); + const bool isExtractOrList = isExtractGroupCommand || options.Command.CommandType == NCommandType::kList; + const bool isRename = options.Command.CommandType == NCommandType::kRename; if ((isExtractOrList || isRename) && options.StdInMode) thereIsArchiveName = false; @@ -1136,9 +1345,11 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) #endif } + nop.Include = true; AddToCensorFromNonSwitchesStrings(isRename ? &options.UpdateOptions.RenamePairs : NULL, curCommandIndex, options.Censor, - nonSwitchStrings, recursedType, wildcardMatching, + nonSwitchStrings, parser.StopSwitchIndex, + nop, thereAreSwitchIncludes, codePage); options.YesToAll = parser[NKey::kYes].ThereIs; @@ -1164,11 +1375,41 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) SetBoolPair(parser, NKey::kAltStreams, options.AltStreams); SetBoolPair(parser, NKey::kHardLinks, options.HardLinks); SetBoolPair(parser, NKey::kSymLinks, options.SymLinks); + + SetBoolPair(parser, NKey::kStoreOwnerId, options.StoreOwnerId); + SetBoolPair(parser, NKey::kStoreOwnerName, options.StoreOwnerName); + + CBoolPair symLinks_AllowDangerous; + SetBoolPair(parser, NKey::kSymLinks_AllowDangerous, symLinks_AllowDangerous); + + + /* + bool supportSymLink = options.SymLinks.Val; + + if (!options.SymLinks.Def) + { + if (isExtractOrList) + supportSymLink = true; + else + supportSymLink = false; + } + + #ifdef ENV_HAVE_LSTAT + if (supportSymLink) + global_use_lstat = 1; + else + global_use_lstat = 0; + #endif + */ + if (isExtractOrList) { CExtractOptionsBase &eo = options.ExtractOptions; + eo.ExcludeDirItems = options.Censor.ExcludeDirItems; + eo.ExcludeFileItems = options.Censor.ExcludeFileItems; + { CExtractNtOptions &nt = eo.NtOptions; nt.NtSecurity = options.NtSecurity; @@ -1185,10 +1426,33 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (!options.SymLinks.Def) nt.SymLinks.Val = true; + nt.SymLinks_AllowDangerous = symLinks_AllowDangerous; + nt.ReplaceColonForAltStream = parser[NKey::kReplaceColonForAltStream].ThereIs; nt.WriteToAltStreamIfColon = parser[NKey::kWriteToAltStreamIfColon].ThereIs; + + nt.ExtractOwner = options.StoreOwnerId.Val; // StoreOwnerName + + if (parser[NKey::kPreserveATime].ThereIs) + nt.PreserveATime = true; + if (parser[NKey::kShareForWrite].ThereIs) + nt.OpenShareForWrite = true; } - + + if (parser[NKey::kZoneFile].ThereIs) + { + eo.ZoneMode = NExtract::NZoneIdMode::kAll; + const UString &s = parser[NKey::kZoneFile].PostStrings[0]; + if (!s.IsEmpty()) + { + if (s == L"0") eo.ZoneMode = NExtract::NZoneIdMode::kNone; + else if (s == L"1") eo.ZoneMode = NExtract::NZoneIdMode::kAll; + else if (s == L"2") eo.ZoneMode = NExtract::NZoneIdMode::kOffice; + else + throw CArcCmdLineException("Unsupported -snz:", s); + } + } + options.Censor.AddPathsToCensor(NWildcard::k_AbsPath); options.Censor.ExtendExclude(); @@ -1198,13 +1462,28 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) NWildcard::CCensor &arcCensor = options.arcCensor; + CNameOption nopArc; + // nopArc.RecursedType = NRecursedType::kNonRecursed; // default: we don't want recursing for archives, if -r specified + // is it OK, external switches can disable WildcardMatching and MarcMode for arc. + nopArc.WildcardMatching = nop.WildcardMatching; + nopArc.MarkMode = nop.MarkMode; + if (parser[NKey::kArInclude].ThereIs) - AddSwitchWildcardsToCensor(arcCensor, parser[NKey::kArInclude].PostStrings, true, NRecursedType::kNonRecursed, wildcardMatching, codePage); + { + nopArc.Include = true; + AddSwitchWildcardsToCensor(arcCensor, parser[NKey::kArInclude].PostStrings, nopArc, codePage); + } if (parser[NKey::kArExclude].ThereIs) - AddSwitchWildcardsToCensor(arcCensor, parser[NKey::kArExclude].PostStrings, false, NRecursedType::kNonRecursed, wildcardMatching, codePage); + { + nopArc.Include = false; + AddSwitchWildcardsToCensor(arcCensor, parser[NKey::kArExclude].PostStrings, nopArc, codePage); + } if (thereIsArchiveName) - AddNameToCensor(arcCensor, options.ArchiveName, true, NRecursedType::kNonRecursed, wildcardMatching); + { + nopArc.Include = true; + AddNameToCensor(arcCensor, nopArc, options.ArchiveName); + } arcCensor.AddPathsToCensor(NWildcard::k_RelatPath); @@ -1242,6 +1521,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kOutputDir].ThereIs) { eo.OutputDir = us2fs(parser[NKey::kOutputDir].PostStrings[0]); + #ifdef _WIN32 + NFile::NName::NormalizeDirSeparators(eo.OutputDir); + #endif NFile::NName::NormalizeDirPathPrefix(eo.OutputDir); } @@ -1281,8 +1563,12 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) updateOptions.MethodMode.Properties = options.Properties; + if (parser[NKey::kPreserveATime].ThereIs) + updateOptions.PreserveATime = true; if (parser[NKey::kShareForWrite].ThereIs) updateOptions.OpenShareForWrite = true; + if (parser[NKey::kStopAfterOpenError].ThereIs) + updateOptions.StopAfterOpenError = true; updateOptions.PathMode = censorPathMode; @@ -1290,6 +1576,9 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) updateOptions.NtSecurity = options.NtSecurity; updateOptions.HardLinks = options.HardLinks; updateOptions.SymLinks = options.SymLinks; + + updateOptions.StoreOwnerId = options.StoreOwnerId; + updateOptions.StoreOwnerName = options.StoreOwnerName; updateOptions.EMailMode = parser[NKey::kEmail].ThereIs; if (updateOptions.EMailMode) @@ -1333,11 +1622,13 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) else if (options.Command.CommandType == NCommandType::kBenchmark) { options.NumIterations = 1; + options.NumIterations_Defined = false; if (curCommandIndex < numNonSwitchStrings) { if (!StringToUInt32(nonSwitchStrings[curCommandIndex], options.NumIterations)) - throw CArcCmdLineException("Incorrect Number of benmchmark iterations", nonSwitchStrings[curCommandIndex]); + throw CArcCmdLineException("Incorrect number of benchmark iterations", nonSwitchStrings[curCommandIndex]); curCommandIndex++; + options.NumIterations_Defined = true; } } else if (options.Command.CommandType == NCommandType::kHash) @@ -1348,10 +1639,14 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) CHashOptions &hashOptions = options.HashOptions; hashOptions.PathMode = censorPathMode; hashOptions.Methods = options.HashMethods; + // hashOptions.HashFilePath = options.HashFilePath; + if (parser[NKey::kPreserveATime].ThereIs) + hashOptions.PreserveATime = true; if (parser[NKey::kShareForWrite].ThereIs) hashOptions.OpenShareForWrite = true; hashOptions.StdInMode = options.StdInMode; hashOptions.AltStreamsMode = options.AltStreams.Val; + hashOptions.SymLinks = options.SymLinks; } else if (options.Command.CommandType == NCommandType::kInfo) { @@ -1359,3 +1654,45 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) else throw 20150919; } + + + +#ifndef _WIN32 + +static AString g_ModuleDirPrefix; + +void Set_ModuleDirPrefix_From_ProgArg0(const char *s); +void Set_ModuleDirPrefix_From_ProgArg0(const char *s) +{ + AString a (s); + int sep = a.ReverseFind_PathSepar(); + a.DeleteFrom((unsigned)(sep + 1)); + g_ModuleDirPrefix = a; +} + +namespace NWindows { +namespace NDLL { + +FString GetModuleDirPrefix(); +FString GetModuleDirPrefix() +{ + FString s; + + s = fas2fs(g_ModuleDirPrefix); + if (s.IsEmpty()) + s = FTEXT(".") FSTRING_PATH_SEPARATOR; + return s; + /* + setenv("_7ZIP_HOME_DIR", "/test/", 0); + const char *home = getenv("_7ZIP_HOME_DIR"); + if (home) + s = home; + else + s = FTEXT(".") FSTRING_PATH_SEPARATOR; + return s; + */ +} + +}} + +#endif // ! _WIN32 diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.h b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.h index a345505cb84..745f999dd7c 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveCommandLine.h @@ -6,14 +6,13 @@ #include "../../../Common/CommandLineParser.h" #include "../../../Common/Wildcard.h" +#include "EnumDirItems.h" + #include "Extract.h" #include "HashCalc.h" #include "Update.h" -struct CArcCmdLineException: public UString -{ - CArcCmdLineException(const char *a, const wchar_t *u = NULL); -}; +typedef CMessagePathException CArcCmdLineException; namespace NCommandType { enum EEnum { @@ -51,8 +50,8 @@ struct CArcCmdLineOptions { bool HelpMode; - bool LargePages; - bool CaseSensitiveChange; + // bool LargePages; + bool CaseSensitive_Change; bool CaseSensitive; bool IsInTerminal; @@ -64,6 +63,13 @@ struct CArcCmdLineOptions bool YesToAll; bool ShowDialog; + bool TechMode; + bool ShowTime; + + AString ListFields; + + int ConsoleCodePage; + NWildcard::CCensor Censor; CArcCommand Command; @@ -74,10 +80,8 @@ struct CArcCmdLineOptions UString Password; #endif - bool TechMode; - bool ShowTime; - UStringVector HashMethods; + // UString HashFilePath; bool AppendName; // UStringVector ArchivePathsSorted; @@ -93,6 +97,9 @@ struct CArcCmdLineOptions CBoolPair AltStreams; CBoolPair HardLinks; CBoolPair SymLinks; + + CBoolPair StoreOwnerId; + CBoolPair StoreOwnerName; CUpdateOptions UpdateOptions; CHashOptions HashOptions; @@ -108,15 +115,30 @@ struct CArcCmdLineOptions // Benchmark UInt32 NumIterations; + bool NumIterations_Defined; CArcCmdLineOptions(): - LargePages(false), - CaseSensitiveChange(false), + HelpMode(false), + // LargePages(false), + CaseSensitive_Change(false), CaseSensitive(false), + IsInTerminal(false), + IsStdOutTerminal(false), + IsStdErrTerminal(false), + StdInMode(false), StdOutMode(false), + EnableHeaders(false), + + YesToAll(false), + ShowDialog(false), + TechMode(false), + ShowTime(false), + + ConsoleCodePage(-1), + Number_for_Out(k_OutStream_stdout), Number_for_Errors(k_OutStream_stderr), Number_for_Percents(k_OutStream_stdout), @@ -130,18 +152,9 @@ class CArcCmdLineParser { NCommandLineParser::CParser parser; public: - CArcCmdLineParser(); + UString Parse1Log; void Parse1(const UStringVector &commandStrings, CArcCmdLineOptions &options); void Parse2(CArcCmdLineOptions &options); }; -HRESULT EnumerateDirItemsAndSort( - NWildcard::CCensor &censor, - NWildcard::ECensorPathMode pathMode, - const UString &addPathPrefix, - UStringVector &sortedPaths, - UStringVector &sortedFullPaths, - CDirItemsStat &st, - IDirItemsCallback *callback); - #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index bc116c93d66..a574f136e7c 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -5,11 +5,17 @@ #undef sprintf #undef printf +// #include +// #include "../../../../C/CpuTicks.h" + #include "../../../../C/Alloc.h" +#include "../../../../C/CpuArch.h" + #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" +#include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" #include "../../../Windows/ErrorMsg.h" @@ -25,7 +31,7 @@ #endif #include "../../Common/FilePathAutoRename.h" -// #include "../../Common/StreamUtils.h" +#include "../../Common/StreamUtils.h" #include "../Common/ExtractingFilePath.h" #include "../Common/PropIDUtils.h" @@ -36,11 +42,17 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static const char *kCantAutoRename = "Can not create file with auto name"; -static const char *kCantRenameFile = "Can not rename existing file"; -static const char *kCantDeleteOutputFile = "Can not delete output file"; -static const char *kCantDeleteOutputDir = "Can not delete output folder"; - +static const char * const kCantAutoRename = "Cannot create file with auto name"; +static const char * const kCantRenameFile = "Cannot rename existing file"; +static const char * const kCantDeleteOutputFile = "Cannot delete output file"; +static const char * const kCantDeleteOutputDir = "Cannot delete output folder"; +static const char * const kCantOpenOutFile = "Cannot open output file"; +static const char * const kCantOpenInFile = "Cannot open input file"; +static const char * const kCantSetFileLen = "Cannot set length for output file"; +#ifdef SUPPORT_LINKS +static const char * const kCantCreateHardLink = "Cannot create hard link"; +static const char * const kCantCreateSymLink = "Cannot create symbolic link"; +#endif #ifndef _SFX @@ -57,9 +69,11 @@ STDMETHODIMP COutStreamWithHash::Write(const void *data, UInt32 size, UInt32 *pr return result; } -#endif +#endif // _SFX + #ifdef _USE_SECURITY_CODE +bool InitLocalPrivileges(); bool InitLocalPrivileges() { NSecurity::CAccessToken token; @@ -78,8 +92,86 @@ bool InitLocalPrivileges() return false; return (GetLastError() == ERROR_SUCCESS); } +#endif // _USE_SECURITY_CODE + + + +#if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX) + +static const char * const kOfficeExtensions = + " doc dot wbk" + " docx docm dotx dotm docb wll wwl" + " xls xlt xlm" + " xlsx xlsm xltx xltm xlsb xla xlam" + " ppt pot pps ppa ppam" + " pptx pptm potx potm ppam ppsx ppsm sldx sldm" + " "; + +static bool FindExt2(const char *p, const UString &name) +{ + const int pathPos = name.ReverseFind_PathSepar(); + const int dotPos = name.ReverseFind_Dot(); + if (dotPos < 0 + || dotPos < pathPos + || dotPos == (int)name.Len() - 1) + return false; + + AString s; + for (unsigned pos = dotPos + 1;; pos++) + { + const wchar_t c = name[pos]; + if (c <= 0) + break; + if (c >= 0x80) + return false; + s += (char)MyCharLower_Ascii((char)c); + } + for (unsigned i = 0; p[i] != 0;) + { + unsigned j; + for (j = i; p[j] != ' '; j++); + if (s.Len() == j - i && memcmp(p + i, (const char *)s, s.Len()) == 0) + return true; + i = j + 1; + } + return false; +} + + +static const FChar * const k_ZoneId_StreamName = FTEXT(":Zone.Identifier"); + +void ReadZoneFile_Of_BaseFile(CFSTR fileName2, CByteBuffer &buf) +{ + FString fileName = fileName2; + fileName += k_ZoneId_StreamName; + + buf.Free(); + NIO::CInFile file; + if (!file.Open(fileName)) + return; + UInt64 fileSize; + if (!file.GetLength(fileSize)) + return; + if (fileSize == 0 || fileSize >= ((UInt32)1 << 16)) + return; + buf.Alloc((size_t)fileSize); + size_t processed; + if (file.ReadFull(buf, (size_t)fileSize, processed) && processed == fileSize) + return; + buf.Free(); +} + +static bool WriteZoneFile(CFSTR fileName, const CByteBuffer &buf) +{ + NIO::COutFile file; + if (!file.Create(fileName, true)) + return false; + return file.WriteFull(buf, buf.Size()); +} + #endif + #ifdef SUPPORT_LINKS int CHardLinkNode::Compare(const CHardLinkNode &a) const @@ -149,7 +241,7 @@ HRESULT CArchiveExtractCallback::PrepareHardLinks(const CRecordVector *r hardIDs.Sort2(); { - // wee keep only items that have 2 or more items + // we keep only items that have 2 or more items unsigned k = 0; unsigned numSame = 1; for (unsigned i = 1; i < hardIDs.Size(); i++) @@ -170,12 +262,14 @@ HRESULT CArchiveExtractCallback::PrepareHardLinks(const CRecordVector *r return S_OK; } -#endif +#endif // SUPPORT_LINKS + CArchiveExtractCallback::CArchiveExtractCallback(): - WriteCTime(true), - WriteATime(true), - WriteMTime(true), + _arc(NULL), + Write_CTime(true), + Write_ATime(true), + Write_MTime(true), _multiArchives(false) { LocalProgressSpec = new CLocalProgress(); @@ -186,6 +280,14 @@ CArchiveExtractCallback::CArchiveExtractCallback(): #endif } + +void CArchiveExtractCallback::InitBeforeNewArchive() +{ + #if defined(_WIN32) && !defined(UNDER_CE) + ZoneBuf.Free(); + #endif +} + void CArchiveExtractCallback::Init( const CExtractNtOptions &ntOptions, const NWildcard::CCensorNode *wildcardCensor, @@ -196,8 +298,9 @@ void CArchiveExtractCallback::Init( const UStringVector &removePathParts, bool removePartsForAltStreams, UInt64 packSize) { - _extractedFolderPaths.Clear(); - _extractedFolderIndices.Clear(); + ClearExtractedDirsInfo(); + _outFileStream.Release(); + _bufPtrSeqOutStream.Release(); #ifdef SUPPORT_LINKS _hardLinks.Clear(); @@ -221,13 +324,19 @@ void CArchiveExtractCallback::Init( _progressTotal_Defined = true; _extractCallback2 = extractCallback2; + _compressProgress.Release(); _extractCallback2.QueryInterface(IID_ICompressProgressInfo, &_compressProgress); + + _callbackMessage.Release(); _extractCallback2.QueryInterface(IID_IArchiveExtractCallbackMessage, &_callbackMessage); + + _folderArchiveExtractCallback2.Release(); _extractCallback2.QueryInterface(IID_IFolderArchiveExtractCallback2, &_folderArchiveExtractCallback2); #ifndef _SFX + ExtractToStreamCallback.Release(); _extractCallback2.QueryInterface(IID_IFolderExtractToStreamCallback, &ExtractToStreamCallback); if (ExtractToStreamCallback) { @@ -264,6 +373,7 @@ void CArchiveExtractCallback::Init( } } + STDMETHODIMP CArchiveExtractCallback::SetTotal(UInt64 size) { COM_TRY_BEGIN @@ -275,6 +385,7 @@ STDMETHODIMP CArchiveExtractCallback::SetTotal(UInt64 size) COM_TRY_END } + static void NormalizeVals(UInt64 &v1, UInt64 &v2) { const UInt64 kMax = (UInt64)1 << 31; @@ -285,6 +396,7 @@ static void NormalizeVals(UInt64 &v1, UInt64 &v2) } } + static UInt64 MyMultDiv64(UInt64 unpCur, UInt64 unpTotal, UInt64 packTotal) { NormalizeVals(packTotal, unpTotal); @@ -294,6 +406,7 @@ static UInt64 MyMultDiv64(UInt64 unpCur, UInt64 unpTotal, UInt64 packTotal) return unpCur * packTotal / unpTotal; } + STDMETHODIMP CArchiveExtractCallback::SetCompleted(const UInt64 *completeValue) { COM_TRY_BEGIN @@ -314,6 +427,7 @@ STDMETHODIMP CArchiveExtractCallback::SetCompleted(const UInt64 *completeValue) COM_TRY_END } + STDMETHODIMP CArchiveExtractCallback::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) { COM_TRY_BEGIN @@ -321,8 +435,11 @@ STDMETHODIMP CArchiveExtractCallback::SetRatioInfo(const UInt64 *inSize, const U COM_TRY_END } + void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath) { + // we use (_item.IsDir) in this function + bool isAbsPath = false; if (!dirPathParts.IsEmpty()) @@ -350,45 +467,72 @@ void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPat fullPath.Add_PathSepar(); const UString &s = dirPathParts[i]; fullPath += us2fs(s); + + const bool isFinalDir = (i == dirPathParts.Size() - 1 && _item.IsDir); + + if (fullPath.IsEmpty()) + { + if (isFinalDir) + _itemFailure = true; + continue; + } + #if defined(_WIN32) && !defined(UNDER_CE) if (_pathMode == NExtract::NPathMode::kAbsPaths) if (i == 0 && s.Len() == 2 && NName::IsDrivePath2(s)) + { + if (isFinalDir) + { + // we don't want to call SetAttrib() for root drive path + _itemFailure = true; + } continue; + } #endif + + // bool res = CreateDir(fullPath); + // if (!res) + if (isFinalDir) + { + if (!NFile::NFind::DoesDirExist(fullPath)) + { + _itemFailure = true; + SendMessageError("Cannot create folder", fullPath); + // SendMessageError_with_LastError() + } + } } } -HRESULT CArchiveExtractCallback::GetTime(int index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined) + +HRESULT CArchiveExtractCallback::GetTime(UInt32 index, PROPID propID, CArcTime &ft) { - filetimeIsDefined = false; + ft.Clear(); NCOM::CPropVariant prop; RINOK(_arc->Archive->GetProperty(index, propID, &prop)); if (prop.vt == VT_FILETIME) - { - filetime = prop.filetime; - filetimeIsDefined = (filetime.dwHighDateTime != 0 || filetime.dwLowDateTime != 0); - } + ft.Set_From_Prop(prop); else if (prop.vt != VT_EMPTY) return E_FAIL; return S_OK; } + HRESULT CArchiveExtractCallback::GetUnpackSize() { - return _arc->GetItemSize(_index, _curSize, _curSizeDefined); + return _arc->GetItem_Size(_index, _curSize, _curSizeDefined); } static void AddPathToMessage(UString &s, const FString &path) { - s.AddAscii(" : "); + s += " : "; s += fs2us(path); } HRESULT CArchiveExtractCallback::SendMessageError(const char *message, const FString &path) { - UString s; - s.AddAscii(message); + UString s (message); AddPathToMessage(s, path); return _extractCallback2->MessageError(s); } @@ -396,21 +540,25 @@ HRESULT CArchiveExtractCallback::SendMessageError(const char *message, const FSt HRESULT CArchiveExtractCallback::SendMessageError_with_LastError(const char *message, const FString &path) { DWORD errorCode = GetLastError(); - UString s; - s.AddAscii(message); - if (errorCode != 0) + if (errorCode == 0) + errorCode = (DWORD)E_FAIL; + UString s (message); { - s.AddAscii(" : "); + s += " : "; s += NError::MyFormatMessage(errorCode); } AddPathToMessage(s, path); return _extractCallback2->MessageError(s); } -HRESULT CArchiveExtractCallback::SendMessageError2(const char *message, const FString &path1, const FString &path2) +HRESULT CArchiveExtractCallback::SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2) { - UString s; - s.AddAscii(message); + UString s (message); + if (errorCode != 0) + { + s += " : "; + s += NError::MyFormatMessage(errorCode); + } AddPathToMessage(s, path1); AddPathToMessage(s, path2); return _extractCallback2->MessageError(s); @@ -433,35 +581,45 @@ STDMETHODIMP CGetProp::GetProp(PROPID propID, PROPVARIANT *value) return Arc->Archive->GetProperty(IndexInArc, propID, value); } -#endif +#endif // _SFX #ifdef SUPPORT_LINKS static UString GetDirPrefixOf(const UString &src) { - UString s = src; + UString s (src); if (!s.IsEmpty()) { if (IsPathSepar(s.Back())) s.DeleteBack(); int pos = s.ReverseFind_PathSepar(); - s.DeleteFrom(pos + 1); + s.DeleteFrom((unsigned)(pos + 1)); } return s; } -#endif +#endif // SUPPORT_LINKS +struct CLinkLevelsInfo +{ + bool IsAbsolute; + int LowLevel; + int FinalLevel; -bool IsSafePath(const UString &path) + void Parse(const UString &path); +}; + +void CLinkLevelsInfo::Parse(const UString &path) { - if (NName::IsAbsolutePath(path)) - return false; + IsAbsolute = NName::IsAbsolutePath(path); + + LowLevel = 0; + FinalLevel = 0; UStringVector parts; SplitPathToParts(path, parts); - unsigned level = 0; + int level = 0; FOR_VECTOR (i, parts) { @@ -469,29 +627,42 @@ bool IsSafePath(const UString &path) if (s.IsEmpty()) { if (i == 0) - return false; + IsAbsolute = true; continue; } if (s == L".") continue; if (s == L"..") { - if (level == 0) - return false; level--; + if (LowLevel > level) + LowLevel = level; } else level++; } - return level > 0; + FinalLevel = level; +} + + +bool IsSafePath(const UString &path); +bool IsSafePath(const UString &path) +{ + CLinkLevelsInfo levelsInfo; + levelsInfo.Parse(path); + return !levelsInfo.IsAbsolute + && levelsInfo.LowLevel >= 0 + && levelsInfo.FinalLevel > 0; } +bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include); bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include) { bool found = false; - + + // CheckPathVect() doesn't check path to Parent nodes if (node.CheckPathVect(item.PathParts, !item.MainIsDir, include)) { if (!include) @@ -514,7 +685,7 @@ bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcIte if (pathParts2.IsEmpty()) pathParts2.AddNew(); UString &back = pathParts2.Back(); - back += L':'; + back += ':'; back += item.AltStreamName; bool include2; @@ -526,11 +697,12 @@ bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcIte return true; } - #endif + #endif // SUPPORT_ALT_STREAMS return found; } + bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item) { bool include; @@ -539,9 +711,10 @@ bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem return false; } + static FString MakePath_from_2_Parts(const FString &prefix, const FString &path) { - FString s = prefix; + FString s (prefix); #if defined(_WIN32) && !defined(UNDER_CE) if (!path.IsEmpty() && path[0] == ':' && !prefix.IsEmpty() && IsPathSepar(prefix.Back())) { @@ -554,9 +727,10 @@ static FString MakePath_from_2_Parts(const FString &prefix, const FString &path) } -/* + #ifdef SUPPORT_LINKS +/* struct CTempMidBuffer { void *Buf; @@ -592,88 +766,24 @@ HRESULT CArchiveExtractCallback::MyCopyFile(ISequentialOutStream *outStream) RINOK(WriteStream(outStream, buf.Buf, num)); } } - -#endif */ -STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStream **outStream, Int32 askExtractMode) -{ - COM_TRY_BEGIN - - *outStream = NULL; - - #ifndef _SFX - if (_hashStream) - _hashStreamSpec->ReleaseStream(); - _hashStreamWasUsed = false; - #endif - - _outFileStream.Release(); - - _encrypted = false; - _position = 0; - _isSplit = false; - - _curSize = 0; - _curSizeDefined = false; - _index = index; - - _diskFilePath.Empty(); - - // _fi.Clear(); - - #ifdef SUPPORT_LINKS - // _CopyFile_Path.Empty(); - linkPath.Empty(); - #endif +HRESULT CArchiveExtractCallback::ReadLink() +{ IInArchive *archive = _arc->Archive; - - #ifndef _SFX - _item._use_baseParentFolder_mode = _use_baseParentFolder_mode; - if (_use_baseParentFolder_mode) - { - _item._baseParentFolder = _baseParentFolder; - if (_pathMode == NExtract::NPathMode::kFullPaths || - _pathMode == NExtract::NPathMode::kAbsPaths) - _item._baseParentFolder = -1; - } - #endif - - #ifdef SUPPORT_ALT_STREAMS - _item.WriteToAltStreamIfColon = _ntOptions.WriteToAltStreamIfColon; - #endif - - RINOK(_arc->GetItem(index, _item)); - - { - NCOM::CPropVariant prop; - RINOK(archive->GetProperty(index, kpidPosition, &prop)); - if (prop.vt != VT_EMPTY) - { - if (prop.vt != VT_UI8) - return E_FAIL; - _position = prop.uhVal.QuadPart; - _isSplit = true; - } - } - - #ifdef SUPPORT_LINKS - - // bool isCopyLink = false; - bool isHardLink = false; - bool isJunction = false; - bool isRelative = false; + const UInt32 index = _index; + _link.Clear(); { NCOM::CPropVariant prop; RINOK(archive->GetProperty(index, kpidHardLink, &prop)); if (prop.vt == VT_BSTR) { - isHardLink = true; - // isCopyLink = false; - isRelative = false; // RAR5, TAR: hard links are from root folder of archive - linkPath.SetFromBstr(prop.bstrVal); + _link.isHardLink = true; + // _link.isCopyLink = false; + _link.isRelative = false; // RAR5, TAR: hard links are from root folder of archive + _link.linkPath.SetFromBstr(prop.bstrVal); } else if (prop.vt != VT_EMPTY) return E_FAIL; @@ -685,10 +795,10 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre RINOK(archive->GetProperty(index, kpidCopyLink, &prop)); if (prop.vt == VT_BSTR) { - isHardLink = false; - isCopyLink = true; - isRelative = false; // RAR5: copy links are from root folder of archive - linkPath.SetFromBstr(prop.bstrVal); + _link.isHardLink = false; + _link.isCopyLink = true; + _link.isRelative = false; // RAR5: copy links are from root folder of archive + _link.linkPath.SetFromBstr(prop.bstrVal); } else if (prop.vt != VT_EMPTY) return E_FAIL; @@ -700,19 +810,19 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre RINOK(archive->GetProperty(index, kpidSymLink, &prop)); if (prop.vt == VT_BSTR) { - isHardLink = false; - // isCopyLink = false; - isRelative = true; // RAR5, TAR: symbolic links can be relative - linkPath.SetFromBstr(prop.bstrVal); + _link.isHardLink = false; + // _link.isCopyLink = false; + _link.isRelative = true; // RAR5, TAR: symbolic links can be relative + _link.linkPath.SetFromBstr(prop.bstrVal); } else if (prop.vt != VT_EMPTY) return E_FAIL; } + NtReparse_Data = NULL; + NtReparse_Size = 0; - bool isOkReparse = false; - - if (linkPath.IsEmpty() && _arc->GetRawProps) + if (_link.linkPath.IsEmpty() && _arc->GetRawProps) { const void *data; UInt32 dataSize; @@ -720,55 +830,79 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre _arc->GetRawProps->GetRawProp(_index, kpidNtReparse, &data, &dataSize, &propType); + // if (dataSize == 1234567) // for debug: unpacking without reparse if (dataSize != 0) { if (propType != NPropDataType::kRaw) return E_FAIL; - UString s; + + // 21.06: we need kpidNtReparse in linux for wim archives created in Windows + // #ifdef _WIN32 + + NtReparse_Data = data; + NtReparse_Size = dataSize; + CReparseAttr reparse; - isOkReparse = reparse.Parse((const Byte *)data, dataSize); + bool isOkReparse = reparse.Parse((const Byte *)data, dataSize); if (isOkReparse) { - isHardLink = false; - // isCopyLink = false; - linkPath = reparse.GetPath(); - isJunction = reparse.IsMountPoint(); - isRelative = reparse.IsRelative(); + _link.isHardLink = false; + // _link.isCopyLink = false; + _link.linkPath = reparse.GetPath(); + _link.isJunction = reparse.IsMountPoint(); + + if (reparse.IsSymLink_WSL()) + { + _link.isWSL = true; + _link.isRelative = reparse.IsRelative_WSL(); + } + else + _link.isRelative = reparse.IsRelative_Win(); + + // const AString s = GetAnsiString(_link.linkPath); + // printf("\n_link.linkPath: %s\n", s.Ptr()); + #ifndef _WIN32 - linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); + _link.linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); #endif } + // #endif } } - if (!linkPath.IsEmpty()) + if (_link.linkPath.IsEmpty()) + return S_OK; + { #ifdef _WIN32 - linkPath.Replace(L'/', WCHAR_PATH_SEPARATOR); + _link.linkPath.Replace(L'/', WCHAR_PATH_SEPARATOR); #endif // rar5 uses "\??\" prefix for absolute links - if (linkPath.IsPrefixedBy(WSTRING_PATH_SEPARATOR L"??" WSTRING_PATH_SEPARATOR)) + if (_link.linkPath.IsPrefixedBy(WSTRING_PATH_SEPARATOR L"??" WSTRING_PATH_SEPARATOR)) { - isRelative = false; - linkPath.DeleteFrontal(4); + _link.isRelative = false; + _link.linkPath.DeleteFrontal(4); } for (;;) // while (NName::IsAbsolutePath(linkPath)) { - unsigned n = NName::GetRootPrefixSize(linkPath); + unsigned n = NName::GetRootPrefixSize(_link.linkPath); if (n == 0) break; - isRelative = false; - linkPath.DeleteFrontal(n); + _link.isRelative = false; + _link.linkPath.DeleteFrontal(n); } } - if (!linkPath.IsEmpty() && !isRelative && _removePathParts.Size() != 0) + if (_link.linkPath.IsEmpty()) + return S_OK; + + if (!_link.isRelative && _removePathParts.Size() != 0) { UStringVector pathParts; - SplitPathToParts(linkPath, pathParts); + SplitPathToParts(_link.linkPath, pathParts); bool badPrefix = false; FOR_VECTOR (i, _removePathParts) { @@ -780,72 +914,765 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre } if (!badPrefix) pathParts.DeleteFrontal(_removePathParts.Size()); - linkPath = MakePathFromParts(pathParts); + _link.linkPath = MakePathFromParts(pathParts); } - #endif - - RINOK(Archive_GetItemBoolProp(archive, index, kpidEncrypted, _encrypted)); - - RINOK(GetUnpackSize()); + /* + if (!_link.linkPath.IsEmpty()) + { + printf("\n_link %s to -> %s\n", GetOemString(_item.Path).Ptr(), GetOemString(_link.linkPath).Ptr()); + } + */ - #ifdef SUPPORT_ALT_STREAMS - - if (!_ntOptions.AltStreams.Val && _item.IsAltStream) - return S_OK; + return S_OK; +} - #endif +#endif // SUPPORT_LINKS - UStringVector &pathParts = _item.PathParts; +#ifndef _WIN32 - if (_wildcardCensor) +static HRESULT GetOwner(IInArchive *archive, + UInt32 index, UInt32 pidName, UInt32 pidId, COwnerInfo &res) +{ { - if (!CensorNode_CheckPath(*_wildcardCensor, _item)) - return S_OK; + NWindows::NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, pidId, &prop)); + if (prop.vt == VT_UI4) + { + res.Id_Defined = true; + res.Id = prop.ulVal; // for debug + // res.Id++; // for debug + // if (pidId == kpidGroupId) res.Id += 7; // for debug + // res.Id = 0; // for debug + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + { + NWindows::NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, pidName, &prop)); + if (prop.vt == VT_BSTR) + { + const UString s = prop.bstrVal; + ConvertUnicodeToUTF8(s, res.Name); + } + else if (prop.vt == VT_UI4) + { + res.Id_Defined = true; + res.Id = prop.ulVal; + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; } + return S_OK; +} + +#endif + + +HRESULT CArchiveExtractCallback::Read_fi_Props() +{ + IInArchive *archive = _arc->Archive; + const UInt32 index = _index; + + _fi.Attrib_Defined = false; + + #ifndef _WIN32 + _fi.Owner.Clear(); + _fi.Group.Clear(); + #endif - #ifndef _SFX - if (_use_baseParentFolder_mode) { - if (!pathParts.IsEmpty()) + NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, kpidPosixAttrib, &prop)); + if (prop.vt == VT_UI4) { - unsigned numRemovePathParts = 0; - - #ifdef SUPPORT_ALT_STREAMS - if (_pathMode == NExtract::NPathMode::kNoPathsAlt && _item.IsAltStream) - numRemovePathParts = pathParts.Size(); - else - #endif - if (_pathMode == NExtract::NPathMode::kNoPaths || - _pathMode == NExtract::NPathMode::kNoPathsAlt) - numRemovePathParts = pathParts.Size() - 1; - pathParts.DeleteFrontal(numRemovePathParts); + _fi.SetFromPosixAttrib(prop.ulVal); } + else if (prop.vt != VT_EMPTY) + return E_FAIL; } - else - #endif + { - if (pathParts.IsEmpty()) + NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, kpidAttrib, &prop)); + if (prop.vt == VT_UI4) { - if (_item.IsDir) - return S_OK; - /* - #ifdef SUPPORT_ALT_STREAMS - if (!_item.IsAltStream) - #endif - return E_FAIL; - */ + _fi.Attrib = prop.ulVal; + _fi.Attrib_Defined = true; } + else if (prop.vt != VT_EMPTY) + return E_FAIL; + } - unsigned numRemovePathParts = 0; - - switch (_pathMode) - { - case NExtract::NPathMode::kFullPaths: - case NExtract::NPathMode::kCurPaths: - { - if (_removePathParts.IsEmpty()) + RINOK(GetTime(index, kpidCTime, _fi.CTime)); + RINOK(GetTime(index, kpidATime, _fi.ATime)); + RINOK(GetTime(index, kpidMTime, _fi.MTime)); + + #ifndef _WIN32 + if (_ntOptions.ExtractOwner) + { + // SendMessageError_with_LastError("_ntOptions.ExtractOwner", _diskFilePath); + GetOwner(archive, index, kpidUser, kpidUserId, _fi.Owner); + GetOwner(archive, index, kpidGroup, kpidGroupId, _fi.Group); + } + #endif + + return S_OK; +} + + + +void CArchiveExtractCallback::CorrectPathParts() +{ + UStringVector &pathParts = _item.PathParts; + + #ifdef SUPPORT_ALT_STREAMS + if (!_item.IsAltStream + || !pathParts.IsEmpty() + || !(_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt)) + #endif + Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, _keepAndReplaceEmptyDirPrefixes, pathParts, _item.MainIsDir); + + #ifdef SUPPORT_ALT_STREAMS + + if (_item.IsAltStream) + { + UString s (_item.AltStreamName); + Correct_AltStream_Name(s); + bool needColon = true; + + if (pathParts.IsEmpty()) + { + pathParts.AddNew(); + if (_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt) + needColon = false; + } + #ifdef _WIN32 + else if (_pathMode == NExtract::NPathMode::kAbsPaths && + NWildcard::GetNumPrefixParts_if_DrivePath(pathParts) == pathParts.Size()) + pathParts.AddNew(); + #endif + + UString &name = pathParts.Back(); + if (needColon) + name += (char)(_ntOptions.ReplaceColonForAltStream ? '_' : ':'); + name += s; + } + + #endif // SUPPORT_ALT_STREAMS +} + + +void CArchiveExtractCallback::GetFiTimesCAM(CFiTimesCAM &pt) +{ + pt.CTime_Defined = false; + pt.ATime_Defined = false; + pt.MTime_Defined = false; + + if (Write_MTime) + { + if (_fi.MTime.Def) + { + _fi.MTime.Write_To_FiTime(pt.MTime); + pt.MTime_Defined = true; + } + else if (_arc->MTime.Def) + { + _arc->MTime.Write_To_FiTime(pt.MTime); + pt.MTime_Defined = true; + } + } + + if (Write_CTime && _fi.CTime.Def) + { + _fi.CTime.Write_To_FiTime(pt.CTime); + pt.CTime_Defined = true; + } + + if (Write_ATime && _fi.ATime.Def) + { + _fi.ATime.Write_To_FiTime(pt.ATime); + pt.ATime_Defined = true; + } +} + + +void CArchiveExtractCallback::CreateFolders() +{ + // 21.04 : we don't change original (_item.PathParts) here + UStringVector pathParts = _item.PathParts; + + if (!_item.IsDir) + { + if (!pathParts.IsEmpty()) + pathParts.DeleteBack(); + } + + if (pathParts.IsEmpty()) + return; + + FString fullPathNew; + CreateComplexDirectory(pathParts, fullPathNew); + + if (!_item.IsDir) + return; + + if (_itemFailure) + return; + + CDirPathTime pt; + GetFiTimesCAM(pt); + + if (pt.IsSomeTimeDefined()) + { + pt.Path = fullPathNew; + pt.SetDirTime(); + _extractedFolders.Add(pt); + } +} + + + +/* + CheckExistFile(fullProcessedPath) + it can change: fullProcessedPath, _isRenamed, _overwriteMode + (needExit = true) means that we must exit GetStream() even for S_OK result. +*/ + +HRESULT CArchiveExtractCallback::CheckExistFile(FString &fullProcessedPath, bool &needExit) +{ + needExit = true; // it was set already before + + NFind::CFileInfo fileInfo; + + if (fileInfo.Find(fullProcessedPath)) + { + if (_overwriteMode == NExtract::NOverwriteMode::kSkip) + return S_OK; + + if (_overwriteMode == NExtract::NOverwriteMode::kAsk) + { + const int slashPos = fullProcessedPath.ReverseFind_PathSepar(); + const FString realFullProcessedPath = fullProcessedPath.Left((unsigned)(slashPos + 1)) + fileInfo.Name; + + /* (fileInfo) can be symbolic link. + we can show final file properties here. */ + + FILETIME ft1; + FiTime_To_FILETIME(fileInfo.MTime, ft1); + + Int32 overwriteResult; + RINOK(_extractCallback2->AskOverwrite( + fs2us(realFullProcessedPath), &ft1, &fileInfo.Size, _item.Path, + _fi.MTime.Def ? &_fi.MTime.FT : NULL, + _curSizeDefined ? &_curSize : NULL, + &overwriteResult)) + + switch (overwriteResult) + { + case NOverwriteAnswer::kCancel: + return E_ABORT; + case NOverwriteAnswer::kNo: + return S_OK; + case NOverwriteAnswer::kNoToAll: + _overwriteMode = NExtract::NOverwriteMode::kSkip; + return S_OK; + + case NOverwriteAnswer::kYes: + break; + case NOverwriteAnswer::kYesToAll: + _overwriteMode = NExtract::NOverwriteMode::kOverwrite; + break; + case NOverwriteAnswer::kAutoRename: + _overwriteMode = NExtract::NOverwriteMode::kRename; + break; + default: + return E_FAIL; + } + } // NExtract::NOverwriteMode::kAsk + + if (_overwriteMode == NExtract::NOverwriteMode::kRename) + { + if (!AutoRenamePath(fullProcessedPath)) + { + RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); + return E_FAIL; + } + _isRenamed = true; + } + else if (_overwriteMode == NExtract::NOverwriteMode::kRenameExisting) + { + FString existPath (fullProcessedPath); + if (!AutoRenamePath(existPath)) + { + RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); + return E_FAIL; + } + // MyMoveFile can rename folders. So it's OK to use it for folders too + if (!MyMoveFile(fullProcessedPath, existPath)) + { + HRESULT errorCode = GetLastError_noZero_HRESULT(); + RINOK(SendMessageError2(errorCode, kCantRenameFile, existPath, fullProcessedPath)); + return E_FAIL; + } + } + else // not Rename* + { + if (fileInfo.IsDir()) + { + // do we need to delete all files in folder? + if (!RemoveDir(fullProcessedPath)) + { + RINOK(SendMessageError_with_LastError(kCantDeleteOutputDir, fullProcessedPath)); + return S_OK; + } + } + else // fileInfo is not Dir + { + if (NFind::DoesFileExist_Raw(fullProcessedPath)) + if (!DeleteFileAlways(fullProcessedPath)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) // check it in linux + { + RINOK(SendMessageError_with_LastError(kCantDeleteOutputFile, fullProcessedPath)); + return S_OK; + // return E_FAIL; + } + } // fileInfo is not Dir + } // not Rename* + } + else // not Find(fullProcessedPath) + { + #if defined(_WIN32) && !defined(UNDER_CE) + // we need to clear READ-ONLY of parent before creating alt stream + int colonPos = NName::FindAltStreamColon(fullProcessedPath); + if (colonPos >= 0 && fullProcessedPath[(unsigned)colonPos + 1] != 0) + { + FString parentFsPath (fullProcessedPath); + parentFsPath.DeleteFrom((unsigned)colonPos); + NFind::CFileInfo parentFi; + if (parentFi.Find(parentFsPath)) + { + if (parentFi.IsReadOnly()) + SetFileAttrib(parentFsPath, parentFi.Attrib & ~(DWORD)FILE_ATTRIBUTE_READONLY); + } + } + #endif // defined(_WIN32) && !defined(UNDER_CE) + } + + needExit = false; + return S_OK; +} + + + + + + +HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr &outStreamLoc, bool &needExit) +{ + needExit = true; + + RINOK(Read_fi_Props()); + + #ifdef SUPPORT_LINKS + IInArchive *archive = _arc->Archive; + #endif + + const UInt32 index = _index; + + bool isAnti = false; + RINOK(_arc->IsItem_Anti(index, isAnti)); + + CorrectPathParts(); + UString processedPath (MakePathFromParts(_item.PathParts)); + + if (!isAnti) + { + // 21.04: CreateFolders doesn't change (_item.PathParts) + CreateFolders(); + } + + FString fullProcessedPath (us2fs(processedPath)); + if (_pathMode != NExtract::NPathMode::kAbsPaths + || !NName::IsAbsolutePath(processedPath)) + { + fullProcessedPath = MakePath_from_2_Parts(_dirPathPrefix, fullProcessedPath); + } + + #ifdef SUPPORT_ALT_STREAMS + if (_item.IsAltStream && _item.ParentIndex != (UInt32)(Int32)-1) + { + const int renIndex = _renamedFiles.FindInSorted(CIndexToPathPair(_item.ParentIndex)); + if (renIndex != -1) + { + const CIndexToPathPair &pair = _renamedFiles[(unsigned)renIndex]; + fullProcessedPath = pair.Path; + fullProcessedPath += ':'; + UString s (_item.AltStreamName); + Correct_AltStream_Name(s); + fullProcessedPath += us2fs(s); + } + } + #endif // SUPPORT_ALT_STREAMS + + if (_item.IsDir) + { + _diskFilePath = fullProcessedPath; + if (isAnti) + RemoveDir(_diskFilePath); + #ifdef SUPPORT_LINKS + if (_link.linkPath.IsEmpty()) + #endif + { + if (!isAnti) + SetAttrib(); + return S_OK; + } + } + else if (!_isSplit) + { + RINOK(CheckExistFile(fullProcessedPath, needExit)); + if (needExit) + return S_OK; + needExit = true; + } + + _diskFilePath = fullProcessedPath; + + + if (isAnti) + { + needExit = false; + return S_OK; + } + + // not anti + + #ifdef SUPPORT_LINKS + + if (!_link.linkPath.IsEmpty()) + { + #ifndef UNDER_CE + { + bool linkWasSet = false; + RINOK(SetFromLinkPath(fullProcessedPath, _link, linkWasSet)); + if (linkWasSet) + { + _isSymLinkCreated = _link.IsSymLink(); + SetAttrib(); + // printf("\nlinkWasSet %s\n", GetAnsiString(_diskFilePath)); + } + } + #endif // UNDER_CE + + // if (_CopyFile_Path.IsEmpty()) + { + needExit = false; + return S_OK; + } + } + + if (!_hardLinks.IDs.IsEmpty() && !_item.IsAltStream) + { + CHardLinkNode h; + bool defined; + RINOK(Archive_Get_HardLinkNode(archive, index, h, defined)); + if (defined) + { + const int linkIndex = _hardLinks.IDs.FindInSorted2(h); + if (linkIndex != -1) + { + FString &hl = _hardLinks.Links[(unsigned)linkIndex]; + if (hl.IsEmpty()) + hl = fullProcessedPath; + else + { + if (!MyCreateHardLink(fullProcessedPath, hl)) + { + HRESULT errorCode = GetLastError_noZero_HRESULT(); + RINOK(SendMessageError2(errorCode, kCantCreateHardLink, fullProcessedPath, hl)); + return S_OK; + } + + // printf("\nHard linkWasSet Archive_Get_HardLinkNode %s\n", GetAnsiString(_diskFilePath)); + // _needSetAttrib = true; // do we need to set attribute ? + SetAttrib(); + needExit = false; + return S_OK; + } + } + } + } + + #endif // SUPPORT_LINKS + + + // ---------- CREATE WRITE FILE ----- + + _outFileStreamSpec = new COutFileStream; + CMyComPtr outFileStream_Loc(_outFileStreamSpec); + + if (!_outFileStreamSpec->Open(fullProcessedPath, _isSplit ? OPEN_ALWAYS: CREATE_ALWAYS)) + { + // if (::GetLastError() != ERROR_FILE_EXISTS || !isSplit) + { + RINOK(SendMessageError_with_LastError(kCantOpenOutFile, fullProcessedPath)); + return S_OK; + } + } + + _needSetAttrib = true; + + bool is_SymLink_in_Data = false; + + if (_curSizeDefined && _curSize > 0 && _curSize < (1 << 12)) + { + if (_fi.IsLinuxSymLink()) + { + is_SymLink_in_Data = true; + _is_SymLink_in_Data_Linux = true; + } + else if (_fi.IsReparse()) + { + is_SymLink_in_Data = true; + _is_SymLink_in_Data_Linux = false; + } + } + + if (is_SymLink_in_Data) + { + _outMemBuf.Alloc((size_t)_curSize); + _bufPtrSeqOutStream_Spec = new CBufPtrSeqOutStream; + _bufPtrSeqOutStream = _bufPtrSeqOutStream_Spec; + _bufPtrSeqOutStream_Spec->Init(_outMemBuf, _outMemBuf.Size()); + outStreamLoc = _bufPtrSeqOutStream; + } + else // not reprase + { + if (_ntOptions.PreAllocateOutFile && !_isSplit && _curSizeDefined && _curSize > (1 << 12)) + { + // UInt64 ticks = GetCpuTicks(); + _fileLength_that_WasSet = _curSize; + bool res = _outFileStreamSpec->File.SetLength(_curSize); + _fileLengthWasSet = res; + + // ticks = GetCpuTicks() - ticks; + // printf("\nticks = %10d\n", (unsigned)ticks); + if (!res) + { + RINOK(SendMessageError_with_LastError(kCantSetFileLen, fullProcessedPath)); + } + + /* + _outFileStreamSpec->File.Close(); + ticks = GetCpuTicks() - ticks; + printf("\nticks = %10d\n", (unsigned)ticks); + return S_FALSE; + */ + + /* + File.SetLength() on FAT (xp64): is fast, but then File.Close() can be slow, + if we don't write any data. + File.SetLength() for remote share file (exFAT) can be slow in some cases, + and the Windows can return "network error" after 1 minute, + while remote file still can grow. + We need some way to detect such bad cases and disable PreAllocateOutFile mode. + */ + + res = _outFileStreamSpec->SeekToBegin_bool(); + if (!res) + { + RINOK(SendMessageError_with_LastError("Cannot seek to begin of file", fullProcessedPath)); + } + } // PreAllocateOutFile + + #ifdef SUPPORT_ALT_STREAMS + if (_isRenamed && !_item.IsAltStream) + { + CIndexToPathPair pair(index, fullProcessedPath); + unsigned oldSize = _renamedFiles.Size(); + unsigned insertIndex = _renamedFiles.AddToUniqueSorted(pair); + if (oldSize == _renamedFiles.Size()) + _renamedFiles[insertIndex].Path = fullProcessedPath; + } + #endif // SUPPORT_ALT_STREAMS + + if (_isSplit) + { + RINOK(_outFileStreamSpec->Seek((Int64)_position, STREAM_SEEK_SET, NULL)); + } + outStreamLoc = outFileStream_Loc; + } // if not reprase + + _outFileStream = outFileStream_Loc; + + needExit = false; + return S_OK; +} + + + +HRESULT CArchiveExtractCallback::GetItem(UInt32 index) +{ + #ifndef _SFX + _item._use_baseParentFolder_mode = _use_baseParentFolder_mode; + if (_use_baseParentFolder_mode) + { + _item._baseParentFolder = (int)_baseParentFolder; + if (_pathMode == NExtract::NPathMode::kFullPaths || + _pathMode == NExtract::NPathMode::kAbsPaths) + _item._baseParentFolder = -1; + } + #endif // _SFX + + #ifdef SUPPORT_ALT_STREAMS + _item.WriteToAltStreamIfColon = _ntOptions.WriteToAltStreamIfColon; + #endif + + return _arc->GetItem(index, _item); +} + + +STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStream **outStream, Int32 askExtractMode) +{ + COM_TRY_BEGIN + + *outStream = NULL; + + #ifndef _SFX + if (_hashStream) + _hashStreamSpec->ReleaseStream(); + _hashStreamWasUsed = false; + #endif + + _outFileStream.Release(); + _bufPtrSeqOutStream.Release(); + + _encrypted = false; + _position = 0; + _isSplit = false; + + _curSize = 0; + _curSizeDefined = false; + _fileLengthWasSet = false; + _fileLength_that_WasSet = 0; + _index = index; + + _diskFilePath.Empty(); + + _isRenamed = false; + + // _fi.Clear(); + + // _is_SymLink_in_Data = false; + _is_SymLink_in_Data_Linux = false; + + _needSetAttrib = false; + _isSymLinkCreated = false; + _itemFailure = false; + + #ifdef SUPPORT_LINKS + // _CopyFile_Path.Empty(); + _link.Clear(); + #endif + + _extractMode = false; + + switch (askExtractMode) + { + case NArchive::NExtract::NAskMode::kExtract: + if (_testMode) + { + // askExtractMode = NArchive::NExtract::NAskMode::kTest; + } + else + _extractMode = true; + break; + }; + + + IInArchive *archive = _arc->Archive; + + RINOK(GetItem(index)); + + { + NCOM::CPropVariant prop; + RINOK(archive->GetProperty(index, kpidPosition, &prop)); + if (prop.vt != VT_EMPTY) + { + if (prop.vt != VT_UI8) + return E_FAIL; + _position = prop.uhVal.QuadPart; + _isSplit = true; + } + } + + #ifdef SUPPORT_LINKS + RINOK(ReadLink()); + #endif // SUPPORT_LINKS + + + RINOK(Archive_GetItemBoolProp(archive, index, kpidEncrypted, _encrypted)); + + RINOK(GetUnpackSize()); + + #ifdef SUPPORT_ALT_STREAMS + if (!_ntOptions.AltStreams.Val && _item.IsAltStream) + return S_OK; + #endif // SUPPORT_ALT_STREAMS + + // we can change (_item.PathParts) in this function + UStringVector &pathParts = _item.PathParts; + + if (_wildcardCensor) + { + if (!CensorNode_CheckPath(*_wildcardCensor, _item)) + return S_OK; + } + + #ifndef _SFX + if (_use_baseParentFolder_mode) + { + if (!pathParts.IsEmpty()) + { + unsigned numRemovePathParts = 0; + + #ifdef SUPPORT_ALT_STREAMS + if (_pathMode == NExtract::NPathMode::kNoPathsAlt && _item.IsAltStream) + numRemovePathParts = pathParts.Size(); + else + #endif + if (_pathMode == NExtract::NPathMode::kNoPaths || + _pathMode == NExtract::NPathMode::kNoPathsAlt) + numRemovePathParts = pathParts.Size() - 1; + pathParts.DeleteFrontal(numRemovePathParts); + } + } + else + #endif // _SFX + { + if (pathParts.IsEmpty()) + { + if (_item.IsDir) + return S_OK; + /* + #ifdef SUPPORT_ALT_STREAMS + if (!_item.IsAltStream) + #endif + return E_FAIL; + */ + } + + unsigned numRemovePathParts = 0; + + switch (_pathMode) + { + case NExtract::NPathMode::kFullPaths: + case NExtract::NPathMode::kCurPaths: + { + if (_removePathParts.IsEmpty()) break; bool badPrefix = false; @@ -912,11 +1739,14 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre case NExtract::NPathMode::kAbsPaths: break; */ + default: + break; } pathParts.DeleteFrontal(numRemovePathParts); } + #ifndef _SFX if (ExtractToStreamCallback) @@ -928,13 +1758,13 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre } GetProp_Spec->Arc = _arc; GetProp_Spec->IndexInArc = index; - UString name = MakePathFromParts(pathParts); + UString name (MakePathFromParts(pathParts)); #ifdef SUPPORT_ALT_STREAMS if (_item.IsAltStream) { if (!pathParts.IsEmpty() || (!_removePartsForAltStreams && _pathMode != NExtract::NPathMode::kNoPathsAlt)) - name += L':'; + name += ':'; name += _item.AltStreamName; } #endif @@ -942,487 +1772,513 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre return ExtractToStreamCallback->GetStream7(name, BoolToInt(_item.IsDir), outStream, askExtractMode, GetProp); } - #endif + #endif // _SFX + CMyComPtr outStreamLoc; -if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) -{ - if (_stdOutMode) + if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) { - outStreamLoc = new CStdOutFileStream; + if (_stdOutMode) + outStreamLoc = new CStdOutFileStream; + else + { + bool needExit = true; + RINOK(GetExtractStream(outStreamLoc, needExit)); + if (needExit) + return S_OK; + } } - else + + #ifndef _SFX + if (_hashStream) { + if (askExtractMode == NArchive::NExtract::NAskMode::kExtract || + askExtractMode == NArchive::NExtract::NAskMode::kTest) { - NCOM::CPropVariant prop; - RINOK(archive->GetProperty(index, kpidAttrib, &prop)); - if (prop.vt == VT_UI4) - { - _fi.Attrib = prop.ulVal; - _fi.AttribDefined = true; - } - else if (prop.vt == VT_EMPTY) - _fi.AttribDefined = false; - else - return E_FAIL; + _hashStreamSpec->SetStream(outStreamLoc); + outStreamLoc = _hashStream; + _hashStreamSpec->Init(true); + _hashStreamWasUsed = true; + } + } + #endif // _SFX + + if (outStreamLoc) + { + /* + #ifdef SUPPORT_LINKS + if (!_CopyFile_Path.IsEmpty()) + { + RINOK(PrepareOperation(askExtractMode)); + RINOK(MyCopyFile(outStreamLoc)); + return SetOperationResult(NArchive::NExtract::NOperationResult::kOK); } + if (_link.isCopyLink && _testMode) + return S_OK; + #endif + */ + *outStream = outStreamLoc.Detach(); + } + + return S_OK; - RINOK(GetTime(index, kpidCTime, _fi.CTime, _fi.CTimeDefined)); - RINOK(GetTime(index, kpidATime, _fi.ATime, _fi.ATimeDefined)); - RINOK(GetTime(index, kpidMTime, _fi.MTime, _fi.MTimeDefined)); + COM_TRY_END +} - bool isAnti = false; - RINOK(_arc->IsItemAnti(index, isAnti)); - #ifdef SUPPORT_ALT_STREAMS - if (!_item.IsAltStream - || !pathParts.IsEmpty() - || !(_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt)) - #endif - Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, pathParts, _item.MainIsDir); - #ifdef SUPPORT_ALT_STREAMS - - if (_item.IsAltStream) - { - UString s = _item.AltStreamName; - Correct_AltStream_Name(s); - bool needColon = true; - if (pathParts.IsEmpty()) - { - pathParts.AddNew(); - if (_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt) - needColon = false; - } - else if (_pathMode == NExtract::NPathMode::kAbsPaths && - NWildcard::GetNumPrefixParts_if_DrivePath(pathParts) == pathParts.Size()) - pathParts.AddNew(); - - UString &name = pathParts.Back(); - if (needColon) - name += (wchar_t)(_ntOptions.ReplaceColonForAltStream ? L'_' : L':'); - name += s; - } - - #endif - UString processedPath = MakePathFromParts(pathParts); - - if (!isAnti) - { - if (!_item.IsDir) - { - if (!pathParts.IsEmpty()) - pathParts.DeleteBack(); - } - - if (!pathParts.IsEmpty()) - { - FString fullPathNew; - CreateComplexDirectory(pathParts, fullPathNew); - if (_item.IsDir) - { - _extractedFolderPaths.Add(fullPathNew); - _extractedFolderIndices.Add(index); - SetDirTime(fullPathNew, - (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, - (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, - (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); - } - } - } - FString fullProcessedPath = us2fs(processedPath); - if (_pathMode != NExtract::NPathMode::kAbsPaths - || !NName::IsAbsolutePath(processedPath)) + + + + +STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) +{ + COM_TRY_BEGIN + + #ifndef _SFX + if (ExtractToStreamCallback) + return ExtractToStreamCallback->PrepareOperation7(askExtractMode); + #endif + + _extractMode = false; + + switch (askExtractMode) + { + case NArchive::NExtract::NAskMode::kExtract: + if (_testMode) + askExtractMode = NArchive::NExtract::NAskMode::kTest; + else + _extractMode = true; + break; + }; + + return _extractCallback2->PrepareOperation(_item.Path, BoolToInt(_item.IsDir), + askExtractMode, _isSplit ? &_position: 0); + + COM_TRY_END +} + + + + + +HRESULT CArchiveExtractCallback::CloseFile() +{ + if (!_outFileStream) + return S_OK; + + HRESULT hres = S_OK; + + const UInt64 processedSize = _outFileStreamSpec->ProcessedSize; + if (_fileLengthWasSet && _fileLength_that_WasSet > processedSize) + { + bool res = _outFileStreamSpec->File.SetLength(processedSize); + _fileLengthWasSet = res; + if (!res) { - fullProcessedPath = MakePath_from_2_Parts(_dirPathPrefix, fullProcessedPath); + HRESULT hres2 = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path)); + if (hres == S_OK) + hres = hres2; } + } - #ifdef SUPPORT_ALT_STREAMS - - if (_item.IsAltStream && _item.ParentIndex != (UInt32)(Int32)-1) + _curSize = processedSize; + _curSizeDefined = true; + + #if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX) + if (ZoneBuf.Size() != 0 + && !_item.IsAltStream) + { + // if (NFind::DoesFileExist_Raw(tempFilePath)) + if (ZoneMode != NExtract::NZoneIdMode::kOffice || + FindExt2(kOfficeExtensions, _diskFilePath)) { - int renIndex = _renamedFiles.FindInSorted(CIndexToPathPair(_item.ParentIndex)); - if (renIndex >= 0) + // we must write zone file before setting of timestamps + const FString path = _diskFilePath + k_ZoneId_StreamName; + if (!WriteZoneFile(path, ZoneBuf)) { - const CIndexToPathPair &pair = _renamedFiles[renIndex]; - fullProcessedPath = pair.Path; - fullProcessedPath += (FChar)':'; - UString s = _item.AltStreamName; - Correct_AltStream_Name(s); - fullProcessedPath += us2fs(s); + // we can't write it in FAT + // SendMessageError_with_LastError("Can't write Zone.Identifier stream", path); } } - - #endif + } + #endif - bool isRenamed = false; + CFiTimesCAM t; + GetFiTimesCAM(t); - if (_item.IsDir) + // #ifdef _WIN32 + if (t.IsSomeTimeDefined()) + _outFileStreamSpec->SetTime( + t.CTime_Defined ? &t.CTime : NULL, + t.ATime_Defined ? &t.ATime : NULL, + t.MTime_Defined ? &t.MTime : NULL); + // #endif + + RINOK(_outFileStreamSpec->Close()); + _outFileStream.Release(); + return hres; +} + + +#ifdef SUPPORT_LINKS + + +HRESULT CArchiveExtractCallback::SetFromLinkPath( + const FString &fullProcessedPath, + const CLinkInfo &linkInfo, + bool &linkWasSet) +{ + linkWasSet = false; + if (!_ntOptions.SymLinks.Val && !linkInfo.isHardLink) + return S_OK; + + UString relatPath; + + /* if (linkInfo.isRelative) + linkInfo.linkPath is final link path that must be stored to file link field + else + linkInfo.linkPath is path from root of archive. So we must add _dirPathPrefix_Full before linkPath. + */ + + if (linkInfo.isRelative) + relatPath = GetDirPrefixOf(_item.Path); + relatPath += linkInfo.linkPath; + + if (!IsSafePath(relatPath)) + { + return SendMessageError2( + 0, // errorCode + "Dangerous link path was ignored", + us2fs(_item.Path), + us2fs(linkInfo.linkPath)); // us2fs(relatPath) + } + + FString existPath; + if (linkInfo.isHardLink /* || linkInfo.IsCopyLink */ || !linkInfo.isRelative) + { + if (!NName::GetFullPath(_dirPathPrefix_Full, us2fs(relatPath), existPath)) { - _diskFilePath = fullProcessedPath; - if (isAnti) - RemoveDir(_diskFilePath); - #ifdef SUPPORT_LINKS - if (linkPath.IsEmpty()) - #endif - return S_OK; + RINOK(SendMessageError("Incorrect path", us2fs(relatPath))); } - else if (!_isSplit) - { + } + else + { + existPath = us2fs(linkInfo.linkPath); + // printf("\nlinkPath = : %s\n", GetOemString(linkInfo.linkPath).Ptr()); + } - // ----- Is file (not split) ----- - NFind::CFileInfo fileInfo; - if (fileInfo.Find(fullProcessedPath)) - { - switch (_overwriteMode) - { - case NExtract::NOverwriteMode::kSkip: - return S_OK; - case NExtract::NOverwriteMode::kAsk: - { - int slashPos = fullProcessedPath.ReverseFind_PathSepar(); - FString realFullProcessedPath = fullProcessedPath.Left(slashPos + 1) + fileInfo.Name; - - Int32 overwriteResult; - RINOK(_extractCallback2->AskOverwrite( - fs2us(realFullProcessedPath), &fileInfo.MTime, &fileInfo.Size, _item.Path, - _fi.MTimeDefined ? &_fi.MTime : NULL, - _curSizeDefined ? &_curSize : NULL, - &overwriteResult)) + if (existPath.IsEmpty()) + return SendMessageError("Empty link", fullProcessedPath); - switch (overwriteResult) - { - case NOverwriteAnswer::kCancel: return E_ABORT; - case NOverwriteAnswer::kNo: return S_OK; - case NOverwriteAnswer::kNoToAll: _overwriteMode = NExtract::NOverwriteMode::kSkip; return S_OK; - case NOverwriteAnswer::kYes: break; - case NOverwriteAnswer::kYesToAll: _overwriteMode = NExtract::NOverwriteMode::kOverwrite; break; - case NOverwriteAnswer::kAutoRename: _overwriteMode = NExtract::NOverwriteMode::kRename; break; - default: - return E_FAIL; - } - } - } - if (_overwriteMode == NExtract::NOverwriteMode::kRename) - { - if (!AutoRenamePath(fullProcessedPath)) - { - RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); - return E_FAIL; - } - isRenamed = true; - } - else if (_overwriteMode == NExtract::NOverwriteMode::kRenameExisting) - { - FString existPath = fullProcessedPath; - if (!AutoRenamePath(existPath)) - { - RINOK(SendMessageError(kCantAutoRename, fullProcessedPath)); - return E_FAIL; - } - // MyMoveFile can raname folders. So it's OK to use it for folders too - if (!MyMoveFile(fullProcessedPath, existPath)) - { - RINOK(SendMessageError2(kCantRenameFile, existPath, fullProcessedPath)); - return E_FAIL; - } - } - else + if (linkInfo.isHardLink /* || linkInfo.IsCopyLink */) + { + // if (linkInfo.isHardLink) + { + if (!MyCreateHardLink(fullProcessedPath, existPath)) { - if (fileInfo.IsDir()) - { - // do we need to delete all files in folder? - if (!RemoveDir(fullProcessedPath)) - { - RINOK(SendMessageError_with_LastError(kCantDeleteOutputDir, fullProcessedPath)); - return S_OK; - } - } - else - { - bool needDelete = true; - if (needDelete) - { - if (!DeleteFileAlways(fullProcessedPath)) - { - RINOK(SendMessageError_with_LastError(kCantDeleteOutputFile, fullProcessedPath)); - return S_OK; - // return E_FAIL; - } - } - } + HRESULT errorCode = GetLastError_noZero_HRESULT(); + RINOK(SendMessageError2(errorCode, kCantCreateHardLink, fullProcessedPath, existPath)); } + linkWasSet = true; + return S_OK; } - else // not Find(fullProcessedPath) + /* + // IsCopyLink { - // we need to clear READ-ONLY of parent before creating alt stream - #if defined(_WIN32) && !defined(UNDER_CE) - int colonPos = NName::FindAltStreamColon(fullProcessedPath); - if (colonPos >= 0 && fullProcessedPath[(unsigned)colonPos + 1] != 0) + NFind::CFileInfo fi; + if (!fi.Find(existPath)) { - FString parentFsPath = fullProcessedPath; - parentFsPath.DeleteFrom(colonPos); - NFind::CFileInfo parentFi; - if (parentFi.Find(parentFsPath)) - { - if (parentFi.IsReadOnly()) - SetFileAttrib(parentFsPath, parentFi.Attrib & ~FILE_ATTRIBUTE_READONLY); - } + RINOK(SendMessageError2("Cannot find the file for copying", existPath, fullProcessedPath)); } - #endif - } - // ----- END of code for Is file (not split) ----- - - } - _diskFilePath = fullProcessedPath; - - - if (!isAnti) - { - #ifdef SUPPORT_LINKS - - if (!linkPath.IsEmpty()) + else { - #ifndef UNDER_CE - - UString relatPath; - if (isRelative) - relatPath = GetDirPrefixOf(_item.Path); - relatPath += linkPath; - - if (!IsSafePath(relatPath)) - { - RINOK(SendMessageError("Dangerous link path was ignored", us2fs(relatPath))); - } + if (_curSizeDefined && _curSize == fi.Size) + _CopyFile_Path = existPath; else { - FString existPath; - if (isHardLink /* || isCopyLink */ || !isRelative) - { - if (!NName::GetFullPath(_dirPathPrefix_Full, us2fs(relatPath), existPath)) - { - RINOK(SendMessageError("Incorrect path", us2fs(relatPath))); - } - } - else - { - existPath = us2fs(linkPath); - } - - if (!existPath.IsEmpty()) - { - if (isHardLink /* || isCopyLink */) - { - // if (isHardLink) - { - if (!MyCreateHardLink(fullProcessedPath, existPath)) - { - RINOK(SendMessageError2("Can not create hard link", fullProcessedPath, existPath)); - // return S_OK; - } - } - /* - else - { - NFind::CFileInfo fi; - if (!fi.Find(existPath)) - { - RINOK(SendMessageError2("Can not find the file for copying", existPath, fullProcessedPath)); - } - else - { - if (_curSizeDefined && _curSize == fi.Size) - _CopyFile_Path = existPath; - else - { - RINOK(SendMessageError2("File size collision for file copying", existPath, fullProcessedPath)); - } - - // RINOK(MyCopyFile(existPath, fullProcessedPath)); - } - } - */ - } - else if (_ntOptions.SymLinks.Val) - { - // bool isSymLink = true; // = false for junction - if (_item.IsDir && !isRelative) - { - // if it's before Vista we use Junction Point - // isJunction = true; - // convertToAbs = true; - } - - CByteBuffer data; - if (FillLinkData(data, fs2us(existPath), !isJunction)) - { - CReparseAttr attr; - if (!attr.Parse(data, data.Size())) - { - RINOK(SendMessageError("Internal error for symbolic link file", us2fs(_item.Path))); - // return E_FAIL; - } - else - if (!NFile::NIO::SetReparseData(fullProcessedPath, _item.IsDir, data, (DWORD)data.Size())) - { - RINOK(SendMessageError_with_LastError("Can not create symbolic link", fullProcessedPath)); - } - } - } - } + RINOK(SendMessageError2("File size collision for file copying", existPath, fullProcessedPath)); } - - #endif + // RINOK(MyCopyFile(existPath, fullProcessedPath)); } - - if (linkPath.IsEmpty() /* || !_CopyFile_Path.IsEmpty() */) - #endif // SUPPORT_LINKS + } + */ + } + + // is Symbolic link + + /* + if (_item.IsDir && !isRelative) + { + // Windows before Vista doesn't support symbolic links. + // we could convert such symbolic links to Junction Points + // isJunction = true; + // convertToAbs = true; + } + */ + + if (!_ntOptions.SymLinks_AllowDangerous.Val) + { + #ifdef _WIN32 + if (_item.IsDir) + #endif + if (linkInfo.isRelative) { - bool needWriteFile = true; - - #ifdef SUPPORT_LINKS - if (!_hardLinks.IDs.IsEmpty() && !_item.IsAltStream) + CLinkLevelsInfo levelsInfo; + levelsInfo.Parse(linkInfo.linkPath); + if (levelsInfo.FinalLevel < 1 || levelsInfo.IsAbsolute) { - CHardLinkNode h; - bool defined; - RINOK(Archive_Get_HardLinkNode(archive, index, h, defined)); - if (defined) - { - { - int linkIndex = _hardLinks.IDs.FindInSorted2(h); - if (linkIndex >= 0) - { - FString &hl = _hardLinks.Links[linkIndex]; - if (hl.IsEmpty()) - hl = fullProcessedPath; - else - { - if (!MyCreateHardLink(fullProcessedPath, hl)) - { - RINOK(SendMessageError2("Can not create hard link", fullProcessedPath, hl)); - return S_OK; - } - needWriteFile = false; - } - } - } - } + return SendMessageError2( + 0, // errorCode + "Dangerous symbolic link path was ignored", + us2fs(_item.Path), + us2fs(linkInfo.linkPath)); } - #endif - - if (needWriteFile) - { - _outFileStreamSpec = new COutFileStream; - CMyComPtr outStreamLoc2(_outFileStreamSpec); - if (!_outFileStreamSpec->Open(fullProcessedPath, _isSplit ? OPEN_ALWAYS: CREATE_ALWAYS)) - { - // if (::GetLastError() != ERROR_FILE_EXISTS || !isSplit) - { - RINOK(SendMessageError_with_LastError("Can not open output file", fullProcessedPath)); - return S_OK; - } - } + } + } + + #ifdef _WIN32 + + CByteBuffer data; + // printf("\nFillLinkData(): %s\n", GetOemString(existPath).Ptr()); + if (!FillLinkData(data, fs2us(existPath), !linkInfo.isJunction, linkInfo.isWSL)) + return SendMessageError("Cannot fill link data", us2fs(_item.Path)); - #ifdef SUPPORT_ALT_STREAMS - if (isRenamed && !_item.IsAltStream) - { - CIndexToPathPair pair(index, fullProcessedPath); - unsigned oldSize = _renamedFiles.Size(); - unsigned insertIndex = _renamedFiles.AddToUniqueSorted(pair); - if (oldSize == _renamedFiles.Size()) - _renamedFiles[insertIndex].Path = fullProcessedPath; - } - #endif + /* + if (NtReparse_Size != data.Size() || memcmp(NtReparse_Data, data, data.Size()) != 0) + { + SendMessageError("reconstructed Reparse is different", fs2us(existPath)); + } + */ + + CReparseAttr attr; + if (!attr.Parse(data, data.Size())) + { + RINOK(SendMessageError("Internal error for symbolic link file", us2fs(_item.Path))); + return S_OK; + } + if (!NFile::NIO::SetReparseData(fullProcessedPath, _item.IsDir, data, (DWORD)data.Size())) + { + RINOK(SendMessageError_with_LastError(kCantCreateSymLink, fullProcessedPath)); + return S_OK; + } + linkWasSet = true; - if (_isSplit) - { - RINOK(_outFileStreamSpec->Seek(_position, STREAM_SEEK_SET, NULL)); - } - - _outFileStream = outStreamLoc2; - } - } - } - - outStreamLoc = _outFileStream; + return S_OK; + + + #else // ! _WIN32 + + if (!NFile::NIO::SetSymLink(fullProcessedPath, existPath)) + { + RINOK(SendMessageError_with_LastError(kCantCreateSymLink, fullProcessedPath)); + return S_OK; } + linkWasSet = true; + + return S_OK; + + #endif // ! _WIN32 } - #ifndef _SFX - if (_hashStream) +bool CLinkInfo::Parse(const Byte *data, size_t dataSize, bool isLinuxData) +{ + Clear(); + // this->isLinux = isLinuxData; + + if (isLinuxData) { - if (askExtractMode == NArchive::NExtract::NAskMode::kExtract || - askExtractMode == NArchive::NExtract::NAskMode::kTest) - { - _hashStreamSpec->SetStream(outStreamLoc); - outStreamLoc = _hashStream; - _hashStreamSpec->Init(true); - _hashStreamWasUsed = true; - } + isJunction = false; + isHardLink = false; + AString utf; + if (dataSize >= (1 << 12)) + return false; + utf.SetFrom_CalcLen((const char *)data, (unsigned)dataSize); + UString u; + if (!ConvertUTF8ToUnicode(utf, u)) + return false; + linkPath = u; + + // in linux symbolic data: we expect that linux separator '/' is used + // if windows link was created, then we also must use linux separator + if (u.IsEmpty()) + return false; + wchar_t c = u[0]; + isRelative = !IS_PATH_SEPAR(c); + return true; } + CReparseAttr reparse; + if (!reparse.Parse(data, dataSize)) + return false; + isHardLink = false; + // isCopyLink = false; + linkPath = reparse.GetPath(); + isJunction = reparse.IsMountPoint(); + + if (reparse.IsSymLink_WSL()) + { + isWSL = true; + isRelative = reparse.IsRelative_WSL(); + } + else + isRelative = reparse.IsRelative_Win(); + + // FIXME !!! + #ifndef _WIN32 + linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); #endif + + return true; +} + +#endif // SUPPORT_LINKS + +HRESULT CArchiveExtractCallback::CloseReparseAndFile() +{ + HRESULT res = S_OK; + + #ifdef SUPPORT_LINKS + + size_t reparseSize = 0; + bool repraseMode = false; + bool needSetReparse = false; + CLinkInfo linkInfo; - if (outStreamLoc) + if (_bufPtrSeqOutStream) { - /* - #ifdef SUPPORT_LINKS - - if (!_CopyFile_Path.IsEmpty()) + repraseMode = true; + reparseSize = _bufPtrSeqOutStream_Spec->GetPos(); + if (_curSizeDefined && reparseSize == _outMemBuf.Size()) { - RINOK(PrepareOperation(askExtractMode)); - RINOK(MyCopyFile(outStreamLoc)); - return SetOperationResult(NArchive::NExtract::NOperationResult::kOK); + /* + CReparseAttr reparse; + DWORD errorCode = 0; + needSetReparse = reparse.Parse(_outMemBuf, reparseSize, errorCode); + if (needSetReparse) + { + UString linkPath = reparse.GetPath(); + #ifndef _WIN32 + linkPath.Replace(L'\\', WCHAR_PATH_SEPARATOR); + #endif + } + */ + needSetReparse = linkInfo.Parse(_outMemBuf, reparseSize, _is_SymLink_in_Data_Linux); + if (!needSetReparse) + res = SendMessageError_with_LastError("Incorrect reparse stream", us2fs(_item.Path)); + } + else + { + res = SendMessageError_with_LastError("Unknown reparse stream", us2fs(_item.Path)); + } + if (!needSetReparse && _outFileStream) + { + HRESULT res2 = WriteStream(_outFileStream, _outMemBuf, reparseSize); + if (res == S_OK) + res = res2; } + _bufPtrSeqOutStream.Release(); + } - if (isCopyLink && _testMode) - return S_OK; + #endif // SUPPORT_LINKS + + + HRESULT res2 = CloseFile(); + + if (res == S_OK) + res = res2; + + RINOK(res); + + #ifdef SUPPORT_LINKS + if (repraseMode) + { + _curSize = reparseSize; + _curSizeDefined = true; + #ifdef SUPPORT_LINKS + if (needSetReparse) + { + // in Linux : we must delete empty file before symbolic link creation + // in Windows : we can create symbolic link even without file deleting + if (!DeleteFileAlways(_diskFilePath)) + { + RINOK(SendMessageError_with_LastError("can't delete file", _diskFilePath)); + } + { + /* + // for DEBUG ONLY: we can extract sym links as WSL links + // to elimanate (non-admin) errors for sym links. + #ifdef _WIN32 + if (!linkInfo.isHardLink && !linkInfo.isJunction) + linkInfo.isWSL = true; + #endif + */ + bool linkWasSet = false; + RINOK(SetFromLinkPath(_diskFilePath, linkInfo, linkWasSet)); + if (linkWasSet) + _isSymLinkCreated = linkInfo.IsSymLink(); + else + _needSetAttrib = false; + } + /* + if (!NFile::NIO::SetReparseData(_diskFilePath, _item.IsDir, )) + { + res = SendMessageError_with_LastError(kCantCreateSymLink, _diskFilePath); + } + */ + } #endif - */ - - *outStream = outStreamLoc.Detach(); } - - return S_OK; - - COM_TRY_END + #endif + return res; } -STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) +void CArchiveExtractCallback::SetAttrib() { - COM_TRY_BEGIN + #ifndef _WIN32 + // Linux now doesn't support permissions for symlinks + if (_isSymLinkCreated) + return; + #endif + + if (_itemFailure + || _diskFilePath.IsEmpty() + || _stdOutMode + || !_extractMode) + return; + + #ifndef _WIN32 + if (_fi.Owner.Id_Defined && + _fi.Group.Id_Defined) + { + if (my_chown(_diskFilePath, _fi.Owner.Id, _fi.Group.Id) != 0) + { + SendMessageError_with_LastError("Cannot set owner", _diskFilePath); + } + } + #endif - #ifndef _SFX - if (ExtractToStreamCallback) - return ExtractToStreamCallback->PrepareOperation7(askExtractMode); - #endif - - _extractMode = false; - - switch (askExtractMode) + if (_fi.Attrib_Defined) { - case NArchive::NExtract::NAskMode::kExtract: - if (_testMode) - askExtractMode = NArchive::NExtract::NAskMode::kTest; - else - _extractMode = true; - break; - }; - - return _extractCallback2->PrepareOperation(_item.Path, BoolToInt(_item.IsDir), - askExtractMode, _isSplit ? &_position: 0); - - COM_TRY_END + // const AString s = GetAnsiString(_diskFilePath); + // printf("\nSetFileAttrib_PosixHighDetect: %s: hex:%x\n", s.Ptr(), _fi.Attrib); + bool res = SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib); + if (!res) + { + // do we need error message here in Windows and in posix? + SendMessageError_with_LastError("Cannot set file attribute", _diskFilePath); + } + } } @@ -1430,9 +2286,14 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) { COM_TRY_BEGIN + // printf("\nCArchiveExtractCallback::SetOperationResult: %d %s\n", opRes, GetAnsiString(_diskFilePath)); + #ifndef _SFX if (ExtractToStreamCallback) - return ExtractToStreamCallback->SetOperationResult7(opRes, BoolToInt(_encrypted)); + { + GetUnpackSize(); + return ExtractToStreamCallback->SetOperationResult8(opRes, BoolToInt(_encrypted), _curSize); + } #endif #ifndef _SFX @@ -1452,19 +2313,9 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) _hashStreamWasUsed = false; } - #endif + #endif // _SFX - if (_outFileStream) - { - _outFileStreamSpec->SetTime( - (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, - (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, - (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); - _curSize = _outFileStreamSpec->ProcessedSize; - _curSizeDefined = true; - RINOK(_outFileStreamSpec->Close()); - _outFileStream.Release(); - } + RINOK(CloseReparseAndFile()); #ifdef _USE_SECURITY_CODE if (!_stdOutMode && _extractMode && _ntOptions.NtSecurity.Val && _arc->GetRawProps) @@ -1482,11 +2333,11 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) SECURITY_INFORMATION securInfo = DACL_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION | OWNER_SECURITY_INFORMATION; if (_saclEnabled) securInfo |= SACL_SECURITY_INFORMATION; - ::SetFileSecurityW(fs2us(_diskFilePath), securInfo, (PSECURITY_DESCRIPTOR)(void *)data); + ::SetFileSecurityW(fs2us(_diskFilePath), securInfo, (PSECURITY_DESCRIPTOR)(void *)(const Byte *)(data)); } } } - #endif + #endif // _USE_SECURITY_CODE if (!_curSizeDefined) GetUnpackSize(); @@ -1510,8 +2361,8 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) else NumFiles++; - if (!_stdOutMode && _extractMode && _fi.AttribDefined) - SetFileAttrib(_diskFilePath, _fi.Attrib); + if (_needSetAttrib) + SetAttrib(); RINOK(_extractCallback2->SetOperationResult(opRes, BoolToInt(_encrypted))); @@ -1520,28 +2371,26 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) COM_TRY_END } + + STDMETHODIMP CArchiveExtractCallback::ReportExtractResult(UInt32 indexType, UInt32 index, Int32 opRes) { if (_folderArchiveExtractCallback2) { bool isEncrypted = false; - wchar_t temp[16]; - UString s2; - const wchar_t *s = NULL; + UString s; if (indexType == NArchive::NEventIndexType::kInArcIndex && index != (UInt32)(Int32)-1) { CReadArcItem item; RINOK(_arc->GetItem(index, item)); - s2 = item.Path; - s = s2; + s = item.Path; RINOK(Archive_GetItemBoolProp(_arc->Archive, index, kpidEncrypted, isEncrypted)); } else { - temp[0] = '#'; - ConvertUInt32ToString(index, temp + 1); - s = temp; + s = '#'; + s.Add_UInt32(index); // if (indexType == NArchive::NEventIndexType::kBlockIndex) {} } @@ -1565,71 +2414,166 @@ STDMETHODIMP CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password) } -struct CExtrRefSortPair +// ---------- HASH functions ---------- + +FString CArchiveExtractCallback::Hash_GetFullFilePath() { - unsigned Len; - unsigned Index; + // this function changes _item.PathParts. + CorrectPathParts(); + const UStringVector &pathParts = _item.PathParts; + const UString processedPath (MakePathFromParts(pathParts)); + FString fullProcessedPath (us2fs(processedPath)); + if (_pathMode != NExtract::NPathMode::kAbsPaths + || !NName::IsAbsolutePath(processedPath)) + { + fullProcessedPath = MakePath_from_2_Parts( + DirPathPrefix_for_HashFiles, + // _dirPathPrefix, + fullProcessedPath); + } + return fullProcessedPath; +} - int Compare(const CExtrRefSortPair &a) const; -}; -#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } +STDMETHODIMP CArchiveExtractCallback::GetDiskProperty(UInt32 index, PROPID propID, PROPVARIANT *value) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + if (propID == kpidSize) + { + RINOK(GetItem(index)); + const FString fullProcessedPath = Hash_GetFullFilePath(); + NFile::NFind::CFileInfo fi; + if (fi.Find_FollowLink(fullProcessedPath)) + if (!fi.IsDir()) + prop = (UInt64)fi.Size; + } + prop.Detach(value); + return S_OK; + COM_TRY_END +} + + +STDMETHODIMP CArchiveExtractCallback::GetStream2(UInt32 index, ISequentialInStream **inStream, UInt32 mode) +{ + COM_TRY_BEGIN + *inStream = NULL; + // if (index != _index) return E_FAIL; + if (mode != NUpdateNotifyOp::kHashRead) + return E_FAIL; + + RINOK(GetItem(index)); + const FString fullProcessedPath = Hash_GetFullFilePath(); + + CInFileStream *inStreamSpec = new CInFileStream; + CMyComPtr inStreamRef = inStreamSpec; + inStreamSpec->Set_PreserveATime(_ntOptions.PreserveATime); + if (!inStreamSpec->OpenShared(fullProcessedPath, _ntOptions.OpenShareForWrite)) + { + RINOK(SendMessageError_with_LastError(kCantOpenInFile, fullProcessedPath)); + return S_OK; + } + *inStream = inStreamRef.Detach(); + return S_OK; + COM_TRY_END +} + -int CExtrRefSortPair::Compare(const CExtrRefSortPair &a) const +STDMETHODIMP CArchiveExtractCallback::ReportOperation( + UInt32 /* indexType */, UInt32 /* index */, UInt32 /* op */) { - RINOZ(-MyCompare(Len, a.Len)); - return MyCompare(Index, a.Index); + // COM_TRY_BEGIN + return S_OK; + // COM_TRY_END } -static unsigned GetNumSlashes(const FChar *s) + +// ------------ After Extracting functions ------------ + +void CDirPathSortPair::SetNumSlashes(const FChar *s) { for (unsigned numSlashes = 0;;) { FChar c = *s++; if (c == 0) - return numSlashes; + { + Len = numSlashes; + return; + } if (IS_PATH_SEPAR(c)) numSlashes++; } } + +bool CDirPathTime::SetDirTime() const +{ + return NDir::SetDirTime(Path, + CTime_Defined ? &CTime : NULL, + ATime_Defined ? &ATime : NULL, + MTime_Defined ? &MTime : NULL); +} + + HRESULT CArchiveExtractCallback::SetDirsTimes() { - CRecordVector pairs; - pairs.ClearAndSetSize(_extractedFolderPaths.Size()); + if (!_arc) + return S_OK; + + CRecordVector pairs; + pairs.ClearAndSetSize(_extractedFolders.Size()); unsigned i; - for (i = 0; i < _extractedFolderPaths.Size(); i++) + for (i = 0; i < _extractedFolders.Size(); i++) { - CExtrRefSortPair &pair = pairs[i]; + CDirPathSortPair &pair = pairs[i]; pair.Index = i; - pair.Len = GetNumSlashes(_extractedFolderPaths[i]); + pair.SetNumSlashes(_extractedFolders[i].Path); } pairs.Sort2(); + HRESULT res = S_OK; + for (i = 0; i < pairs.Size(); i++) { - int pairIndex = pairs[i].Index; - int index = _extractedFolderIndices[pairIndex]; + const CDirPathTime &dpt = _extractedFolders[pairs[i].Index]; + if (!dpt.SetDirTime()) + { + // result = E_FAIL; + // do we need error message here in Windows and in posix? + // SendMessageError_with_LastError("Cannot set directory time", dpt.Path); + } + } - FILETIME CTime; - FILETIME ATime; - FILETIME MTime; - - bool CTimeDefined; - bool ATimeDefined; - bool MTimeDefined; + /* + #ifndef _WIN32 + for (i = 0; i < _delayedSymLinks.Size(); i++) + { + const CDelayedSymLink &link = _delayedSymLinks[i]; + if (!link.Create()) + { + if (res == S_OK) + res = GetLastError_noZero_HRESULT(); + // res = E_FAIL; + // do we need error message here in Windows and in posix? + SendMessageError_with_LastError("Cannot create Symbolic Link", link._source); + } + } + #endif // _WIN32 + */ + + ClearExtractedDirsInfo(); + return res; +} - RINOK(GetTime(index, kpidCTime, CTime, CTimeDefined)); - RINOK(GetTime(index, kpidATime, ATime, ATimeDefined)); - RINOK(GetTime(index, kpidMTime, MTime, MTimeDefined)); - // printf("\n%S", _extractedFolderPaths[pairIndex]); - SetDirTime(_extractedFolderPaths[pairIndex], - (WriteCTime && CTimeDefined) ? &CTime : NULL, - (WriteATime && ATimeDefined) ? &ATime : NULL, - (WriteMTime && MTimeDefined) ? &MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); - } - return S_OK; +HRESULT CArchiveExtractCallback::CloseArc() +{ + HRESULT res = CloseReparseAndFile(); + HRESULT res2 = SetDirsTimes(); + if (res == S_OK) + res = res2; + _arc = NULL; + return res; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.h b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.h index 7d4155fe2e7..fe70bc92dff 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveExtractCallback.h @@ -4,12 +4,14 @@ #define __ARCHIVE_EXTRACT_CALLBACK_H #include "../../../Common/MyCom.h" +#include "../../../Common/MyLinux.h" #include "../../../Common/Wildcard.h" #include "../../IPassword.h" #include "../../Common/FileStreams.h" #include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" #include "../../Archive/IArchive.h" @@ -52,18 +54,38 @@ struct CExtractNtOptions { CBoolPair NtSecurity; CBoolPair SymLinks; + CBoolPair SymLinks_AllowDangerous; CBoolPair HardLinks; CBoolPair AltStreams; bool ReplaceColonForAltStream; bool WriteToAltStreamIfColon; + bool ExtractOwner; + + bool PreAllocateOutFile; + + // used for hash arcs only, when we open external files + bool PreserveATime; + bool OpenShareForWrite; + CExtractNtOptions(): ReplaceColonForAltStream(false), - WriteToAltStreamIfColon(false) + WriteToAltStreamIfColon(false), + ExtractOwner(false), + PreserveATime(false), + OpenShareForWrite(false) { SymLinks.Val = true; + SymLinks_AllowDangerous.Val = false; HardLinks.Val = true; AltStreams.Val = true; + + PreAllocateOutFile = + #ifdef _WIN32 + true; + #else + false; + #endif } }; @@ -142,11 +164,98 @@ struct CIndexToPathPair #endif + + +struct CFiTimesCAM +{ + CFiTime CTime; + CFiTime ATime; + CFiTime MTime; + + bool CTime_Defined; + bool ATime_Defined; + bool MTime_Defined; + + bool IsSomeTimeDefined() const + { + return + CTime_Defined | + ATime_Defined | + MTime_Defined; + } +}; + +struct CDirPathTime: public CFiTimesCAM +{ + FString Path; + + bool SetDirTime() const; +}; + + +#ifdef SUPPORT_LINKS + +struct CLinkInfo +{ + // bool isCopyLink; + bool isHardLink; + bool isJunction; + bool isRelative; + bool isWSL; + UString linkPath; + + bool IsSymLink() const { return !isHardLink; } + + CLinkInfo(): + // IsCopyLink(false), + isHardLink(false), + isJunction(false), + isRelative(false), + isWSL(false) + {} + + void Clear() + { + // IsCopyLink = false; + isHardLink = false; + isJunction = false; + isRelative = false; + isWSL = false; + linkPath.Empty(); + } + + bool Parse(const Byte *data, size_t dataSize, bool isLinuxData); +}; + +#endif // SUPPORT_LINKS + + +#ifndef _WIN32 + +struct COwnerInfo +{ + bool Id_Defined; + UInt32 Id; + AString Name; + + void Clear() + { + Id_Defined = false; + Id = 0; + Name.Empty(); + } +}; + +#endif + + class CArchiveExtractCallback: public IArchiveExtractCallback, public IArchiveExtractCallbackMessage, public ICryptoGetTextPassword, public ICompressProgressInfo, + public IArchiveUpdateCallbackFile, + public IArchiveGetDiskProperty, public CMyUnknownImp { const CArc *_arc; @@ -163,6 +272,7 @@ class CArchiveExtractCallback: FString _dirPathPrefix_Full; NExtract::NPathMode::EEnum _pathMode; NExtract::NOverwriteMode::EEnum _overwriteMode; + bool _keepAndReplaceEmptyDirPrefixes; // replace them to "_"; #ifndef _SFX @@ -179,31 +289,77 @@ class CArchiveExtractCallback: bool _extractMode; - bool WriteCTime; - bool WriteATime; - bool WriteMTime; + bool Write_CTime; + bool Write_ATime; + bool Write_MTime; bool _encrypted; struct CProcessedFileInfo { - FILETIME CTime; - FILETIME ATime; - FILETIME MTime; + CArcTime CTime; + CArcTime ATime; + CArcTime MTime; UInt32 Attrib; - - bool CTimeDefined; - bool ATimeDefined; - bool MTimeDefined; - bool AttribDefined; + bool Attrib_Defined; + + #ifndef _WIN32 + COwnerInfo Owner; + COwnerInfo Group; + #endif + + bool IsReparse() const + { + return (Attrib_Defined && (Attrib & FILE_ATTRIBUTE_REPARSE_POINT) != 0); + } + + bool IsLinuxSymLink() const + { + return (Attrib_Defined && MY_LIN_S_ISLNK(Attrib >> 16)); + } + + void SetFromPosixAttrib(UInt32 a) + { + // here we set only part of combined attribute required by SetFileAttrib() call + #ifdef _WIN32 + // Windows sets FILE_ATTRIBUTE_NORMAL, if we try to set 0 as attribute. + Attrib = MY_LIN_S_ISDIR(a) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + if ((a & 0222) == 0) // (& S_IWUSR) in p7zip + Attrib |= FILE_ATTRIBUTE_READONLY; + // 22.00 : we need type bits for (MY_LIN_S_IFLNK) for IsLinuxSymLink() + a &= MY_LIN_S_IFMT; + if (a == MY_LIN_S_IFLNK) + Attrib |= (a << 16); + #else + Attrib = (a << 16) | FILE_ATTRIBUTE_UNIX_EXTENSION; + #endif + Attrib_Defined = true; + } } _fi; + // bool _is_SymLink_in_Data; + bool _is_SymLink_in_Data_Linux; // false = WIN32, true = LINUX + + bool _needSetAttrib; + bool _isSymLinkCreated; + bool _itemFailure; + UInt32 _index; UInt64 _curSize; bool _curSizeDefined; + bool _fileLengthWasSet; + UInt64 _fileLength_that_WasSet; + COutFileStream *_outFileStreamSpec; CMyComPtr _outFileStream; + CByteBuffer _outMemBuf; + CBufPtrSeqOutStream *_bufPtrSeqOutStream_Spec; + CMyComPtr _bufPtrSeqOutStream; + + #ifndef _SFX COutStreamWithHash *_hashStreamSpec; @@ -230,22 +386,34 @@ class CArchiveExtractCallback: UInt64 _progressTotal; bool _progressTotal_Defined; - FStringVector _extractedFolderPaths; - CRecordVector _extractedFolderIndices; + CObjectVector _extractedFolders; + + #ifndef _WIN32 + // CObjectVector _delayedSymLinks; + #endif #if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX) bool _saclEnabled; #endif void CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath); - HRESULT GetTime(int index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined); + HRESULT GetTime(UInt32 index, PROPID propID, CArcTime &ft); HRESULT GetUnpackSize(); + FString Hash_GetFullFilePath(); + + void SetAttrib(); + +public: HRESULT SendMessageError(const char *message, const FString &path); HRESULT SendMessageError_with_LastError(const char *message, const FString &path); - HRESULT SendMessageError2(const char *message, const FString &path1, const FString &path2); + HRESULT SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2); public: + #if defined(_WIN32) && !defined(UNDER_CE) + NExtract::NZoneIdMode::EEnum ZoneMode; + CByteBuffer ZoneBuf; + #endif CLocalProgress *LocalProgressSpec; @@ -255,10 +423,20 @@ public: UInt64 UnpackSize; UInt64 AltStreams_UnpackSize; - MY_UNKNOWN_IMP3(IArchiveExtractCallbackMessage, ICryptoGetTextPassword, ICompressProgressInfo) + FString DirPathPrefix_for_HashFiles; + + MY_UNKNOWN_IMP5( + IArchiveExtractCallbackMessage, + ICryptoGetTextPassword, + ICompressProgressInfo, + IArchiveUpdateCallbackFile, + IArchiveGetDiskProperty + ) INTERFACE_IArchiveExtractCallback(;) INTERFACE_IArchiveExtractCallbackMessage(;) + INTERFACE_IArchiveUpdateCallbackFile(;) + INTERFACE_IArchiveGetDiskProperty(;) STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); @@ -268,11 +446,19 @@ public: void InitForMulti(bool multiArchives, NExtract::NPathMode::EEnum pathMode, - NExtract::NOverwriteMode::EEnum overwriteMode) + NExtract::NOverwriteMode::EEnum overwriteMode, + NExtract::NZoneIdMode::EEnum zoneMode, + bool keepAndReplaceEmptyDirPrefixes) { _multiArchives = multiArchives; _pathMode = pathMode; _overwriteMode = overwriteMode; + #if defined(_WIN32) && !defined(UNDER_CE) + ZoneMode = zoneMode; + #else + UNUSED_VAR(zoneMode) + #endif + _keepAndReplaceEmptyDirPrefixes = keepAndReplaceEmptyDirPrefixes; NumFolders = NumFiles = NumAltStreams = UnpackSize = AltStreams_UnpackSize = 0; } @@ -289,6 +475,8 @@ public: #endif + void InitBeforeNewArchive(); + void Init( const CExtractNtOptions &ntOptions, const NWildcard::CCensorNode *wildcardCensor, @@ -304,10 +492,12 @@ public: private: CHardLinks _hardLinks; - UString linkPath; + CLinkInfo _link; // FString _CopyFile_Path; // HRESULT MyCopyFile(ISequentialOutStream *outStream); + HRESULT Link(const FString &fullProcessedPath); + HRESULT ReadLink(); public: // call PrepareHardLinks() after Init() @@ -330,9 +520,70 @@ public: } #endif + HRESULT CloseArc(); + +private: + void ClearExtractedDirsInfo() + { + _extractedFolders.Clear(); + #ifndef _WIN32 + // _delayedSymLinks.Clear(); + #endif + } + + HRESULT Read_fi_Props(); + void CorrectPathParts(); + void GetFiTimesCAM(CFiTimesCAM &pt); + void CreateFolders(); + + bool _isRenamed; + HRESULT CheckExistFile(FString &fullProcessedPath, bool &needExit); + HRESULT GetExtractStream(CMyComPtr &outStreamLoc, bool &needExit); + HRESULT GetItem(UInt32 index); + + HRESULT CloseFile(); + HRESULT CloseReparseAndFile(); + HRESULT CloseReparseAndFile2(); HRESULT SetDirsTimes(); + + const void *NtReparse_Data; + UInt32 NtReparse_Size; + + #ifdef SUPPORT_LINKS + HRESULT SetFromLinkPath( + const FString &fullProcessedPath, + const CLinkInfo &linkInfo, + bool &linkWasSet); + #endif +}; + + +struct CArchiveExtractCallback_Closer +{ + CArchiveExtractCallback *_ref; + + CArchiveExtractCallback_Closer(CArchiveExtractCallback *ref): _ref(ref) {} + + HRESULT Close() + { + HRESULT res = S_OK; + if (_ref) + { + res = _ref->CloseArc(); + _ref = NULL; + } + return res; + } + + ~CArchiveExtractCallback_Closer() + { + Close(); + } }; + bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item); +void ReadZoneFile_Of_BaseFile(CFSTR fileName2, CByteBuffer &buf); + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.cpp index 06d2ab80895..1baf3e1e300 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../Common/Wildcard.h" + #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" @@ -11,7 +13,7 @@ using namespace NWindows; using namespace NFile; -UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) +static UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) { FString resultName = fi.Name; if (!fi.IsDir() && !keepName) @@ -19,7 +21,7 @@ UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) int dotPos = resultName.ReverseFind_Dot(); if (dotPos > 0) { - FString archiveName2 = resultName.Left(dotPos); + FString archiveName2 = resultName.Left((unsigned)dotPos); if (archiveName2.ReverseFind_Dot() < 0) resultName = archiveName2; } @@ -29,7 +31,7 @@ UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) static FString CreateArchiveName2(const FString &path, bool fromPrev, bool keepName) { - FString resultName = FTEXT("Archive"); + FString resultName ("Archive"); if (fromPrev) { FString dirPrefix; @@ -62,7 +64,7 @@ static FString CreateArchiveName2(const FString &path, bool fromPrev, bool keepN int dotPos = resultName.ReverseFind_Dot(); if (dotPos > 0) { - FString name2 = resultName.Left(dotPos); + FString name2 = resultName.Left((unsigned)dotPos); if (name2.ReverseFind_Dot() < 0) resultName = name2; } @@ -72,7 +74,82 @@ static FString CreateArchiveName2(const FString &path, bool fromPrev, bool keepN return resultName; } -UString CreateArchiveName(const UString &path, bool fromPrev, bool keepName) + +UString CreateArchiveName(const UStringVector &paths, const NFind::CFileInfo *fi) { - return Get_Correct_FsFile_Name(fs2us(CreateArchiveName2(us2fs(path), fromPrev, keepName))); + bool keepName = false; + /* + if (paths.Size() == 1) + { + const UString &name = paths[0]; + if (name.Len() > 4) + if (CompareFileNames(name.RightPtr(4), L".tar") == 0) + keepName = true; + } + */ + + UString name; + if (fi) + name = CreateArchiveName(*fi, keepName); + else + { + if (paths.IsEmpty()) + return L"archive"; + bool fromPrev = (paths.Size() > 1); + name = Get_Correct_FsFile_Name(fs2us(CreateArchiveName2(us2fs(paths.Front()), fromPrev, keepName))); + } + + UStringVector names; + + { + FOR_VECTOR (i, paths) + { + NFind::CFileInfo fi2; + const NFind::CFileInfo *fp; + if (fi && paths.Size() == 1) + fp = fi; + else + { + if (!fi2.Find(us2fs(paths[i]))) + continue; + fp = &fi2; + } + names.Add(fs2us(fp->Name)); + } + } + + UString postfix; + UInt32 index = 1; + + for (;;) + { + // we don't want cases when we include archive to itself. + // so we find first available name for archive + const UString name2 = name + postfix; + const UString name2_zip = name2 + L".zip"; + const UString name2_7z = name2 + L".7z"; + const UString name2_tar = name2 + L".tar"; + const UString name2_wim = name2 + L".wim"; + + unsigned i = 0; + + for (i = 0; i < names.Size(); i++) + { + const UString &fname = names[i]; + if ( 0 == CompareFileNames(fname, name2_zip) + || 0 == CompareFileNames(fname, name2_7z) + || 0 == CompareFileNames(fname, name2_tar) + || 0 == CompareFileNames(fname, name2_wim)) + break; + } + + if (i == names.Size()) + break; + index++; + postfix = "_"; + postfix.Add_UInt32(index); + } + + name += postfix; + return name; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.h b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.h index ec2f1e8d1d9..0d32645f479 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveName.h @@ -3,11 +3,8 @@ #ifndef __ARCHIVE_NAME_H #define __ARCHIVE_NAME_H -#include "../../../Common/MyString.h" - #include "../../../Windows/FileFind.h" -UString CreateArchiveName(const UString &path, bool fromPrev, bool keepName); -UString CreateArchiveName(const NWindows::NFile::NFind::CFileInfo &fileInfo, bool keepName); +UString CreateArchiveName(const UStringVector &paths, const NWindows::NFile::NFind::CFileInfo *fi = NULL); #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp index fd2807ac149..64aa9878567 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp @@ -34,7 +34,8 @@ STDMETHODIMP COpenCallbackImp::SetCompleted(const UInt64 *files, const UInt64 *b return Callback->Open_SetCompleted(files, bytes); COM_TRY_END } - + + STDMETHODIMP COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value) { COM_TRY_BEGIN @@ -48,13 +49,14 @@ STDMETHODIMP COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value) else switch (propID) { - case kpidName: prop = _fileInfo.Name; break; + case kpidName: prop = fs2us(_fileInfo.Name); break; case kpidIsDir: prop = _fileInfo.IsDir(); break; case kpidSize: prop = _fileInfo.Size; break; - case kpidAttrib: prop = (UInt32)_fileInfo.Attrib; break; - case kpidCTime: prop = _fileInfo.CTime; break; - case kpidATime: prop = _fileInfo.ATime; break; - case kpidMTime: prop = _fileInfo.MTime; break; + case kpidAttrib: prop = (UInt32)_fileInfo.GetWinAttrib(); break; + case kpidPosixAttrib: prop = (UInt32)_fileInfo.GetPosixAttrib(); break; + case kpidCTime: PropVariant_SetFrom_FiTime(prop, _fileInfo.CTime); break; + case kpidATime: PropVariant_SetFrom_FiTime(prop, _fileInfo.ATime); break; + case kpidMTime: PropVariant_SetFrom_FiTime(prop, _fileInfo.MTime); break; } prop.Detach(value); return S_OK; @@ -63,7 +65,7 @@ STDMETHODIMP COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value) struct CInFileStreamVol: public CInFileStream { - int FileNameIndex; + unsigned FileNameIndex; COpenCallbackImp *OpenCallbackImp; CMyComPtr OpenCallbackRef; @@ -102,14 +104,29 @@ STDMETHODIMP COpenCallbackImp::GetStream(const wchar_t *name, IInStream **inStre // if (!allowAbsVolPaths) if (!IsSafePath(name2)) return S_FALSE; - + + #ifdef _WIN32 + /* WIN32 allows wildcards in Find() function + and doesn't allow wildcard in File.Open() + so we can work without the following wildcard check here */ + if (name2.Find(L'*') >= 0) + return S_FALSE; + { + int startPos = 0; + if (name2.IsPrefixedBy_Ascii_NoCase("\\\\?\\")) + startPos = 3; + if (name2.Find(L'?', startPos) >= 0) + return S_FALSE; + } + #endif + #endif FString fullPath; if (!NFile::NName::GetFullPath(_folderPrefix, us2fs(name2), fullPath)) return S_FALSE; - if (!_fileInfo.Find(fullPath)) + if (!_fileInfo.Find_FollowLink(fullPath)) return S_FALSE; if (_fileInfo.IsDir()) return S_FALSE; @@ -117,10 +134,7 @@ STDMETHODIMP COpenCallbackImp::GetStream(const wchar_t *name, IInStream **inStre CMyComPtr inStreamTemp = inFile; if (!inFile->Open(fullPath)) { - DWORD lastError = ::GetLastError(); - if (lastError == 0) - return E_FAIL; - return HRESULT_FROM_WIN32(lastError); + return GetLastError_noZero_HRESULT(); } FileSizes.Add(_fileInfo.Size); diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.h b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.h index e9575f56123..46b267683b3 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ArchiveOpenCallback.h @@ -6,6 +6,7 @@ #include "../../../Common/MyCom.h" #include "../../../Windows/FileFind.h" +#include "../../../Windows/FileIO.h" #ifndef _NO_CRYPTO #include "../../IPassword.h" @@ -88,24 +89,28 @@ public: CMyComPtr ReOpenCallback; // UInt64 TotalSize; - COpenCallbackImp(): Callback(NULL), _subArchiveMode(false) {} + COpenCallbackImp(): _subArchiveMode(false), Callback(NULL) {} - void Init(const FString &folderPrefix, const FString &fileName) + HRESULT Init2(const FString &folderPrefix, const FString &fileName) { - _folderPrefix = folderPrefix; - if (!_fileInfo.Find(_folderPrefix + fileName)) - throw 20121118; FileNames.Clear(); FileNames_WasUsed.Clear(); FileSizes.Clear(); _subArchiveMode = false; // TotalSize = 0; PasswordWasAsked = false; + _folderPrefix = folderPrefix; + if (!_fileInfo.Find_FollowLink(_folderPrefix + fileName)) + { + // throw 20121118; + return GetLastError_noZero_HRESULT(); + } + return S_OK; } bool SetSecondFileInfo(CFSTR newName) { - return _fileInfo.Find(newName) && !_fileInfo.IsDir(); + return _fileInfo.Find_FollowLink(newName) && !_fileInfo.IsDir(); } }; diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/Bench.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/Bench.cpp index ecbacbc9c8e..cc0cfd0c433 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/Bench.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/Bench.cpp @@ -2,19 +2,24 @@ #include "StdAfx.h" -#include +#include "../../../../C/CpuArch.h" + +// #include #ifndef _WIN32 + #define USE_POSIX_TIME #define USE_POSIX_TIME2 -#endif +#endif // _WIN32 #ifdef USE_POSIX_TIME #include +#include #ifdef USE_POSIX_TIME2 #include +#include #endif -#endif +#endif // USE_POSIX_TIME #ifdef _WIN32 #define USE_ALLOCA @@ -29,37 +34,33 @@ #endif #include "../../../../C/7zCrc.h" -#include "../../../../C/Alloc.h" -#include "../../../../C/CpuArch.h" - -#include "../../../Windows/System.h" +#include "../../../../C/RotateDefs.h" #ifndef _7ZIP_ST #include "../../../Windows/Synchronization.h" #include "../../../Windows/Thread.h" #endif -#if defined(_WIN32) || defined(UNIX_USE_WIN_FILE) -#define USE_WIN_FILE -#endif - -#ifdef USE_WIN_FILE #include "../../../Windows/FileIO.h" -#endif - +#include "../../../Windows/FileFind.h" +#include "../../../Windows/SystemInfo.h" #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" #include "../../Common/MethodProps.h" +#include "../../Common/StreamObjects.h" #include "../../Common/StreamUtils.h" #include "Bench.h" using namespace NWindows; +#ifndef _7ZIP_ST static const UInt32 k_LZMA = 0x030101; +#endif static const UInt64 kComplexInCommands = (UInt64)1 << #ifdef UNDER_CE @@ -68,9 +69,9 @@ static const UInt64 kComplexInCommands = (UInt64)1 << 34; #endif -static const UInt32 kComplexInSeconds = 4; +static const UInt32 kComplexInMs = 4000; -static void SetComplexCommands(UInt32 complexInSeconds, +static void SetComplexCommandsMs(UInt32 complexInMs, bool isSpecifiedFreq, UInt64 cpuFreq, UInt64 &complexInCommands) { complexInCommands = kComplexInCommands; @@ -80,96 +81,80 @@ static void SetComplexCommands(UInt32 complexInSeconds, cpuFreq = kMinFreq; if (cpuFreq < kMaxFreq || isSpecifiedFreq) { - if (complexInSeconds != 0) - complexInCommands = complexInSeconds * cpuFreq; + if (complexInMs != 0) + complexInCommands = complexInMs * cpuFreq / 1000; else complexInCommands = cpuFreq >> 2; } } +// const UInt64 kBenchmarkUsageMult = 1000000; // for debug +static const unsigned kBenchmarkUsageMultBits = 16; +static const UInt64 kBenchmarkUsageMult = 1 << kBenchmarkUsageMultBits; + +UInt64 Benchmark_GetUsage_Percents(UInt64 usage) +{ + return (100 * usage + kBenchmarkUsageMult / 2) / kBenchmarkUsageMult; +} + static const unsigned kNumHashDictBits = 17; -static const UInt32 kFilterUnpackSize = (48 << 10); +static const UInt32 kFilterUnpackSize = (47 << 10); // + 5; // for test -static const unsigned kOldLzmaDictBits = 30; +static const unsigned kOldLzmaDictBits = 32; -static const UInt32 kAdditionalSize = (1 << 16); +// static const size_t kAdditionalSize = (size_t)1 << 32; // for debug +static const size_t kAdditionalSize = (size_t)1 << 16; static const UInt32 kCompressedAdditionalSize = (1 << 10); -static const UInt32 kMaxLzmaPropSize = 5; + +static const UInt32 kMaxMethodPropSize = (1 << 6); + + +#define ALLOC_WITH_HRESULT(_buffer_, _size_) \ + { (_buffer_)->Alloc(_size_); \ + if (_size_ && !(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; } + class CBaseRandomGenerator { UInt32 A1; UInt32 A2; + UInt32 Salt; public: - CBaseRandomGenerator() { Init(); } + CBaseRandomGenerator(UInt32 salt = 0): Salt(salt) { Init(); } void Init() { A1 = 362436069; A2 = 521288629;} + MY_FORCE_INLINE UInt32 GetRnd() { - return + return Salt ^ + ( ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) + - ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ); + ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ) + ); } }; -static const unsigned kBufferAlignment = 1 << 4; - -struct CBenchBuffer +MY_NO_INLINE +static void RandGen(Byte *buf, size_t size) { - size_t BufferSize; - - #ifdef _WIN32 - - Byte *Buffer; - - CBenchBuffer(): BufferSize(0), Buffer(NULL) {} - ~CBenchBuffer() { ::MidFree(Buffer); } - - void AllocAlignedMask(size_t size, size_t) - { - ::MidFree(Buffer); - BufferSize = 0; - Buffer = (Byte *)::MidAlloc(size); - if (Buffer) - BufferSize = size; - } - - #else - - Byte *Buffer; - Byte *_bufBase; - - CBenchBuffer(): BufferSize(0), Buffer(NULL), _bufBase(NULL){} - ~CBenchBuffer() { ::MidFree(_bufBase); } - - void AllocAlignedMask(size_t size, size_t alignMask) + CBaseRandomGenerator RG; + const size_t size4 = size & ~((size_t)3); + size_t i; + for (i = 0; i < size4; i += 4) { - ::MidFree(_bufBase); - Buffer = NULL; - BufferSize = 0; - _bufBase = (Byte *)::MidAlloc(size + alignMask); - - if (_bufBase) - { - // Buffer = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - Buffer = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - BufferSize = size; - } + const UInt32 v = RG.GetRnd(); + SetUi32(buf + i, v); } - - #endif - - bool Alloc(size_t size) + UInt32 v = RG.GetRnd(); + for (; i < size; i++) { - if (Buffer && BufferSize == size) - return true; - AllocAlignedMask(size, kBufferAlignment - 1); - return (Buffer != NULL || size == 0); + buf[i] = (Byte)v; + v >>= 8; } -}; +} -class CBenchRandomGenerator: public CBenchBuffer +class CBenchRandomGenerator: public CMidAlignedBuffer { static UInt32 GetVal(UInt32 &res, unsigned numBits) { @@ -186,27 +171,33 @@ class CBenchRandomGenerator: public CBenchBuffer public: - void GenerateSimpleRandom(CBaseRandomGenerator *_RG_) + void GenerateSimpleRandom(UInt32 salt) { - CBaseRandomGenerator rg = *_RG_; - const size_t bufSize = BufferSize; - Byte *buf = Buffer; + CBaseRandomGenerator rg(salt); + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; for (size_t i = 0; i < bufSize; i++) buf[i] = (Byte)rg.GetRnd(); - *_RG_ = rg; } - void GenerateLz(unsigned dictBits, CBaseRandomGenerator *_RG_) + void GenerateLz(unsigned dictBits, UInt32 salt) { - CBaseRandomGenerator rg = *_RG_; - UInt32 pos = 0; - UInt32 rep0 = 1; - const size_t bufSize = BufferSize; - Byte *buf = Buffer; + CBaseRandomGenerator rg(salt); + size_t pos = 0; + size_t rep0 = 1; + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; unsigned posBits = 1; + + // printf("\n dictBits = %d\n", (UInt32)dictBits); + // printf("\n bufSize = 0x%p\n", (const void *)bufSize); while (pos < bufSize) { + /* + if (pos >= ((UInt32)1 << 31)) + printf(" %x\n", pos); + */ UInt32 r = rg.GetRnd(); if (GetVal(r, 1) == 0 || pos < 1024) buf[pos++] = (Byte)(r & 0xFF); @@ -219,7 +210,7 @@ public: { len += GetLen(r); - while (((UInt32)1 << posBits) < pos) + while (((size_t)1 << posBits) < pos) posBits++; unsigned numBitsMax = dictBits; @@ -233,11 +224,12 @@ public: for (;;) { - UInt32 ppp = GetVal(r, numLogBits) + kAddBits; + const UInt32 ppp = GetVal(r, numLogBits) + kAddBits; r = rg.GetRnd(); if (ppp > numBitsMax) continue; - rep0 = GetVal(r, ppp); + // rep0 = GetVal(r, ppp); + rep0 = r & (((size_t)1 << ppp) - 1); if (rep0 < pos) break; r = rg.GetRnd(); @@ -245,10 +237,11 @@ public: rep0++; } + // len *= 300; // for debug { - UInt32 rem = (UInt32)bufSize - pos; + const size_t rem = bufSize - pos; if (len > rem) - len = rem; + len = (UInt32)rem; } Byte *dest = buf + pos; const Byte *src = dest - rep0; @@ -257,8 +250,7 @@ public: *dest++ = *src++; } } - - *_RG_ = rg; + // printf("\n CRC = %x\n", CrcCalc(buf, bufSize)); } }; @@ -270,6 +262,7 @@ class CBenchmarkInStream: const Byte *Data; size_t Pos; size_t Size; + public: MY_UNKNOWN_IMP void Init(const Byte *data, size_t size) @@ -278,19 +271,22 @@ public: Size = size; Pos = 0; } + bool WasFinished() const { return Pos == Size; } STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); }; STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processedSize) { - size_t remain = Size - Pos; - UInt32 kMaxBlockSize = (1 << 20); + const UInt32 kMaxBlockSize = (1 << 20); if (size > kMaxBlockSize) size = kMaxBlockSize; + const size_t remain = Size - Pos; if (size > remain) size = (UInt32)remain; - for (UInt32 i = 0; i < size; i++) - ((Byte *)data)[i] = Data[Pos + i]; + + if (size != 0) + memcpy(data, Data + Pos, size); + Pos += size; if (processedSize) *processedSize = size; @@ -299,7 +295,7 @@ STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processed class CBenchmarkOutStream: public ISequentialOutStream, - public CBenchBuffer, + public CMidAlignedBuffer, public CMyUnknownImp { // bool _overflow; @@ -319,6 +315,18 @@ public: Pos = 0; } + void InitCrc() + { + Crc = CRC_INIT_VAL; + } + + void Calc(const void *data, size_t size) + { + Crc = CrcUpdate(Crc, data, size); + } + + size_t GetPos() const { return Pos; } + // void Print() { printf("\n%8d %8d\n", (unsigned)BufferSize, (unsigned)Pos); } MY_UNKNOWN_IMP @@ -327,15 +335,15 @@ public: STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { - size_t curSize = BufferSize - Pos; + size_t curSize = Size() - Pos; if (curSize > size) curSize = size; if (curSize != 0) { if (RealCopy) - memcpy(Buffer + Pos, data, curSize); + memcpy(((Byte *)*this) + Pos, data, curSize); if (CalcCrc) - Crc = CrcUpdate(Crc, data, curSize); + Calc(data, curSize); Pos += curSize; } if (processedSize) @@ -348,6 +356,7 @@ STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *p return S_OK; } + class CCrcOutStream: public ISequentialOutStream, public CMyUnknownImp @@ -355,39 +364,46 @@ class CCrcOutStream: public: bool CalcCrc; UInt32 Crc; + UInt64 Pos; + MY_UNKNOWN_IMP CCrcOutStream(): CalcCrc(true) {}; - void Init() { Crc = CRC_INIT_VAL; } + void Init() { Crc = CRC_INIT_VAL; Pos = 0; } + void Calc(const void *data, size_t size) + { + Crc = CrcUpdate(Crc, data, size); + } STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; STDMETHODIMP CCrcOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { if (CalcCrc) - Crc = CrcUpdate(Crc, data, size); + Calc(data, size); + Pos += size; if (processedSize) *processedSize = size; return S_OK; } +// #include "../../../../C/My_sys_time.h" + static UInt64 GetTimeCount() { #ifdef USE_POSIX_TIME #ifdef USE_POSIX_TIME2 timeval v; if (gettimeofday(&v, 0) == 0) - return (UInt64)(v.tv_sec) * 1000000 + v.tv_usec; + return (UInt64)(v.tv_sec) * 1000000 + (UInt64)v.tv_usec; return (UInt64)time(NULL) * 1000000; #else return time(NULL); #endif #else - /* LARGE_INTEGER value; if (::QueryPerformanceCounter(&value)) - return value.QuadPart; - */ + return (UInt64)value.QuadPart; return GetTickCount(); #endif } @@ -401,15 +417,14 @@ static UInt64 GetFreq() return 1; #endif #else - /* LARGE_INTEGER value; if (::QueryPerformanceFrequency(&value)) - return value.QuadPart; - */ + return (UInt64)value.QuadPart; return 1000; #endif } + #ifdef USE_POSIX_TIME struct CUserTime @@ -419,50 +434,98 @@ struct CUserTime void Init() { - Prev = clock(); + // Prev = clock(); + Sum = 0; + Prev = 0; + Update(); Sum = 0; } - UInt64 GetUserTime() + void Update() { + tms t; + /* clock_t res = */ times(&t); + clock_t newVal = t.tms_utime + t.tms_stime; + Sum += (UInt64)(newVal - Prev); + Prev = newVal; + + /* clock_t v = clock(); - Sum += v - Prev; - Prev = v; + if (v != -1) + { + Sum += v - Prev; + Prev = v; + } + */ + } + UInt64 GetUserTime() + { + Update(); return Sum; } }; #else -static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } -UInt64 GetWinUserTime() -{ - FILETIME creationTime, exitTime, kernelTime, userTime; - if ( - #ifdef UNDER_CE - ::GetThreadTimes(::GetCurrentThread() - #else - ::GetProcessTimes(::GetCurrentProcess() - #endif - , &creationTime, &exitTime, &kernelTime, &userTime) != 0) - return GetTime64(userTime) + GetTime64(kernelTime); - return (UInt64)GetTickCount() * 10000; -} struct CUserTime { - UInt64 StartTime; + bool UseTick; + DWORD Prev_Tick; + UInt64 Prev; + UInt64 Sum; - void Init() { StartTime = GetWinUserTime(); } - UInt64 GetUserTime() { return GetWinUserTime() - StartTime; } + void Init() + { + UseTick = false; + Prev_Tick = 0; + Prev = 0; + Sum = 0; + Update(); + Sum = 0; + } + UInt64 GetUserTime() + { + Update(); + return Sum; + } + void Update(); }; +static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } + +void CUserTime::Update() +{ + DWORD new_Tick = GetTickCount(); + FILETIME creationTime, exitTime, kernelTime, userTime; + if (!UseTick && + #ifdef UNDER_CE + ::GetThreadTimes(::GetCurrentThread() + #else + ::GetProcessTimes(::GetCurrentProcess() + #endif + , &creationTime, &exitTime, &kernelTime, &userTime)) + { + UInt64 newVal = GetTime64(userTime) + GetTime64(kernelTime); + Sum += newVal - Prev; + Prev = newVal; + } + else + { + UseTick = true; + Sum += (UInt64)(new_Tick - (DWORD)Prev_Tick) * 10000; + } + Prev_Tick = new_Tick; +} + + #endif static UInt64 GetUserFreq() { #ifdef USE_POSIX_TIME - return CLOCKS_PER_SEC; + // return CLOCKS_PER_SEC; + return (UInt64)sysconf(_SC_CLK_TCK); #else return 10000000; #endif @@ -524,14 +587,14 @@ class CBenchProgressInfo: { public: CBenchProgressStatus *Status; - HRESULT Res; IBenchCallback *Callback; - CBenchProgressInfo(): Callback(0) {} + CBenchProgressInfo(): Callback(NULL) {} MY_UNKNOWN_IMP STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; + STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) { HRESULT res = Status->GetResult(); @@ -539,6 +602,22 @@ STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 return res; if (!Callback) return res; + + /* + static UInt64 inSizePrev = 0; + static UInt64 outSizePrev = 0; + UInt64 delta1 = 0, delta2 = 0, val1 = 0, val2 = 0; + if (inSize) { val1 = *inSize; delta1 = val1 - inSizePrev; inSizePrev = val1; } + if (outSize) { val2 = *outSize; delta2 = val2 - outSizePrev; outSizePrev = val2; } + UInt64 percents = delta2 * 1000; + if (delta1 != 0) + percents /= delta1; + printf("=== %7d %7d %7d %7d ratio = %4d\n", + (unsigned)(val1 >> 10), (unsigned)(delta1 >> 10), + (unsigned)(val2 >> 10), (unsigned)(delta2 >> 10), + (unsigned)percents); + */ + CBenchInfo info; SetFinishTime(info); if (Status->EncodeMode) @@ -560,66 +639,125 @@ STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 static const unsigned kSubBits = 8; -static UInt32 GetLogSize(UInt32 size) +static unsigned GetLogSize(UInt64 size) +{ + unsigned i = 0; + for (;;) + { + i++; size >>= 1; if (size == 0) break; + } + return i; +} + + +static UInt32 GetLogSize_Sub(UInt64 size) +{ + if (size <= 1) + return 0; + const unsigned i = GetLogSize(size) - 1; + UInt32 v; + if (i <= kSubBits) + v = (UInt32)(size) << (kSubBits - i); + else + v = (UInt32)(size >> (i - kSubBits)); + return ((UInt32)i << kSubBits) + (v & (((UInt32)1 << kSubBits) - 1)); +} + + +static UInt64 Get_UInt64_from_double(double v) { - for (unsigned i = kSubBits; i < 32; i++) - for (UInt32 j = 0; j < (1 << kSubBits); j++) - if (size <= (((UInt32)1) << i) + (j << (i - kSubBits))) - return (i << kSubBits) + j; - return (32 << kSubBits); + const UInt64 kMaxVal = (UInt64)1 << 62; + if (v > (double)(Int64)kMaxVal) + return kMaxVal; + return (UInt64)v; } -static void NormalizeVals(UInt64 &v1, UInt64 &v2) +static UInt64 MyMultDiv64(UInt64 m1, UInt64 m2, UInt64 d) { - while (v1 > 1000000) + if (d == 0) + d = 1; + const double v = + (double)(Int64)m1 * + (double)(Int64)m2 / + (double)(Int64)d; + return Get_UInt64_from_double(v); + /* + unsigned n1 = GetLogSize(m1); + unsigned n2 = GetLogSize(m2); + while (n1 + n2 > 64) { - v1 >>= 1; - v2 >>= 1; + if (n1 >= n2) + { + m1 >>= 1; + n1--; + } + else + { + m2 >>= 1; + n2--; + } + d >>= 1; } + + if (d == 0) + d = 1; + return m1 * m2 / d; + */ } + UInt64 CBenchInfo::GetUsage() const { UInt64 userTime = UserTime; UInt64 userFreq = UserFreq; UInt64 globalTime = GlobalTime; UInt64 globalFreq = GlobalFreq; - NormalizeVals(userTime, userFreq); - NormalizeVals(globalFreq, globalTime); + if (userFreq == 0) userFreq = 1; if (globalTime == 0) globalTime = 1; - return userTime * globalFreq * 1000000 / userFreq / globalTime; + + const double v = + ((double)(Int64)userTime / (double)(Int64)userFreq) + * ((double)(Int64)globalFreq / (double)(Int64)globalTime) + * (double)(Int64)kBenchmarkUsageMult; + return Get_UInt64_from_double(v); + /* + return MyMultDiv64( + MyMultDiv64(kBenchmarkUsageMult, userTime, userFreq), + globalFreq, globalTime); + */ } + UInt64 CBenchInfo::GetRatingPerUsage(UInt64 rating) const { - UInt64 userTime = UserTime; - UInt64 userFreq = UserFreq; - UInt64 globalTime = GlobalTime; + if (UserTime == 0) + { + return 0; + // userTime = 1; + } UInt64 globalFreq = GlobalFreq; - NormalizeVals(userFreq, userTime); - NormalizeVals(globalTime, globalFreq); if (globalFreq == 0) globalFreq = 1; - if (userTime == 0) - userTime = 1; - return userFreq * globalTime / globalFreq * rating / userTime; -} -static UInt64 MyMultDiv64(UInt64 value, UInt64 elapsedTime, UInt64 freq) -{ - UInt64 elTime = elapsedTime; - NormalizeVals(freq, elTime); - if (elTime == 0) - elTime = 1; - return value * freq / elTime; + const double v = + ((double)(Int64)GlobalTime / (double)(Int64)globalFreq) + * ((double)(Int64)UserFreq / (double)(Int64)UserTime) + * (double)(Int64)rating; + return Get_UInt64_from_double(v); + /* + return MyMultDiv64( + MyMultDiv64(rating, UserFreq, UserTime), + GlobalTime, globalFreq); + */ } -UInt64 CBenchInfo::GetSpeed(UInt64 numCommands) const + +UInt64 CBenchInfo::GetSpeed(UInt64 numUnits) const { - return MyMultDiv64(numCommands, GlobalTime, GlobalFreq); + return MyMultDiv64(numUnits, GlobalFreq, GlobalTime); } struct CBenchProps @@ -630,11 +768,20 @@ struct CBenchProps UInt32 DecComplexCompr; UInt32 DecComplexUnc; - CBenchProps(): LzmaRatingMode(false) {} + unsigned KeySize; + + CBenchProps(): + LzmaRatingMode(false), + KeySize(0) + {} + void SetLzmaCompexity(); UInt64 GeComprCommands(UInt64 unpackSize) { + const UInt32 kMinSize = 100; + if (unpackSize < kMinSize) + unpackSize = kMinSize; return unpackSize * EncComplex; } @@ -643,7 +790,7 @@ struct CBenchProps return (packSize * DecComplexCompr + unpackSize * DecComplexUnc); } - UInt64 GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size); + UInt64 GetCompressRating(UInt64 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size); UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations); }; @@ -655,61 +802,234 @@ void CBenchProps::SetLzmaCompexity() LzmaRatingMode = true; } -UInt64 CBenchProps::GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) +UInt64 CBenchProps::GetCompressRating(UInt64 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) { if (dictSize < (1 << kBenchMinDicLogSize)) dictSize = (1 << kBenchMinDicLogSize); UInt64 encComplex = EncComplex; if (LzmaRatingMode) { - UInt64 t = GetLogSize(dictSize) - (kBenchMinDicLogSize << kSubBits); + /* + for (UInt64 uu = 0; uu < (UInt64)0xf << 60;) + { + unsigned rr = GetLogSize_Sub(uu); + printf("\n%16I64x , log = %4x", uu, rr); + uu += 1; + uu += uu / 50; + } + */ + // throw 1; + const UInt32 t = GetLogSize_Sub(dictSize) - (kBenchMinDicLogSize << kSubBits); encComplex = 870 + ((t * t * 5) >> (2 * kSubBits)); } - UInt64 numCommands = (UInt64)size * encComplex; - return MyMultDiv64(numCommands, elapsedTime, freq); + const UInt64 numCommands = (UInt64)size * encComplex; + return MyMultDiv64(numCommands, freq, elapsedTime); } UInt64 CBenchProps::GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations) { - UInt64 numCommands = (inSize * DecComplexCompr + outSize * DecComplexUnc) * numIterations; - return MyMultDiv64(numCommands, elapsedTime, freq); + const UInt64 numCommands = (inSize * DecComplexCompr + outSize * DecComplexUnc) * numIterations; + return MyMultDiv64(numCommands, freq, elapsedTime); } -UInt64 GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) + + +UInt64 CBenchInfo::GetRating_LzmaEnc(UInt64 dictSize) const { CBenchProps props; props.SetLzmaCompexity(); - return props.GetCompressRating(dictSize, elapsedTime, freq, size); + return props.GetCompressRating(dictSize, GlobalTime, GlobalFreq, UnpackSize * NumIterations); } -UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations) +UInt64 CBenchInfo::GetRating_LzmaDec() const { CBenchProps props; props.SetLzmaCompexity(); - return props.GetDecompressRating(elapsedTime, freq, outSize, inSize, numIterations); + return props.GetDecompressRating(GlobalTime, GlobalFreq, UnpackSize, PackSize, NumIterations); +} + + +#ifndef _7ZIP_ST + +#define NUM_CPU_LEVELS_MAX 3 + +struct CAffinityMode +{ + unsigned NumBundleThreads; + unsigned NumLevels; + unsigned NumCoreThreads; + unsigned NumCores; + // unsigned DivideNum; + UInt32 Sizes[NUM_CPU_LEVELS_MAX]; + + void SetLevels(unsigned numCores, unsigned numCoreThreads); + DWORD_PTR GetAffinityMask(UInt32 bundleIndex, CCpuSet *cpuSet) const; + bool NeedAffinity() const { return NumBundleThreads != 0; } + + WRes CreateThread_WithAffinity(NWindows::CThread &thread, THREAD_FUNC_TYPE startAddress, LPVOID parameter, UInt32 bundleIndex) const + { + if (NeedAffinity()) + { + CCpuSet cpuSet; + GetAffinityMask(bundleIndex, &cpuSet); + return thread.Create_With_CpuSet(startAddress, parameter, &cpuSet); + } + return thread.Create(startAddress, parameter); + } + + CAffinityMode(): + NumBundleThreads(0), + NumLevels(0), + NumCoreThreads(1) + // DivideNum(1) + {} +}; + +void CAffinityMode::SetLevels(unsigned numCores, unsigned numCoreThreads) +{ + NumCores = numCores; + NumCoreThreads = numCoreThreads; + NumLevels = 0; + if (numCoreThreads == 0 || numCores == 0 || numCores % numCoreThreads != 0) + return; + UInt32 c = numCores / numCoreThreads; + UInt32 c2 = 1; + while ((c & 1) == 0) + { + c >>= 1; + c2 <<= 1; + } + if (c2 != 1) + Sizes[NumLevels++] = c2; + if (c != 1) + Sizes[NumLevels++] = c; + if (numCoreThreads != 1) + Sizes[NumLevels++] = numCoreThreads; + if (NumLevels == 0) + Sizes[NumLevels++] = 1; + + /* + printf("\n Cores:"); + for (unsigned i = 0; i < NumLevels; i++) + { + printf(" %d", Sizes[i]); + } + printf("\n"); + */ +} + + +DWORD_PTR CAffinityMode::GetAffinityMask(UInt32 bundleIndex, CCpuSet *cpuSet) const +{ + CpuSet_Zero(cpuSet); + + if (NumLevels == 0) + return 0; + + // printf("\n%2d", bundleIndex); + + /* + UInt32 low = 0; + if (DivideNum != 1) + { + low = bundleIndex % DivideNum; + bundleIndex /= DivideNum; + } + */ + + UInt32 numGroups = NumCores / NumBundleThreads; + UInt32 m = bundleIndex % numGroups; + UInt32 v = 0; + for (unsigned i = 0; i < NumLevels; i++) + { + UInt32 size = Sizes[i]; + while ((size & 1) == 0) + { + v *= 2; + v |= (m & 1); + m >>= 1; + size >>= 1; + } + v *= size; + v += m % size; + m /= size; + } + + // UInt32 nb = NumBundleThreads / DivideNum; + UInt32 nb = NumBundleThreads; + + DWORD_PTR mask = ((DWORD_PTR)1 << nb) - 1; + // v += low; + mask <<= v; + + // printf(" %2d %8x \n ", v, (unsigned)mask); + #ifdef _WIN32 + *cpuSet = mask; + #else + { + for (unsigned k = 0; k < nb; k++) + CpuSet_Set(cpuSet, v + k); + } + #endif + + return mask; } -struct CEncoderInfo; -struct CEncoderInfo +struct CBenchSyncCommon +{ + bool ExitMode; + NSynchronization::CManualResetEvent StartEvent; + + CBenchSyncCommon(): ExitMode(false) {} +}; + +#endif + + + +class CEncoderInfo; + +class CEncoderInfo { + CLASS_NO_COPY(CEncoderInfo) + +public: + #ifndef _7ZIP_ST NWindows::CThread thread[2]; + NSynchronization::CManualResetEvent ReadyEvent; UInt32 NumDecoderSubThreads; + CBenchSyncCommon *Common; + UInt32 EncoderIndex; + UInt32 NumEncoderInternalThreads; + CAffinityMode AffinityMode; #endif + CMyComPtr _encoder; CMyComPtr _encoderFilter; CBenchProgressInfo *progressInfoSpec[2]; CMyComPtr progressInfo[2]; UInt64 NumIterations; + UInt32 Salt; + #ifdef USE_ALLOCA size_t AllocaSize; #endif + unsigned KeySize; Byte _key[32]; Byte _iv[16]; + + HRESULT Set_Key_and_IV(ICryptoProperties *cp) + { + RINOK(cp->SetKey(_key, KeySize)); + return cp->SetInitVector(_iv, sizeof(_iv)); + } + Byte _psw[16]; + bool CheckCrc_Enc; bool CheckCrc_Dec; @@ -741,26 +1061,37 @@ struct CEncoderInfo const Byte *fileData; CBenchRandomGenerator rg; - CBenchBuffer rgCopy; // it must be 16-byte aligned !!! - CBenchmarkOutStream *propStreamSpec; + CMidAlignedBuffer rgCopy; // it must be 16-byte aligned !!! + + // CBenchmarkOutStream *propStreamSpec; + Byte propsData[kMaxMethodPropSize]; + CBufPtrSeqOutStream *propStreamSpec; CMyComPtr propStream; - // for decode + unsigned generateDictBits; COneMethodInfo _method; + + // for decode size_t _uncompressedDataSize; - HRESULT Init( - const COneMethodInfo &method, - unsigned generateDictBits, - CBaseRandomGenerator *rg); + HRESULT Generate(); HRESULT Encode(); HRESULT Decode(UInt32 decoderIndex); CEncoderInfo(): - fileData(NULL), + #ifndef _7ZIP_ST + Common(NULL), + #endif + Salt(0), + KeySize(0), CheckCrc_Enc(true), CheckCrc_Dec(true), - outStreamSpec(0), callback(0), printCallback(0), propStreamSpec(0) {} + outStreamSpec(NULL), + callback(NULL), + printCallback(NULL), + fileData(NULL), + propStreamSpec(NULL) + {} #ifndef _7ZIP_ST @@ -775,14 +1106,15 @@ struct CEncoderInfo #endif res = encoder->Encode(); - encoder->Results[0] = res; } catch(...) { res = E_FAIL; } + encoder->Results[0] = res; if (res != S_OK) encoder->progressInfoSpec[0]->Status->SetResult(res); + encoder->ReadyEvent.Set(); return 0; } @@ -801,7 +1133,13 @@ struct CEncoderInfo HRESULT CreateEncoderThread() { - return thread[0].Create(EncodeThreadFunction, this); + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = AffinityMode.CreateThread_WithAffinity(thread[0], EncodeThreadFunction, this, + EncoderIndex); + return HRESULT_FROM_WIN32(res); } HRESULT CreateDecoderThread(unsigned index, bool callbackMode @@ -819,23 +1157,34 @@ struct CEncoderInfo #endif decoder.CallbackMode = callbackMode; - return thread[index].Create(DecodeThreadFunction, &decoder); + + WRes res = AffinityMode.CreateThread_WithAffinity(thread[index], DecodeThreadFunction, &decoder, + // EncoderIndex * NumEncoderInternalThreads + index + EncoderIndex + ); + + return HRESULT_FROM_WIN32(res); } #endif }; -HRESULT CEncoderInfo::Init( - const COneMethodInfo &method, - unsigned generateDictBits, - CBaseRandomGenerator *rgLoc) + + +static size_t GetBenchCompressedSize(size_t bufferSize) +{ + return kCompressedAdditionalSize + bufferSize + bufferSize / 16; + // kBufferSize / 2; +} + + +HRESULT CEncoderInfo::Generate() { + const COneMethodInfo &method = _method; + // we need extra space, if input data is already compressed - const size_t kCompressedBufferSize = - kCompressedAdditionalSize + - kBufferSize + kBufferSize / 16; - // kBufferSize / 2; + const size_t kCompressedBufferSize = GetBenchCompressedSize(kBufferSize); if (kCompressedBufferSize < kBufferSize) return E_FAIL; @@ -844,41 +1193,47 @@ HRESULT CEncoderInfo::Init( if (!fileData) { - if (!rg.Alloc(kBufferSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&rg, kBufferSize); // DWORD ttt = GetTickCount(); if (generateDictBits == 0) - rg.GenerateSimpleRandom(rgLoc); + rg.GenerateSimpleRandom(Salt); else - rg.GenerateLz(generateDictBits, rgLoc); + { + if (generateDictBits >= sizeof(size_t) * 8 + && kBufferSize > ((size_t)1 << (sizeof(size_t) * 8 - 1))) + return E_INVALIDARG; + rg.GenerateLz(generateDictBits, Salt); + // return E_ABORT; // for debug + } // printf("\n%d\n ", GetTickCount() - ttt); - crc = CrcCalc(rg.Buffer, rg.BufferSize); - uncompressedDataPtr = rg.Buffer; + crc = CrcCalc((const Byte *)rg, rg.Size()); + uncompressedDataPtr = (const Byte *)rg; } if (_encoderFilter) { - if (!rgCopy.Alloc(kBufferSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&rgCopy, kBufferSize); } - outStreamSpec = new CBenchmarkOutStream; - outStream = outStreamSpec; - if (!outStreamSpec->Alloc(kCompressedBufferSize)) - return E_OUTOFMEMORY; + if (!outStream) + { + outStreamSpec = new CBenchmarkOutStream; + outStream = outStreamSpec; + } + + ALLOC_WITH_HRESULT(outStreamSpec, kCompressedBufferSize) - propStreamSpec = 0; if (!propStream) { - propStreamSpec = new CBenchmarkOutStream; + propStreamSpec = new CBufPtrSeqOutStream; // CBenchmarkOutStream; propStream = propStreamSpec; } - if (!propStreamSpec->Alloc(kMaxLzmaPropSize)) - return E_OUTOFMEMORY; - propStreamSpec->Init(true, false); + // ALLOC_WITH_HRESULT_2(propStreamSpec, kMaxMethodPropSize); + // propStreamSpec->Init(true, false); + propStreamSpec->Init(propsData, sizeof(propsData)); CMyComPtr coder; @@ -891,10 +1246,25 @@ HRESULT CEncoderInfo::Init( coder.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = kBufferSize; - RINOK(method.SetCoderProps(scp, &reduceSize)); - } - else + const UInt64 reduceSize = kBufferSize; + + /* in posix new thread uses same affinity as parent thread, + so we don't need to send affinity to coder in posix */ + UInt64 affMask; + #if !defined(_7ZIP_ST) && defined(_WIN32) + { + CCpuSet cpuSet; + affMask = AffinityMode.GetAffinityMask(EncoderIndex, &cpuSet); + } + #else + affMask = 0; + #endif + // affMask <<= 3; // debug line: to test no affinity in coder; + // affMask = 0; + + RINOK(method.SetCoderProps_DSReduce_Aff(scp, &reduceSize, (affMask != 0 ? &affMask : NULL))); + } + else { if (method.AreThereNonOptionalProps()) return E_INVALIDARG; @@ -964,6 +1334,32 @@ static void My_FilterBench(ICompressFilter *filter, Byte *data, size_t size) HRESULT CEncoderInfo::Encode() { + // printf("\nCEncoderInfo::Generate\n"); + + RINOK(Generate()); + + // printf("\n2222\n"); + + #ifndef _7ZIP_ST + if (Common) + { + Results[0] = S_OK; + WRes wres = ReadyEvent.Set(); + if (wres == 0) + wres = Common->StartEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + if (Common->ExitMode) + return S_OK; + } + else + #endif + { + CBenchProgressInfo *bpi = progressInfoSpec[0]; + bpi->SetStartTime(); + } + + CBenchInfo &bi = progressInfoSpec[0]->BenchInfo; bi.UnpackSize = 0; bi.PackSize = 0; @@ -976,52 +1372,100 @@ HRESULT CEncoderInfo::Encode() coder.QueryInterface(IID_ICryptoProperties, &cp); CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream; CMyComPtr inStream = inStreamSpec; - UInt64 prev = 0; - - UInt32 crcPrev = 0; if (cp) { - RINOK(cp->SetKey(_key, sizeof(_key))); - RINOK(cp->SetInitVector(_iv, sizeof(_iv))); + RINOK(Set_Key_and_IV(cp)); } - for (UInt64 i = 0; i < NumIterations; i++) + compressedSize = 0; + if (_encoderFilter) + compressedSize = kBufferSize; + + // CBenchmarkOutStream *outStreamSpec = this->outStreamSpec; + UInt64 prev = 0; + const UInt32 mask = (CheckCrc_Enc ? 0 : 0xFFF); + bool useCrc = (mask < NumIterations); + bool crcPrev_defined = false; + UInt32 crcPrev = 0; + UInt64 i = NumIterations; + // printCallback->NewLine(); + + while (i != 0) { - if (printCallback && bi.UnpackSize - prev > (1 << 20)) + i--; + if (printCallback && bi.UnpackSize - prev >= (1 << 24)) { RINOK(printCallback->CheckBreak()); prev = bi.UnpackSize; } + + /* + CBenchInfo info; + progressInfoSpec[0]->SetStartTime(); + */ - bool isLast = (i == NumIterations - 1); - bool calcCrc = ((isLast || (i & 0x7F) == 0 || CheckCrc_Enc) && NumIterations != 1); - outStreamSpec->Init(isLast, calcCrc); - + bool calcCrc = false; + if (useCrc) + calcCrc = (((UInt32)i & mask) == 0); + if (_encoderFilter) { - memcpy(rgCopy.Buffer, uncompressedDataPtr, kBufferSize); + // if (needRealData) + memcpy((Byte *)*outStreamSpec, uncompressedDataPtr, kBufferSize); _encoderFilter->Init(); - My_FilterBench(_encoderFilter, rgCopy.Buffer, kBufferSize); - RINOK(WriteStream(outStream, rgCopy.Buffer, kBufferSize)); + My_FilterBench(_encoderFilter, (Byte *)*outStreamSpec, kBufferSize); + if (calcCrc) + { + outStreamSpec->InitCrc(); + outStreamSpec->Calc((Byte *)*outStreamSpec, kBufferSize); + } } else { + outStreamSpec->Init(true, calcCrc); // write real data for speed consistency at any number of iterations inStreamSpec->Init(uncompressedDataPtr, kBufferSize); RINOK(_encoder->Code(inStream, outStream, NULL, NULL, progressInfo[0])); + if (!inStreamSpec->WasFinished()) + return E_FAIL; + if (compressedSize != outStreamSpec->Pos) + { + if (compressedSize != 0) + return E_FAIL; + compressedSize = outStreamSpec->Pos; + } } // outStreamSpec->Print(); - UInt32 crcNew = CRC_GET_DIGEST(outStreamSpec->Crc); - if (i == 0) - crcPrev = crcNew; - else if (calcCrc && crcPrev != crcNew) - return E_FAIL; + if (calcCrc) + { + const UInt32 crc2 = CRC_GET_DIGEST(outStreamSpec->Crc); + if (crcPrev_defined && crcPrev != crc2) + return E_FAIL; + crcPrev = crc2; + crcPrev_defined = true; + } - compressedSize = outStreamSpec->Pos; bi.UnpackSize += kBufferSize; bi.PackSize += compressedSize; + + /* + { + progressInfoSpec[0]->SetFinishTime(info); + info.UnpackSize = 0; + info.PackSize = 0; + info.NumIterations = 1; + + info.UnpackSize = kBufferSize; + info.PackSize = compressedSize; + // printf("\n%7d\n", encoder.compressedSize); + + RINOK(callback->SetEncodeResult(info, true)); + printCallback->NewLine(); + } + */ + } _encoder.Release(); @@ -1047,7 +1491,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) CMyComPtr setDecProps; coder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecProps); - if (!setDecProps && propStreamSpec->Pos != 0) + if (!setDecProps && propStreamSpec->GetPos() != 0) return E_FAIL; CCrcOutStream *crcOutStreamSpec = new CCrcOutStream; @@ -1072,7 +1516,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) coder.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = _uncompressedDataSize; + const UInt64 reduceSize = _uncompressedDataSize; RINOK(_method.SetCoderProps(scp, &reduceSize)); } @@ -1081,7 +1525,10 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (setDecProps) { - RINOK(setDecProps->SetDecoderProperties2(propStreamSpec->Buffer, (UInt32)propStreamSpec->Pos)); + RINOK(setDecProps->SetDecoderProperties2( + /* (const Byte *)*propStreamSpec, */ + propsData, + (UInt32)propStreamSpec->GetPos())); } { @@ -1097,40 +1544,84 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (cp) { - RINOK(cp->SetKey(_key, sizeof(_key))); - RINOK(cp->SetInitVector(_iv, sizeof(_iv))); + RINOK(Set_Key_and_IV(cp)); + } + + CMyComPtr setFinishMode; + + if (_decoderFilter) + { + if (compressedSize > rgCopy.Size()) + return E_FAIL; + } + else + { + decoder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); } - for (UInt64 i = 0; i < NumIterations; i++) + const UInt64 numIterations = this->NumIterations; + const UInt32 mask = (CheckCrc_Dec ? 0 : 0xFFF); + + for (UInt64 i = 0; i < numIterations; i++) { - if (printCallback && pi->BenchInfo.UnpackSize - prev > (1 << 20)) + if (printCallback && pi->BenchInfo.UnpackSize - prev >= (1 << 24)) { RINOK(printCallback->CheckBreak()); prev = pi->BenchInfo.UnpackSize; } - inStreamSpec->Init(outStreamSpec->Buffer, compressedSize); + const UInt64 outSize = kBufferSize; + bool calcCrc = false; + if (((UInt32)i & mask) == 0) + calcCrc = true; crcOutStreamSpec->Init(); - UInt64 outSize = kBufferSize; - crcOutStreamSpec->CalcCrc = ((i & 0x7F) == 0 || CheckCrc_Dec); - if (_decoderFilter) { - if (compressedSize > rgCopy.BufferSize) - return E_FAIL; - memcpy(rgCopy.Buffer, outStreamSpec->Buffer, compressedSize); + if (calcCrc) // for pure filter speed test without multi-iteration consistency + // if (needRealData) + memcpy((Byte *)rgCopy, (const Byte *)*outStreamSpec, compressedSize); _decoderFilter->Init(); - My_FilterBench(_decoderFilter, rgCopy.Buffer, compressedSize); - RINOK(WriteStream(crcOutStream, rgCopy.Buffer, compressedSize)); + My_FilterBench(_decoderFilter, (Byte *)rgCopy, compressedSize); + if (calcCrc) + crcOutStreamSpec->Calc((const Byte *)rgCopy, compressedSize); } else { + crcOutStreamSpec->CalcCrc = calcCrc; + inStreamSpec->Init((const Byte *)*outStreamSpec, compressedSize); + + if (setFinishMode) + { + RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))); + } + RINOK(decoder->Code(inStream, crcOutStream, 0, &outSize, progressInfo[decoderIndex])); + + if (setFinishMode) + { + if (!inStreamSpec->WasFinished()) + return S_FALSE; + + CMyComPtr getInStreamProcessedSize; + decoder.QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); + + if (getInStreamProcessedSize) + { + UInt64 processed; + RINOK(getInStreamProcessedSize->GetInStreamProcessedSize(&processed)); + if (processed != compressedSize) + return S_FALSE; + } + } + + if (crcOutStreamSpec->Pos != outSize) + return S_FALSE; } - if (crcOutStreamSpec->CalcCrc && CRC_GET_DIGEST(crcOutStreamSpec->Crc) != crc) + if (calcCrc && CRC_GET_DIGEST(crcOutStreamSpec->Crc) != crc) return S_FALSE; + pi->BenchInfo.UnpackSize += kBufferSize; pi->BenchInfo.PackSize += compressedSize; } @@ -1146,7 +1637,7 @@ static const UInt32 kNumThreadsMax = (1 << 12); struct CBenchEncoders { CEncoderInfo *encoders; - CBenchEncoders(UInt32 num): encoders(0) { encoders = new CEncoderInfo[num]; } + CBenchEncoders(UInt32 num): encoders(NULL) { encoders = new CEncoderInfo[num]; } ~CBenchEncoders() { delete []encoders; } }; @@ -1160,19 +1651,74 @@ static UInt64 GetNumIterations(UInt64 numCommands, UInt64 complexInCommands) } + +#ifndef _7ZIP_ST + +// ---------- CBenchThreadsFlusher ---------- + +struct CBenchThreadsFlusher +{ + CBenchEncoders *EncodersSpec; + CBenchSyncCommon Common; + unsigned NumThreads; + bool NeedClose; + + CBenchThreadsFlusher(): NumThreads(0), NeedClose(false) {} + + ~CBenchThreadsFlusher() + { + StartAndWait(true); + } + + WRes StartAndWait(bool exitMode = false); +}; + + +WRes CBenchThreadsFlusher::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes res = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + NWindows::CThread &t = EncodersSpec->encoders[i].thread[0]; + if (t.IsCreated()) + { + WRes res2 = t.Wait_Close(); + if (res == 0) + res = res2; + } + } + NeedClose = false; + return res; +} + +#endif // _7ZIP_ST + + + +static void SetPseudoRand(Byte *data, size_t size, UInt32 startValue) +{ + for (size_t i = 0; i < size; i++) + { + data[i] = (Byte)startValue; + startValue++; + } +} + + + static HRESULT MethodBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, - bool - #ifndef _7ZIP_ST - oldLzmaBenchMode - #endif - , - UInt32 - #ifndef _7ZIP_ST - numThreads - #endif - , + #ifndef _7ZIP_ST + bool oldLzmaBenchMode, + UInt32 numThreads, + const CAffinityMode *affinityMode, + #endif const COneMethodInfo &method2, size_t uncompressedDataSize, const Byte *fileData, @@ -1185,9 +1731,11 @@ static HRESULT MethodBench( COneMethodInfo method = method2; UInt64 methodId; UInt32 numStreams; - if (!FindMethod( + const int codecIndex = FindMethod_Index( EXTERNAL_CODECS_LOC_VARS - method.MethodName, methodId, numStreams)) + method.MethodName, true, + methodId, numStreams); + if (codecIndex < 0) return E_NOTIMPL; if (numStreams != 1) return E_INVALIDARG; @@ -1198,18 +1746,21 @@ static HRESULT MethodBench( #ifndef _7ZIP_ST numEncoderThreads = numThreads; - if (oldLzmaBenchMode && methodId == k_LZMA) + if (oldLzmaBenchMode) + if (methodId == k_LZMA) { - bool fixedNumber; - UInt32 numLzmaThreads = method.Get_Lzma_NumThreads(fixedNumber); - if (!fixedNumber && numThreads == 1) + if (numThreads == 1 && method.Get_NumThreads() < 0) method.AddProp_NumThreads(1); + const UInt32 numLzmaThreads = method.Get_Lzma_NumThreads(); if (numThreads > 1 && numLzmaThreads > 1) { - numEncoderThreads = numThreads / 2; + numEncoderThreads = (numThreads + 1) / 2; // 20.03 numSubDecoderThreads = 2; } } + + bool mtEncMode = (numEncoderThreads > 1) || affinityMode->NeedAffinity(); + #endif CBenchEncoders encodersSpec(numEncoderThreads); @@ -1223,35 +1774,50 @@ static HRESULT MethodBench( encoder.callback = (i == 0) ? callback : 0; encoder.printCallback = printCallback; + #ifndef _7ZIP_ST + encoder.EncoderIndex = i; + encoder.NumEncoderInternalThreads = numSubDecoderThreads; + encoder.AffinityMode = *affinityMode; + + /* + if (numSubDecoderThreads > 1) + if (encoder.AffinityMode.NeedAffinity() + && encoder.AffinityMode.NumBundleThreads == 1) + { + // if old LZMA benchmark uses two threads in coder, we increase (NumBundleThreads) for old LZMA benchmark uses two threads instead of one + if (encoder.AffinityMode.NumBundleThreads * 2 <= encoder.AffinityMode.NumCores) + encoder.AffinityMode.NumBundleThreads *= 2; + } + */ + + #endif + { CCreatedCoder cod; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, true, encoder._encoderFilter, cod)); + RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS (unsigned)codecIndex, true, encoder._encoderFilter, cod)); encoder._encoder = cod.Coder; if (!encoder._encoder && !encoder._encoderFilter) return E_NOTIMPL; } - encoder.CheckCrc_Enc = (benchProps->EncComplex) > 30 ; - encoder.CheckCrc_Dec = (benchProps->DecComplexCompr + benchProps->DecComplexUnc) > 30 ; + encoder.CheckCrc_Enc = (benchProps->EncComplex) > 30; + encoder.CheckCrc_Dec = (benchProps->DecComplexCompr + benchProps->DecComplexUnc) > 30; - memset(encoder._iv, 0, sizeof(encoder._iv)); - memset(encoder._key, 0, sizeof(encoder._key)); - memset(encoder._psw, 0, sizeof(encoder._psw)); + SetPseudoRand(encoder._iv, sizeof(encoder._iv), 17); + SetPseudoRand(encoder._key, sizeof(encoder._key), 51); + SetPseudoRand(encoder._psw, sizeof(encoder._psw), 123); for (UInt32 j = 0; j < numSubDecoderThreads; j++) { CCreatedCoder cod; CMyComPtr &decoder = encoder._decoders[j]; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); decoder = cod.Coder; if (!encoder._decoderFilter && !decoder) return E_NOTIMPL; } } - CBaseRandomGenerator rg; - rg.Init(); - UInt32 crc = 0; if (fileData) crc = CrcCalc(fileData, uncompressedDataSize); @@ -1260,22 +1826,41 @@ static HRESULT MethodBench( { CEncoderInfo &encoder = encoders[i]; encoder._method = method; + encoder.generateDictBits = generateDictBits; encoder._uncompressedDataSize = uncompressedDataSize; encoder.kBufferSize = uncompressedDataSize; encoder.fileData = fileData; encoder.crc = crc; - - RINOK(encoders[i].Init(method, generateDictBits, &rg)); } CBenchProgressStatus status; status.Res = S_OK; status.EncodeMode = true; + #ifndef _7ZIP_ST + CBenchThreadsFlusher encoderFlusher; + if (mtEncMode) + { + WRes wres = encoderFlusher.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + encoderFlusher.NumThreads = numEncoderThreads; + encoderFlusher.EncodersSpec = &encodersSpec; + encoderFlusher.NeedClose = true; + } + #endif + for (i = 0; i < numEncoderThreads; i++) { CEncoderInfo &encoder = encoders[i]; encoder.NumIterations = GetNumIterations(benchProps->GeComprCommands(uncompressedDataSize), complexInCommands); + // encoder.NumIterations = 3; + encoder.Salt = g_CrcTable[i & 0xFF]; + encoder.Salt ^= (g_CrcTable[(i >> 8) & 0xFF] << 3); + // (g_CrcTable[0] == 0), and (encoder.Salt == 0) for first thread + // printf(" %8x", encoder.Salt); + + encoder.KeySize = benchProps->KeySize; for (int j = 0; j < 2; j++) { @@ -1290,30 +1875,50 @@ static HRESULT MethodBench( CBenchProgressInfo *bpi = encoder.progressInfoSpec[0]; bpi->Callback = callback; bpi->BenchInfo.NumIterations = numEncoderThreads; - bpi->SetStartTime(); } #ifndef _7ZIP_ST - if (numEncoderThreads > 1) + if (mtEncMode) { #ifdef USE_ALLOCA encoder.AllocaSize = (i * 16 * 21) & 0x7FF; #endif + encoder.Common = &encoderFlusher.Common; RINOK(encoder.CreateEncoderThread()) } - else #endif - { - RINOK(encoder.Encode()); - } } - + + if (printCallback) + { + RINOK(printCallback->CheckBreak()); + } + #ifndef _7ZIP_ST - if (numEncoderThreads > 1) + if (mtEncMode) + { for (i = 0; i < numEncoderThreads; i++) - encoders[i].thread[0].Wait(); + { + CEncoderInfo &encoder = encoders[i]; + WRes wres = encoder.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(encoder.Results[0]); + } + + CBenchProgressInfo *bpi = encoders[0].progressInfoSpec[0]; + bpi->SetStartTime(); + + WRes wres = encoderFlusher.StartAndWait(); + if (status.Res == 0 && wres != 0) + return HRESULT_FROM_WIN32(wres); + } + else #endif + { + RINOK(encoders[0].Encode()); + } RINOK(status.Res); @@ -1329,20 +1934,37 @@ static HRESULT MethodBench( CEncoderInfo &encoder = encoders[i]; info.UnpackSize += encoder.kBufferSize; info.PackSize += encoder.compressedSize; + // printf("\n%7d\n", encoder.compressedSize); } RINOK(callback->SetEncodeResult(info, true)); + + + // ---------- Decode ---------- + status.Res = S_OK; status.EncodeMode = false; - UInt32 numDecoderThreads = numEncoderThreads * numSubDecoderThreads; + const UInt32 numDecoderThreads = numEncoderThreads * numSubDecoderThreads; + #ifndef _7ZIP_ST + const bool mtDecoderMode = (numDecoderThreads > 1) || affinityMode->NeedAffinity(); + #endif for (i = 0; i < numEncoderThreads; i++) { CEncoderInfo &encoder = encoders[i]; + /* + #ifndef _7ZIP_ST + // encoder.affinityMode = *affinityMode; + if (encoder.NumEncoderInternalThreads != 1) + encoder.AffinityMode.DivideNum = encoder.NumEncoderInternalThreads; + #endif + */ + + if (i == 0) { encoder.NumIterations = GetNumIterations(benchProps->GeDecomprCommands(encoder.compressedSize, encoder.kBufferSize), complexInCommands); @@ -1357,9 +1979,9 @@ static HRESULT MethodBench( #ifndef _7ZIP_ST { int numSubThreads = method.Get_NumThreads(); - encoder.NumDecoderSubThreads = (numSubThreads <= 0) ? 1 : numSubThreads; + encoder.NumDecoderSubThreads = (numSubThreads <= 0) ? 1 : (unsigned)numSubThreads; } - if (numDecoderThreads > 1) + if (mtDecoderMode) { for (UInt32 j = 0; j < numSubDecoderThreads; j++) { @@ -1379,25 +2001,36 @@ static HRESULT MethodBench( } #ifndef _7ZIP_ST - HRESULT res = S_OK; - if (numDecoderThreads > 1) + if (mtDecoderMode) + { + WRes wres = 0; + HRESULT res = S_OK; for (i = 0; i < numEncoderThreads; i++) for (UInt32 j = 0; j < numSubDecoderThreads; j++) { CEncoderInfo &encoder = encoders[i]; - encoder.thread[j].Wait(); - if (encoder.Results[j] != S_OK) - res = encoder.Results[j]; + WRes wres2 = encoder.thread[j]. + // Wait(); // later we can get thread times from thread in UNDER_CE + Wait_Close(); + if (wres == 0 && wres2 != 0) + wres = wres2; + HRESULT res2 = encoder.Results[j]; + if (res == 0 && res2 != 0) + res = res2; } - RINOK(res); - #endif - + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(res); + } + #endif // _7ZIP_ST + RINOK(status.Res); encoders[0].progressInfoSpec[0]->SetFinishTime(info); - + + /* #ifndef _7ZIP_ST #ifdef UNDER_CE - if (numDecoderThreads > 1) + if (mtDecoderMode) for (i = 0; i < numEncoderThreads; i++) for (UInt32 j = 0; j < numSubDecoderThreads; j++) { @@ -1407,6 +2040,7 @@ static HRESULT MethodBench( } #endif #endif + */ info.UnpackSize = 0; info.PackSize = 0; @@ -1419,16 +2053,36 @@ static HRESULT MethodBench( info.PackSize += encoder.compressedSize; } - RINOK(callback->SetDecodeResult(info, false)); + // RINOK(callback->SetDecodeResult(info, false)); // why we called before 21.03 ?? RINOK(callback->SetDecodeResult(info, true)); return S_OK; } -static inline UInt64 GetLZMAUsage(bool multiThread, UInt32 dictionary) + +static inline UInt64 GetDictSizeFromLog(unsigned dictSizeLog) +{ + /* + if (dictSizeLog < 32) + return (UInt32)1 << dictSizeLog; + else + return (UInt32)(Int32)-1; + */ + return (UInt64)1 << dictSizeLog; +} + + +// it's limit of current LZMA implementation that can be changed later +#define kLzmaMaxDictSize ((UInt32)15 << 28) + +static inline UInt64 GetLZMAUsage(bool multiThread, int btMode, UInt64 dict) { - UInt32 hs = dictionary - 1; + if (dict == 0) + dict = 1; + if (dict > kLzmaMaxDictSize) + dict = kLzmaMaxDictSize; + UInt32 hs = (UInt32)dict - 1; hs |= (hs >> 1); hs |= (hs >> 2); hs |= (hs >> 4); @@ -1438,53 +2092,158 @@ static inline UInt64 GetLZMAUsage(bool multiThread, UInt32 dictionary) if (hs > (1 << 24)) hs >>= 1; hs++; - return ((hs + (1 << 16)) + (UInt64)dictionary * 2) * 4 + (UInt64)dictionary * 3 / 2 + + hs += (1 << 16); + + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)(1 << 16); + UInt64 blockSize = (UInt64)dict + (1 << 16) + + (multiThread ? (1 << 20) : 0); + blockSize += (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)); + if (blockSize >= kBlockSizeMax) + blockSize = kBlockSizeMax; + + UInt64 son = (UInt64)dict; + if (btMode) + son *= 2; + const UInt64 v = (hs + son) * 4 + blockSize + (1 << 20) + (multiThread ? (6 << 20) : 0); + + // printf("\nGetLZMAUsage = %d\n", (UInt32)(v >> 20)); + // printf("\nblockSize = %d\n", (UInt32)(blockSize >> 20)); + return v; } -UInt64 GetBenchMemoryUsage(UInt32 numThreads, UInt32 dictionary, bool totalBench) + +UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool totalBench) { - const UInt32 kBufferSize = dictionary; - const UInt32 kCompressedBufferSize = kBufferSize; // / 2; - bool lzmaMt = (totalBench || numThreads > 1); + const size_t kBufferSize = (size_t)dictionary + kAdditionalSize; + const UInt64 kCompressedBufferSize = GetBenchCompressedSize(kBufferSize); // / 2; + if (level < 0) + level = 5; + const int algo = (level < 5 ? 0 : 1); + const int btMode = (algo == 0 ? 0 : 1); + UInt32 numBigThreads = numThreads; - if (!totalBench && lzmaMt) - numBigThreads /= 2; + bool lzmaMt = (totalBench || (numThreads > 1 && btMode)); + if (btMode) + { + if (!totalBench && lzmaMt) + numBigThreads /= 2; + } return ((UInt64)kBufferSize + kCompressedBufferSize + - GetLZMAUsage(lzmaMt, dictionary) + (2 << 20)) * numBigThreads; + GetLZMAUsage(lzmaMt, btMode, dictionary) + (2 << 20)) * numBigThreads; +} + +static UInt64 GetBenchMemoryUsage_Hash(UInt32 numThreads, UInt64 dictionary) +{ + // dictionary += (dictionary >> 9); // for page tables (virtual memory) + return (UInt64)(dictionary + (1 << 15)) * numThreads + (2 << 20); +} + + +// ---------- CRC and HASH ---------- + +struct CCrcInfo_Base +{ + CMidAlignedBuffer Buffer; + const Byte *Data; + size_t Size; + bool CreateLocalBuf; + UInt32 CheckSum_Res; + + CCrcInfo_Base(): CreateLocalBuf(true), CheckSum_Res(0) {} + + HRESULT Generate(const Byte *data, size_t size); + HRESULT CrcProcess(UInt64 numIterations, + const UInt32 *checkSum, IHasher *hf, + IBenchPrintCallback *callback); +}; + + +HRESULT CCrcInfo_Base::Generate(const Byte *data, size_t size) +{ + Size = size; + Data = data; + if (!data || CreateLocalBuf) + { + ALLOC_WITH_HRESULT(&Buffer, size) + Data = Buffer; + } + if (!data) + RandGen(Buffer, size); + else if (CreateLocalBuf && size != 0) + memcpy(Buffer, data, size); + return S_OK; } -static HRESULT CrcBig(const void *data, UInt32 size, UInt64 numIterations, + +HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, const UInt32 *checkSum, IHasher *hf, IBenchPrintCallback *callback) { + MY_ALIGN(16) Byte hash[64]; - UInt64 i; - for (i = 0; i < sizeof(hash); i++) - hash[i] = 0; - for (i = 0; i < numIterations; i++) + memset(hash, 0, sizeof(hash)); + + CheckSum_Res = 0; + + const UInt32 hashSize = hf->GetDigestSize(); + if (hashSize > sizeof(hash)) + return S_FALSE; + + const Byte *buf = Data; + const size_t size = Size; + UInt32 checkSum_Prev = 0; + + UInt64 prev = 0; + UInt64 cur = 0; + + for (UInt64 i = 0; i < numIterations; i++) { - if (callback && (i & 0xFF) == 0) + hf->Init(); + size_t pos = 0; + do { - RINOK(callback->CheckBreak()); + const size_t rem = size - pos; + const UInt32 kStep = ((UInt32)1 << 31); + const UInt32 curSize = (rem < kStep) ? (UInt32)rem : kStep; + hf->Update(buf + pos, curSize); + pos += curSize; } - hf->Init(); - hf->Update(data, size); + while (pos != size); + hf->Final(hash); - UInt32 hashSize = hf->GetDigestSize(); - if (hashSize > sizeof(hash)) - return S_FALSE; UInt32 sum = 0; for (UInt32 j = 0; j < hashSize; j += 4) - sum ^= GetUi32(hash + j); - if (checkSum && sum != *checkSum) { - return S_FALSE; + sum = rotlFixed(sum, 11); + sum += GetUi32(hash + j); + } + if (checkSum) + { + if (sum != *checkSum) + return S_FALSE; + } + else + { + checkSum_Prev = sum; + checkSum = &checkSum_Prev; + } + if (callback) + { + cur += size; + if (cur - prev >= ((UInt32)1 << 30)) + { + prev = cur; + RINOK(callback->CheckBreak()); + } } } + CheckSum_Res = checkSum_Prev; return S_OK; } +extern +UInt32 g_BenchCpuFreqTemp; // we need non-static variavble to disable compiler optimization UInt32 g_BenchCpuFreqTemp = 1; #define YY1 sum += val; sum ^= val; @@ -1509,22 +2268,27 @@ EXTERN_C_END #ifndef _7ZIP_ST -struct CFreqInfo +struct CBaseThreadInfo { NWindows::CThread Thread; IBenchPrintCallback *Callback; HRESULT CallbackRes; - UInt32 ValRes; - UInt32 Size; - UInt64 NumIterations; - void Wait() + WRes Wait_If_Created() { - Thread.Wait(); - Thread.Close(); + if (!Thread.IsCreated()) + return 0; + return Thread.Wait_Close(); } }; +struct CFreqInfo: public CBaseThreadInfo +{ + UInt32 ValRes; + UInt32 Size; + UInt64 NumIterations; +}; + static THREAD_FUNC_DECL FreqThreadFunction(void *param) { CFreqInfo *p = (CFreqInfo *)param; @@ -1532,9 +2296,12 @@ static THREAD_FUNC_DECL FreqThreadFunction(void *param) UInt32 sum = g_BenchCpuFreqTemp; for (UInt64 k = p->NumIterations; k > 0; k--) { - p->CallbackRes = p->Callback->CheckBreak(); - if (p->CallbackRes != S_OK) - return 0; + if (p->Callback) + { + p->CallbackRes = p->Callback->CheckBreak(); + if (p->CallbackRes != S_OK) + return 0; + } sum = CountCpuFreq(sum, p->Size, g_BenchCpuFreqTemp); } p->ValRes = sum; @@ -1546,13 +2313,21 @@ struct CFreqThreads CFreqInfo *Items; UInt32 NumThreads; - CFreqThreads(): Items(0), NumThreads(0) {} - void WaitAll() + CFreqThreads(): Items(NULL), NumThreads(0) {} + + WRes WaitAll() { + WRes wres = 0; for (UInt32 i = 0; i < NumThreads; i++) - Items[i].Wait(); + { + WRes wres2 = Items[i].Wait_If_Created(); + if (wres == 0 && wres2 != 0) + wres = wres2; + } NumThreads = 0; + return wres; } + ~CFreqThreads() { WaitAll(); @@ -1560,105 +2335,184 @@ struct CFreqThreads } }; -struct CCrcInfo -{ - NWindows::CThread Thread; - IBenchPrintCallback *Callback; - HRESULT CallbackRes; +static THREAD_FUNC_DECL CrcThreadFunction(void *param); + +struct CCrcInfo: public CBaseThreadInfo +{ const Byte *Data; - UInt32 Size; + size_t Size; UInt64 NumIterations; bool CheckSumDefined; UInt32 CheckSum; CMyComPtr Hasher; HRESULT Res; + UInt32 CheckSum_Res; - #ifdef USE_ALLOCA - size_t AllocaSize; + #ifndef _7ZIP_ST + NSynchronization::CManualResetEvent ReadyEvent; + UInt32 ThreadIndex; + CBenchSyncCommon *Common; + CAffinityMode AffinityMode; #endif - void Wait() + // we want to call CCrcInfo_Base::Buffer.Free() in main thread. + // so we uses non-local CCrcInfo_Base. + CCrcInfo_Base crcib; + + HRESULT CreateThread() { - Thread.Wait(); - Thread.Close(); + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = AffinityMode.CreateThread_WithAffinity(Thread, CrcThreadFunction, this, + ThreadIndex); + return HRESULT_FROM_WIN32(res); } -}; -static THREAD_FUNC_DECL CrcThreadFunction(void *param) -{ - CCrcInfo *p = (CCrcInfo *)param; - #ifdef USE_ALLOCA - alloca(p->AllocaSize); + size_t AllocaSize; #endif - p->Res = CrcBig(p->Data, p->Size, p->NumIterations, - p->CheckSumDefined ? &p->CheckSum : NULL, p->Hasher, - p->Callback); + void Process(); + + CCrcInfo(): Res(E_FAIL) {} +}; + +static const bool k_Crc_CreateLocalBuf_For_File = true; // for total BW test +// static const bool k_Crc_CreateLocalBuf_For_File = false; // for shared memory read test + +void CCrcInfo::Process() +{ + crcib.CreateLocalBuf = k_Crc_CreateLocalBuf_For_File; + // we can use additional Generate() passes to reduce some time effects for new page allocation + // for (unsigned y = 0; y < 10; y++) + Res = crcib.Generate(Data, Size); + + // if (Common) + { + WRes wres = ReadyEvent.Set(); + if (wres != 0) + { + if (Res == 0) + Res = HRESULT_FROM_WIN32(wres); + return; + } + if (Res != 0) + return; + + wres = Common->StartEvent.Lock(); + + if (wres != 0) + { + Res = HRESULT_FROM_WIN32(wres); + return; + } + if (Common->ExitMode) + return; + } + + Res = crcib.CrcProcess(NumIterations, + CheckSumDefined ? &CheckSum : NULL, Hasher, + Callback); + CheckSum_Res = crcib.CheckSum_Res; + /* + We don't want to include the time of slow CCrcInfo_Base::Buffer.Free() + to time of benchmark. So we don't free Buffer here + */ + // crcib.Buffer.Free(); +} + + +static THREAD_FUNC_DECL CrcThreadFunction(void *param) +{ + CCrcInfo *p = (CCrcInfo *)param; + + #ifdef USE_ALLOCA + alloca(p->AllocaSize); + #endif + p->Process(); return 0; } + struct CCrcThreads { CCrcInfo *Items; - UInt32 NumThreads; + unsigned NumThreads; + CBenchSyncCommon Common; + bool NeedClose; + + CCrcThreads(): Items(NULL), NumThreads(0), NeedClose(false) {} + + WRes StartAndWait(bool exitMode = false); - CCrcThreads(): Items(0), NumThreads(0) {} - void WaitAll() - { - for (UInt32 i = 0; i < NumThreads; i++) - Items[i].Wait(); - NumThreads = 0; - } ~CCrcThreads() { - WaitAll(); + StartAndWait(true); delete []Items; } }; + +WRes CCrcThreads::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes wres = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + WRes wres2 = Items[i].Wait_If_Created(); + if (wres == 0 && wres2 != 0) + wres = wres2; + } + NumThreads = 0; + NeedClose = false; + return wres; +} + #endif -static UInt32 CrcCalc1(const Byte *buf, UInt32 size) + +static UInt32 CrcCalc1(const Byte *buf, size_t size) { UInt32 crc = CRC_INIT_VAL;; - for (UInt32 i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) crc = CRC_UPDATE_BYTE(crc, buf[i]); return CRC_GET_DIGEST(crc); } -static void RandGen(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) -{ - for (UInt32 i = 0; i < size; i++) - buf[i] = (Byte)RG.GetRnd(); -} - -static UInt32 RandGenCrc(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) +/* +static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) { RandGen(buf, size, RG); return CrcCalc1(buf, size); } +*/ -bool CrcInternalTest() +static bool CrcInternalTest() { - CBenchBuffer buffer; - const UInt32 kBufferSize0 = (1 << 8); - const UInt32 kBufferSize1 = (1 << 10); - const UInt32 kCheckSize = (1 << 5); - if (!buffer.Alloc(kBufferSize0 + kBufferSize1)) + CAlignedBuffer buffer; + const size_t kBufferSize0 = (1 << 8); + const size_t kBufferSize1 = (1 << 10); + const unsigned kCheckSize = (1 << 5); + buffer.Alloc(kBufferSize0 + kBufferSize1); + if (!buffer.IsAllocated()) return false; - Byte *buf = buffer.Buffer; - UInt32 i; + Byte *buf = (Byte *)buffer; + size_t i; for (i = 0; i < kBufferSize0; i++) buf[i] = (Byte)i; UInt32 crc1 = CrcCalc1(buf, kBufferSize0); if (crc1 != 0x29058C73) return false; - CBaseRandomGenerator RG; - RandGen(buf + kBufferSize0, kBufferSize1, RG); + RandGen(buf + kBufferSize0, kBufferSize1); for (i = 0; i < kBufferSize0 + kBufferSize1 - kCheckSize; i++) - for (UInt32 j = 0; j < kCheckSize; j++) + for (unsigned j = 0; j < kCheckSize; j++) if (CrcCalc1(buf + i, j) != CrcCalc(buf + i, j)) return false; return true; @@ -1672,11 +2526,25 @@ struct CBenchMethod UInt32 DecComplexCompr; UInt32 DecComplexUnc; const char *Name; + // unsigned KeySize; }; +// #define USE_SW_CMPLX + +#ifdef USE_SW_CMPLX +#define CMPLX(x) ((x) * 1000) +#else +#define CMPLX(x) (x) +#endif + static const CBenchMethod g_Bench[] = { - { 40, 17, 357, 145, 20, "LZMA:x1" }, + // { 40, 17, 357, 145, 20, "LZMA:x1" }, + // { 20, 18, 360, 145, 20, "LZMA2:x1:mt2" }, + + { 20, 18, 360, 145, 20, "LZMA:x1" }, + { 20, 22, 600, 145, 20, "LZMA:x3" }, + { 80, 24, 1220, 145, 20, "LZMA:x5:mt1" }, { 80, 24, 1220, 145, 20, "LZMA:x5:mt2" }, @@ -1690,14 +2558,42 @@ static const CBenchMethod g_Bench[] = { 10, 19, 815, 122, 122, "BZip2:x5:mt2" }, { 10, 19, 2530, 122, 122, "BZip2:x7" }, + // { 10, 18, 1010, 0, 1150, "PPMDZip:x1" }, { 10, 18, 1010, 0, 1150, "PPMD:x1" }, + // { 10, 22, 1655, 0, 1830, "PPMDZip:x5" }, { 10, 22, 1655, 0, 1830, "PPMD:x5" }, - { 2, 0, 6, 0, 6, "Delta:4" }, + // { 2, 0, 3, 0, 4, "Delta:1" }, + // { 2, 0, 3, 0, 4, "Delta:2" }, + // { 2, 0, 3, 0, 4, "Delta:3" }, + { 2, 0, 3, 0, 4, "Delta:4" }, + // { 2, 0, 3, 0, 4, "Delta:8" }, + // { 2, 0, 3, 0, 4, "Delta:32" }, + { 2, 0, 4, 0, 4, "BCJ" }, + // { 10, 0, 18, 0, 18, "AES128CBC:1" }, + // { 10, 0, 21, 0, 21, "AES192CBC:1" }, { 10, 0, 24, 0, 24, "AES256CBC:1" }, - { 2, 0, 8, 0, 2, "AES256CBC:2" } + + // { 10, 0, 18, 0, 18, "AES128CTR:1" }, + // { 10, 0, 21, 0, 21, "AES192CTR:1" }, + // { 10, 0, 24, 0, 24, "AES256CTR:1" }, + // { 2, 0, CMPLX(6), 0, CMPLX(1), "AES128CBC:2" }, + // { 2, 0, CMPLX(7), 0, CMPLX(1), "AES192CBC:2" }, + { 2, 0, CMPLX(8), 0, CMPLX(1), "AES256CBC:2" }, + + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES128CTR:2" }, + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:2" }, + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:2" }, + + // { 1, 0, CMPLX(6), 0, CMPLX(1), "AES128CBC:3" }, + // { 1, 0, CMPLX(7), 0, CMPLX(1), "AES192CBC:3" }, + { 1, 0, CMPLX(8), 0, CMPLX(1), "AES256CBC:3" } + + // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES128CTR:3" }, + // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:3" }, + // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:3" }, }; struct CBenchHash @@ -1708,36 +2604,25 @@ struct CBenchHash const char *Name; }; -static const CBenchHash g_Hash[] = -{ - { 1, 1820, 0x8F8FEDAB, "CRC32:1" }, - { 10, 558, 0x8F8FEDAB, "CRC32:4" }, - { 10, 339, 0x8F8FEDAB, "CRC32:8" }, - { 10, 512, 0xDF1C17CC, "CRC64" }, - { 10, 5100, 0x2D79FF2E, "SHA256" }, - { 10, 2340, 0x4C25132B, "SHA1" }, - { 2, 5500, 0xE084E913, "BLAKE2sp" } -}; +// #define ARM_CRC_MUL 100 +#define ARM_CRC_MUL 1 -struct CTotalBenchRes +static const CBenchHash g_Hash[] = { - // UInt64 NumIterations1; // for Usage - UInt64 NumIterations2; // for Rating / RPU - - UInt64 Rating; - UInt64 Usage; - UInt64 RPU; + { 1, 1820, 0x21e207bb, "CRC32:1" }, + { 10, 558, 0x21e207bb, "CRC32:4" }, + { 10, 339, 0x21e207bb, "CRC32:8" } , + { 2, 128 *ARM_CRC_MUL, 0x21e207bb, "CRC32:32" }, + { 2, 64 *ARM_CRC_MUL, 0x21e207bb, "CRC32:64" }, + { 10, 512, 0x41b901d1, "CRC64" }, - void Init() { /* NumIterations1 = 0; */ NumIterations2 = 0; Rating = 0; Usage = 0; RPU = 0; } - - void SetSum(const CTotalBenchRes &r1, const CTotalBenchRes &r2) - { - Rating = (r1.Rating + r2.Rating); - Usage = (r1.Usage + r2.Usage); - RPU = (r1.RPU + r2.RPU); - // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); - NumIterations2 = (r1.NumIterations2 + r2.NumIterations2); - } + { 10, 5100, 0x7913ba03, "SHA256:1" }, + { 2, CMPLX((32 * 4 + 1) * 4 + 4), 0x7913ba03, "SHA256:2" }, + + { 10, 2340, 0xff769021, "SHA1:1" }, + { 2, CMPLX((20 * 6 + 1) * 4 + 4), 0xff769021, "SHA1:2" }, + + { 2, 5500, 0x85189d02, "BLAKE2sp" } }; static void PrintNumber(IBenchPrintCallback &f, UInt64 value, unsigned size) @@ -1754,9 +2639,11 @@ static void PrintNumber(IBenchPrintCallback &f, UInt64 value, unsigned size) unsigned len = (unsigned)strlen(s + startPos); if (size > len) { - startPos -= (size - len); - if (startPos < 0) + size -= len; + if (startPos < size) startPos = 0; + else + startPos -= size; } f.Print(s + startPos); } @@ -1782,7 +2669,10 @@ static void PrintRating(IBenchPrintCallback &f, UInt64 rating, unsigned size) static void PrintPercents(IBenchPrintCallback &f, UInt64 val, UInt64 divider, unsigned size) { - PrintNumber(f, (val * 100 + divider / 2) / divider, size); + UInt64 v = 0; + if (divider != 0) + v = (val * 100 + divider / 2) / divider; + PrintNumber(f, v, size); } static void PrintChars(IBenchPrintCallback &f, char c, unsigned size) @@ -1798,9 +2688,14 @@ static void PrintSpaces(IBenchPrintCallback &f, unsigned size) PrintChars(f, ' ', size); } +static void PrintUsage(IBenchPrintCallback &f, UInt64 usage, unsigned size) +{ + PrintNumber(f, Benchmark_GetUsage_Percents(usage), size); +} + static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt64 rating, bool showFreq, UInt64 cpuFreq) { - PrintNumber(f, (usage + 5000) / 10000, kFieldSize_Usage); + PrintUsage(f, usage, kFieldSize_Usage); PrintRating(f, rpu, kFieldSize_RU); PrintRating(f, rating, kFieldSize_Rating); if (showFreq) @@ -1809,15 +2704,39 @@ static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt6 PrintSpaces(f, kFieldSize_EUAndEffec); else { - UInt64 ddd = cpuFreq * usage / 100; - if (ddd == 0) - ddd = 1; - PrintPercents(f, (rating * 10000), ddd, kFieldSize_EU); + PrintPercents(f, rating, cpuFreq * usage / kBenchmarkUsageMult, kFieldSize_EU); PrintPercents(f, rating, cpuFreq, kFieldSize_Effec); } } } + +void CTotalBenchRes::Generate_From_BenchInfo(const CBenchInfo &info) +{ + Speed = info.GetUnpackSizeSpeed(); + Usage = info.GetUsage(); + RPU = info.GetRatingPerUsage(Rating); +} + +void CTotalBenchRes::Mult_For_Weight(unsigned weight) +{ + NumIterations2 *= weight; + RPU *= weight; + Rating *= weight; + Usage *= weight; + Speed *= weight; +} + +void CTotalBenchRes::Update_With_Res(const CTotalBenchRes &r) +{ + Rating += r.Rating; + Usage += r.Usage; + RPU += r.RPU; + Speed += r.Speed; + // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); + NumIterations2 += r.NumIterations2; +} + static void PrintResults(IBenchPrintCallback *f, const CBenchInfo &info, unsigned weight, @@ -1825,39 +2744,126 @@ static void PrintResults(IBenchPrintCallback *f, bool showFreq, UInt64 cpuFreq, CTotalBenchRes *res) { - UInt64 speed = info.GetSpeed(info.UnpackSize * info.NumIterations); + CTotalBenchRes t; + t.Rating = rating; + t.NumIterations2 = 1; + t.Generate_From_BenchInfo(info); + if (f) { - if (speed != 0) - PrintNumber(*f, speed / 1024, kFieldSize_Speed); + if (t.Speed != 0) + PrintNumber(*f, t.Speed / 1024, kFieldSize_Speed); else PrintSpaces(*f, 1 + kFieldSize_Speed); } - UInt64 usage = info.GetUsage(); - UInt64 rpu = info.GetRatingPerUsage(rating); if (f) { - PrintResults(*f, usage, rpu, rating, showFreq, cpuFreq); + PrintResults(*f, t.Usage, t.RPU, rating, showFreq, cpuFreq); } if (res) { // res->NumIterations1++; - res->NumIterations2 += weight; - res->RPU += (rpu * weight); - res->Rating += (rating * weight); - res->Usage += (usage * weight); + t.Mult_For_Weight(weight); + res->Update_With_Res(t); } } -static void PrintTotals(IBenchPrintCallback &f, bool showFreq, UInt64 cpuFreq, const CTotalBenchRes &res) +static void PrintTotals(IBenchPrintCallback &f, + bool showFreq, UInt64 cpuFreq, bool showSpeed, const CTotalBenchRes &res) { - PrintSpaces(f, 1 + kFieldSize_Speed); + const UInt64 numIterations2 = res.NumIterations2 ? res.NumIterations2 : 1; + const UInt64 speed = res.Speed / numIterations2; + if (showSpeed && speed != 0) + PrintNumber(f, speed / 1024, kFieldSize_Speed); + else + PrintSpaces(f, 1 + kFieldSize_Speed); + + // PrintSpaces(f, 1 + kFieldSize_Speed); // UInt64 numIterations1 = res.NumIterations1; if (numIterations1 == 0) numIterations1 = 1; - UInt64 numIterations2 = res.NumIterations2; if (numIterations2 == 0) numIterations2 = 1; PrintResults(f, res.Usage / numIterations2, res.RPU / numIterations2, res.Rating / numIterations2, showFreq, cpuFreq); } + +static void PrintHex(AString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} + +AString GetProcessThreadsInfo(const NSystem::CProcessAffinity &ti) +{ + AString s; + // s.Add_UInt32(ti.numProcessThreads); + unsigned numSysThreads = ti.GetNumSystemThreads(); + if (ti.GetNumProcessThreads() != numSysThreads) + { + // if (ti.numProcessThreads != ti.numSysThreads) + { + s += " / "; + s.Add_UInt32(numSysThreads); + } + s += " : "; + #ifdef _WIN32 + PrintHex(s, ti.processAffinityMask); + s += " / "; + PrintHex(s, ti.systemAffinityMask); + #else + unsigned i = (numSysThreads + 3) & ~(unsigned)3; + if (i == 0) + i = 4; + for (; i >= 4; ) + { + i -= 4; + unsigned val = 0; + for (unsigned k = 0; k < 4; k++) + { + const unsigned bit = (ti.IsCpuSet(i + k) ? 1 : 0); + val += (bit << k); + } + PrintHex(s, val); + } + #endif + } + return s; +} + + +#ifdef _7ZIP_LARGE_PAGES + +#ifdef _WIN32 +extern bool g_LargePagesMode; +extern "C" +{ + extern SIZE_T g_LargePageSize; +} +#endif + +void Add_LargePages_String(AString &s) +{ + #ifdef _WIN32 + if (g_LargePagesMode || g_LargePageSize != 0) + { + s.Add_OptSpaced("(LP-"); + PrintSize_KMGT_Or_Hex(s, g_LargePageSize); + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_PageGB()) + s += "-1G"; + #endif + if (!g_LargePagesMode) + s += "-NA"; + s += ")"; + } + #else + s += ""; + #endif +} + +#endif + + + static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, bool size_Defined, UInt64 size, const char *threadsString, UInt32 numThreads) { @@ -1867,19 +2873,30 @@ static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, PrintNumber(f, (size >> 20), 6); else f.Print(" ?"); - f.Print(" MB, # "); + f.Print(" MB"); + + #ifdef _7ZIP_LARGE_PAGES + { + AString s; + Add_LargePages_String(s); + f.Print(s); + } + #endif + + f.Print(", # "); f.Print(threadsString); PrintNumber(f, numThreads, 3); - f.NewLine(); } + + struct CBenchCallbackToPrint: public IBenchCallback { CBenchProps BenchProps; CTotalBenchRes EncodeRes; CTotalBenchRes DecodeRes; IBenchPrintCallback *_file; - UInt32 DictSize; + UInt64 DictSize; bool Use2Columns; unsigned NameFieldSize; @@ -1930,7 +2947,7 @@ HRESULT CBenchCallbackToPrint::SetEncodeResult(const CBenchInfo &info, bool fina return S_OK; } -static const char *kSep = " | "; +static const char * const kSep = " | "; HRESULT CBenchCallbackToPrint::SetDecodeResult(const CBenchInfo &info, bool final) { @@ -1963,26 +2980,29 @@ void CBenchCallbackToPrint::NewLine() _file->NewLine(); } -void PrintLeft(IBenchPrintCallback &f, const char *s, unsigned size) +static void PrintLeft(IBenchPrintCallback &f, const char *s, unsigned size) { f.Print(s); - int numSpaces = size - MyStringLen(s); + int numSpaces = (int)size - (int)MyStringLen(s); if (numSpaces > 0) - PrintSpaces(f, numSpaces); + PrintSpaces(f, (unsigned)numSpaces); } -void PrintRight(IBenchPrintCallback &f, const char *s, unsigned size) +static void PrintRight(IBenchPrintCallback &f, const char *s, unsigned size) { - int numSpaces = size - MyStringLen(s); + int numSpaces = (int)size - (int)MyStringLen(s); if (numSpaces > 0) - PrintSpaces(f, numSpaces); + PrintSpaces(f, (unsigned)numSpaces); f.Print(s); } static HRESULT TotalBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, + #ifndef _7ZIP_ST UInt32 numThreads, + const CAffinityMode *affinityMode, + #endif bool forceUnpackSize, size_t unpackSize, const Byte *fileData, @@ -1992,6 +3012,12 @@ static HRESULT TotalBench( { const CBenchMethod &bench = g_Bench[i]; PrintLeft(*callback->_file, bench.Name, kFieldSize_Name); + { + unsigned keySize = 32; + if (IsString1PrefixedByString2(bench.Name, "AES128")) keySize = 16; + else if (IsString1PrefixedByString2(bench.Name, "AES192")) keySize = 24; + callback->BenchProps.KeySize = keySize; + } callback->BenchProps.DecComplexUnc = bench.DecComplexUnc; callback->BenchProps.DecComplexCompr = bench.DecComplexCompr; callback->BenchProps.EncComplex = bench.EncComplex; @@ -2011,7 +3037,10 @@ static HRESULT TotalBench( HRESULT res = MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, - false, numThreads, method, + #ifndef _7ZIP_ST + false, numThreads, affinityMode, + #endif + method, unpackSize2, fileData, bench.DictBits, printCallback, callback, &callback->BenchProps); @@ -2034,20 +3063,40 @@ static HRESULT TotalBench( } -static HRESULT FreqBench( - UInt64 complexInCommands, - UInt32 numThreads, - IBenchPrintCallback *_file, - bool showFreq, - UInt64 specifiedFreq, - UInt64 &cpuFreq, - UInt32 &res) +struct CFreqBench +{ + // in: + UInt64 complexInCommands; + UInt32 numThreads; + bool showFreq; + UInt64 specifiedFreq; + + // out: + UInt64 CpuFreqRes; + UInt64 UsageRes; + UInt32 res; + + CFreqBench() + {} + + HRESULT FreqBench(IBenchPrintCallback *_file + #ifndef _7ZIP_ST + , const CAffinityMode *affinityMode + #endif + ); +}; + + +HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file + #ifndef _7ZIP_ST + , const CAffinityMode *affinityMode + #endif + ) { res = 0; - cpuFreq = 0; + CpuFreqRes = 0; + UsageRes = 0; - UInt32 bufferSize = 1 << 20; - UInt32 complexity = kNumFreqCommands; if (numThreads == 0) numThreads = 1; @@ -2055,17 +3104,26 @@ static HRESULT FreqBench( numThreads = 1; #endif - UInt32 bsize = (bufferSize == 0 ? 1 : bufferSize); - UInt64 numIterations = complexInCommands / complexity / bsize; - if (numIterations == 0) + const UInt32 complexity = kNumFreqCommands; + UInt64 numIterations = complexInCommands / complexity; + UInt32 numIterations2 = 1 << 30; + if (numIterations > numIterations2) + numIterations /= numIterations2; + else + { + numIterations2 = (UInt32)numIterations; numIterations = 1; + } CBenchInfoCalc progressInfoSpec; #ifndef _7ZIP_ST - CFreqThreads threads; - if (numThreads > 1) + + bool mtMode = (numThreads > 1) || affinityMode->NeedAffinity(); + + if (mtMode) { + CFreqThreads threads; threads.Items = new CFreqInfo[numThreads]; UInt32 i; for (i = 0; i < numThreads; i++) @@ -2074,16 +3132,22 @@ static HRESULT FreqBench( info.Callback = _file; info.CallbackRes = S_OK; info.NumIterations = numIterations; - info.Size = bufferSize; + info.Size = numIterations2; } progressInfoSpec.SetStartTime(); for (i = 0; i < numThreads; i++) { + // Sleep(10); CFreqInfo &info = threads.Items[i]; - RINOK(info.Thread.Create(FreqThreadFunction, &info)); - threads.NumThreads++; + WRes wres = affinityMode->CreateThread_WithAffinity(info.Thread, FreqThreadFunction, &info, i); + if (info.Thread.IsCreated()) + threads.NumThreads++; + if (wres != 0) + return HRESULT_FROM_WIN32(wres); } - threads.WaitAll(); + WRes wres = threads.WaitAll(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); for (i = 0; i < numThreads; i++) { RINOK(threads.Items[i].CallbackRes); @@ -2096,11 +3160,20 @@ static HRESULT FreqBench( UInt32 sum = g_BenchCpuFreqTemp; for (UInt64 k = numIterations; k > 0; k--) { - RINOK(_file->CheckBreak()); - sum = CountCpuFreq(sum, bufferSize, g_BenchCpuFreqTemp); + sum = CountCpuFreq(sum, numIterations2, g_BenchCpuFreqTemp); + if (_file) + { + RINOK(_file->CheckBreak()); + } } res += sum; } + + if (res == 0x12345678) + if (_file) + { + RINOK(_file->CheckBreak()); + } CBenchInfo info; progressInfoSpec.SetFinishTime(info); @@ -2109,17 +3182,17 @@ static HRESULT FreqBench( info.PackSize = 0; info.NumIterations = 1; + const UInt64 numCommands = (UInt64)numIterations * numIterations2 * numThreads * complexity; + const UInt64 rating = info.GetSpeed(numCommands); + CpuFreqRes = rating / numThreads; + UsageRes = info.GetUsage(); + if (_file) { - { - UInt64 numCommands = (UInt64)numIterations * bufferSize * numThreads * complexity; - UInt64 rating = info.GetSpeed(numCommands); - cpuFreq = rating / numThreads; - PrintResults(_file, info, + PrintResults(_file, info, 0, // weight rating, - showFreq, showFreq ? (specifiedFreq != 0 ? specifiedFreq : cpuFreq) : 0, NULL); - } + showFreq, showFreq ? (specifiedFreq != 0 ? specifiedFreq : CpuFreqRes) : 0, NULL); RINOK(_file->CheckBreak()); } @@ -2131,12 +3204,21 @@ static HRESULT FreqBench( static HRESULT CrcBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, - UInt32 numThreads, UInt32 bufferSize, + UInt32 numThreads, + const size_t bufferSize, + const Byte *fileData, + UInt64 &speed, + UInt64 &usage, + UInt32 complexity, unsigned benchWeight, const UInt32 *checkSum, const COneMethodInfo &method, IBenchPrintCallback *_file, + #ifndef _7ZIP_ST + const CAffinityMode *affinityMode, + #endif + bool showRating, CTotalBenchRes *encodeRes, bool showFreq, UInt64 cpuFreq) { @@ -2147,7 +3229,7 @@ static HRESULT CrcBench( numThreads = 1; #endif - AString methodName = method.MethodName; + const AString &methodName = method.MethodName; // methodName.RemoveChar(L'-'); CMethodId hashID; if (!FindHashMethod( @@ -2155,81 +3237,112 @@ static HRESULT CrcBench( methodName, hashID)) return E_NOTIMPL; - CBenchBuffer buffer; - size_t totalSize = (size_t)bufferSize * numThreads; - if (totalSize / numThreads != bufferSize) - return E_OUTOFMEMORY; - if (!buffer.Alloc(totalSize)) - return E_OUTOFMEMORY; + /* + // if will generate random data in each thread, instead of global data + CMidAlignedBuffer buffer; + if (!fileData) + { + ALLOC_WITH_HRESULT(&buffer, bufferSize) + RandGen(buffer, bufferSize); + fileData = buffer; + } + */ - Byte *buf = buffer.Buffer; - CBaseRandomGenerator RG; - UInt32 bsize = (bufferSize == 0 ? 1 : bufferSize); + const size_t bsize = (bufferSize == 0 ? 1 : bufferSize); UInt64 numIterations = complexInCommands * 256 / complexity / bsize; if (numIterations == 0) numIterations = 1; CBenchInfoCalc progressInfoSpec; + CBenchInfo info; #ifndef _7ZIP_ST - CCrcThreads threads; - if (numThreads > 1) + bool mtEncMode = (numThreads > 1) || affinityMode->NeedAffinity(); + + if (mtEncMode) { + CCrcThreads threads; threads.Items = new CCrcInfo[numThreads]; - + { + WRes wres = threads.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + threads.NeedClose = true; + } + UInt32 i; for (i = 0; i < numThreads; i++) { - CCrcInfo &info = threads.Items[i]; + CCrcInfo &ci = threads.Items[i]; AString name; - RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, info.Hasher)); - if (!info.Hasher) + RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, ci.Hasher)); + if (!ci.Hasher) return E_NOTIMPL; CMyComPtr scp; - info.Hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); + ci.Hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = 1; - RINOK(method.SetCoderProps(scp, &reduceSize)); + RINOK(method.SetCoderProps(scp)); } - Byte *data = buf + (size_t)bufferSize * i; - info.Callback = _file; - info.Data = data; - info.NumIterations = numIterations; - info.Size = bufferSize; - /* info.Crc = */ RandGenCrc(data, bufferSize, RG); - info.CheckSumDefined = false; + ci.Callback = _file; + ci.Data = fileData; + ci.NumIterations = numIterations; + ci.Size = bufferSize; + ci.CheckSumDefined = false; if (checkSum) { - info.CheckSum = *checkSum; - info.CheckSumDefined = (checkSum && (i == 0)); + ci.CheckSum = *checkSum; + ci.CheckSumDefined = true; } #ifdef USE_ALLOCA - info.AllocaSize = (i * 16 * 21) & 0x7FF; + ci.AllocaSize = (i * 16 * 21) & 0x7FF; #endif } - progressInfoSpec.SetStartTime(); + for (i = 0; i < numThreads; i++) + { + CCrcInfo &ci = threads.Items[i]; + ci.ThreadIndex = i; + ci.Common = &threads.Common; + ci.AffinityMode = *affinityMode; + HRESULT hres = ci.CreateThread(); + if (ci.Thread.IsCreated()) + threads.NumThreads++; + if (hres != 0) + return hres; + } for (i = 0; i < numThreads; i++) { - CCrcInfo &info = threads.Items[i]; - RINOK(info.Thread.Create(CrcThreadFunction, &info)); - threads.NumThreads++; + CCrcInfo &ci = threads.Items[i]; + WRes wres = ci.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(ci.Res); } - threads.WaitAll(); + + progressInfoSpec.SetStartTime(); + + WRes wres = threads.StartAndWait(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + + progressInfoSpec.SetFinishTime(info); + for (i = 0; i < numThreads; i++) { RINOK(threads.Items[i].Res); + if (i != 0) + if (threads.Items[i].CheckSum_Res != + threads.Items[i - 1].CheckSum_Res) + return S_FALSE; } } else #endif { - /* UInt32 crc = */ RandGenCrc(buf, bufferSize, RG); - progressInfoSpec.SetStartTime(); CMyComPtr hasher; AString name; RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, hasher)); @@ -2239,14 +3352,16 @@ static HRESULT CrcBench( hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); if (scp) { - UInt64 reduceSize = 1; - RINOK(method.SetCoderProps(scp, &reduceSize)); + RINOK(method.SetCoderProps(scp)); } - RINOK(CrcBig(buf, bufferSize, numIterations, checkSum, hasher, _file)); + CCrcInfo_Base crcib; + crcib.CreateLocalBuf = false; + RINOK(crcib.Generate(fileData, bufferSize)); + progressInfoSpec.SetStartTime(); + RINOK(crcib.CrcProcess(numIterations, checkSum, hasher, _file)); + progressInfoSpec.SetFinishTime(info); } - CBenchInfo info; - progressInfoSpec.SetFinishTime(info); UInt64 unpSize = numIterations * bufferSize; UInt64 unpSizeThreads = unpSize * numThreads; @@ -2256,9 +3371,13 @@ static HRESULT CrcBench( if (_file) { + if (showRating) { - UInt64 numCommands = unpSizeThreads * complexity / 256; - UInt64 rating = info.GetSpeed(numCommands); + UInt64 unpSizeThreads2 = unpSizeThreads; + if (unpSizeThreads2 == 0) + unpSizeThreads2 = numIterations * 1 * numThreads; + const UInt64 numCommands = unpSizeThreads2 * complexity / 256; + const UInt64 rating = info.GetSpeed(numCommands); PrintResults(_file, info, benchWeight, rating, showFreq, cpuFreq, encodeRes); @@ -2267,15 +3386,23 @@ static HRESULT CrcBench( } speed = info.GetSpeed(unpSizeThreads); + usage = info.GetUsage(); return S_OK; } + + static HRESULT TotalBench_Hash( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, - UInt32 numThreads, UInt32 bufSize, + UInt32 numThreads, + size_t bufSize, + const Byte *fileData, IBenchPrintCallback *printCallback, CBenchCallbackToPrint *callback, + #ifndef _7ZIP_ST + const CAffinityMode *affinityMode, + #endif CTotalBenchRes *encodeRes, bool showFreq, UInt64 cpuFreq) { @@ -2292,15 +3419,22 @@ static HRESULT TotalBench_Hash( propVariant = bench.Name; RINOK(method.ParseMethodFromPROPVARIANT(UString(), propVariant)); - UInt64 speed; + UInt64 speed, usage; + HRESULT res = CrcBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, - numThreads, bufSize, - speed, + numThreads, bufSize, fileData, + speed, usage, bench.Complex, bench.Weight, - &bench.CheckSum, method, - printCallback, encodeRes, showFreq, cpuFreq); + (!fileData && bufSize == (1 << kNumHashDictBits)) ? &bench.CheckSum : NULL, + method, + printCallback, + #ifndef _7ZIP_ST + affinityMode, + #endif + true, // showRating + encodeRes, showFreq, cpuFreq); if (res == E_NOTIMPL) { // callback->Print(" ---"); @@ -2333,110 +3467,18 @@ static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) prop = result; } -static UInt32 GetNumThreadsNext(unsigned i, UInt32 numThreads) -{ - if (i < 2) - return i + 1; - i -= 1; - UInt32 num = (UInt32)(2 + (i & 1)) << (i >> 1); - return (num <= numThreads) ? num : numThreads; -} static bool AreSameMethodNames(const char *fullName, const char *shortName) { - for (;;) - { - char c2 = *shortName++; - if (c2 == 0) - return true; - char c1 = *fullName++; - if (MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) - return false; - } + return StringsAreEqualNoCase_Ascii(fullName, shortName); } -#ifdef MY_CPU_X86_OR_AMD64 -static void PrintCpuChars(AString &s, UInt32 v) -{ - for (int j = 0; j < 4; j++) - { - Byte b = (Byte)(v & 0xFF); - v >>= 8; - if (b == 0) - break; - s += (char)b; - } -} -static void x86cpuid_to_String(const Cx86cpuid &c, AString &s) +static void Print_Usage_and_Threads(IBenchPrintCallback &f, UInt64 usage, UInt32 threads) { - s.Empty(); - - UInt32 maxFunc2 = 0; - UInt32 t[3]; - - MyCPUID(0x80000000, &maxFunc2, &t[0], &t[1], &t[2]); - - bool fullNameIsAvail = (maxFunc2 >= 0x80000004); - - if (!fullNameIsAvail) - { - for (int i = 0; i < 3; i++) - PrintCpuChars(s, c.vendor[i]); - } - else - { - for (int i = 0; i < 3; i++) - { - UInt32 d[4] = { 0 }; - MyCPUID(0x80000002 + i, &d[0], &d[1], &d[2], &d[3]); - for (int j = 0; j < 4; j++) - PrintCpuChars(s, d[j]); - } - } - - s.Add_Space_if_NotEmpty(); - { - char temp[32]; - ConvertUInt32ToHex(c.ver, temp); - s += '('; - s += temp; - s += ')'; - } -} - -#endif - - -void GetCpuName(AString &s) -{ - s.Empty(); - - #ifdef MY_CPU_X86_OR_AMD64 - { - Cx86cpuid cpuid; - if (x86cpuid_CheckAndRead(&cpuid)) - { - x86cpuid_to_String(cpuid, s); - return; - } - #ifdef MY_CPU_AMD64 - s = "x64"; - #else - s = "x86"; - #endif - } - #else - - #ifdef MY_CPU_LE - s = "LE"; - #elif defined(MY_CPU_BE) - s = "BE"; - #endif - - #endif + PrintRequirements(f, "usage:", true, usage, "Benchmark threads: ", threads); } @@ -2446,38 +3488,87 @@ HRESULT Bench( IBenchCallback *benchCallback, const CObjectVector &props, UInt32 numIterations, - bool multiDict) + bool multiDict, + IBenchFreqCallback *freqCallback) { if (!CrcInternalTest()) - return S_FALSE; + return E_FAIL; UInt32 numCPUs = 1; UInt64 ramSize = (UInt64)(sizeof(size_t)) << 29; + NSystem::CProcessAffinity threadsInfo; + threadsInfo.InitST(); + #ifndef _7ZIP_ST - numCPUs = NSystem::GetNumberOfProcessors(); + + if (threadsInfo.Get() && threadsInfo.GetNumProcessThreads() != 0) + numCPUs = threadsInfo.GetNumProcessThreads(); + else + numCPUs = NSystem::GetNumberOfProcessors(); + #endif + + // numCPUs = 24; + /* + { + DWORD_PTR mask = (1 << 0); + DWORD_PTR old = SetThreadAffinityMask(GetCurrentThread(), mask); + old = old; + DWORD_PTR old2 = SetThreadAffinityMask(GetCurrentThread(), mask); + old2 = old2; + return 0; + } + */ bool ramSize_Defined = NSystem::GetRamSize(ramSize); UInt32 numThreadsSpecified = numCPUs; - - UInt32 testTime = kComplexInSeconds; - + bool needSetComplexity = false; + UInt32 testTimeMs = kComplexInMs; + UInt32 startDicLog = 22; + bool startDicLog_Defined = false; UInt64 specifiedFreq = 0; - bool multiThreadTests = false; + UInt64 complexInCommands = kComplexInCommands; + UInt32 numThreads_Start = 1; + + #ifndef _7ZIP_ST + CAffinityMode affinityMode; + #endif + COneMethodInfo method; - CBenchBuffer fileDataBuffer; + CMidAlignedBuffer fileDataBuffer; + bool use_fileData = false; + bool isFixedDict = false; { unsigned i; + + if (printCallback) + { + for (i = 0; i < props.Size(); i++) + { + const CProperty &property = props[i]; + printCallback->Print(" "); + printCallback->Print(GetAnsiString(property.Name)); + if (!property.Value.IsEmpty()) + { + printCallback->Print("="); + printCallback->Print(GetAnsiString(property.Value)); + } + } + if (!props.IsEmpty()) + printCallback->NewLine(); + } + + for (i = 0; i < props.Size(); i++) { const CProperty &property = props[i]; - UString name = property.Name; + UString name (property.Name); name.MakeLower_Ascii(); if (name.IsEqualTo("file")) @@ -2485,36 +3576,38 @@ HRESULT Bench( if (property.Value.IsEmpty()) return E_INVALIDARG; - #ifdef USE_WIN_FILE - NFile::NIO::CInFile file; if (!file.Open(us2fs(property.Value))) - return E_INVALIDARG; - UInt64 len; - if (!file.GetLength(len)) - return E_FAIL; - if (len >= ((UInt32)1 << 31) || len == 0) - return E_INVALIDARG; - if (!fileDataBuffer.Alloc((size_t)len)) - return E_OUTOFMEMORY; - UInt32 processedSize; - file.Read(fileDataBuffer.Buffer, (UInt32)len, processedSize); - if (processedSize != len) - return E_FAIL; - if (printCallback) + return GetLastError_noZero_HRESULT(); + size_t len; { - printCallback->Print("file size ="); - // printCallback->Print(GetOemString(property.Value)); - PrintNumber(*printCallback, len, 0); - printCallback->NewLine(); + UInt64 len64; + if (!file.GetLength(len64)) + return GetLastError_noZero_HRESULT(); + if (printCallback) + { + printCallback->Print("file size ="); + PrintNumber(*printCallback, len64, 0); + printCallback->NewLine(); + } + len = (size_t)len64; + if (len != len64) + return E_INVALIDARG; } - continue; - #else + // (len == 0) is allowed. Also it's allowed if Alloc(0) returns NULL here - return E_NOTIMPL; + ALLOC_WITH_HRESULT(&fileDataBuffer, len); + use_fileData = true; - #endif + { + size_t processed; + if (!file.ReadFull((Byte *)fileDataBuffer, len, processed)) + return GetLastError_noZero_HRESULT(); + if (processed != len) + return E_FAIL; + } + continue; } NCOM::CPropVariant propVariant; @@ -2523,14 +3616,65 @@ HRESULT Bench( if (name.IsEqualTo("time")) { - RINOK(ParsePropToUInt32(L"", propVariant, testTime)); + RINOK(ParsePropToUInt32(UString(), propVariant, testTimeMs)); + needSetComplexity = true; + testTimeMs *= 1000; continue; } - + + if (name.IsEqualTo("timems")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, testTimeMs)); + needSetComplexity = true; + continue; + } + + if (name.IsEqualTo("tic")) + { + UInt32 v; + RINOK(ParsePropToUInt32(UString(), propVariant, v)); + if (v >= 64) + return E_INVALIDARG; + complexInCommands = (UInt64)1 << v; + continue; + } + + const bool isCurrent_fixedDict = name.IsEqualTo("df"); + if (isCurrent_fixedDict) + isFixedDict = true; + if (isCurrent_fixedDict || name.IsEqualTo("ds")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, startDicLog)); + if (startDicLog > 32) + return E_INVALIDARG; + startDicLog_Defined = true; + continue; + } + + if (name.IsEqualTo("mts")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, numThreads_Start)); + continue; + } + + if (name.IsEqualTo("af")) + { + UInt32 bundle; + RINOK(ParsePropToUInt32(UString(), propVariant, bundle)); + if (bundle > 0 && bundle < numCPUs) + { + #ifndef _7ZIP_ST + affinityMode.SetLevels(numCPUs, 2); + affinityMode.NumBundleThreads = bundle; + #endif + } + continue; + } + if (name.IsEqualTo("freq")) { UInt32 freq32 = 0; - RINOK(ParsePropToUInt32(L"", propVariant, freq32)); + RINOK(ParsePropToUInt32(UString(), propVariant, freq32)); if (freq32 == 0) return E_INVALIDARG; specifiedFreq = (UInt64)freq32 * 1000000; @@ -2548,19 +3692,14 @@ HRESULT Bench( if (name.IsPrefixedBy_Ascii_NoCase("mt")) { UString s = name.Ptr(2); - if (s == L"*") + if (s.IsEqualTo("*") + || (s.IsEmpty() + && propVariant.vt == VT_BSTR + && StringsAreEqual_Ascii(propVariant.bstrVal, "*"))) { multiThreadTests = true; continue; } - if (s.IsEmpty() && propVariant.vt == VT_BSTR) - { - if (wcscmp(propVariant.bstrVal, L"*") == 0) - { - multiThreadTests = true; - continue; - } - } #ifndef _7ZIP_ST RINOK(ParseMtProp(s, propVariant, numCPUs, numThreadsSpecified)); #endif @@ -2574,19 +3713,26 @@ HRESULT Bench( if (printCallback) { AString s; - GetCpuName(s); + + #ifndef _WIN32 + s += "Compiler: "; + GetCompiler(s); + printCallback->Print(s); + printCallback->NewLine(); + s.Empty(); + #endif + + GetSystemInfoText(s); printCallback->Print(s); printCallback->NewLine(); } if (printCallback) { - printCallback->Print("CPU Freq:"); + printCallback->Print("1T CPU Freq (MHz):"); } - UInt64 complexInCommands = kComplexInCommands; - - if (printCallback /* || benchCallback */) + if (printCallback || freqCallback) { UInt64 numMilCommands = 1 << 6; if (specifiedFreq != 0) @@ -2603,12 +3749,18 @@ HRESULT Bench( UInt64 start = ::GetTimeCount(); UInt32 sum = (UInt32)start; sum = CountCpuFreq(sum, (UInt32)(numMilCommands * 1000000 / kNumFreqCommands), g_BenchCpuFreqTemp); + if (sum == 0xF1541213) + if (printCallback) + printCallback->Print(""); const UInt64 realDelta = ::GetTimeCount() - start; start = realDelta; if (start == 0) start = 1; - UInt64 freq = GetFreq(); + if (start > (UInt64)1 << 61) + start = 1; + const UInt64 freq = GetFreq(); // mips is constant in some compilers + const UInt64 hz = MyMultDiv64(numMilCommands * 1000000, freq, start); const UInt64 mipsVal = numMilCommands * freq / start; if (printCallback) { @@ -2619,37 +3771,159 @@ HRESULT Bench( else { // PrintNumber(*printCallback, start, 0); - PrintNumber(*printCallback, mipsVal, 5 + ((sum == 0xF1541213) ? 1 : 0)); + PrintNumber(*printCallback, mipsVal, 5); } } - /* - if (benchCallback) - benchCallback->AddCpuFreq(mipsVal); - */ + if (freqCallback) + { + RINOK(freqCallback->AddCpuFreq(1, hz, kBenchmarkUsageMult)); + } - if (jj >= 3) + if (jj >= 1) { - SetComplexCommands(testTime, false, mipsVal * 1000000, complexInCommands); - if (jj >= 8 || start >= freq) + bool needStop = (numMilCommands >= (1 << + #ifdef _DEBUG + 7 + #else + 11 + #endif + )); + if (start >= freq * 16) + { + printCallback->Print(" (Cmplx)"); + if (!freqCallback) // we don't want complexity change for old gui lzma benchmark + { + needSetComplexity = true; + } + needStop = true; + } + if (needSetComplexity) + SetComplexCommandsMs(testTimeMs, false, mipsVal * 1000000, complexInCommands); + if (needStop) break; - // break; // change it numMilCommands <<= 1; } } + if (freqCallback) + { + RINOK(freqCallback->FreqsFinished(1)); + } } + if (numThreadsSpecified >= 2) + if (printCallback || freqCallback) + { + if (printCallback) + printCallback->NewLine(); + + /* it can show incorrect frequency for HT threads. + so we reduce freq test to (numCPUs / 2) */ + + UInt32 numThreads = numThreadsSpecified >= numCPUs / 2 ? numCPUs / 2: numThreadsSpecified; + if (numThreads < 1) + numThreads = 1; + + if (printCallback) + { + char s[128]; + ConvertUInt64ToString(numThreads, s); + printCallback->Print(s); + printCallback->Print("T CPU Freq (MHz):"); + } + UInt64 numMilCommands = 1 << 10; + if (specifiedFreq != 0) + { + while (numMilCommands > 1 && specifiedFreq < (numMilCommands * 1000000)) + numMilCommands >>= 1; + } + + for (int jj = 0;; jj++) + { + if (printCallback) + RINOK(printCallback->CheckBreak()); + + { + // PrintLeft(f, "CPU", kFieldSize_Name); + + // UInt32 resVal; + + CFreqBench fb; + fb.complexInCommands = numMilCommands * 1000000; + fb.numThreads = numThreads; + // showFreq; + // fb.showFreq = (freqTest == kNumCpuTests - 1 || specifiedFreq != 0); + fb.showFreq = true; + fb.specifiedFreq = 1; + + HRESULT res = fb.FreqBench(NULL /* printCallback */ + #ifndef _7ZIP_ST + , &affinityMode + #endif + ); + RINOK(res); + + if (freqCallback) + { + RINOK(freqCallback->AddCpuFreq(numThreads, fb.CpuFreqRes, fb.UsageRes)); + } + + if (printCallback) + { + /* + if (realDelta == 0) + { + printCallback->Print(" -"); + } + else + */ + { + // PrintNumber(*printCallback, start, 0); + PrintUsage(*printCallback, fb.UsageRes, 3); + printCallback->Print("%"); + PrintNumber(*printCallback, fb.CpuFreqRes / 1000000, 0); + printCallback->Print(" "); + + // PrintNumber(*printCallback, fb.UsageRes, 5); + } + } + } + // if (jj >= 1) + { + bool needStop = (numMilCommands >= (1 << + #ifdef _DEBUG + 7 + #else + 11 + #endif + )); + if (needStop) + break; + numMilCommands <<= 1; + } + } + if (freqCallback) + { + RINOK(freqCallback->FreqsFinished(numThreads)); + } + } + + if (printCallback) { printCallback->NewLine(); printCallback->NewLine(); PrintRequirements(*printCallback, "size: ", ramSize_Defined, ramSize, "CPU hardware threads:", numCPUs); + printCallback->Print(GetProcessThreadsInfo(threadsInfo)); + printCallback->NewLine(); } if (numThreadsSpecified < 1 || numThreadsSpecified > kNumThreadsMax) return E_INVALIDARG; - UInt32 dict; - bool dictIsDefined = method.Get_DicSize(dict); + UInt64 dict = (UInt64)1 << startDicLog; + const bool dictIsDefined = (isFixedDict || method.Get_DicSize(dict)); + + const int level = method.GetLevel(); if (method.MethodName.IsEmpty()) method.MethodName = "LZMA"; @@ -2658,18 +3932,33 @@ HRESULT Bench( { CBenchProps benchProps; benchProps.SetLzmaCompexity(); - UInt32 dictSize = method.Get_Lzma_DicSize(); - UInt32 uncompressedDataSize = kAdditionalSize + dictSize; + const UInt64 dictSize = method.Get_Lzma_DicSize(); + + size_t uncompressedDataSize; + if (use_fileData) + { + uncompressedDataSize = fileDataBuffer.Size(); + } + else + { + uncompressedDataSize = kAdditionalSize + (size_t)dictSize; + if (uncompressedDataSize < dictSize) + return E_INVALIDARG; + } + return MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, + #ifndef _7ZIP_ST true, numThreadsSpecified, + &affinityMode, + #endif method, - uncompressedDataSize, fileDataBuffer.Buffer, + uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, benchCallback, &benchProps); } - AString methodName = method.MethodName; + AString methodName (method.MethodName); if (methodName.IsEqualTo_Ascii_NoCase("CRC")) methodName = "crc32"; method.MethodName = methodName; @@ -2680,96 +3969,199 @@ HRESULT Bench( if (!printCallback) return S_FALSE; IBenchPrintCallback &f = *printCallback; - if (!dictIsDefined) - dict = (1 << 24); + UInt64 dict64 = dict; + if (!dictIsDefined) + dict64 = (1 << 27); + if (use_fileData) + { + if (!dictIsDefined) + dict64 = fileDataBuffer.Size(); + else if (dict64 > fileDataBuffer.Size()) + dict64 = fileDataBuffer.Size(); + } // methhodName.RemoveChar(L'-'); UInt32 complexity = 10000; const UInt32 *checkSum = NULL; { - for (unsigned i = 0; i < ARRAY_SIZE(g_Hash); i++) + unsigned i; + for (i = 0; i < ARRAY_SIZE(g_Hash); i++) { const CBenchHash &h = g_Hash[i]; - AString s = h.Name; - AString hProp; - int propPos = s.Find(':'); + AString benchMethod (h.Name); + AString benchProps; + int propPos = benchMethod.Find(':'); if (propPos >= 0) { - hProp = s.Ptr(propPos + 1); - s.DeleteFrom(propPos); + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); } - if (AreSameMethodNames(s, methodName)) + if (AreSameMethodNames(benchMethod, methodName)) { - complexity = h.Complex; - checkSum = &h.CheckSum; - if (method.PropsString.IsEqualTo_Ascii_NoCase(hProp)) - break; + bool isMainMathed = method.PropsString.IsEmpty(); + if (isMainMathed) + isMainMathed = !checkSum + || (benchMethod.IsEqualTo_Ascii_NoCase("crc32") && benchProps.IsEqualTo_Ascii_NoCase("8")); + const bool sameProps = method.PropsString.IsEqualTo_Ascii_NoCase(benchProps); + if (sameProps || isMainMathed) + { + complexity = h.Complex; + checkSum = &h.CheckSum; + if (sameProps) + break; + } } } + if (!checkSum) + return E_NOTIMPL; } + { + UInt64 usage = 1 << 20; + UInt64 bufSize = dict64; + if (use_fileData) + { + usage += fileDataBuffer.Size(); + if (bufSize > fileDataBuffer.Size()) + bufSize = fileDataBuffer.Size(); + #ifndef _7ZIP_ST + if (numThreadsSpecified != 1) + usage += bufSize * numThreadsSpecified * (k_Crc_CreateLocalBuf_For_File ? 1 : 0); + #endif + } + else + usage += numThreadsSpecified * bufSize; + Print_Usage_and_Threads(f, usage, numThreadsSpecified); + } + f.NewLine(); - f.Print("Size"); - const unsigned kFieldSize_CrcSpeed = 6; - unsigned numThreadsTests = 0; - for (;;) - { - UInt32 t = GetNumThreadsNext(numThreadsTests, numThreadsSpecified); - PrintNumber(f, t, kFieldSize_CrcSpeed); - numThreadsTests++; - if (t >= numThreadsSpecified) - break; + + const unsigned kFieldSize_CrcSpeed = 7; + CUIntVector numThreadsVector; + { + unsigned nt = numThreads_Start; + for (;;) + { + if (nt > numThreadsSpecified) + break; + numThreadsVector.Add(nt); + unsigned next = nt * 2; + UInt32 ntHalf= numThreadsSpecified / 2; + if (ntHalf > nt && ntHalf < next) + numThreadsVector.Add(ntHalf); + if (numThreadsSpecified > nt && numThreadsSpecified < next) + numThreadsVector.Add(numThreadsSpecified); + nt = next; + } + { + f.NewLine(); + f.Print("THRD"); + FOR_VECTOR (ti, numThreadsVector) + { + PrintNumber(f, numThreadsVector[ti], 1 + kFieldSize_Usage + kFieldSize_CrcSpeed); + } + } + { + f.NewLine(); + f.Print(" "); + FOR_VECTOR (ti, numThreadsVector) + { + PrintRight(f, "Usage", kFieldSize_Usage + 1); + PrintRight(f, "BW", kFieldSize_CrcSpeed + 1); + } + } + { + f.NewLine(); + f.Print("Size"); + FOR_VECTOR (ti, numThreadsVector) + { + PrintRight(f, "%", kFieldSize_Usage + 1); + PrintRight(f, "MB/s", kFieldSize_CrcSpeed + 1); + } + } } + f.NewLine(); f.NewLine(); - CTempValues speedTotals(numThreadsTests); + + CTempValues speedTotals(numThreadsVector.Size()); + CTempValues usageTotals(numThreadsVector.Size()); { - for (unsigned ti = 0; ti < numThreadsTests; ti++) + FOR_VECTOR (ti, numThreadsVector) + { speedTotals.Values[ti] = 0; + usageTotals.Values[ti] = 0; + } } UInt64 numSteps = 0; + for (UInt32 i = 0; i < numIterations; i++) { - for (unsigned pow = 10; pow < 32; pow++) + unsigned pow = 10; // kNumHashDictBits + if (startDicLog_Defined) + pow = startDicLog; + for (;; pow++) { - UInt32 bufSize = (UInt32)1 << pow; - if (bufSize > dict) - break; + const UInt64 bufSize = (UInt64)1 << pow; char s[16]; ConvertUInt32ToString(pow, s); unsigned pos = MyStringLen(s); s[pos++] = ':'; s[pos++] = ' '; s[pos] = 0; - f.Print(s); + PrintRight(f, s, 4); + + size_t dataSize = fileDataBuffer.Size(); + if (dataSize > bufSize || !use_fileData) + dataSize = (size_t)bufSize; - for (unsigned ti = 0; ti < numThreadsTests; ti++) + FOR_VECTOR (ti, numThreadsVector) { RINOK(f.CheckBreak()); - UInt32 t = GetNumThreadsNext(ti, numThreadsSpecified); + const UInt32 t = numThreadsVector[ti]; UInt64 speed = 0; - RINOK(CrcBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, - t, bufSize, speed, + UInt64 usage = 0; + + HRESULT res = CrcBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, + t, + dataSize, (const Byte *)fileDataBuffer, + speed, usage, complexity, 1, // benchWeight, - (pow == kNumHashDictBits) ? checkSum : NULL, method, NULL, NULL, false, 0)); - PrintNumber(f, (speed >> 20), kFieldSize_CrcSpeed); + (pow == kNumHashDictBits && !use_fileData) ? checkSum : NULL, + method, + &f, + #ifndef _7ZIP_ST + &affinityMode, + #endif + false, // showRating + NULL, false, 0); + + RINOK(res); + + PrintUsage(f, usage, kFieldSize_Usage); + PrintNumber(f, speed / 1000000, kFieldSize_CrcSpeed); speedTotals.Values[ti] += speed; + usageTotals.Values[ti] += usage; } + f.NewLine(); numSteps++; + if (dataSize >= dict64) + break; } } if (numSteps != 0) { f.NewLine(); f.Print("Avg:"); - for (unsigned ti = 0; ti < numThreadsTests; ti++) + for (unsigned ti = 0; ti < numThreadsVector.Size(); ti++) { - PrintNumber(f, ((speedTotals.Values[ti] / numSteps) >> 20), kFieldSize_CrcSpeed); + PrintUsage(f, usageTotals.Values[ti] / numSteps, kFieldSize_Usage); + PrintNumber(f, speedTotals.Values[ti] / numSteps / 1000000, kFieldSize_CrcSpeed); } f.NewLine(); } @@ -2827,7 +4219,7 @@ HRESULT Bench( f.NewLine(); } - if (!dictIsDefined) + if (!dictIsDefined && !onlyHashBench) { const unsigned dicSizeLog_Main = (totalBenchMode ? 24 : 25); unsigned dicSizeLog = dicSizeLog_Main; @@ -2838,10 +4230,10 @@ HRESULT Bench( if (ramSize_Defined) for (; dicSizeLog > kBenchMinDicLogSize; dicSizeLog--) - if (GetBenchMemoryUsage(numThreads, ((UInt32)1 << dicSizeLog), totalBenchMode) + (8 << 20) <= ramSize) + if (GetBenchMemoryUsage(numThreads, level, ((UInt64)1 << dicSizeLog), totalBenchMode) + (8 << 20) <= ramSize) break; - dict = (UInt32)1 << dicSizeLog; + dict = (UInt64)1 << dicSizeLog; if (totalBenchMode && dicSizeLog != dicSizeLog_Main) { @@ -2851,7 +4243,13 @@ HRESULT Bench( } } - PrintRequirements(f, "usage:", true, GetBenchMemoryUsage(numThreads, dict, totalBenchMode), "Benchmark threads: ", numThreads); + Print_Usage_and_Threads(f, + onlyHashBench ? + GetBenchMemoryUsage_Hash(numThreads, dict) : + GetBenchMemoryUsage(numThreads, level, dict, totalBenchMode), + numThreads); + + f.NewLine(); f.NewLine(); @@ -2934,6 +4332,7 @@ HRESULT Bench( if (specifiedFreq != 0) cpuFreq = specifiedFreq; + // bool showTotalSpeed = false; if (totalBenchMode) { @@ -2941,88 +4340,164 @@ HRESULT Bench( { if (i != 0) printCallback->NewLine(); - HRESULT res; const unsigned kNumCpuTests = 3; for (unsigned freqTest = 0; freqTest < kNumCpuTests; freqTest++) { PrintLeft(f, "CPU", kFieldSize_Name); - UInt32 resVal; - RINOK(FreqBench(complexInCommands, numThreads, printCallback, - (freqTest == kNumCpuTests - 1 || specifiedFreq != 0), // showFreq - specifiedFreq, - cpuFreq, resVal)); + + // UInt32 resVal; + + CFreqBench fb; + fb.complexInCommands = complexInCommands; + fb.numThreads = numThreads; + // showFreq; + fb.showFreq = (freqTest == kNumCpuTests - 1 || specifiedFreq != 0); + fb.specifiedFreq = specifiedFreq; + + HRESULT res = fb.FreqBench(printCallback + #ifndef _7ZIP_ST + , &affinityMode + #endif + ); + RINOK(res); + + cpuFreq = fb.CpuFreqRes; callback.NewLine(); if (specifiedFreq != 0) cpuFreq = specifiedFreq; + if (testTimeMs >= 1000) if (freqTest == kNumCpuTests - 1) - SetComplexCommands(testTime, specifiedFreq != 0, cpuFreq, complexInCommands); + { + // SetComplexCommandsMs(testTimeMs, specifiedFreq != 0, cpuFreq, complexInCommands); + } } callback.NewLine(); + // return S_OK; // change it + callback.SetFreq(true, cpuFreq); if (!onlyHashBench) { - res = TotalBench(EXTERNAL_CODECS_LOC_VARS - complexInCommands, numThreads, - dictIsDefined || fileDataBuffer.Buffer, // forceUnpackSize - fileDataBuffer.Buffer ? fileDataBuffer.BufferSize : dict, - fileDataBuffer.Buffer, + size_t dataSize = (size_t)dict; + if (use_fileData) + { + dataSize = fileDataBuffer.Size(); + if (dictIsDefined && dataSize > dict) + dataSize = (size_t)dict; + } + + HRESULT res = TotalBench(EXTERNAL_CODECS_LOC_VARS + complexInCommands, + #ifndef _7ZIP_ST + numThreads, + &affinityMode, + #endif + dictIsDefined || use_fileData, // forceUnpackSize + dataSize, + (const Byte *)fileDataBuffer, printCallback, &callback); RINOK(res); } - res = TotalBench_Hash(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, - 1 << kNumHashDictBits, printCallback, &callback, &callback.EncodeRes, true, cpuFreq); - RINOK(res); + { + size_t dataSize = (size_t)1 << kNumHashDictBits; + if (dictIsDefined) + { + dataSize = (size_t)dict; + if (dataSize != dict) + return E_OUTOFMEMORY; + } + if (use_fileData) + { + dataSize = fileDataBuffer.Size(); + if (dictIsDefined && dataSize > dict) + dataSize = (size_t)dict; + } + + HRESULT res = TotalBench_Hash(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, + dataSize, (const Byte *)fileDataBuffer, + printCallback, &callback, + #ifndef _7ZIP_ST + &affinityMode, + #endif + &callback.EncodeRes, true, cpuFreq); + RINOK(res); + } callback.NewLine(); { PrintLeft(f, "CPU", kFieldSize_Name); - UInt32 resVal; - UInt64 cpuFreqLastTemp = cpuFreq; - RINOK(FreqBench(complexInCommands, numThreads, printCallback, - specifiedFreq != 0, // showFreq - specifiedFreq, - cpuFreqLastTemp, resVal)); + + CFreqBench fb; + fb.complexInCommands = complexInCommands; + fb.numThreads = numThreads; + // showFreq; + fb.showFreq = (specifiedFreq != 0); + fb.specifiedFreq = specifiedFreq; + + HRESULT res = fb.FreqBench(printCallback + #ifndef _7ZIP_ST + , &affinityMode + #endif + ); + RINOK(res); callback.NewLine(); } } } else { - bool needSetComplexity = true; + needSetComplexity = true; if (!methodName.IsEqualTo_Ascii_NoCase("LZMA")) { - for (unsigned i = 0; i < ARRAY_SIZE(g_Bench); i++) + unsigned i; + for (i = 0; i < ARRAY_SIZE(g_Bench); i++) { const CBenchMethod &h = g_Bench[i]; - AString s = h.Name; - if (AreSameMethodNames(h.Name, methodName)) + AString benchMethod (h.Name); + AString benchProps; + int propPos = benchMethod.Find(':'); + if (propPos >= 0) { - callback.BenchProps.EncComplex = h.EncComplex; - callback.BenchProps.DecComplexCompr = h.DecComplexCompr; - callback.BenchProps.DecComplexUnc = h.DecComplexUnc;; - needSetComplexity = false; - break; + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); + } + + if (AreSameMethodNames(benchMethod, methodName)) + { + if (benchProps.IsEmpty() + || (benchProps == "x5" && method.PropsString.IsEmpty()) + || method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps)) + { + callback.BenchProps.EncComplex = h.EncComplex; + callback.BenchProps.DecComplexCompr = h.DecComplexCompr; + callback.BenchProps.DecComplexUnc = h.DecComplexUnc;; + needSetComplexity = false; + break; + } } } + if (i == ARRAY_SIZE(g_Bench)) + return E_NOTIMPL; } if (needSetComplexity) callback.BenchProps.SetLzmaCompexity(); + if (startDicLog < kBenchMinDicLogSize) + startDicLog = kBenchMinDicLogSize; + for (unsigned i = 0; i < numIterations; i++) { - const unsigned kStartDicLog = 22; - unsigned pow = (dict < ((UInt32)1 << kStartDicLog)) ? kBenchMinDicLogSize : kStartDicLog; + unsigned pow = (dict < GetDictSizeFromLog(startDicLog)) ? kBenchMinDicLogSize : (unsigned)startDicLog; if (!multiDict) - pow = 31; - while (((UInt32)1 << pow) > dict && pow > 0) + pow = 32; + while (GetDictSizeFromLog(pow) > dict && pow > 0) pow--; - for (; ((UInt32)1 << pow) <= dict; pow++) + for (; GetDictSizeFromLog(pow) <= dict; pow++) { char s[16]; ConvertUInt32ToString(pow, s); @@ -3030,7 +4505,7 @@ HRESULT Bench( s[pos++] = ':'; s[pos] = 0; PrintLeft(f, s, kFieldSize_SmallName); - callback.DictSize = (UInt32)1 << pow; + callback.DictSize = (UInt64)1 << pow; COneMethodInfo method2 = method; @@ -3040,17 +4515,19 @@ HRESULT Bench( // method2 can have two different dictionary size properties. // And last property is main. NCOM::CPropVariant propVariant = (UInt32)pow; - RINOK(method2.ParseMethodFromPROPVARIANT(L"d", propVariant)); + RINOK(method2.ParseMethodFromPROPVARIANT((UString)"d", propVariant)); } size_t uncompressedDataSize; - if (fileDataBuffer.Buffer) + if (use_fileData) { - uncompressedDataSize = fileDataBuffer.BufferSize; + uncompressedDataSize = fileDataBuffer.Size(); } else { - uncompressedDataSize = callback.DictSize; + uncompressedDataSize = (size_t)callback.DictSize; + if (uncompressedDataSize != callback.DictSize) + return E_OUTOFMEMORY; if (uncompressedDataSize >= (1 << 18)) uncompressedDataSize += kAdditionalSize; } @@ -3058,9 +4535,12 @@ HRESULT Bench( HRESULT res = MethodBench( EXTERNAL_CODECS_LOC_VARS complexInCommands, + #ifndef _7ZIP_ST true, numThreads, + &affinityMode, + #endif method2, - uncompressedDataSize, fileDataBuffer.Buffer, + uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, &callback, &callback.BenchProps); f.NewLine(); RINOK(res); @@ -3083,16 +4563,19 @@ HRESULT Bench( if (use2Columns) { PrintLeft(f, "Avr:", callback.NameFieldSize); - PrintTotals(f, showFreq, cpuFreq, callback.EncodeRes); + PrintTotals(f, showFreq, cpuFreq, !totalBenchMode, callback.EncodeRes); f.Print(kSep); - PrintTotals(f, showFreq, cpuFreq, callback.DecodeRes); + PrintTotals(f, showFreq, cpuFreq, !totalBenchMode, callback.DecodeRes); f.NewLine(); } PrintLeft(f, "Tot:", callback.NameFieldSize); CTotalBenchRes midRes; - midRes.SetSum(callback.EncodeRes, callback.DecodeRes); - PrintTotals(f, showFreq, cpuFreq, midRes); + midRes = callback.EncodeRes; + midRes.Update_With_Res(callback.DecodeRes); + + // midRes.SetSum(callback.EncodeRes, callback.DecodeRes); + PrintTotals(f, showFreq, cpuFreq, false, midRes); f.NewLine(); } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/Bench.h b/3rdparty/lzma/CPP/7zip/UI/Common/Bench.h index 841250ef2cc..ab0c30487c4 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/Bench.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/Bench.h @@ -3,9 +3,13 @@ #ifndef __7ZIP_BENCH_H #define __7ZIP_BENCH_H +#include "../../../Windows/System.h" + #include "../../Common/CreateCoder.h" #include "../../UI/Common/Property.h" +UInt64 Benchmark_GetUsage_Percents(UInt64 usage); + struct CBenchInfo { UInt64 GlobalTime; @@ -15,26 +19,71 @@ struct CBenchInfo UInt64 UnpackSize; UInt64 PackSize; UInt64 NumIterations; + + /* + during Code(): we track benchInfo only from one thread (theads with index[0]) + NumIterations means number of threads + UnpackSize and PackSize are total sizes of all iterations of current thread + after Code(): + NumIterations means the number of Iterations + UnpackSize and PackSize are total sizes of all threads + */ CBenchInfo(): NumIterations(0) {} + UInt64 GetUsage() const; UInt64 GetRatingPerUsage(UInt64 rating) const; - UInt64 GetSpeed(UInt64 numCommands) const; + UInt64 GetSpeed(UInt64 numUnits) const; + UInt64 GetUnpackSizeSpeed() const { return GetSpeed(UnpackSize * NumIterations); } + + UInt64 Get_UnpackSize_Full() const { return UnpackSize * NumIterations; } + + UInt64 GetRating_LzmaEnc(UInt64 dictSize) const; + UInt64 GetRating_LzmaDec() const; }; + +struct CTotalBenchRes +{ + // UInt64 NumIterations1; // for Usage + UInt64 NumIterations2; // for Rating / RPU + + UInt64 Rating; + UInt64 Usage; + UInt64 RPU; + UInt64 Speed; + + void Init() { /* NumIterations1 = 0; */ NumIterations2 = 0; Rating = 0; Usage = 0; RPU = 0; Speed = 0; } + + void SetSum(const CTotalBenchRes &r1, const CTotalBenchRes &r2) + { + Rating = (r1.Rating + r2.Rating); + Usage = (r1.Usage + r2.Usage); + RPU = (r1.RPU + r2.RPU); + Speed = (r1.Speed + r2.Speed); + // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); + NumIterations2 = (r1.NumIterations2 + r2.NumIterations2); + } + + void Generate_From_BenchInfo(const CBenchInfo &info); + void Mult_For_Weight(unsigned weight); + void Update_With_Res(const CTotalBenchRes &r); +}; + + + struct IBenchCallback { - virtual HRESULT SetFreq(bool showFreq, UInt64 cpuFreq) = 0; + // virtual HRESULT SetFreq(bool showFreq, UInt64 cpuFreq) = 0; virtual HRESULT SetEncodeResult(const CBenchInfo &info, bool final) = 0; virtual HRESULT SetDecodeResult(const CBenchInfo &info, bool final) = 0; }; -UInt64 GetCompressRating(UInt32 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size); -UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations); + const unsigned kBenchMinDicLogSize = 18; -UInt64 GetBenchMemoryUsage(UInt32 numThreads, UInt32 dictionary, bool totalBench = false); +UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool totalBench); struct IBenchPrintCallback { @@ -43,13 +92,31 @@ struct IBenchPrintCallback virtual HRESULT CheckBreak() = 0; }; +struct IBenchFreqCallback +{ + virtual HRESULT AddCpuFreq(unsigned numThreads, UInt64 freq, UInt64 usage) = 0; + virtual HRESULT FreqsFinished(unsigned numThreads) = 0; +}; + HRESULT Bench( DECL_EXTERNAL_CODECS_LOC_VARS IBenchPrintCallback *printCallback, IBenchCallback *benchCallback, const CObjectVector &props, UInt32 numIterations, - bool multiDict - ); + bool multiDict, + IBenchFreqCallback *freqCallback = NULL); + +AString GetProcessThreadsInfo(const NWindows::NSystem::CProcessAffinity &ti); + +void GetSysInfo(AString &s1, AString &s2); +void GetCpuName(AString &s); +void AddCpuFeatures(AString &s); + +#ifdef _7ZIP_LARGE_PAGES +void Add_LargePages_String(AString &s); +#else +// #define Add_LargePages_String +#endif #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/DefaultName.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/DefaultName.cpp index 02f611c3d18..8c34ffc7348 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/DefaultName.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/DefaultName.cpp @@ -20,7 +20,7 @@ static UString GetDefaultName3(const UString &fileName, int dotPos = fileName.ReverseFind_Dot(); if (dotPos > 0) - return fileName.Left(dotPos) + addSubExtension; + return fileName.Left((unsigned)dotPos) + addSubExtension; if (addSubExtension.IsEmpty()) return fileName + L'~'; diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/DirItem.h b/3rdparty/lzma/CPP/7zip/UI/Common/DirItem.h index 4286b4e8db0..86e385fe7e0 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/DirItem.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/DirItem.h @@ -3,9 +3,15 @@ #ifndef __DIR_ITEM_H #define __DIR_ITEM_H +#ifdef _WIN32 +#include "../../../Common/MyLinux.h" +#endif + #include "../../../Common/MyString.h" #include "../../../Windows/FileFind.h" +#include "../../../Windows/PropVariant.h" +#include "../../../Windows/TimeUtils.h" #include "../../Common/UniqBlocks.h" @@ -20,9 +26,18 @@ struct CDirItemsStat UInt64 AltStreamsSize; UInt64 NumErrors; - // UInt64 GetTotalItems() const { return NumDirs + NumFiles + NumAltStreams; } + // UInt64 Get_NumItems() const { return NumDirs + NumFiles + NumAltStreams; } + UInt64 Get_NumDataItems() const { return NumFiles + NumAltStreams; } UInt64 GetTotalBytes() const { return FilesSize + AltStreamsSize; } + + bool IsEmpty() const { return + 0 == NumDirs + && 0 == NumFiles + && 0 == NumAltStreams + && 0 == FilesSize + && 0 == AltStreamsSize + && 0 == NumErrors; } CDirItemsStat(): NumDirs(0), @@ -34,6 +49,30 @@ struct CDirItemsStat {} }; + +struct CDirItemsStat2: public CDirItemsStat +{ + UInt64 Anti_NumDirs; + UInt64 Anti_NumFiles; + UInt64 Anti_NumAltStreams; + + // UInt64 Get_NumItems() const { return Anti_NumDirs + Anti_NumFiles + Anti_NumAltStreams + CDirItemsStat::Get_NumItems(); } + UInt64 Get_NumDataItems2() const { return Anti_NumFiles + Anti_NumAltStreams + CDirItemsStat::Get_NumDataItems(); } + + bool IsEmpty() const { return CDirItemsStat::IsEmpty() + && 0 == Anti_NumDirs + && 0 == Anti_NumFiles + && 0 == Anti_NumAltStreams; } + + CDirItemsStat2(): + Anti_NumDirs(0), + Anti_NumFiles(0), + Anti_NumAltStreams(0) + {} +}; + + + #define INTERFACE_IDirItemsCallback(x) \ virtual HRESULT ScanError(const FString &path, DWORD systemError) x; \ virtual HRESULT ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) x; \ @@ -43,33 +82,217 @@ struct IDirItemsCallback INTERFACE_IDirItemsCallback(=0) }; -struct CDirItem + +struct CArcTime +{ + FILETIME FT; + UInt16 Prec; + Byte Ns100; + bool Def; + + CArcTime() + { + Clear(); + } + + void Clear() + { + FT.dwHighDateTime = FT.dwLowDateTime = 0; + Prec = 0; + Ns100 = 0; + Def = false; + } + + bool IsZero() const + { + return FT.dwLowDateTime == 0 && FT.dwHighDateTime == 0 && Ns100 == 0; + } + + int CompareWith(const CArcTime &a) const + { + const int res = CompareFileTime(&FT, &a.FT); + if (res != 0) + return res; + if (Ns100 < a.Ns100) return -1; + if (Ns100 > a.Ns100) return 1; + return 0; + } + + UInt64 Get_FILETIME_as_UInt64() const + { + return (((UInt64)FT.dwHighDateTime) << 32) + FT.dwLowDateTime; + } + + UInt32 Get_DosTime() const + { + FILETIME ft2 = FT; + if ((Prec == k_PropVar_TimePrec_Base + 8 || + Prec == k_PropVar_TimePrec_Base + 9) + && Ns100 != 0) + { + UInt64 u64 = Get_FILETIME_as_UInt64(); + // we round up even small (ns < 100ns) as FileTimeToDosTime() + if (u64 % 20000000 == 0) + { + u64++; + ft2.dwHighDateTime = (DWORD)(u64 >> 32); + ft2.dwHighDateTime = (DWORD)u64; + } + } + // FileTimeToDosTime() is expected to round up in Windows + UInt32 dosTime; + // we use simplified code with utctime->dos. + // do we need local time instead here? + NWindows::NTime::FileTime_To_DosTime(ft2, dosTime); + return dosTime; + } + + int GetNumDigits() const + { + if (Prec == k_PropVar_TimePrec_Unix || + Prec == k_PropVar_TimePrec_DOS) + return 0; + if (Prec == k_PropVar_TimePrec_HighPrec) + return 9; + if (Prec == k_PropVar_TimePrec_0) + return 7; + int digits = (int)Prec - (int)k_PropVar_TimePrec_Base; + if (digits < 0) + digits = 0; + return digits; + } + + void Write_To_FiTime(CFiTime &dest) const + { + #ifdef _WIN32 + dest = FT; + #else + if (FILETIME_To_timespec(FT, dest)) + if ((Prec == k_PropVar_TimePrec_Base + 8 || + Prec == k_PropVar_TimePrec_Base + 9) + && Ns100 != 0) + { + dest.tv_nsec += Ns100; + } + #endif + } + + // (Def) is not set + void Set_From_FILETIME(const FILETIME &ft) + { + FT = ft; + // Prec = k_PropVar_TimePrec_CompatNTFS; + Prec = k_PropVar_TimePrec_Base + 7; + Ns100 = 0; + } + + // (Def) is not set + // it set full form precision: k_PropVar_TimePrec_Base + numDigits + void Set_From_FiTime(const CFiTime &ts) + { + #ifdef _WIN32 + FT = ts; + Prec = k_PropVar_TimePrec_Base + 7; + // Prec = k_PropVar_TimePrec_Base; // for debug + // Prec = 0; // for debug + Ns100 = 0; + #else + unsigned ns100; + FiTime_To_FILETIME_ns100(ts, FT, ns100); + Ns100 = (Byte)ns100; + Prec = k_PropVar_TimePrec_Base + 9; + #endif + } + + void Set_From_Prop(const PROPVARIANT &prop) + { + FT = prop.filetime; + unsigned prec = 0; + unsigned ns100 = 0; + const unsigned prec_Temp = prop.wReserved1; + if (prec_Temp != 0 + && prec_Temp <= k_PropVar_TimePrec_1ns + && prop.wReserved3 == 0) + { + const unsigned ns100_Temp = prop.wReserved2; + if (ns100_Temp < 100) + { + ns100 = ns100_Temp; + prec = prec_Temp; + } + } + Prec = (UInt16)prec; + Ns100 = (Byte)ns100; + Def = true; + } +}; + + +struct CDirItem: public NWindows::NFile::NFind::CFileInfoBase { - UInt64 Size; - FILETIME CTime; - FILETIME ATime; - FILETIME MTime; UString Name; - #if defined(_WIN32) && !defined(UNDER_CE) - // UString ShortName; + #ifndef UNDER_CE CByteBuffer ReparseData; - CByteBuffer ReparseData2; // fixed (reduced) absolute links + #ifdef _WIN32 + // UString ShortName; + CByteBuffer ReparseData2; // fixed (reduced) absolute links for WIM format bool AreReparseData() const { return ReparseData.Size() != 0 || ReparseData2.Size() != 0; } - #endif + #else + bool AreReparseData() const { return ReparseData.Size() != 0; } + #endif // _WIN32 + + #endif // !UNDER_CE - UInt32 Attrib; + void Copy_From_FileInfoBase(const NWindows::NFile::NFind::CFileInfoBase &fi) + { + (NWindows::NFile::NFind::CFileInfoBase &)*this = fi; + } + int PhyParent; int LogParent; int SecureIndex; - bool IsAltStream; - - CDirItem(): PhyParent(-1), LogParent(-1), SecureIndex(-1), IsAltStream(false) {} - bool IsDir() const { return (Attrib & FILE_ATTRIBUTE_DIRECTORY) != 0 ; } + #ifdef _WIN32 + #else + int OwnerNameIndex; + int OwnerGroupIndex; + #endif + + CDirItem(): + PhyParent(-1) + , LogParent(-1) + , SecureIndex(-1) + #ifdef _WIN32 + #else + , OwnerNameIndex(-1) + , OwnerGroupIndex(-1) + #endif + { + } + + + CDirItem(const NWindows::NFile::NFind::CFileInfo &fi, + int phyParent, int logParent, int secureIndex): + CFileInfoBase(fi) + , Name(fs2us(fi.Name)) + #if defined(_WIN32) && !defined(UNDER_CE) + // , ShortName(fs2us(fi.ShortName)) + #endif + , PhyParent(phyParent) + , LogParent(logParent) + , SecureIndex(secureIndex) + #ifdef _WIN32 + #else + , OwnerNameIndex(-1) + , OwnerGroupIndex(-1) + #endif + {} }; + + class CDirItems { UStringVector Prefixes; @@ -84,18 +307,26 @@ public: CObjectVector Items; bool SymLinks; - bool ScanAltStreams; - + bool ExcludeDirItems; + bool ExcludeFileItems; + bool ShareForWrite; + + /* it must be called after anotrher checks */ + bool CanIncludeItem(bool isDir) const + { + return isDir ? !ExcludeDirItems : !ExcludeFileItems; + } + + CDirItemsStat Stat; - #ifndef UNDER_CE + #if !defined(UNDER_CE) HRESULT SetLinkInfo(CDirItem &dirItem, const NWindows::NFile::NFind::CFileInfo &fi, const FString &phyPrefix); #endif - - #if defined(_WIN32) && !defined(UNDER_CE) + #if defined(_WIN32) && !defined(UNDER_CE) CUniqBlocks SecureBlocks; CByteBuffer TempSecureBuf; @@ -103,8 +334,19 @@ public: bool ReadSecure; HRESULT AddSecurityItem(const FString &path, int &secureIndex); + HRESULT FillFixedReparse(); - #endif + #endif + + #ifndef _WIN32 + + C_UInt32_UString_Map OwnerNameMap; + C_UInt32_UString_Map OwnerGroupMap; + bool StoreOwnerName; + + HRESULT FillDeviceSizes(); + + #endif IDirItemsCallback *Callback; @@ -124,6 +366,9 @@ public: unsigned AddPrefix(int phyParent, int logParent, const UString &prefix); void DeleteLastPrefix(); + + // HRESULT EnumerateOneDir(const FString &phyPrefix, CObjectVector &files); + HRESULT EnumerateOneDir(const FString &phyPrefix, CObjectVector &files); HRESULT EnumerateItems2( const FString &phyPrefix, @@ -131,27 +376,29 @@ public: const FStringVector &filePaths, FStringVector *requestedPaths); - #if defined(_WIN32) && !defined(UNDER_CE) - void FillFixedReparse(); - #endif - void ReserveDown(); }; + + + struct CArcItem { UInt64 Size; - FILETIME MTime; UString Name; + CArcTime MTime; // it can be mtime of archive file, if MTime is not defined for item in archive bool IsDir; bool IsAltStream; - bool SizeDefined; - bool MTimeDefined; + bool Size_Defined; bool Censored; UInt32 IndexInServer; - int TimeType; - CArcItem(): IsDir(false), IsAltStream(false), SizeDefined(false), MTimeDefined(false), Censored(false), TimeType(-1) {} + CArcItem(): + IsDir(false), + IsAltStream(false), + Size_Defined(false), + Censored(false) + {} }; #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.cpp index 3ebf25aad8f..dab372543ed 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -3,6 +3,13 @@ #include "StdAfx.h" #include +// #include + +#ifndef _WIN32 +#include +#include +#include "../../../Common/UTFConvert.h" +#endif #include "../../../Common/Wildcard.h" @@ -16,37 +23,66 @@ #endif #include "EnumDirItems.h" +#include "SortUtils.h" using namespace NWindows; using namespace NFile; using namespace NName; + +static bool FindFile_KeepDots(NFile::NFind::CFileInfo &fi, const FString &path, bool followLink) +{ + const bool res = fi.Find(path, followLink); + if (!res) + return res; + if (path.IsEmpty()) + return res; + // we keep name "." and "..", if it's without tail slash + const FChar *p = path.RightPtr(1); + if (*p != '.') + return res; + if (p != path.Ptr()) + { + FChar c = p[-1]; + if (!IS_PATH_SEPAR(c)) + { + if (c != '.') + return res; + p--; + if (p != path.Ptr()) + { + c = p[-1]; + if (!IS_PATH_SEPAR(c)) + return res; + } + } + } + fi.Name = p; + return res; +} + + void CDirItems::AddDirFileInfo(int phyParent, int logParent, int secureIndex, const NFind::CFileInfo &fi) { - CDirItem di; - di.Size = fi.Size; - di.CTime = fi.CTime; - di.ATime = fi.ATime; - di.MTime = fi.MTime; - di.Attrib = fi.Attrib; - di.IsAltStream = fi.IsAltStream; + /* + CDirItem di(fi); di.PhyParent = phyParent; di.LogParent = logParent; di.SecureIndex = secureIndex; - di.Name = fs2us(fi.Name); - #if defined(_WIN32) && !defined(UNDER_CE) - // di.ShortName = fs2us(fi.ShortName); - #endif Items.Add(di); + */ + VECTOR_ADD_NEW_OBJECT (Items, CDirItem(fi, phyParent, logParent, secureIndex)) if (fi.IsDir()) Stat.NumDirs++; + #ifdef _WIN32 else if (fi.IsAltStream) { Stat.NumAltStreams++; Stat.AltStreamsSize += fi.Size; } + #endif else { Stat.NumFiles++; @@ -54,8 +90,13 @@ void CDirItems::AddDirFileInfo(int phyParent, int logParent, int secureIndex, } } +// (DWORD)E_FAIL +#define DI_DEFAULT_ERROR ERROR_INVALID_FUNCTION + HRESULT CDirItems::AddError(const FString &path, DWORD errorCode) { + if (errorCode == 0) + errorCode = DI_DEFAULT_ERROR; Stat.NumErrors++; if (Callback) return Callback->ScanError(path, errorCode); @@ -82,17 +123,17 @@ UString CDirItems::GetPrefixesPath(const CIntVector &parents, int index, const U unsigned len = name.Len(); int i; - for (i = index; i >= 0; i = parents[i]) - len += Prefixes[i].Len(); + for (i = index; i >= 0; i = parents[(unsigned)i]) + len += Prefixes[(unsigned)i].Len(); wchar_t *p = path.GetBuf_SetEnd(len) + len; p -= name.Len(); wmemcpy(p, (const wchar_t *)name, name.Len()); - for (i = index; i >= 0; i = parents[i]) + for (i = index; i >= 0; i = parents[(unsigned)i]) { - const UString &s = Prefixes[i]; + const UString &s = Prefixes[(unsigned)i]; p -= s.Len(); wmemcpy(p, (const wchar_t *)s, s.Len()); } @@ -139,9 +180,15 @@ bool InitLocalPrivileges(); CDirItems::CDirItems(): SymLinks(false), ScanAltStreams(false) - #ifdef _USE_SECURITY_CODE + , ExcludeDirItems(false) + , ExcludeFileItems(false) + , ShareForWrite(false) + #ifdef _USE_SECURITY_CODE , ReadSecure(false) - #endif + #endif + #ifndef _WIN32 + , StoreOwnerName(false) + #endif , Callback(NULL) { #ifdef _USE_SECURITY_CODE @@ -149,6 +196,7 @@ CDirItems::CDirItems(): #endif } + #ifdef _USE_SECURITY_CODE HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) @@ -165,7 +213,7 @@ HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) DWORD errorCode = 0; DWORD secureSize; - BOOL res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); + BOOL res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(void *)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); if (res) { @@ -184,7 +232,7 @@ HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) else { TempSecureBuf.Alloc(secureSize); - res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); + res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(void *)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); if (res) { if (secureSize != TempSecureBuf.Size()) @@ -198,33 +246,120 @@ HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) if (res) { - secureIndex = SecureBlocks.AddUniq(TempSecureBuf, secureSize); + secureIndex = (int)SecureBlocks.AddUniq(TempSecureBuf, secureSize); return S_OK; } - if (errorCode == 0) - errorCode = ERROR_INVALID_FUNCTION; return AddError(path, errorCode); } -#endif +#endif // _USE_SECURITY_CODE -HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phyPrefix) + +HRESULT CDirItems::EnumerateOneDir(const FString &phyPrefix, CObjectVector &files) { - RINOK(ScanProgress(phyPrefix)); + NFind::CEnumerator enumerator; + // printf("\n enumerator.SetDirPrefix(phyPrefix) \n"); + + enumerator.SetDirPrefix(phyPrefix); + + #ifdef _WIN32 + + NFind::CFileInfo fi; - NFind::CEnumerator enumerator(phyPrefix + FCHAR_ANY_MASK); for (unsigned ttt = 0; ; ttt++) { - NFind::CFileInfo fi; bool found; if (!enumerator.Next(fi, found)) + return AddError(phyPrefix); + if (!found) + return S_OK; + files.Add(fi); + if (Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)); + } + } + + #else // _WIN32 + + // enumerator.SolveLinks = !SymLinks; + + CObjectVector entries; + + for (unsigned ttt = 0; ; ttt++) + { + bool found; + NFind::CDirEntry de; + if (!enumerator.Next(de, found)) { return AddError(phyPrefix); } if (!found) - return S_OK; + break; + entries.Add(de); + } + FOR_VECTOR(i, entries) + { + const NFind::CDirEntry &de = entries[i]; + NFind::CFileInfo fi; + if (!enumerator.Fill_FileInfo(de, fi, !SymLinks)) + // if (!fi.Find_AfterEnumerator(path)) + { + const FString path = phyPrefix + de.Name; + { + RINOK(AddError(path)); + continue; + } + } + + files.Add(fi); + + if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)); + } + } + + return S_OK; + + #endif // _WIN32 +} + + + + +HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phyPrefix) +{ + RINOK(ScanProgress(phyPrefix)); + + CObjectVector files; + RINOK(EnumerateOneDir(phyPrefix, files)); + + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + const NFind::CFileInfo &fi = files[i]; + #else + const NFind::CFileInfo &fi = files[i]; + /* + NFind::CFileInfo fi; + { + const NFind::CDirEntry &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(AddError(path)); + continue; + } + fi.Name = di.Name; + } + */ + #endif + + if (CanIncludeItem(fi.IsDir())) + { int secureIndex = -1; #ifdef _USE_SECURITY_CODE if (ReadSecure) @@ -232,10 +367,10 @@ HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phy RINOK(AddSecurityItem(phyPrefix + fi.Name, secureIndex)); } #endif - AddDirFileInfo(phyParent, logParent, secureIndex, fi); + } - if (Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) { RINOK(ScanProgress(phyPrefix)); } @@ -244,26 +379,45 @@ HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phy { const FString name2 = fi.Name + FCHAR_PATH_SEPARATOR; unsigned parent = AddPrefix(phyParent, logParent, fs2us(name2)); - RINOK(EnumerateDir(parent, parent, phyPrefix + name2)); + RINOK(EnumerateDir((int)parent, (int)parent, phyPrefix + name2)); } } + return S_OK; } + +/* +EnumerateItems2() + const FStringVector &filePaths - are path without tail slashes. + All dir prefixes of filePaths will be not stores in logical paths +fix it: we can scan AltStream also. +*/ + +#ifdef _WIN32 +// #define FOLLOW_LINK_PARAM +// #define FOLLOW_LINK_PARAM2 +#define FOLLOW_LINK_PARAM , (!SymLinks) +#define FOLLOW_LINK_PARAM2 , (!dirItems.SymLinks) +#else +#define FOLLOW_LINK_PARAM , (!SymLinks) +#define FOLLOW_LINK_PARAM2 , (!dirItems.SymLinks) +#endif + HRESULT CDirItems::EnumerateItems2( const FString &phyPrefix, const UString &logPrefix, const FStringVector &filePaths, FStringVector *requestedPaths) { - int phyParent = phyPrefix.IsEmpty() ? -1 : AddPrefix(-1, -1, fs2us(phyPrefix)); - int logParent = logPrefix.IsEmpty() ? -1 : AddPrefix(-1, -1, logPrefix); + const int phyParent = phyPrefix.IsEmpty() ? -1 : (int)AddPrefix(-1, -1, fs2us(phyPrefix)); + const int logParent = logPrefix.IsEmpty() ? -1 : (int)AddPrefix(-1, -1, logPrefix); FOR_VECTOR (i, filePaths) { const FString &filePath = filePaths[i]; NFind::CFileInfo fi; const FString phyPath = phyPrefix + filePath; - if (!fi.Find(phyPath)) + if (!FindFile_KeepDots(fi, phyPath FOLLOW_LINK_PARAM)) { RINOK(AddError(phyPath)); continue; @@ -271,15 +425,17 @@ HRESULT CDirItems::EnumerateItems2( if (requestedPaths) requestedPaths->Add(phyPath); - int delimiter = filePath.ReverseFind_PathSepar(); + const int delimiter = filePath.ReverseFind_PathSepar(); FString phyPrefixCur; int phyParentCur = phyParent; if (delimiter >= 0) { - phyPrefixCur.SetFrom(filePath, delimiter + 1); - phyParentCur = AddPrefix(phyParent, logParent, fs2us(phyPrefixCur)); + phyPrefixCur.SetFrom(filePath, (unsigned)(delimiter + 1)); + phyParentCur = (int)AddPrefix(phyParent, logParent, fs2us(phyPrefixCur)); } + if (CanIncludeItem(fi.IsDir())) + { int secureIndex = -1; #ifdef _USE_SECURITY_CODE if (ReadSecure) @@ -287,14 +443,14 @@ HRESULT CDirItems::EnumerateItems2( RINOK(AddSecurityItem(phyPath, secureIndex)); } #endif - AddDirFileInfo(phyParentCur, logParent, secureIndex, fi); + } if (fi.IsDir()) { const FString name2 = fi.Name + FCHAR_PATH_SEPARATOR; unsigned parent = AddPrefix(phyParentCur, logParent, fs2us(name2)); - RINOK(EnumerateDir(parent, parent, phyPrefix + phyPrefixCur + name2)); + RINOK(EnumerateDir((int)parent, (int)parent, phyPrefix + phyPrefixCur + name2)); } } @@ -305,34 +461,46 @@ HRESULT CDirItems::EnumerateItems2( - - static HRESULT EnumerateDirItems( const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &phyPrefix, - const UStringVector &addArchivePrefix, + const int phyParent, const int logParent, + const FString &phyPrefix, + const UStringVector &addParts, // additional parts from curNode CDirItems &dirItems, bool enterToSubFolders); + +/* EnumerateDirItems_Spec() + adds new Dir item prefix, and enumerates dir items, + then it can remove that Dir item prefix, if there are no items in that dir. +*/ + + +/* + EnumerateDirItems_Spec() + it's similar to EnumerateDirItems, but phyPrefix doesn't include (curFolderName) +*/ + static HRESULT EnumerateDirItems_Spec( const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &curFolderName, - const FString &phyPrefix, - const UStringVector &addArchivePrefix, + const int phyParent, const int logParent, const FString &curFolderName, + const FString &phyPrefix, // without (curFolderName) + const UStringVector &addParts, // (curNode + addParts) includes (curFolderName) CDirItems &dirItems, bool enterToSubFolders) { const FString name2 = curFolderName + FCHAR_PATH_SEPARATOR; - unsigned parent = dirItems.AddPrefix(phyParent, logParent, fs2us(name2)); - unsigned numItems = dirItems.Items.Size(); + const unsigned parent = dirItems.AddPrefix(phyParent, logParent, fs2us(name2)); + const unsigned numItems = dirItems.Items.Size(); HRESULT res = EnumerateDirItems( - curNode, parent, parent, phyPrefix + name2, - addArchivePrefix, dirItems, enterToSubFolders); + curNode, (int)parent, (int)parent, phyPrefix + name2, + addParts, dirItems, enterToSubFolders); if (numItems == dirItems.Items.Size()) dirItems.DeleteLastPrefix(); return res; } + #ifndef UNDER_CE #ifdef _WIN32 @@ -340,136 +508,222 @@ static HRESULT EnumerateDirItems_Spec( static HRESULT EnumerateAltStreams( const NFind::CFileInfo &fi, const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &fullPath, - const UStringVector &addArchivePrefix, // prefix from curNode + const int phyParent, const int logParent, + const FString &phyPath, // with (fi.Name), without tail slash for folders + const UStringVector &addParts, // with (fi.Name), prefix parts from curNode + bool addAllSubStreams, CDirItems &dirItems) { - NFind::CStreamEnumerator enumerator(fullPath); + // we don't use (ExcludeFileItems) rules for AltStreams + // if (dirItems.ExcludeFileItems) return S_OK; + + NFind::CStreamEnumerator enumerator(phyPath); for (;;) { NFind::CStreamInfo si; bool found; if (!enumerator.Next(si, found)) { - return dirItems.AddError(fullPath + FTEXT(":*")); // , (DWORD)E_FAIL + return dirItems.AddError(phyPath + FTEXT(":*")); // , (DWORD)E_FAIL } if (!found) return S_OK; if (si.IsMainStream()) continue; - UStringVector addArchivePrefixNew = addArchivePrefix; - UString reducedName = si.GetReducedName(); - addArchivePrefixNew.Back() += reducedName; - if (curNode.CheckPathToRoot(false, addArchivePrefixNew, true)) + UStringVector parts = addParts; + const UString reducedName = si.GetReducedName(); + parts.Back() += reducedName; + if (curNode.CheckPathToRoot(false, parts, true)) continue; + if (!addAllSubStreams) + if (!curNode.CheckPathToRoot(true, parts, true)) + continue; + NFind::CFileInfo fi2 = fi; fi2.Name += us2fs(reducedName); fi2.Size = si.Size; - fi2.Attrib &= ~FILE_ATTRIBUTE_DIRECTORY; + fi2.Attrib &= ~(DWORD)(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); fi2.IsAltStream = true; dirItems.AddDirFileInfo(phyParent, logParent, -1, fi2); } } -#endif +#endif // _WIN32 + + +/* We get Reparse data and parse it. + If there is Reparse error, we free dirItem.Reparse data. + Do we need to work with empty reparse data? +*/ HRESULT CDirItems::SetLinkInfo(CDirItem &dirItem, const NFind::CFileInfo &fi, const FString &phyPrefix) { - if (!SymLinks || !fi.HasReparsePoint()) + if (!SymLinks) return S_OK; + + #ifdef _WIN32 + if (!fi.HasReparsePoint() || fi.IsAltStream) + #else // _WIN32 + if (!fi.IsPosixLink()) + #endif // _WIN32 + return S_OK; + const FString path = phyPrefix + fi.Name; CByteBuffer &buf = dirItem.ReparseData; if (NIO::GetReparseData(path, buf)) { - CReparseAttr attr; - if (attr.Parse(buf, buf.Size())) - return S_OK; + // if (dirItem.ReparseData.Size() != 0) + Stat.FilesSize -= fi.Size; + return S_OK; } + DWORD res = ::GetLastError(); buf.Free(); - return AddError(path , res); + return AddError(path, res); } -#endif +#endif // UNDER_CE + + static HRESULT EnumerateForItem( - NFind::CFileInfo &fi, + const NFind::CFileInfo &fi, const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &phyPrefix, - const UStringVector &addArchivePrefix, // prefix from curNode + const int phyParent, const int logParent, const FString &phyPrefix, + const UStringVector &addParts, // additional parts from curNode, without (fi.Name) CDirItems &dirItems, bool enterToSubFolders) { const UString name = fs2us(fi.Name); - bool enterToSubFolders2 = enterToSubFolders; - UStringVector addArchivePrefixNew = addArchivePrefix; - addArchivePrefixNew.Add(name); - { - UStringVector addArchivePrefixNewTemp(addArchivePrefixNew); - if (curNode.CheckPathToRoot(false, addArchivePrefixNewTemp, !fi.IsDir())) - return S_OK; - } - int dirItemIndex = -1; + UStringVector newParts = addParts; + newParts.Add(name); - if (curNode.CheckPathToRoot(true, addArchivePrefixNew, !fi.IsDir())) + // check the path in exclude rules + if (curNode.CheckPathToRoot(false, newParts, !fi.IsDir())) + return S_OK; + + #if !defined(UNDER_CE) + int dirItemIndex = -1; + #if defined(_WIN32) + bool addAllSubStreams = false; + bool needAltStreams = true; + #endif // _WIN32 + #endif // !defined(UNDER_CE) + + // check the path in inlcude rules + if (curNode.CheckPathToRoot(true, newParts, !fi.IsDir())) { - int secureIndex = -1; + #if !defined(UNDER_CE) + // dirItemIndex = (int)dirItems.Items.Size(); + #if defined(_WIN32) + // we will not check include rules for substreams. + addAllSubStreams = true; + #endif // _WIN32 + #endif // !defined(UNDER_CE) + + if (dirItems.CanIncludeItem(fi.IsDir())) + { + int secureIndex = -1; #ifdef _USE_SECURITY_CODE - if (dirItems.ReadSecure) + if (dirItems.ReadSecure) + { + RINOK(dirItems.AddSecurityItem(phyPrefix + fi.Name, secureIndex)); + } + #endif + #if !defined(UNDER_CE) + dirItemIndex = (int)dirItems.Items.Size(); + #endif // !defined(UNDER_CE) + dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); + } + else { - RINOK(dirItems.AddSecurityItem(phyPrefix + fi.Name, secureIndex)); + #if defined(_WIN32) && !defined(UNDER_CE) + needAltStreams = false; + #endif } - #endif - dirItemIndex = dirItems.Items.Size(); - dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); if (fi.IsDir()) - enterToSubFolders2 = true; + enterToSubFolders = true; } - #ifndef UNDER_CE - if (dirItems.ScanAltStreams) - { - RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, - phyPrefix + fi.Name, - addArchivePrefixNew, dirItems)); - } + #if !defined(UNDER_CE) + + // we don't scan AltStreams for link files if (dirItemIndex >= 0) { - CDirItem &dirItem = dirItems.Items[dirItemIndex]; + CDirItem &dirItem = dirItems.Items[(unsigned)dirItemIndex]; RINOK(dirItems.SetLinkInfo(dirItem, fi, phyPrefix)); if (dirItem.ReparseData.Size() != 0) return S_OK; } - #endif + #if defined(_WIN32) + if (needAltStreams && dirItems.ScanAltStreams) + { + RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, + phyPrefix + fi.Name, // with (fi.Name) + newParts, // with (fi.Name) + addAllSubStreams, + dirItems)); + } + #endif + + #endif // !defined(UNDER_CE) + + + #ifndef _WIN32 + if (!fi.IsPosixLink()) // posix link can follow to dir + #endif if (!fi.IsDir()) return S_OK; - - const NWildcard::CCensorNode *nextNode = 0; - if (addArchivePrefix.IsEmpty()) + + const NWildcard::CCensorNode *nextNode = NULL; + + if (addParts.IsEmpty()) { int index = curNode.FindSubNode(name); if (index >= 0) - nextNode = &curNode.SubNodes[index]; + { + nextNode = &curNode.SubNodes[(unsigned)index]; + newParts.Clear(); + } } - if (!enterToSubFolders2 && nextNode == 0) - return S_OK; - - addArchivePrefixNew = addArchivePrefix; - if (nextNode == 0) + + if (!nextNode) { + if (!enterToSubFolders) + return S_OK; + + #ifndef _WIN32 + if (fi.IsPosixLink()) + { + // here we can try to resolve posix link + // if the link to dir, then can we follow it + return S_OK; // we don't follow posix link + } + #else + if (dirItems.SymLinks && fi.HasReparsePoint()) + { + /* 20.03: in SymLinks mode: we don't enter to directory that + has reparse point and has no CCensorNode + NOTE: (curNode and parent nodes) still can have wildcard rules + to include some items of target directory (of reparse point), + but we ignore these rules here. + */ + return S_OK; + } + #endif nextNode = &curNode; - addArchivePrefixNew.Add(name); } return EnumerateDirItems_Spec( - *nextNode, phyParent, logParent, fi.Name, phyPrefix, - addArchivePrefixNew, + *nextNode, phyParent, logParent, fi.Name, + phyPrefix, // without (fi.Name) + newParts, // relative to (*nextNode). (*nextNode + newParts) includes (fi.Name) dirItems, - enterToSubFolders2); + enterToSubFolders); } @@ -488,10 +742,13 @@ static bool CanUseFsDirect(const NWildcard::CCensorNode &curNode) /* Windows doesn't support file name with wildcard But if another system supports file name with wildcard, - and wildcard mode is disabled, we can ignore wildcard in name */ + and wildcard mode is disabled, we can ignore wildcard in name + */ /* + #ifndef _WIN32 if (!item.WildcardParsing) continue; + #endif */ if (DoesNameContainWildcard(name)) return false; @@ -507,26 +764,33 @@ static bool IsVirtualFsFolder(const FString &prefix, const UString &name) UString s = fs2us(prefix); s += name; s.Add_PathSepar(); + // it returns (true) for non real FS folder path like - "\\SERVER\" return IsPathSepar(s[0]) && GetRootPrefixSize(s) == 0; } #endif + + static HRESULT EnumerateDirItems( const NWildcard::CCensorNode &curNode, - int phyParent, int logParent, const FString &phyPrefix, - const UStringVector &addArchivePrefix, // prefix from curNode + const int phyParent, const int logParent, const FString &phyPrefix, + const UStringVector &addParts, // prefix from curNode including CDirItems &dirItems, bool enterToSubFolders) { if (!enterToSubFolders) + { + /* if there are IncludeItems censor rules that affect items in subdirs, + then we will enter to all subfolders */ if (curNode.NeedCheckSubDirs()) enterToSubFolders = true; + } RINOK(dirItems.ScanProgress(phyPrefix)); // try direct_names case at first - if (addArchivePrefix.IsEmpty() && !enterToSubFolders) + if (addParts.IsEmpty() && !enterToSubFolders) { if (CanUseFsDirect(curNode)) { @@ -541,6 +805,15 @@ static HRESULT EnumerateDirItems( const UString &name = item.PathParts.Front(); FString fullPath = phyPrefix + us2fs(name); + /* + // not possible now + if (!item.ForDir && !item.ForFile) + { + RINOK(dirItems.AddError(fullPath, ERROR_INVALID_PARAMETER)); + continue; + } + */ + #if defined(_WIN32) && !defined(UNDER_CE) bool needAltStreams = true; #endif @@ -570,7 +843,7 @@ static HRESULT EnumerateDirItems( #endif */ - fullPath = FCHAR_PATH_SEPARATOR; + fullPath = CHAR_PATH_SEPARATOR; } #if defined(_WIN32) && !defined(UNDER_CE) else if (item.IsDriveItem()) @@ -591,16 +864,27 @@ static HRESULT EnumerateDirItems( } else #endif - if (!fi.Find(fullPath)) + if (!FindFile_KeepDots(fi, fullPath FOLLOW_LINK_PARAM2)) { RINOK(dirItems.AddError(fullPath)); continue; } - bool isDir = fi.IsDir(); - if (isDir && !item.ForDir || !isDir && !item.ForFile) + /* + #ifdef _WIN32 + #define MY_ERROR_IS_DIR ERROR_FILE_NOT_FOUND + #define MY_ERROR_NOT_DIR DI_DEFAULT_ERROR + #else + #define MY_ERROR_IS_DIR EISDIR + #define MY_ERROR_NOT_DIR ENOTDIR + #endif + */ + + const bool isDir = fi.IsDir(); + if (isDir ? !item.ForDir : !item.ForFile) { - RINOK(dirItems.AddError(fullPath, (DWORD)E_FAIL)); + // RINOK(dirItems.AddError(fullPath, isDir ? MY_ERROR_IS_DIR: MY_ERROR_NOT_DIR)); + RINOK(dirItems.AddError(fullPath, DI_DEFAULT_ERROR)); continue; } { @@ -610,6 +894,9 @@ static HRESULT EnumerateDirItems( continue; } + + if (dirItems.CanIncludeItem(fi.IsDir())) + { int secureIndex = -1; #ifdef _USE_SECURITY_CODE if (needSecurity && dirItems.ReadSecure) @@ -620,53 +907,75 @@ static HRESULT EnumerateDirItems( dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); - #ifndef UNDER_CE + // we don't scan AltStreams for link files + + #if !defined(UNDER_CE) { CDirItem &dirItem = dirItems.Items.Back(); RINOK(dirItems.SetLinkInfo(dirItem, fi, phyPrefix)); if (dirItem.ReparseData.Size() != 0) - { - if (fi.IsAltStream) - dirItems.Stat.AltStreamsSize -= fi.Size; - else - dirItems.Stat.FilesSize -= fi.Size; continue; - } } - #endif - - - #ifndef UNDER_CE + + #if defined(_WIN32) if (needAltStreams && dirItems.ScanAltStreams) { UStringVector pathParts; pathParts.Add(fs2us(fi.Name)); RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, - fullPath, pathParts, dirItems)); + fullPath, // including (name) + pathParts, // including (fi.Name) + true, /* addAllSubStreams */ + dirItems)); } - #endif + #endif // defined(_WIN32) + + #endif // !defined(UNDER_CE) + } + + #ifndef _WIN32 + if (!fi.IsPosixLink()) // posix link can follow to dir + #endif if (!isDir) continue; - - UStringVector addArchivePrefixNew; - const NWildcard::CCensorNode *nextNode = 0; + + UStringVector newParts; + const NWildcard::CCensorNode *nextNode = NULL; int index = curNode.FindSubNode(name); if (index >= 0) { - for (int t = needEnterVector.Size(); t <= index; t++) + for (int t = (int)needEnterVector.Size(); t <= index; t++) needEnterVector.Add(true); - needEnterVector[index] = false; - nextNode = &curNode.SubNodes[index]; + needEnterVector[(unsigned)index] = false; + nextNode = &curNode.SubNodes[(unsigned)index]; } else { + #ifndef _WIN32 + if (fi.IsPosixLink()) + { + // here we can try to resolve posix link + // if the link to dir, then can we follow it + continue; // we don't follow posix link + } + #else + if (dirItems.SymLinks) + { + if (fi.HasReparsePoint()) + { + /* 20.03: in SymLinks mode: we don't enter to directory that + has reparse point and has no CCensorNode */ + continue; + } + } + #endif nextNode = &curNode; - addArchivePrefixNew.Add(name); // don't change it to fi.Name. It's for shortnames support + newParts.Add(name); // don't change it to fi.Name. It's for shortnames support } RINOK(EnumerateDirItems_Spec(*nextNode, phyParent, logParent, fi.Name, phyPrefix, - addArchivePrefixNew, dirItems, true)); + newParts, dirItems, true)); } for (i = 0; i < curNode.SubNodes.Size(); i++) @@ -682,7 +991,7 @@ static HRESULT EnumerateDirItems( { { if (nextNode.Name.IsEmpty()) - fullPath = FCHAR_PATH_SEPARATOR; + fullPath = CHAR_PATH_SEPARATOR; #ifdef _WIN32 else if (NWildcard::IsDriveColonName(nextNode.Name)) fullPath.Add_PathSepar(); @@ -691,7 +1000,7 @@ static HRESULT EnumerateDirItems( } // we don't want to call fi.Find() for root folder or virtual folder - if (phyPrefix.IsEmpty() && nextNode.Name.IsEmpty() + if ((phyPrefix.IsEmpty() && nextNode.Name.IsEmpty()) #if defined(_WIN32) && !defined(UNDER_CE) || IsVirtualFsFolder(phyPrefix, nextNode.Name) #endif @@ -702,7 +1011,7 @@ static HRESULT EnumerateDirItems( } else { - if (!fi.Find(fullPath)) + if (!FindFile_KeepDots(fi, fullPath FOLLOW_LINK_PARAM2)) { if (!nextNode.AreThereIncludeItems()) continue; @@ -712,7 +1021,7 @@ static HRESULT EnumerateDirItems( if (!fi.IsDir()) { - RINOK(dirItems.AddError(fullPath, (DWORD)E_FAIL)); + RINOK(dirItems.AddError(fullPath, DI_DEFAULT_ERROR)); continue; } } @@ -764,7 +1073,7 @@ static HRESULT EnumerateDirItems( fi.Name = driveName; RINOK(EnumerateForItem(fi, curNode, phyParent, logParent, phyPrefix, - addArchivePrefix, dirItems, enterToSubFolders)); + addParts, dirItems, enterToSubFolders)); } return S_OK; } @@ -773,41 +1082,81 @@ static HRESULT EnumerateDirItems( #endif #endif - NFind::CEnumerator enumerator(phyPrefix + FCHAR_ANY_MASK); - for (unsigned ttt = 0; ; ttt++) + + CObjectVector files; + + // for (int y = 0; y < 1; y++) { - NFind::CFileInfo fi; - bool found; - if (!enumerator.Next(fi, found)) + // files.Clear(); + RINOK(dirItems.EnumerateOneDir(phyPrefix, files)); + /* + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + // const NFind::CFileInfo &fi = files[i]; + #else + NFind::CFileInfo &fi = files[i]; { - RINOK(dirItems.AddError(phyPrefix)); - break; + const NFind::CFileInfo &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(dirItems.AddError(path)); + continue; + } + fi.Name = di.Name; } - if (!found) - break; + #endif + + } + */ + } - if (dirItems.Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + const NFind::CFileInfo &fi = files[i]; + #else + const NFind::CFileInfo &fi = files[i]; + /* + NFind::CFileInfo fi; { - RINOK(dirItems.ScanProgress(phyPrefix)); + const NFind::CDirEntry &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(dirItems.AddError(path)); + continue; + } + fi.Name = di.Name; } + */ + #endif RINOK(EnumerateForItem(fi, curNode, phyParent, logParent, phyPrefix, - addArchivePrefix, dirItems, enterToSubFolders)); + addParts, dirItems, enterToSubFolders)); + if (dirItems.Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(dirItems.ScanProgress(phyPrefix)); + } } return S_OK; } + + + HRESULT EnumerateItems( const NWildcard::CCensor &censor, const NWildcard::ECensorPathMode pathMode, - const UString &addPathPrefix, + const UString &addPathPrefix, // prefix that will be added to Logical Path CDirItems &dirItems) { FOR_VECTOR (i, censor.Pairs) { const NWildcard::CPair &pair = censor.Pairs[i]; - int phyParent = pair.Prefix.IsEmpty() ? -1 : dirItems.AddPrefix(-1, -1, pair.Prefix); + const int phyParent = pair.Prefix.IsEmpty() ? -1 : (int)dirItems.AddPrefix(-1, -1, pair.Prefix); int logParent = -1; if (pathMode == NWildcard::k_AbsPath) @@ -815,7 +1164,7 @@ HRESULT EnumerateItems( else { if (!addPathPrefix.IsEmpty()) - logParent = dirItems.AddPrefix(-1, -1, addPathPrefix); + logParent = (int)dirItems.AddPrefix(-1, -1, addPathPrefix); } RINOK(EnumerateDirItems(pair.Head, phyParent, logParent, us2fs(pair.Prefix), UStringVector(), @@ -825,34 +1174,107 @@ HRESULT EnumerateItems( } dirItems.ReserveDown(); - #if defined(_WIN32) && !defined(UNDER_CE) - dirItems.FillFixedReparse(); - #endif + #if defined(_WIN32) && !defined(UNDER_CE) + RINOK(dirItems.FillFixedReparse()); + #endif + + #ifndef _WIN32 + RINOK(dirItems.FillDeviceSizes()); + #endif return S_OK; } + #if defined(_WIN32) && !defined(UNDER_CE) -void CDirItems::FillFixedReparse() +HRESULT CDirItems::FillFixedReparse() { - /* imagex/WIM reduces absolute pathes in links (raparse data), - if we archive non root folder. We do same thing here */ - - if (!SymLinks) - return; - FOR_VECTOR(i, Items) { CDirItem &item = Items[i]; + + if (!SymLinks) + { + // continue; // for debug + if (!item.Has_Attrib_ReparsePoint()) + continue; + + // if (item.IsDir()) continue; + + const FString phyPath = GetPhyPath(i); + + NFind::CFileInfo fi; + if (fi.Fill_From_ByHandleFileInfo(phyPath)) // item.IsDir() + { + item.Size = fi.Size; + item.CTime = fi.CTime; + item.ATime = fi.ATime; + item.MTime = fi.MTime; + item.Attrib = fi.Attrib; + continue; + } + + /* + // we request properties of target file instead of properies of symbolic link + // here we also can manually parse unsupported links (like WSL links) + NIO::CInFile inFile; + if (inFile.Open(phyPath)) + { + BY_HANDLE_FILE_INFORMATION info; + if (inFile.GetFileInformation(&info)) + { + // Stat.FilesSize doesn't contain item.Size already + // Stat.FilesSize -= item.Size; + item.Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + Stat.FilesSize += item.Size; + item.CTime = info.ftCreationTime; + item.ATime = info.ftLastAccessTime; + item.MTime = info.ftLastWriteTime; + item.Attrib = info.dwFileAttributes; + continue; + } + } + */ + + RINOK(AddError(phyPath)); + continue; + } + + // (SymLinks == true) here + if (item.ReparseData.Size() == 0) continue; + + // if (item.Size == 0) + { + // 20.03: we use Reparse Data instead of real data + item.Size = item.ReparseData.Size(); + } CReparseAttr attr; if (!attr.Parse(item.ReparseData, item.ReparseData.Size())) + { + const FString phyPath = GetPhyPath(i); + AddError(phyPath, attr.ErrorCode); continue; - if (attr.IsRelative()) + } + + /* imagex/WIM reduces absolute paths in links (raparse data), + if we archive non root folder. We do same thing here */ + + bool isWSL = false; + if (attr.IsSymLink_WSL()) + { + // isWSL = true; + // we don't change WSL symlinks continue; + } + else + { + if (attr.IsRelative_Win()) + continue; + } const UString &link = attr.GetPath(); if (!IsDrivePath(link)) @@ -862,7 +1284,7 @@ void CDirItems::FillFixedReparse() FString fullPathF; if (!NDir::MyGetFullPathName(GetPhyPath(i), fullPathF)) continue; - UString fullPath = fs2us(fullPathF); + const UString fullPath = fs2us(fullPathF); const UString logPath = GetLogPath(i); if (logPath.Len() >= fullPath.Len()) continue; @@ -873,7 +1295,7 @@ void CDirItems::FillFixedReparse() if (!IsPathSepar(prefix.Back())) continue; - unsigned rootPrefixSize = GetRootPrefixSize(prefix); + const unsigned rootPrefixSize = GetRootPrefixSize(prefix); if (rootPrefixSize == 0) continue; if (rootPrefixSize == prefix.Len()) @@ -889,10 +1311,330 @@ void CDirItems::FillFixedReparse() newLink += link.Ptr(prefix.Len()); CByteBuffer data; - if (!FillLinkData(data, newLink, attr.IsSymLink())) + bool isSymLink = !attr.IsMountPoint(); + if (!FillLinkData(data, newLink, isSymLink, isWSL)) continue; item.ReparseData2 = data; } + return S_OK; } #endif + + +#ifndef _WIN32 + +HRESULT CDirItems::FillDeviceSizes() +{ + { + FOR_VECTOR (i, Items) + { + CDirItem &item = Items[i]; + + if (S_ISBLK(item.mode) && item.Size == 0) + { + const FString phyPath = GetPhyPath(i); + NIO::CInFile inFile; + inFile.PreserveATime = true; + if (inFile.OpenShared(phyPath, ShareForWrite)) // fixme: OpenShared ?? + { + UInt64 size = 0; + if (inFile.GetLength(size)) + item.Size = size; + } + } + if (StoreOwnerName) + { + OwnerNameMap.Add_UInt32(item.uid); + OwnerGroupMap.Add_UInt32(item.gid); + } + } + } + + if (StoreOwnerName) + { + UString u; + AString a; + { + FOR_VECTOR (i, OwnerNameMap.Numbers) + { + // 200K/sec speed + u.Empty(); + const passwd *pw = getpwuid(OwnerNameMap.Numbers[i]); + // printf("\ngetpwuid=%s\n", pw->pw_name); + if (pw) + { + a = pw->pw_name; + ConvertUTF8ToUnicode(a, u); + } + OwnerNameMap.Strings.Add(u); + } + } + { + FOR_VECTOR (i, OwnerGroupMap.Numbers) + { + u.Empty(); + const group *gr = getgrgid(OwnerGroupMap.Numbers[i]); + if (gr) + { + // printf("\ngetgrgid %d %s\n", OwnerGroupMap.Numbers[i], gr->gr_name); + a = gr->gr_name; + ConvertUTF8ToUnicode(a, u); + } + OwnerGroupMap.Strings.Add(u); + } + } + + FOR_VECTOR (i, Items) + { + CDirItem &item = Items[i]; + { + const int index = OwnerNameMap.Find(item.uid); + if (index < 0) throw 1; + item.OwnerNameIndex = index; + } + { + const int index = OwnerGroupMap.Find(item.gid); + if (index < 0) throw 1; + item.OwnerGroupIndex = index; + } + } + } + + + // if (NeedOwnerNames) + { + /* + { + for (unsigned i = 0 ; i < 10000; i++) + { + const passwd *pw = getpwuid(i); + if (pw) + { + UString u; + ConvertUTF8ToUnicode(AString(pw->pw_name), u); + OwnerNameMap.Add(i, u); + OwnerNameMap.Add(i, u); + OwnerNameMap.Add(i, u); + } + const group *gr = getgrgid(i); + if (gr) + { + // we can use utf-8 here. + UString u; + ConvertUTF8ToUnicode(AString(gr->gr_name), u); + OwnerGroupMap.Add(i, u); + } + } + } + */ + /* + { + FOR_VECTOR (i, OwnerNameMap.Strings) + { + AString s; + ConvertUnicodeToUTF8(OwnerNameMap.Strings[i], s); + printf("\n%5d %s", (unsigned)OwnerNameMap.Numbers[i], s.Ptr()); + } + } + { + printf("\n\n=========Groups\n"); + FOR_VECTOR (i, OwnerGroupMap.Strings) + { + AString s; + ConvertUnicodeToUTF8(OwnerGroupMap.Strings[i], s); + printf("\n%5d %s", (unsigned)OwnerGroupMap.Numbers[i], s.Ptr()); + } + } + */ + } + /* + for (unsigned i = 0 ; i < 100000000; i++) + { + // const passwd *pw = getpwuid(1000); + // pw = pw; + int pos = OwnerNameMap.Find(1000); + if (pos < 0 - (int)i) + throw 1; + } + */ + + return S_OK; +} + +#endif + + + +static const char * const kCannotFindArchive = "Cannot find archive"; + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode censorPathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback) +{ + FStringVector paths; + + { + CDirItems dirItems; + dirItems.Callback = callback; + { + HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); + st = dirItems.Stat; + RINOK(res); + } + + FOR_VECTOR (i, dirItems.Items) + { + const CDirItem &dirItem = dirItems.Items[i]; + if (!dirItem.IsDir()) + paths.Add(dirItems.GetPhyPath(i)); + } + } + + if (paths.Size() == 0) + { + // return S_OK; + throw CMessagePathException(kCannotFindArchive); + } + + UStringVector fullPaths; + + unsigned i; + + for (i = 0; i < paths.Size(); i++) + { + FString fullPath; + NFile::NDir::MyGetFullPathName(paths[i], fullPath); + fullPaths.Add(fs2us(fullPath)); + } + + CUIntVector indices; + SortFileNames(fullPaths, indices); + sortedPaths.ClearAndReserve(indices.Size()); + sortedFullPaths.ClearAndReserve(indices.Size()); + + for (i = 0; i < indices.Size(); i++) + { + unsigned index = indices[i]; + sortedPaths.AddInReserved(fs2us(paths[index])); + sortedFullPaths.AddInReserved(fullPaths[index]); + if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) + throw CMessagePathException("Duplicate archive path:", sortedFullPaths[i]); + } + + return S_OK; +} + + + + +#ifdef _WIN32 + +static bool IsDotsName(const wchar_t *s) +{ + return s[0] == '.' && (s[1] == 0 || (s[1] == '.' && s[2] == 0)); +} + +// This code converts all short file names to long file names. + +static void ConvertToLongName(const UString &prefix, UString &name) +{ + if (name.IsEmpty() + || DoesNameContainWildcard(name) + || IsDotsName(name)) + return; + NFind::CFileInfo fi; + const FString path (us2fs(prefix + name)); + #ifndef UNDER_CE + if (NFile::NName::IsDevicePath(path)) + return; + #endif + if (fi.Find(path)) + name = fs2us(fi.Name); +} + +static void ConvertToLongNames(const UString &prefix, CObjectVector &items) +{ + FOR_VECTOR (i, items) + { + NWildcard::CItem &item = items[i]; + if (item.Recursive || item.PathParts.Size() != 1) + continue; + if (prefix.IsEmpty() && item.IsDriveItem()) + continue; + ConvertToLongName(prefix, item.PathParts.Front()); + } +} + +static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) +{ + ConvertToLongNames(prefix, node.IncludeItems); + ConvertToLongNames(prefix, node.ExcludeItems); + unsigned i; + for (i = 0; i < node.SubNodes.Size(); i++) + { + UString &name = node.SubNodes[i].Name; + if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) + continue; + ConvertToLongName(prefix, name); + } + // mix folders with same name + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; + for (unsigned j = i + 1; j < node.SubNodes.Size();) + { + const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; + if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) + { + nextNode1.IncludeItems += nextNode2.IncludeItems; + nextNode1.ExcludeItems += nextNode2.ExcludeItems; + node.SubNodes.Delete(j); + } + else + j++; + } + } + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode = node.SubNodes[i]; + ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); + } +} + +void ConvertToLongNames(NWildcard::CCensor &censor) +{ + FOR_VECTOR (i, censor.Pairs) + { + NWildcard::CPair &pair = censor.Pairs[i]; + ConvertToLongNames(pair.Prefix, pair.Head); + } +} + +#endif + + +CMessagePathException::CMessagePathException(const char *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} + +CMessagePathException::CMessagePathException(const wchar_t *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.h b/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.h index 7afb800a033..9b17c6006e8 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/EnumDirItems.h @@ -5,12 +5,8 @@ #include "../../../Common/Wildcard.h" -#include "../../../Windows/FileFind.h" - #include "DirItem.h" -void AddDirFileInfo(int phyParent, int logParent, int secureIndex, - const NWindows::NFile::NFind::CFileInfo &fi, CObjectVector &dirItems); HRESULT EnumerateItems( const NWildcard::CCensor &censor, @@ -18,4 +14,25 @@ HRESULT EnumerateItems( const UString &addPathPrefix, CDirItems &dirItems); + +struct CMessagePathException: public UString +{ + CMessagePathException(const char *a, const wchar_t *u = NULL); + CMessagePathException(const wchar_t *a, const wchar_t *u = NULL); +}; + + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode pathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback); + +#ifdef _WIN32 +void ConvertToLongNames(NWildcard::CCensor &censor); +#endif + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/Extract.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/Extract.cpp index 3da0aadb813..58f5218b625 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/Extract.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/Extract.cpp @@ -7,10 +7,13 @@ #include "../../../Common/StringConvert.h" #include "../../../Windows/FileDir.h" +#include "../../../Windows/FileName.h" +#include "../../../Windows/ErrorMsg.h" #include "../../../Windows/PropVariant.h" #include "../../../Windows/PropVariantConv.h" #include "../Common/ExtractingFilePath.h" +#include "../Common/HashCalc.h" #include "Extract.h" #include "SetProperties.h" @@ -19,6 +22,19 @@ using namespace NWindows; using namespace NFile; using namespace NDir; + +static void SetErrorMessage(const char *message, + const FString &path, HRESULT errorCode, + UString &s) +{ + s = message; + s += " : "; + s += NError::MyFormatMessage(errorCode); + s += " : "; + s += fs2us(path); +} + + static HRESULT DecompressArchive( CCodecs *codecs, const CArchiveLink &arcLink, @@ -47,11 +63,11 @@ static HRESULT DecompressArchive( // So it extracts different archives to one folder. // We will use top level archive name const CArc &arc0 = arcLink.Arcs[0]; - if (StringsAreEqualNoCase_Ascii(codecs->Formats[arc0.FormatIndex].Name, "pe")) + if (arc0.FormatIndex >= 0 && StringsAreEqualNoCase_Ascii(codecs->Formats[(unsigned)arc0.FormatIndex].Name, "pe")) replaceName = arc0.DefaultName; } - outDir.Replace(FSTRING_ANY_MASK, us2fs(Get_Correct_FsFile_Name(replaceName))); + outDir.Replace(FString("*"), us2fs(Get_Correct_FsFile_Name(replaceName))); bool elimIsPossible = false; UString elimPrefix; // only pure name without dir delimiter @@ -73,7 +89,7 @@ static HRESULT DecompressArchive( } } - bool allFilesAreAllowed = wildcardCensor.AreAllAllowed(); + const bool allFilesAreAllowed = wildcardCensor.AreAllAllowed(); if (!options.StdInMode) { @@ -84,9 +100,14 @@ static HRESULT DecompressArchive( for (UInt32 i = 0; i < numItems; i++) { - if (elimIsPossible || !allFilesAreAllowed) + if (elimIsPossible + || !allFilesAreAllowed + || options.ExcludeDirItems + || options.ExcludeFileItems) { RINOK(arc.GetItem(i, item)); + if (item.IsDir ? options.ExcludeDirItems : options.ExcludeFileItems) + continue; } else { @@ -156,7 +177,7 @@ static HRESULT DecompressArchive( #endif if (outDir.IsEmpty()) - outDir = FTEXT(".") FSTRING_PATH_SEPARATOR; + outDir = "." STRING_PATH_SEPARATOR; /* #ifdef _WIN32 else if (NName::IsAltPathPrefix(outDir)) {} @@ -164,11 +185,8 @@ static HRESULT DecompressArchive( */ else if (!CreateComplexDir(outDir)) { - HRESULT res = ::GetLastError(); - if (res == S_OK) - res = E_FAIL; - errorMessage.SetFromAscii("Can not create output directory: "); - errorMessage += fs2us(outDir); + const HRESULT res = GetLastError_noZero_HRESULT(); + SetErrorMessage("Cannot create output directory", outDir, res, errorMessage); return res; } @@ -197,6 +215,9 @@ static HRESULT DecompressArchive( HRESULT result; Int32 testMode = (options.TestMode && !calcCrc) ? 1: 0; + + CArchiveExtractCallback_Closer ecsCloser(ecs); + if (options.StdInMode) { result = archive->Extract(NULL, (UInt32)(Int32)-1, testMode, ecs); @@ -206,8 +227,11 @@ static HRESULT DecompressArchive( } else result = archive->Extract(&realIndices.Front(), realIndices.Size(), testMode, ecs); - if (result == S_OK && !options.StdInMode) - result = ecs->SetDirsTimes(); + + HRESULT res2 = ecsCloser.Close(); + if (result == S_OK) + result = res2; + return callback->ExtractResult(result); } @@ -215,17 +239,18 @@ static HRESULT DecompressArchive( Sorted list for file paths was sorted with case insensitive compare function. But FindInSorted function did binary search via case sensitive compare function */ -int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &name) +int Find_FileName_InSortedVector(const UStringVector &fileNames, const UString &name); +int Find_FileName_InSortedVector(const UStringVector &fileNames, const UString &name) { - unsigned left = 0, right = fileName.Size(); + unsigned left = 0, right = fileNames.Size(); while (left != right) { - unsigned mid = (left + right) / 2; - const UString &midValue = fileName[mid]; - int compare = CompareFileNames(name, midValue); - if (compare == 0) - return mid; - if (compare < 0) + const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const UString &midVal = fileNames[mid]; + const int comp = CompareFileNames(name, midVal); + if (comp == 0) + return (int)mid; + if (comp < 0) right = mid; else left = mid + 1; @@ -233,7 +258,10 @@ int Find_FileName_InSortedVector(const UStringVector &fileName, const UString &n return -1; } + + HRESULT Extract( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector &types, const CIntVector &excludedFormats, @@ -262,11 +290,19 @@ HRESULT Extract( fi.Size = 0; if (!options.StdInMode) { - const FString &arcPath = us2fs(arcPaths[i]); - if (!fi.Find(arcPath)) - throw "there is no such archive"; + const FString arcPath = us2fs(arcPaths[i]); + if (!fi.Find_FollowLink(arcPath)) + { + const HRESULT errorCode = GetLastError_noZero_HRESULT(); + SetErrorMessage("Cannot find archive file", arcPath, errorCode, errorMessage); + return errorCode; + } if (fi.IsDir()) - throw "can't decompress folder"; + { + HRESULT errorCode = E_FAIL; + SetErrorMessage("The item is a directory", arcPath, errorCode, errorMessage); + return errorCode; + } } arcSizes.Add(fi.Size); totalPackSize += fi.Size; @@ -278,8 +314,15 @@ HRESULT Extract( CArchiveExtractCallback *ecs = new CArchiveExtractCallback; CMyComPtr ec(ecs); - bool multi = (numArcs > 1); - ecs->InitForMulti(multi, options.PathMode, options.OverwriteMode); + + const bool multi = (numArcs > 1); + + ecs->InitForMulti(multi, + options.PathMode, + options.OverwriteMode, + options.ZoneMode, + false // keepEmptyDirParts + ); #ifndef _SFX ecs->SetHashMethods(hash); #endif @@ -297,17 +340,27 @@ HRESULT Extract( if (skipArcs[i]) continue; + ecs->InitBeforeNewArchive(); + const UString &arcPath = arcPaths[i]; NFind::CFileInfo fi; if (options.StdInMode) { - fi.Size = 0; - fi.Attrib = 0; + // do we need ctime and mtime? + fi.ClearBase(); + fi.Size = 0; // (UInt64)(Int64)-1; + fi.SetAsFile(); + // NTime::GetCurUtc_FiTime(fi.MTime); + // fi.CTime = fi.ATime = fi.MTime; } else { - if (!fi.Find(us2fs(arcPath)) || fi.IsDir()) - throw "there is no such archive"; + if (!fi.Find_FollowLink(us2fs(arcPath)) || fi.IsDir()) + { + const HRESULT errorCode = GetLastError_noZero_HRESULT(); + SetErrorMessage("Cannot find archive file", us2fs(arcPath), errorCode, errorMessage); + return errorCode; + } } /* @@ -371,13 +424,42 @@ HRESULT Extract( { thereAreNotOpenArcs = true; if (!options.StdInMode) + totalPackProcessed += fi.Size; + continue; + } + + #if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX) + if (options.ZoneMode != NExtract::NZoneIdMode::kNone + && !options.StdInMode) + { + ReadZoneFile_Of_BaseFile(us2fs(arcPath), ecs->ZoneBuf); + } + #endif + + + if (arcLink.Arcs.Size() != 0) + { + if (arcLink.GetArc()->IsHashHandler(op)) { - NFind::CFileInfo fi2; - if (fi2.Find(us2fs(arcPath))) - if (!fi2.IsDir()) - totalPackProcessed += fi2.Size; + if (!options.TestMode) + { + /* real Extracting to files is possible. + But user can think that hash archive contains real files. + So we block extracting here. */ + return E_NOTIMPL; + } + FString dirPrefix = us2fs(options.HashDir); + if (dirPrefix.IsEmpty()) + { + if (!NFile::NDir::GetOnlyDirPrefix(us2fs(arcPath), dirPrefix)) + { + // return GetLastError_noZero_HRESULT(); + } + } + if (!dirPrefix.IsEmpty()) + NName::NormalizeDirPathPrefix(dirPrefix); + ecs->DirPathPrefix_for_HashFiles = dirPrefix; } - continue; } if (!options.StdInMode) @@ -389,7 +471,7 @@ HRESULT Extract( // numArcs = arcPaths.Size(); if (arcLink.VolumePaths.Size() != 0) { - Int64 correctionSize = arcLink.VolumesSize; + Int64 correctionSize = (Int64)arcLink.VolumesSize; FOR_VECTOR (v, arcLink.VolumePaths) { int index = Find_FileName_InSortedVector(arcPathsFull, arcLink.VolumePaths[v]); @@ -407,7 +489,7 @@ HRESULT Extract( Int64 newPackSize = (Int64)totalPackSize + correctionSize; if (newPackSize < 0) newPackSize = 0; - totalPackSize = newPackSize; + totalPackSize = (UInt64)newPackSize; RINOK(extractCallback->SetTotal(totalPackSize)); } } @@ -428,11 +510,16 @@ HRESULT Extract( */ CArc &arc = arcLink.Arcs.Back(); - arc.MTimeDefined = (!options.StdInMode && !fi.IsDevice); - arc.MTime = fi.MTime; + arc.MTime.Def = !options.StdInMode + #ifdef _WIN32 + && !fi.IsDevice + #endif + ; + if (arc.MTime.Def) + arc.MTime.Set_From_FiTime(fi.MTime); UInt64 packProcessed; - bool calcCrc = + const bool calcCrc = #ifndef _SFX (hash != NULL); #else diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/Extract.h b/3rdparty/lzma/CPP/7zip/UI/Common/Extract.h index 03ac74b2d0b..f3d1126b779 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/Extract.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/Extract.h @@ -17,20 +17,28 @@ struct CExtractOptionsBase { CBoolPair ElimDup; - + + bool ExcludeDirItems; + bool ExcludeFileItems; + bool PathMode_Force; bool OverwriteMode_Force; NExtract::NPathMode::EEnum PathMode; NExtract::NOverwriteMode::EEnum OverwriteMode; + NExtract::NZoneIdMode::EEnum ZoneMode; FString OutputDir; CExtractNtOptions NtOptions; + UString HashDir; CExtractOptionsBase(): + ExcludeDirItems(false), + ExcludeFileItems(false), PathMode_Force(false), OverwriteMode_Force(false), PathMode(NExtract::NPathMode::kFullPaths), - OverwriteMode(NExtract::NOverwriteMode::kAsk) + OverwriteMode(NExtract::NOverwriteMode::kAsk), + ZoneMode(NExtract::NZoneIdMode::kNone) {} }; @@ -48,15 +56,17 @@ struct CExtractOptions: public CExtractOptionsBase CObjectVector Properties; #endif + /* #ifdef EXTERNAL_CODECS CCodecs *Codecs; #endif + */ CExtractOptions(): - TestMode(false), StdInMode(false), StdOutMode(false), - YesToAll(false) + YesToAll(false), + TestMode(false) {} }; @@ -77,6 +87,7 @@ struct CDecompressStat }; HRESULT Extract( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector &types, const CIntVector &excludedFormats, diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ExtractMode.h b/3rdparty/lzma/CPP/7zip/UI/Common/ExtractMode.h index 3b2b9a02e81..9ad831eb708 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ExtractMode.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ExtractMode.h @@ -29,6 +29,16 @@ namespace NOverwriteMode }; } +namespace NZoneIdMode +{ + enum EEnum + { + kNone, + kAll, + kOffice + }; +} + } #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.cpp index 39e67b7f2ed..a1282b72229 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.cpp @@ -8,6 +8,18 @@ #include "ExtractingFilePath.h" +extern +bool g_PathTrailReplaceMode; +bool g_PathTrailReplaceMode = + #ifdef _WIN32 + true + #else + false + #endif + ; + + +#ifdef _WIN32 static void ReplaceIncorrectChars(UString &s) { { @@ -22,24 +34,60 @@ static void ReplaceIncorrectChars(UString &s) || #endif c == WCHAR_PATH_SEPARATOR) - s.ReplaceOneCharAtPos(i, '_'); + { + #if WCHAR_PATH_SEPARATOR != L'/' + // 22.00 : WSL replacement for backslash + if (c == WCHAR_PATH_SEPARATOR) + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; + else + #endif + c = '_'; + s.ReplaceOneCharAtPos(i, + c + // (wchar_t)(0xf000 + c) // 21.02 debug: WSL encoding for unsupported characters + ); + } } } - #ifdef _WIN32 + if (g_PathTrailReplaceMode) { - for (unsigned i = s.Len(); i != 0;) + /* + // if (g_PathTrailReplaceMode == 1) { - wchar_t c = s[--i]; - if (c != '.' && c != ' ') - break; - s.ReplaceOneCharAtPos(i, '_'); + if (!s.IsEmpty()) + { + wchar_t c = s.Back(); + if (c == '.' || c == ' ') + { + // s += (wchar_t)(0x9c); // STRING TERMINATOR + s += (wchar_t)'_'; + } + } + } + else + */ + { + unsigned i; + for (i = s.Len(); i != 0;) + { + wchar_t c = s[i - 1]; + if (c != '.' && c != ' ') + break; + i--; + s.ReplaceOneCharAtPos(i, '_'); + // s.ReplaceOneCharAtPos(i, (c == ' ' ? (wchar_t)(0x2423) : (wchar_t)0x00B7)); + } + /* + if (g_PathTrailReplaceMode > 1 && i != s.Len()) + { + s.DeleteFrom(i); + } + */ } } - #endif } - -#ifdef _WIN32 +#endif /* WinXP-64 doesn't support ':', '\\' and '/' symbols in name of alt stream. But colon in postfix ":$DATA" is allowed. @@ -61,9 +109,11 @@ void Correct_AltStream_Name(UString &s) s.ReplaceOneCharAtPos(i, '_'); } if (s.IsEmpty()) - s = L'_'; + s = '_'; } +#ifdef _WIN32 + static const unsigned g_ReservedWithNum_Index = 4; static const char * const g_ReservedNames[] = @@ -112,7 +162,10 @@ static void CorrectUnsupportedName(UString &name) static void Correct_PathPart(UString &s) { // "." and ".." - if (s[0] == '.' && (s[1] == 0 || s[1] == '.' && s[2] == 0)) + if (s.IsEmpty()) + return; + + if (s[0] == '.' && (s[1] == 0 || (s[1] == '.' && s[2] == 0))) s.Empty(); #ifdef _WIN32 else @@ -120,8 +173,8 @@ static void Correct_PathPart(UString &s) #endif } -// static const wchar_t *k_EmptyReplaceName = L"[]"; -static const wchar_t k_EmptyReplaceName = L'_'; +// static const char * const k_EmptyReplaceName = "[]"; +static const char k_EmptyReplaceName = '_'; UString Get_Correct_FsFile_Name(const UString &name) { @@ -138,7 +191,7 @@ UString Get_Correct_FsFile_Name(const UString &name) } -void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) +void Correct_FsPath(bool absIsAllowed, bool keepAndReplaceEmptyPrefixes, UStringVector &parts, bool isDir) { unsigned i = 0; @@ -147,6 +200,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) #if defined(_WIN32) && !defined(UNDER_CE) bool isDrive = false; #endif + if (parts[0].IsEmpty()) { i = 1; @@ -157,7 +211,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) if (parts.Size() > 2 && parts[2] == L"?") { i = 3; - if (parts.Size() > 3 && NWindows::NFile::NName::IsDrivePath2(parts[3])) + if (parts.Size() > 3 && NWindows::NFile::NName::IsDrivePath2(parts[3])) { isDrive = true; i = 4; @@ -176,16 +230,19 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) if (isDrive) { // we convert "c:name" to "c:\name", if absIsAllowed path. - const UString &ds = parts[i - 1]; - if (ds.Len() != 2) + UString &ds = parts[i - 1]; + if (ds.Len() > 2) { - UString s = ds.Ptr(2); - parts.Insert(i, s); + parts.Insert(i, ds.Ptr(2)); + ds.DeleteFrom(2); } } #endif } + if (i != 0) + keepAndReplaceEmptyPrefixes = false; + for (; i < parts.Size();) { UString &s = parts[i]; @@ -194,15 +251,17 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) if (s.IsEmpty()) { - if (isDir || i != parts.Size() - 1) - { - parts.Delete(i); - continue; - } + if (!keepAndReplaceEmptyPrefixes) + if (isDir || i != parts.Size() - 1) + { + parts.Delete(i); + continue; + } s = k_EmptyReplaceName; } else { + keepAndReplaceEmptyPrefixes = false; #ifdef _WIN32 CorrectUnsupportedName(s); #endif @@ -214,7 +273,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir) if (!isDir) { if (parts.IsEmpty()) - parts.Add(k_EmptyReplaceName); + parts.Add((UString)k_EmptyReplaceName); else { UString &s = parts.Back(); diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.h b/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.h index a52a1f4a5b4..8f8f9f1b2a0 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ExtractingFilePath.h @@ -5,14 +5,26 @@ #include "../../../Common/MyString.h" -#ifdef _WIN32 +// #ifdef _WIN32 void Correct_AltStream_Name(UString &s); -#endif +// #endif // replaces unsuported characters, and replaces "." , ".." and "" to "[]" UString Get_Correct_FsFile_Name(const UString &name); -void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir); +/* + Correct_FsPath() corrects path parts to prepare it for File System operations. + It also corrects empty path parts like "\\\\": + - frontal empty path parts : it removes them or changes them to "_" + - another empty path parts : it removes them + if (absIsAllowed && path is absolute) : it removes empty path parts after start absolute path prefix marker + else + { + if (!keepAndReplaceEmptyPrefixes) : it removes empty path parts + if ( keepAndReplaceEmptyPrefixes) : it changes each empty frontal path part to "_" + } +*/ +void Correct_FsPath(bool absIsAllowed, bool keepAndReplaceEmptyPrefixes, UStringVector &parts, bool isDir); UString MakePathFromParts(const UStringVector &parts); diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.cpp index 60a747d5590..f0aa4bd14ff 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.cpp @@ -3,44 +3,52 @@ #include "StdAfx.h" #include "../../../../C/Alloc.h" +#include "../../../../C/CpuArch.h" +#include "../../../Common/DynLimBuf.h" +#include "../../../Common/IntToString.h" #include "../../../Common/StringToInt.h" #include "../../Common/FileStreams.h" +#include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" #include "../../Common/StreamUtils.h" +#include "../../Archive/Common/ItemNameUtils.h" +#include "../../Archive/IArchive.h" + #include "EnumDirItems.h" #include "HashCalc.h" using namespace NWindows; +#ifdef EXTERNAL_CODECS +extern const CExternalCodecs *g_ExternalCodecs_Ptr; +#endif + class CHashMidBuf { void *_data; public: - CHashMidBuf(): _data(0) {} + CHashMidBuf(): _data(NULL) {} operator void *() { return _data; } bool Alloc(size_t size) { - if (_data != 0) + if (_data) return false; _data = ::MidAlloc(size); - return _data != 0; + return _data != NULL; } ~CHashMidBuf() { ::MidFree(_data); } }; -static const char *k_DefaultHashMethod = "CRC32"; +static const char * const k_DefaultHashMethod = "CRC32"; HRESULT CHashBundle::SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVector &hashMethods) { UStringVector names = hashMethods; if (names.IsEmpty()) - { - UString s; - s.SetFromAscii(k_DefaultHashMethod); - names.Add(s); - } + names.Add(UString(k_DefaultHashMethod)); CRecordVector ids; CObjectVector methods; @@ -94,15 +102,15 @@ HRESULT CHashBundle::SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVecto if (scp) RINOK(m.SetCoderProps(scp, NULL)); } - UInt32 digestSize = hasher->GetDigestSize(); + const UInt32 digestSize = hasher->GetDigestSize(); if (digestSize > k_HashCalc_DigestSize_Max) return E_NOTIMPL; CHasherState &h = Hashers.AddNew(); + h.DigestSize = digestSize; h.Hasher = hasher; h.Name = name; - h.DigestSize = digestSize; for (unsigned k = 0; k < k_HashCalc_NumGroups; k++) - memset(h.Digests[k], 0, digestSize); + h.InitDigestGroup(k); } return S_OK; @@ -115,7 +123,7 @@ void CHashBundle::InitForNewFile() { CHasherState &h = Hashers[i]; h.Hasher->Init(); - memset(h.Digests[k_HashCalc_Index_Current], 0, h.DigestSize); + h.InitDigestGroup(k_HashCalc_Index_Current); } } @@ -134,14 +142,46 @@ void CHashBundle::SetSize(UInt64 size) static void AddDigests(Byte *dest, const Byte *src, UInt32 size) { unsigned next = 0; - for (UInt32 i = 0; i < size; i++) + /* + // we could use big-endian addition for sha-1 and sha-256 + // but another hashers are little-endian + if (size > 8) + { + for (unsigned i = size; i != 0;) + { + i--; + next += (unsigned)dest[i] + (unsigned)src[i]; + dest[i] = (Byte)next; + next >>= 8; + } + } + else + */ + { + for (unsigned i = 0; i < size; i++) + { + next += (unsigned)dest[i] + (unsigned)src[i]; + dest[i] = (Byte)next; + next >>= 8; + } + } + + // we use little-endian to store extra bytes + dest += k_HashCalc_DigestSize_Max; + for (unsigned i = 0; i < k_HashCalc_ExtraSize; i++) { - next += (unsigned)dest[i] + (unsigned)src[i]; + next += (unsigned)dest[i]; dest[i] = (Byte)next; next >>= 8; } } +void CHasherState::AddDigest(unsigned groupIndex, const Byte *data) +{ + NumSums[groupIndex]++; + AddDigests(Digests[groupIndex], data, DigestSize); +} + void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) { if (isDir) @@ -167,9 +207,9 @@ void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) CHasherState &h = Hashers[i]; if (!isDir) { - h.Hasher->Final(h.Digests[0]); + h.Hasher->Final(h.Digests[0]); // k_HashCalc_Index_Current if (!isAltStream) - AddDigests(h.Digests[k_HashCalc_Index_DataSum], h.Digests[0], h.DigestSize); + h.AddDigest(k_HashCalc_Index_DataSum, h.Digests[0]); } h.Hasher->Init(); @@ -179,6 +219,15 @@ void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) for (unsigned k = 0; k < path.Len(); k++) { wchar_t c = path[k]; + + // 21.04: we want same hash for linux and windows paths + #if CHAR_PATH_SEPARATOR != '/' + if (c == CHAR_PATH_SEPARATOR) + c = '/'; + // if (c == (wchar_t)('\\' + 0xf000)) c = '\\'; // to debug WSL + // if (c > 0xf000 && c < 0xf080) c -= 0xf000; // to debug WSL + #endif + Byte temp[2] = { (Byte)(c & 0xFF), (Byte)((c >> 8) & 0xFF) }; h.Hasher->Update(temp, 2); } @@ -186,9 +235,216 @@ void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) Byte tempDigest[k_HashCalc_DigestSize_Max]; h.Hasher->Final(tempDigest); if (!isAltStream) - AddDigests(h.Digests[k_HashCalc_Index_NamesSum], tempDigest, h.DigestSize); - AddDigests(h.Digests[k_HashCalc_Index_StreamsSum], tempDigest, h.DigestSize); + h.AddDigest(k_HashCalc_Index_NamesSum, tempDigest); + h.AddDigest(k_HashCalc_Index_StreamsSum, tempDigest); + } +} + + +static void CSum_Name_OriginalToEscape(const AString &src, AString &dest) +{ + dest.Empty(); + for (unsigned i = 0; i < src.Len();) + { + char c = src[i++]; + if (c == '\n') + { + dest += '\\'; + c = 'n'; + } + else if (c == '\\') + dest += '\\'; + dest += c; + } +} + + +static bool CSum_Name_EscapeToOriginal(const char *s, AString &dest) +{ + bool isOK = true; + dest.Empty(); + for (;;) + { + char c = *s++; + if (c == 0) + break; + if (c == '\\') + { + const char c1 = *s; + if (c1 == 'n') + { + c = '\n'; + s++; + } + else if (c1 == '\\') + { + c = c1; + s++; + } + else + { + // original md5sum returns NULL for such bad strings + isOK = false; + } + } + dest += c; + } + return isOK; +} + + + +static void SetSpacesAndNul(char *s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + +static const unsigned kHashColumnWidth_Min = 4 * 2; + +static unsigned GetColumnWidth(unsigned digestSize) +{ + const unsigned width = digestSize * 2; + return width < kHashColumnWidth_Min ? kHashColumnWidth_Min: width; +} + + +static void AddHashResultLine( + AString &_s, + // bool showHash, + // UInt64 fileSize, bool showSize, + const CObjectVector &hashers + // unsigned digestIndex, = k_HashCalc_Index_Current + ) +{ + FOR_VECTOR (i, hashers) + { + const CHasherState &h = hashers[i]; + char s[k_HashCalc_DigestSize_Max * 2 + 64]; + s[0] = 0; + // if (showHash) + HashHexToString(s, h.Digests[k_HashCalc_Index_Current], h.DigestSize); + const unsigned pos = (unsigned)strlen(s); + const int numSpaces = (int)GetColumnWidth(h.DigestSize) - (int)pos; + if (numSpaces > 0) + SetSpacesAndNul(s + pos, (unsigned)numSpaces); + if (i != 0) + _s += ' '; + _s += s; + } + + /* + if (showSize) + { + _s += ' '; + static const unsigned kSizeField_Len = 13; // same as in HashCon.cpp + char s[kSizeField_Len + 32]; + char *p = s; + SetSpacesAndNul(s, kSizeField_Len); + p = s + kSizeField_Len; + ConvertUInt64ToString(fileSize, p); + int numSpaces = (int)kSizeField_Len - (int)strlen(p); + if (numSpaces > 0) + p -= (unsigned)numSpaces; + _s += p; + } + */ +} + + +static void Add_LF(CDynLimBuf &hashFileString, const CHashOptionsLocal &options) +{ + hashFileString += (char)(options.HashMode_Zero.Val ? 0 : '\n'); +} + + + + +static void WriteLine(CDynLimBuf &hashFileString, + const CHashOptionsLocal &options, + const UString &path2, + bool isDir, + const AString &methodName, + const AString &hashesString) +{ + if (options.HashMode_OnlyHash.Val) + { + hashFileString += hashesString; + Add_LF(hashFileString, options); + return; + } + + UString path = path2; + + bool isBin = false; + const bool zeroMode = options.HashMode_Zero.Val; + const bool tagMode = options.HashMode_Tag.Val; + +#if CHAR_PATH_SEPARATOR != '/' + path.Replace(WCHAR_PATH_SEPARATOR, L'/'); + // path.Replace((wchar_t)('\\' + 0xf000), L'\\'); // to debug WSL +#endif + + AString utf8; + ConvertUnicodeToUTF8(path, utf8); + + AString esc; + CSum_Name_OriginalToEscape(utf8, esc); + + if (!zeroMode) + { + if (esc != utf8) + { + /* Original md5sum writes escape in that case. + We do same for compatibility with original md5sum. */ + hashFileString += '\\'; + } + } + + if (isDir && !esc.IsEmpty() && esc.Back() != '/') + esc += '/'; + + if (tagMode) + { + if (!methodName.IsEmpty()) + { + hashFileString += methodName; + hashFileString += ' '; + } + hashFileString += '('; + hashFileString += esc; + hashFileString += ')'; + hashFileString += " = "; } + + hashFileString += hashesString; + + if (!tagMode) + { + hashFileString += ' '; + hashFileString += (char)(isBin ? '*' : ' '); + hashFileString += esc; + } + + Add_LF(hashFileString, options); +} + + + +static void WriteLine(CDynLimBuf &hashFileString, + const CHashOptionsLocal &options, + const UString &path, + bool isDir, + const CHashBundle &hb) +{ + AString methodName; + if (!hb.Hashers.IsEmpty()) + methodName = hb.Hashers[0].Name; + + AString hashesString; + AddHashResultLine(hashesString, hb.Hashers); + WriteLine(hashFileString, options, path, isDir, methodName, hashesString); } @@ -206,16 +462,19 @@ HRESULT HashCalc( { CDirItem di; di.Size = (UInt64)(Int64)-1; - di.Attrib = 0; - di.MTime.dwLowDateTime = 0; - di.MTime.dwHighDateTime = 0; - di.CTime = di.ATime = di.MTime; + di.SetAsFile(); dirItems.Items.Add(di); } else { RINOK(callback->StartScanning()); + + dirItems.SymLinks = options.SymLinks.Val; dirItems.ScanAltStreams = options.AltStreamsMode; + dirItems.ExcludeDirItems = censor.ExcludeDirItems; + dirItems.ExcludeFileItems = censor.ExcludeFileItems; + + dirItems.ShareForWrite = options.OpenShareForWrite; HRESULT res = EnumerateItems(censor, options.PathMode, @@ -234,17 +493,19 @@ HRESULT HashCalc( unsigned i; CHashBundle hb; RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS options.Methods)); - hb.Init(); + // hb.Init(); hb.NumErrors = dirItems.Stat.NumErrors; - + + UInt64 totalSize = 0; if (options.StdInMode) { RINOK(callback->SetNumFiles(1)); } else { - RINOK(callback->SetTotal(dirItems.Stat.GetTotalBytes())); + totalSize = dirItems.Stat.GetTotalBytes(); + RINOK(callback->SetTotal(totalSize)); } const UInt32 kBufSize = 1 << 15; @@ -256,47 +517,88 @@ HRESULT HashCalc( RINOK(callback->BeforeFirstFile(hb)); + /* + CDynLimBuf hashFileString((size_t)1 << 31); + const bool needGenerate = !options.HashFilePath.IsEmpty(); + */ + for (i = 0; i < dirItems.Items.Size(); i++) { CMyComPtr inStream; UString path; bool isDir = false; bool isAltStream = false; + if (options.StdInMode) { inStream = new CStdInFileStream; } else { - CInFileStream *inStreamSpec = new CInFileStream; - inStream = inStreamSpec; - const CDirItem &dirItem = dirItems.Items[i]; - isDir = dirItem.IsDir(); - isAltStream = dirItem.IsAltStream; path = dirItems.GetLogPath(i); - if (!isDir) + const CDirItem &di = dirItems.Items[i]; + #ifdef _WIN32 + isAltStream = di.IsAltStream; + #endif + + #ifndef UNDER_CE + // if (di.AreReparseData()) + if (di.ReparseData.Size() != 0) + { + CBufInStream *inStreamSpec = new CBufInStream(); + inStream = inStreamSpec; + inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); + } + else + #endif { - FString phyPath = dirItems.GetPhyPath(i); - if (!inStreamSpec->OpenShared(phyPath, options.OpenShareForWrite)) + CInFileStream *inStreamSpec = new CInFileStream; + inStreamSpec->Set_PreserveATime(options.PreserveATime); + inStream = inStreamSpec; + isDir = di.IsDir(); + if (!isDir) { - HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); - hb.NumErrors++; - if (res != S_FALSE) - return res; - continue; + const FString phyPath = dirItems.GetPhyPath(i); + if (!inStreamSpec->OpenShared(phyPath, options.OpenShareForWrite)) + { + HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); + hb.NumErrors++; + if (res != S_FALSE) + return res; + continue; + } + if (!options.StdInMode) + { + UInt64 curSize = 0; + if (inStreamSpec->GetSize(&curSize) == S_OK) + { + if (curSize > di.Size) + { + totalSize += curSize - di.Size; + RINOK(callback->SetTotal(totalSize)); + // printf("\ntotal = %d MiB\n", (unsigned)(totalSize >> 20)); + } + } + } + // inStreamSpec->ReloadProps(); } } } + RINOK(callback->GetStream(path, isDir)); UInt64 fileSize = 0; hb.InitForNewFile(); + if (!isDir) { for (UInt32 step = 0;; step++) { if ((step & 0xFF) == 0) + { + // printf("\ncompl = %d\n", (unsigned)(completeValue >> 20)); RINOK(callback->SetCompleted(&completeValue)); + } UInt32 size; RINOK(inStream->Read(buf, kBufSize, &size)); if (size == 0) @@ -306,20 +608,54 @@ HRESULT HashCalc( completeValue += size; } } + hb.Final(isDir, isAltStream, path); + + /* + if (needGenerate + && (options.HashMode_Dirs.Val || !isDir)) + { + WriteLine(hashFileString, + options, + path, // change it + isDir, + hb); + + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + */ + RINOK(callback->SetOperationResult(fileSize, hb, !isDir)); RINOK(callback->SetCompleted(&completeValue)); } + + /* + if (needGenerate) + { + NFile::NIO::COutFile file; + if (!file.Create(us2fs(options.HashFilePath), true)) // createAlways + return GetLastError_noZero_HRESULT(); + if (!file.WriteFull(hashFileString, hashFileString.Len())) + return GetLastError_noZero_HRESULT(); + } + */ + return callback->AfterLastFile(hb); } -static inline char GetHex(unsigned v) +static inline char GetHex_Upper(unsigned v) { return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); } -void AddHashHexToString(char *dest, const Byte *data, UInt32 size) +static inline char GetHex_Lower(unsigned v) +{ + return (char)((v < 10) ? ('0' + v) : ('a' + (v - 10))); +} + +void HashHexToString(char *dest, const Byte *data, UInt32 size) { dest[size * 2] = 0; @@ -334,18 +670,1437 @@ void AddHashHexToString(char *dest, const Byte *data, UInt32 size) return; } - int step = 2; if (size <= 8) { - step = -2; - dest += size * 2 - 2; + dest += size * 2; + for (UInt32 i = 0; i < size; i++) + { + const unsigned b = data[i]; + dest -= 2; + dest[0] = GetHex_Upper((b >> 4) & 0xF); + dest[1] = GetHex_Upper(b & 0xF); + } + } + else + { + for (UInt32 i = 0; i < size; i++) + { + const unsigned b = data[i]; + dest[0] = GetHex_Lower((b >> 4) & 0xF); + dest[1] = GetHex_Lower(b & 0xF); + dest += 2; + } + } +} + +void CHasherState::WriteToString(unsigned digestIndex, char *s) const +{ + HashHexToString(s, Digests[digestIndex], DigestSize); + + if (digestIndex != 0 && NumSums[digestIndex] != 1) + { + unsigned numExtraBytes = GetNumExtraBytes_for_Group(digestIndex); + if (numExtraBytes > 4) + numExtraBytes = 8; + else // if (numExtraBytes >= 0) + numExtraBytes = 4; + // if (numExtraBytes != 0) + { + s += strlen(s); + *s++ = '-'; + // *s = 0; + HashHexToString(s, GetExtraData_for_Group(digestIndex), numExtraBytes); + } + } +} + + + +// ---------- Hash Handler ---------- + +namespace NHash { + +static size_t ParseHexString(const char *s, Byte *dest) throw() +{ + size_t num; + for (num = 0;; num++, s += 2) + { + unsigned c = (Byte)s[0]; + unsigned v0; + if (c >= '0' && c <= '9') v0 = (c - '0'); + else if (c >= 'A' && c <= 'F') v0 = 10 + (c - 'A'); + else if (c >= 'a' && c <= 'f') v0 = 10 + (c - 'a'); + else + return num; + c = (Byte)s[1]; + unsigned v1; + if (c >= '0' && c <= '9') v1 = (c - '0'); + else if (c >= 'A' && c <= 'F') v1 = 10 + (c - 'A'); + else if (c >= 'a' && c <= 'f') v1 = 10 + (c - 'a'); + else + return num; + if (dest) + dest[num] = (Byte)(v1 | (v0 << 4)); + } +} + + +#define IsWhite(c) ((c) == ' ' || (c) == '\t') + +bool CHashPair::IsDir() const +{ + if (Name.IsEmpty() || Name.Back() != '/') + return false; + // here we expect that Dir items contain only zeros or no Hash + for (size_t i = 0; i < Hash.Size(); i++) + if (Hash[i] != 0) + return false; + return true; +} + + +bool CHashPair::ParseCksum(const char *s) +{ + const char *end; + + const UInt32 crc = ConvertStringToUInt32(s, &end); + if (*end != ' ') + return false; + end++; + + const UInt64 size = ConvertStringToUInt64(end, &end); + if (*end != ' ') + return false; + end++; + + Name = end; + + Hash.Alloc(4); + SetBe32(Hash, crc); + + Size_from_Arc = size; + Size_from_Arc_Defined = true; + + return true; +} + + + +static const char *SkipWhite(const char *s) +{ + while (IsWhite(*s)) + s++; + return s; +} + +static const char * const k_CsumMethodNames[] = +{ + "sha256" + , "sha224" +// , "sha512/224" +// , "sha512/256" + , "sha512" + , "sha384" + , "sha1" + , "md5" + , "blake2b" + , "crc64" + , "crc32" + , "cksum" +}; + +static UString GetMethod_from_FileName(const UString &name) +{ + AString s; + ConvertUnicodeToUTF8(name, s); + const int dotPos = s.ReverseFind_Dot(); + const char *src = s.Ptr(); + bool isExtension = false; + if (dotPos >= 0) + { + isExtension = true; + src = s.Ptr(dotPos + 1); + } + const char *m = ""; + unsigned i; + for (i = 0; i < ARRAY_SIZE(k_CsumMethodNames); i++) + { + m = k_CsumMethodNames[i]; + if (isExtension) + { + if (StringsAreEqual_Ascii(src, m)) + break; + } + else if (IsString1PrefixedByString2_NoCase_Ascii(src, m)) + if (StringsAreEqual_Ascii(src + strlen(m), "sums")) + break; + } + UString res; + if (i != ARRAY_SIZE(k_CsumMethodNames)) + res = m; + return res; +} + + +bool CHashPair::Parse(const char *s) +{ + // here we keep compatibility with original md5sum / shasum + bool escape = false; + + s = SkipWhite(s); + + if (*s == '\\') + { + s++; + escape = true; + } + + // const char *kMethod = GetMethod_from_FileName(s); + // if (kMethod) + if (ParseHexString(s, NULL) < 4) + { + // BSD-style checksum line + { + const char *s2 = s; + for (; *s2 != 0; s2++) + { + const char c = *s2; + if (c == 0) + return false; + if (c == ' ' || c == '(') + break; + } + Method.SetFrom(s, (unsigned)(s2 - s)); + s = s2; + } + IsBSD = true; + if (*s == ' ') + s++; + if (*s != '(') + return false; + s++; + { + const char *s2 = s; + for (; *s2 != 0; s2++) + {} + for (;;) + { + s2--; + if (s2 < s) + return false; + if (*s2 == ')') + break; + } + Name.SetFrom(s, (unsigned)(s2 - s)); + s = s2 + 1; + } + + s = SkipWhite(s); + if (*s != '=') + return false; + s++; + s = SkipWhite(s); + } + + { + const size_t num = ParseHexString(s, NULL); + Hash.Alloc(num); + ParseHexString(s, Hash); + const size_t numChars = num * 2; + HashString.SetFrom(s, (unsigned)numChars); + s += numChars; } - for (UInt32 i = 0; i < size; i++) + if (IsBSD) + { + if (*s != 0) + return false; + if (escape) + { + const AString temp (Name); + return CSum_Name_EscapeToOriginal(temp, Name); + } + return true; + } + + if (*s == 0) + return true; + + if (*s != ' ') + return false; + s++; + const char c = *s; + if (c != ' ' + && c != '*' + && c != 'U' // shasum Universal + && c != '^' // shasum 0/1 + ) + return false; + Mode = c; + s++; + if (escape) + return CSum_Name_EscapeToOriginal(s, Name); + Name = s; + return true; +} + + +static bool GetLine(CByteBuffer &buf, bool zeroMode, bool cr_lf_Mode, size_t &posCur, AString &s) +{ + s.Empty(); + size_t pos = posCur; + const Byte *p = buf; + unsigned numDigits = 0; + for (; pos < buf.Size(); pos++) + { + const Byte b = p[pos]; + if (b == 0) + { + numDigits = 1; + break; + } + if (zeroMode) + continue; + if (b == 0x0a) + { + numDigits = 1; + break; + } + if (!cr_lf_Mode) + continue; + if (b == 0x0d) + { + if (pos + 1 >= buf.Size()) + { + numDigits = 1; + break; + // return false; + } + if (p[pos + 1] == 0x0a) + { + numDigits = 2; + break; + } + } + } + s.SetFrom((const char *)(p + posCur), (unsigned)(pos - posCur)); + posCur = pos + numDigits; + return true; +} + + +static bool Is_CR_LF_Data(const Byte *buf, size_t size) +{ + bool isCrLf = false; + for (size_t i = 0; i < size;) { - unsigned b = data[i]; - dest[0] = GetHex((b >> 4) & 0xF); - dest[1] = GetHex(b & 0xF); - dest += step; + const Byte b = buf[i]; + if (b == 0x0a) + return false; + if (b == 0x0d) + { + if (i == size - 1) + return false; + if (buf[i + 1] != 0x0a) + return false; + isCrLf = true; + i += 2; + } + else + i++; + } + return isCrLf; +} + + +static const Byte kArcProps[] = +{ + // kpidComment, + kpidCharacts +}; + +static const Byte kProps[] = +{ + kpidPath, + kpidSize, + kpidPackSize, + kpidMethod +}; + +static const Byte kRawProps[] = +{ + kpidChecksum +}; + + +STDMETHODIMP CHandler::GetParent(UInt32 /* index */ , UInt32 *parent, UInt32 *parentType) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + return S_OK; +} + +STDMETHODIMP CHandler::GetNumRawProps(UInt32 *numProps) +{ + *numProps = ARRAY_SIZE(kRawProps); + return S_OK; +} + +STDMETHODIMP CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +{ + *propID = kRawProps[index]; + *name = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType) +{ + *data = NULL; + *dataSize = 0; + *propType = 0; + + if (propID == kpidChecksum) + { + const CHashPair &hp = HashPairs[index]; + if (hp.Hash.Size() > 0) + { + *data = hp.Hash; + *dataSize = (UInt32)hp.Hash.Size(); + *propType = NPropDataType::kRaw; + } + return S_OK; + } + + return S_OK; +} + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = HashPairs.Size(); + return S_OK; +} + +static void Add_OptSpace_String(UString &dest, const char *src) +{ + dest.Add_Space_if_NotEmpty(); + dest += src; +} + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) +{ + NWindows::NCOM::CPropVariant prop; + switch (propID) + { + case kpidPhySize: if (_phySize != 0) prop = _phySize; break; + /* + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + // if (_sres == k_Base64_RES_NeedMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; + if (v != 0) + prop = v; + break; + } + */ + case kpidCharacts: + { + UString s; + if (_hashSize_Defined) + { + s.Add_Space_if_NotEmpty(); + s.Add_UInt32(_hashSize * 8); + s += "-bit"; + } + if (!_nameExtenstion.IsEmpty()) + { + s.Add_Space_if_NotEmpty(); + s += _nameExtenstion; + } + if (_is_PgpMethod) + { + Add_OptSpace_String(s, "PGP"); + if (!_pgpMethod.IsEmpty()) + { + s += ":"; + s += _pgpMethod; + } + } + if (_is_ZeroMode) + Add_OptSpace_String(s, "ZERO"); + if (_are_there_Tags) + Add_OptSpace_String(s, "TAG"); + if (_are_there_Dirs) + Add_OptSpace_String(s, "DIRS"); + prop = s; + break; + } + + case kpidReadOnly: + { + if (_isArc) + if (!CanUpdate()) + prop = true; + break; + } + } + prop.Detach(value); + return S_OK; +} + + +STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value) +{ + // COM_TRY_BEGIN + NWindows::NCOM::CPropVariant prop; + CHashPair &hp = HashPairs[index]; + switch (propID) + { + case kpidIsDir: + { + prop = hp.IsDir(); + break; + } + case kpidPath: + { + UString path; + hp.Get_UString_Path(path); + + NArchive::NItemName::ReplaceToOsSlashes_Remove_TailSlash(path, + true); // useBackslashReplacement + + prop = path; + break; + } + case kpidSize: + { + // client needs processed size of last file + if (hp.Size_from_Disk_Defined) + prop = (UInt64)hp.Size_from_Disk; + else if (hp.Size_from_Arc_Defined) + prop = (UInt64)hp.Size_from_Arc; + break; + } + case kpidPackSize: + { + prop = (UInt64)hp.Hash.Size(); + break; + } + case kpidMethod: + { + if (!hp.Method.IsEmpty()) + prop = hp.Method; + break; + } + } + prop.Detach(value); + return S_OK; + // COM_TRY_END +} + + +static HRESULT ReadStream_to_Buf(IInStream *stream, CByteBuffer &buf, IArchiveOpenCallback *openCallback) +{ + buf.Free(); + UInt64 len; + RINOK(stream->Seek(0, STREAM_SEEK_END, &len)); + if (len == 0 || len >= ((UInt64)1 << 31)) + return S_FALSE; + RINOK(stream->Seek(0, STREAM_SEEK_SET, NULL)); + buf.Alloc((size_t)len); + UInt64 pos = 0; + // return ReadStream_FALSE(stream, buf, (size_t)len); + for (;;) + { + const UInt32 kBlockSize = ((UInt32)1 << 24); + const UInt32 curSize = (len < kBlockSize) ? (UInt32)len : kBlockSize; + UInt32 processedSizeLoc; + RINOK(stream->Read((Byte *)buf + pos, curSize, &processedSizeLoc)); + if (processedSizeLoc == 0) + return E_FAIL; + len -= processedSizeLoc; + pos += processedSizeLoc; + if (len == 0) + return S_OK; + if (openCallback) + { + const UInt64 files = 0; + RINOK(openCallback->SetCompleted(&files, &pos)); + } + } +} + + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *openCallback) +{ + COM_TRY_BEGIN + { + Close(); + + CByteBuffer buf; + RINOK(ReadStream_to_Buf(stream, buf, openCallback)) + + CObjectVector &pairs = HashPairs; + + bool zeroMode = false; + bool cr_lf_Mode = false; + { + for (size_t i = 0; i < buf.Size(); i++) + if (buf[i] == 0) + { + zeroMode = true; + break; + } + } + _is_ZeroMode = zeroMode; + if (!zeroMode) + cr_lf_Mode = Is_CR_LF_Data(buf, buf.Size()); + + if (openCallback) + { + CMyComPtr openVolumeCallback; + openCallback->QueryInterface(IID_IArchiveOpenVolumeCallback, (void **)&openVolumeCallback); + NCOM::CPropVariant prop; + if (openVolumeCallback) + { + RINOK(openVolumeCallback->GetProperty(kpidName, &prop)); + if (prop.vt == VT_BSTR) + _nameExtenstion = GetMethod_from_FileName(prop.bstrVal); + } + } + + bool cksumMode = false; + if (_nameExtenstion.IsEqualTo_Ascii_NoCase("cksum")) + cksumMode = true; + _is_CksumMode = cksumMode; + + size_t pos = 0; + AString s; + bool minusMode = false; + unsigned numLines = 0; + + while (pos < buf.Size()) + { + if (!GetLine(buf, zeroMode, cr_lf_Mode, pos, s)) + return S_FALSE; + numLines++; + if (s.IsEmpty()) + continue; + + if (s.IsPrefixedBy_Ascii_NoCase("; ")) + { + if (numLines != 1) + return S_FALSE; + // comment line of FileVerifier++ + continue; + } + + if (s.IsPrefixedBy_Ascii_NoCase("-----")) + { + if (minusMode) + break; // end of pgp mode + minusMode = true; + if (s.IsPrefixedBy_Ascii_NoCase("-----BEGIN PGP SIGNED MESSAGE")) + { + if (_is_PgpMethod) + return S_FALSE; + if (!GetLine(buf, zeroMode, cr_lf_Mode, pos, s)) + return S_FALSE; + const char *kStart = "Hash: "; + if (!s.IsPrefixedBy_Ascii_NoCase(kStart)) + return S_FALSE; + _pgpMethod = s.Ptr((unsigned)strlen(kStart)); + _is_PgpMethod = true; + } + continue; + } + + CHashPair pair; + pair.FullLine = s; + if (cksumMode) + { + if (!pair.ParseCksum(s)) + return S_FALSE; + } + else if (!pair.Parse(s)) + return S_FALSE; + pairs.Add(pair); + } + + { + unsigned hashSize = 0; + bool hashSize_Dismatch = false; + for (unsigned i = 0; i < HashPairs.Size(); i++) + { + const CHashPair &hp = HashPairs[i]; + if (i == 0) + hashSize = (unsigned)hp.Hash.Size(); + else + if (hashSize != hp.Hash.Size()) + hashSize_Dismatch = true; + + if (hp.IsBSD) + _are_there_Tags = true; + if (!_are_there_Dirs && hp.IsDir()) + _are_there_Dirs = true; + } + if (!hashSize_Dismatch && hashSize != 0) + { + _hashSize = hashSize; + _hashSize_Defined = true; + } + } + + _phySize = buf.Size(); + _isArc = true; + return S_OK; + } + COM_TRY_END +} + + +void CHandler::ClearVars() +{ + _phySize = 0; + _isArc = false; + _is_CksumMode = false; + _is_PgpMethod = false; + _is_ZeroMode = false; + _are_there_Tags = false; + _are_there_Dirs = false; + _hashSize_Defined = false; + _hashSize = 0; +} + + +STDMETHODIMP CHandler::Close() +{ + ClearVars(); + _nameExtenstion.Empty(); + _pgpMethod.Empty(); + HashPairs.Clear(); + return S_OK; +} + + +static bool CheckDigests(const Byte *a, const Byte *b, size_t size) +{ + if (size <= 8) + { + /* we use reversed order for one digest, when text representation + uses big-order for crc-32 and crc-64 */ + for (size_t i = 0; i < size; i++) + if (a[i] != b[size - 1 - i]) + return false; + return true; + } + { + for (size_t i = 0; i < size; i++) + if (a[i] != b[i]) + return false; + return true; + } +} + + +static void AddDefaultMethod(UStringVector &methods, unsigned size) +{ + const char *m = NULL; + if (size == 32) m = "sha256"; + else if (size == 20) m = "sha1"; + else if (size == 16) m = "md5"; + else if (size == 8) m = "crc64"; + else if (size == 4) m = "crc32"; + else + return; + #ifdef EXTERNAL_CODECS + const CExternalCodecs *__externalCodecs = g_ExternalCodecs_Ptr; + #endif + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS + AString(m), id)) + methods.Add(UString(m)); +} + + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + + /* + if (testMode == 0) + return E_NOTIMPL; + */ + + const bool allFilesMode = (numItems == (UInt32)(Int32)-1); + if (allFilesMode) + numItems = HashPairs.Size(); + if (numItems == 0) + return S_OK; + + #ifdef EXTERNAL_CODECS + const CExternalCodecs *__externalCodecs = g_ExternalCodecs_Ptr; + #endif + + CHashBundle hb_Glob; + // UStringVector methods = options.Methods; + UStringVector methods; + + if (methods.IsEmpty() && !_nameExtenstion.IsEmpty()) + { + AString utf; + ConvertUnicodeToUTF8(_nameExtenstion, utf); + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS utf, id)) + methods.Add(_nameExtenstion); + } + + if (methods.IsEmpty() && !_pgpMethod.IsEmpty()) + { + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS _pgpMethod, id)) + methods.Add(UString(_pgpMethod)); + } + + if (methods.IsEmpty() && _pgpMethod.IsEmpty() && _hashSize_Defined) + AddDefaultMethod(methods, _hashSize); + + RINOK(hb_Glob.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods)); + + CMyComPtr updateCallbackFile; + extractCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&updateCallbackFile); + if (!updateCallbackFile) + return E_NOTIMPL; + { + CMyComPtr GetDiskProperty; + extractCallback->QueryInterface(IID_IArchiveGetDiskProperty, (void **)&GetDiskProperty); + if (GetDiskProperty) + { + UInt64 totalSize = 0; + UInt32 i; + for (i = 0; i < numItems; i++) + { + const UInt32 index = allFilesMode ? i : indices[i]; + const CHashPair &hp = HashPairs[index]; + if (hp.IsDir()) + continue; + { + NCOM::CPropVariant prop; + RINOK(GetDiskProperty->GetDiskProperty(index, kpidSize, &prop)); + if (prop.vt != VT_UI8) + continue; + totalSize += prop.uhVal.QuadPart; + } + } + RINOK(extractCallback->SetTotal(totalSize)); + // RINOK(Hash_SetTotalUnpacked->Hash_SetTotalUnpacked(indices, numItems)); + } + } + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, false); + lps->InSize = lps->OutSize = 0; + + UInt32 i; + for (i = 0; i < numItems; i++) + { + RINOK(lps->SetCur()); + const UInt32 index = allFilesMode ? i : indices[i]; + + CHashPair &hp = HashPairs[index]; + + UString path; + hp.Get_UString_Path(path); + + CMyComPtr inStream; + const bool isDir = hp.IsDir(); + if (!isDir) + { + RINOK(updateCallbackFile->GetStream2(index, &inStream, NUpdateNotifyOp::kHashRead)); + if (!inStream) + { + continue; // we have shown error in GetStream2() + } + // askMode = NArchive::NExtract::NAskMode::kSkip; + } + + Int32 askMode = testMode ? + NArchive::NExtract::NAskMode::kTest : + NArchive::NExtract::NAskMode::kExtract; + + CMyComPtr realOutStream; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)); + + /* PrepareOperation() can expect kExtract to set + Attrib and security of output file */ + askMode = NArchive::NExtract::NAskMode::kReadExternal; + + extractCallback->PrepareOperation(askMode); + + const bool isAltStream = false; + + UInt64 fileSize = 0; + + CHashBundle hb_Loc; + + CHashBundle *hb_Use = &hb_Glob; + + HRESULT res_SetMethods = S_OK; + + UStringVector methods_loc; + + if (!hp.Method.IsEmpty()) + { + hb_Use = &hb_Loc; + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS hp.Method, id)) + { + methods_loc.Add(UString(hp.Method)); + RINOK(hb_Loc.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods_loc)); + } + else + res_SetMethods = E_NOTIMPL; + } + else if (methods.IsEmpty()) + { + AddDefaultMethod(methods_loc, (unsigned)hp.Hash.Size()); + if (!methods_loc.IsEmpty()) + { + hb_Use = &hb_Loc; + RINOK(hb_Loc.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods_loc)); + } + } + + const bool isSupportedMode = hp.IsSupportedMode(); + hb_Use->InitForNewFile(); + + if (inStream) + { + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + RINOK(lps->SetRatioInfo(NULL, &fileSize)); + } + UInt32 size; + RINOK(inStream->Read(buf, kBufSize, &size)); + if (size == 0) + break; + hb_Use->Update(buf, size); + if (realOutStream) + { + RINOK(WriteStream(realOutStream, buf, size)); + } + fileSize += size; + } + + hp.Size_from_Disk = fileSize; + hp.Size_from_Disk_Defined = true; + } + + realOutStream.Release(); + inStream.Release(); + + lps->InSize += hp.Hash.Size(); + lps->OutSize += fileSize; + + hb_Use->Final(isDir, isAltStream, path); + + Int32 opRes = NArchive::NExtract::NOperationResult::kUnsupportedMethod; + if (isSupportedMode + && res_SetMethods != E_NOTIMPL + && hb_Use->Hashers.Size() > 0 + ) + { + const CHasherState &hs = hb_Use->Hashers[0]; + if (hs.DigestSize == hp.Hash.Size()) + { + opRes = NArchive::NExtract::NOperationResult::kCRCError; + if (CheckDigests(hp.Hash, hs.Digests[0], hs.DigestSize)) + if (!hp.Size_from_Arc_Defined || hp.Size_from_Arc == fileSize) + opRes = NArchive::NExtract::NOperationResult::kOK; + } + } + + RINOK(extractCallback->SetOperationResult(opRes)); + } + + return lps->SetCur(); + + COM_TRY_END +} + + +// ---------- UPDATE ---------- + +struct CUpdateItem +{ + int IndexInArc; + unsigned IndexInClient; + UInt64 Size; + bool NewData; + bool NewProps; + bool IsDir; + UString Path; + + CUpdateItem(): Size(0), IsDir(false) {} +}; + + +static HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, + UString &res, + bool convertSlash) +{ + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(index, propId, &prop)); + if (prop.vt == VT_BSTR) + { + res = prop.bstrVal; + if (convertSlash) + NArchive::NItemName::ReplaceSlashes_OsToUnix(res); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + return S_OK; +} + + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *callback) +{ + COM_TRY_BEGIN + + if (_isArc && !CanUpdate()) + return E_NOTIMPL; + + /* + CMyComPtr reportArcProp; + callback->QueryInterface(IID_IArchiveUpdateCallbackArcProp, (void **)&reportArcProp); + */ + + CObjectVector updateItems; + + UInt64 complexity = 0; + + UInt32 i; + for (i = 0; i < numItems; i++) + { + CUpdateItem ui; + Int32 newData; + Int32 newProps; + UInt32 indexInArc; + + if (!callback) + return E_FAIL; + + RINOK(callback->GetUpdateItemInfo(i, &newData, &newProps, &indexInArc)); + + ui.NewProps = IntToBool(newProps); + ui.NewData = IntToBool(newData); + ui.IndexInArc = (int)indexInArc; + ui.IndexInClient = i; + if (IntToBool(newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsDir, &prop)); + if (prop.vt == VT_EMPTY) + ui.IsDir = false; + else if (prop.vt != VT_BOOL) + return E_INVALIDARG; + else + ui.IsDir = (prop.boolVal != VARIANT_FALSE); + } + + RINOK(GetPropString(callback, i, kpidPath, ui.Path, + true)); // convertSlash + /* + if (ui.IsDir && !ui.Name.IsEmpty() && ui.Name.Back() != '/') + ui.Name += '/'; + */ + } + + if (IntToBool(newData)) + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidSize, &prop)); + if (prop.vt == VT_UI8) + { + ui.Size = prop.uhVal.QuadPart; + complexity += ui.Size; + } + else if (prop.vt == VT_EMPTY) + ui.Size = (UInt64)(Int64)-1; + else + return E_INVALIDARG; + } + + updateItems.Add(ui); + } + + if (complexity != 0) + { + RINOK(callback->SetTotal(complexity)); + } + + #ifdef EXTERNAL_CODECS + const CExternalCodecs *__externalCodecs = g_ExternalCodecs_Ptr; + #endif + + CHashBundle hb; + UStringVector methods; + if (!_methods.IsEmpty()) + { + FOR_VECTOR(k, _methods) + { + methods.Add(_methods[k]); + } + } + else if (_crcSize_WasSet) + { + AddDefaultMethod(methods, _crcSize); + } + else + { + CMyComPtr getRootProps; + callback->QueryInterface(IID_IArchiveGetRootProps, (void **)&getRootProps); + + NCOM::CPropVariant prop; + if (getRootProps) + { + RINOK(getRootProps->GetRootProp(kpidArcFileName, &prop)); + if (prop.vt == VT_BSTR) + { + const UString method = GetMethod_from_FileName(prop.bstrVal); + if (!method.IsEmpty()) + methods.Add(method); + } + } + } + + RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS methods)); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(callback, true); + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + CDynLimBuf hashFileString((size_t)1 << 31); + + CHashOptionsLocal options = _options; + + if (_isArc) + { + if (!options.HashMode_Zero.Def && _is_ZeroMode) + options.HashMode_Zero.Val = true; + if (!options.HashMode_Tag.Def && _are_there_Tags) + options.HashMode_Tag.Val = true; + if (!options.HashMode_Dirs.Def && _are_there_Dirs) + options.HashMode_Dirs.Val = true; + } + if (options.HashMode_OnlyHash.Val && updateItems.Size() != 1) + options.HashMode_OnlyHash.Val = false; + + lps->OutSize = 0; + complexity = 0; + + for (i = 0; i < updateItems.Size(); i++) + { + lps->InSize = complexity; + RINOK(lps->SetCur()); + + const CUpdateItem &ui = updateItems[i]; + + /* + CHashPair item; + if (!ui.NewProps) + item = HashPairs[(unsigned)ui.IndexInArc]; + */ + + if (ui.NewData) + { + UInt64 currentComplexity = ui.Size; + UInt64 fileSize = 0; + + CMyComPtr fileInStream; + bool needWrite = true; + { + HRESULT res = callback->GetStream(ui.IndexInClient, &fileInStream); + + if (res == S_FALSE) + needWrite = false; + else + { + RINOK(res); + + if (fileInStream) + { + CMyComPtr streamGetSize; + fileInStream->QueryInterface(IID_IStreamGetSize, (void **)&streamGetSize); + if (streamGetSize) + { + UInt64 size; + if (streamGetSize->GetSize(&size) == S_OK) + currentComplexity = size; + } + /* + CMyComPtr getProps; + fileInStream->QueryInterface(IID_IStreamGetProps, (void **)&getProps); + if (getProps) + { + FILETIME mTime; + UInt64 size2; + if (getProps->GetProps(&size2, NULL, NULL, &mTime, NULL) == S_OK) + { + currentComplexity = size2; + // item.MTime = NWindows::NTime::FileTimeToUnixTime64(mTime);; + } + } + */ + } + else + { + currentComplexity = 0; + } + } + } + + hb.InitForNewFile(); + const bool isDir = ui.IsDir; + + if (needWrite && fileInStream && !isDir) + { + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + RINOK(lps->SetRatioInfo(&fileSize, NULL)); + // RINOK(callback->SetCompleted(&completeValue)); + } + UInt32 size; + RINOK(fileInStream->Read(buf, kBufSize, &size)); + if (size == 0) + break; + hb.Update(buf, size); + fileSize += size; + } + currentComplexity = fileSize; + } + + fileInStream.Release(); + const bool isAltStream = false; + hb.Final(isDir, isAltStream, ui.Path); + + if (options.HashMode_Dirs.Val || !isDir) + { + if (!hb.Hashers.IsEmpty()) + lps->OutSize += hb.Hashers[0].DigestSize; + WriteLine(hashFileString, + options, + ui.Path, + isDir, + hb); + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + + complexity += currentComplexity; + + /* + if (reportArcProp) + { + PROPVARIANT prop; + prop.vt = VT_EMPTY; + prop.wReserved1 = 0; + + NCOM::PropVarEm_Set_UInt64(&prop, fileSize); + RINOK(reportArcProp->ReportProp(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, kpidSize, &prop)); + + for (unsigned k = 0; k < hb.Hashers.Size(); k++) + { + const CHasherState &hs = hb.Hashers[k]; + + if (hs.DigestSize == 4 && hs.Name.IsEqualTo_Ascii_NoCase("crc32")) + { + NCOM::PropVarEm_Set_UInt32(&prop, GetUi32(hs.Digests[k_HashCalc_Index_Current])); + RINOK(reportArcProp->ReportProp(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, kpidCRC, &prop)); + } + else + { + RINOK(reportArcProp->ReportRawProp(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, + kpidChecksum, hs.Digests[k_HashCalc_Index_Current], + hs.DigestSize, NPropDataType::kRaw)); + } + RINOK(reportArcProp->ReportFinished(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, NArchive::NUpdate::NOperationResult::kOK)); + } + } + */ + RINOK(callback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); + } + else + { + // old data + const CHashPair &existItem = HashPairs[(unsigned)ui.IndexInArc]; + if (ui.NewProps) + { + WriteLine(hashFileString, + options, + ui.Path, + ui.IsDir, + existItem.Method, existItem.HashString + ); + } + else + { + hashFileString += existItem.FullLine; + Add_LF(hashFileString, options); + } + } + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + + RINOK(WriteStream(outStream, hashFileString, hashFileString.Len())); + + return S_OK; + COM_TRY_END +} + + + +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) +{ + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + if (name.IsEqualTo("m")) // "hm" hash method + { + // COneMethodInfo omi; + // RINOK(omi.ParseMethodFromPROPVARIANT(L"", value)); + // _methods.Add(omi.MethodName); // change it. use omi.PropsString + if (value.vt != VT_BSTR) + return E_INVALIDARG; + UString s (value.bstrVal); + _methods.Add(s); + return S_OK; + } + + if (name.IsEqualTo("flags")) + { + if (value.vt != VT_BSTR) + return E_INVALIDARG; + if (!_options.ParseString(value.bstrVal)) + return E_INVALIDARG; + return S_OK; + } + + if (name.IsPrefixedBy_Ascii_NoCase("crc")) + { + name.Delete(0, 3); + _crcSize = 4; + _crcSize_WasSet = true; + return ParsePropToUInt32(name, value, _crcSize); + } + + // common properties + if (name.IsPrefixedBy_Ascii_NoCase("mt") + || name.IsPrefixedBy_Ascii_NoCase("memuse")) + return S_OK; + + return E_INVALIDARG; +} + + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + COM_TRY_BEGIN + + InitProps(); + + for (UInt32 i = 0; i < numProps; i++) + { + RINOK(SetProperty(names[i], values[i])); + } + return S_OK; + COM_TRY_END +} + +CHandler::CHandler() +{ + ClearVars(); + InitProps(); +} + +} + + + +static IInArchive *CreateHashHandler_In() { return new NHash::CHandler; } +static IOutArchive *CreateHashHandler_Out() { return new NHash::CHandler; } + +void Codecs_AddHashArcHandler(CCodecs *codecs) +{ + { + CArcInfoEx item; + + item.Name = "Hash"; + item.CreateInArchive = CreateHashHandler_In; + item.CreateOutArchive = CreateHashHandler_Out; + item.IsArcFunc = NULL; + item.Flags = + NArcInfoFlags::kKeepName + | NArcInfoFlags::kStartOpen + | NArcInfoFlags::kByExtOnlyOpen + // | NArcInfoFlags::kPureStartOpen + | NArcInfoFlags::kHashHandler + ; + + // ubuntu uses "SHA256SUMS" file + item.AddExts(UString ( + "sha256 sha512 sha224 sha384 sha1 sha md5" + // "b2sum" + " crc32 crc64" + " asc" + " cksum" + ), + UString()); + + item.UpdateEnabled = (item.CreateOutArchive != NULL); + item.SignatureOffset = 0; + // item.Version = MY_VER_MIX; + item.NewInterface = true; + + item.Signatures.AddNew().CopyFrom(NULL, 0); + + codecs->Formats.Add(item); } } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.h b/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.h index 38908e2e8b3..c566caa881f 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/HashCalc.h @@ -3,18 +3,25 @@ #ifndef __HASH_CALC_H #define __HASH_CALC_H +#include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" #include "../../Common/CreateCoder.h" #include "../../Common/MethodProps.h" #include "DirItem.h" -#include "Property.h" +#include "IFileExtractCallback.h" const unsigned k_HashCalc_DigestSize_Max = 64; - +const unsigned k_HashCalc_ExtraSize = 8; const unsigned k_HashCalc_NumGroups = 4; +/* + if (size <= 8) : upper case : reversed byte order : it shows 32-bit/64-bit number, if data contains little-endian number + if (size > 8) : lower case : original byte order (as big-endian byte sequence) +*/ +void HashHexToString(char *dest, const Byte *data, UInt32 size); + enum { k_HashCalc_Index_Current, @@ -28,9 +35,37 @@ struct CHasherState CMyComPtr Hasher; AString Name; UInt32 DigestSize; - Byte Digests[k_HashCalc_NumGroups][k_HashCalc_DigestSize_Max]; + UInt64 NumSums[k_HashCalc_NumGroups]; + Byte Digests[k_HashCalc_NumGroups][k_HashCalc_DigestSize_Max + k_HashCalc_ExtraSize]; + + void InitDigestGroup(unsigned groupIndex) + { + NumSums[groupIndex] = 0; + memset(Digests[groupIndex], 0, sizeof(Digests[groupIndex])); + } + + const Byte *GetExtraData_for_Group(unsigned groupIndex) const + { + return Digests[groupIndex] + k_HashCalc_DigestSize_Max; + } + + unsigned GetNumExtraBytes_for_Group(unsigned groupIndex) const + { + const Byte *p = GetExtraData_for_Group(groupIndex); + // we use little-endian to read extra bytes + for (unsigned i = k_HashCalc_ExtraSize; i != 0; i--) + if (p[i - 1] != 0) + return i; + return 0; + } + + void AddDigest(unsigned groupIndex, const Byte *data); + + void WriteToString(unsigned digestIndex, char *s) const; }; + + struct IHashCalc { virtual void InitForNewFile() = 0; @@ -52,13 +87,19 @@ struct CHashBundle: public IHashCalc UInt64 CurSize; + UString MainName; + UString FirstFileName; + HRESULT SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVector &methods); - void Init() + // void Init() {} + CHashBundle() { NumDirs = NumFiles = NumAltStreams = FilesSize = AltStreamsSize = NumErrors = 0; } + virtual ~CHashBundle() {}; + void InitForNewFile(); void Update(const void *data, UInt32 size); void SetSize(UInt64 size); @@ -77,24 +118,92 @@ struct CHashBundle: public IHashCalc virtual HRESULT GetStream(const wchar_t *name, bool isFolder) x; \ virtual HRESULT OpenFileError(const FString &path, DWORD systemError) x; \ virtual HRESULT SetOperationResult(UInt64 fileSize, const CHashBundle &hb, bool showHash) x; \ - virtual HRESULT AfterLastFile(const CHashBundle &hb) x; \ + virtual HRESULT AfterLastFile(CHashBundle &hb) x; \ struct IHashCallbackUI: public IDirItemsCallback { INTERFACE_IHashCallbackUI(=0) }; + +struct CHashOptionsLocal +{ + CBoolPair HashMode_Zero; + CBoolPair HashMode_Tag; + CBoolPair HashMode_Dirs; + CBoolPair HashMode_OnlyHash; + + void Init_HashOptionsLocal() + { + HashMode_Zero.Init(); + HashMode_Tag.Init(); + HashMode_Dirs.Init(); + HashMode_OnlyHash.Init(); + // HashMode_Dirs = true; // for debug + } + + CHashOptionsLocal() + { + Init_HashOptionsLocal(); + } + + bool ParseFlagCharOption(wchar_t c, bool val) + { + c = MyCharLower_Ascii(c); + if (c == 'z') HashMode_Zero.SetVal_as_Defined(val); + else if (c == 't') HashMode_Tag.SetVal_as_Defined(val); + else if (c == 'd') HashMode_Dirs.SetVal_as_Defined(val); + else if (c == 'h') HashMode_OnlyHash.SetVal_as_Defined(val); + else return false; + return true; + } + + bool ParseString(const UString &s) + { + for (unsigned i = 0; i < s.Len();) + { + const wchar_t c = s[i++]; + bool val = true; + if (i < s.Len()) + { + const wchar_t next = s[i]; + if (next == '-') + { + val = false; + i++; + } + } + if (!ParseFlagCharOption(c, val)) + return false; + } + return true; + } +}; + + struct CHashOptions + // : public CHashOptionsLocal { UStringVector Methods; + // UString HashFilePath; + + bool PreserveATime; bool OpenShareForWrite; bool StdInMode; bool AltStreamsMode; + CBoolPair SymLinks; + NWildcard::ECensorPathMode PathMode; - - CHashOptions(): StdInMode(false), OpenShareForWrite(false), AltStreamsMode(false), PathMode(NWildcard::k_RelatPath) {}; + + CHashOptions(): + PreserveATime(false), + OpenShareForWrite(false), + StdInMode(false), + AltStreamsMode(false), + PathMode(NWildcard::k_RelatPath) {}; }; + HRESULT HashCalc( DECL_EXTERNAL_CODECS_LOC_VARS const NWildcard::CCensor &censor, @@ -102,6 +211,130 @@ HRESULT HashCalc( AString &errorInfo, IHashCallbackUI *callback); -void AddHashHexToString(char *dest, const Byte *data, UInt32 size); + + +#ifndef _SFX + +namespace NHash { + +struct CHashPair +{ + CByteBuffer Hash; + char Mode; + bool IsBSD; + bool Size_from_Arc_Defined; + bool Size_from_Disk_Defined; + AString Method; + AString Name; + + AString FullLine; + AString HashString; + // unsigned HashLengthInBits; + + // AString MethodName; + UInt64 Size_from_Arc; + UInt64 Size_from_Disk; + + bool IsDir() const; + + void Get_UString_Path(UString &path) const + { + path.Empty(); + if (!ConvertUTF8ToUnicode(Name, path)) + return; + } + + bool ParseCksum(const char *s); + bool Parse(const char *s); + + bool IsSupportedMode() const + { + return Mode != 'U' && Mode != '^'; + } + + CHashPair(): + Mode(0) + , IsBSD(false) + , Size_from_Arc_Defined(false) + , Size_from_Disk_Defined(false) + // , HashLengthInBits(0) + , Size_from_Arc(0) + , Size_from_Disk(0) + {} +}; + + +class CHandler: + public IInArchive, + public IArchiveGetRawProps, + // public IGetArchiveHashHandler, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + bool _isArc; + UInt64 _phySize; + CObjectVector HashPairs; + UString _nameExtenstion; + // UString _method_fromName; + AString _pgpMethod; + bool _is_CksumMode; + bool _is_PgpMethod; + bool _is_ZeroMode; + bool _are_there_Tags; + bool _are_there_Dirs; + bool _hashSize_Defined; + unsigned _hashSize; + + bool _crcSize_WasSet; + UInt32 _crcSize; + UStringVector _methods; + + void ClearVars(); + + void InitProps() + { + _crcSize_WasSet = false; + _crcSize = 4; + _methods.Clear(); + _options.Init_HashOptionsLocal(); + } + + CHashOptionsLocal _options; + + bool CanUpdate() const + { + if (!_isArc || _is_PgpMethod || _is_CksumMode) + return false; + return true; + + } + + HRESULT SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value); + +public: + + CHandler(); + + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveGetRawProps, + IOutArchive, + ISetProperties + /*, IGetArchiveHashHandler */ + ) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + INTERFACE_IArchiveGetRawProps(;) + // STDMETHOD(GetArchiveHashHandler)(CHandler **handler); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); +}; + +} + +void Codecs_AddHashArcHandler(CCodecs *codecs); + +#endif + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/IFileExtractCallback.h b/3rdparty/lzma/CPP/7zip/UI/Common/IFileExtractCallback.h index c456c862737..e6a85c6d06f 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/IFileExtractCallback.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/IFileExtractCallback.h @@ -103,9 +103,9 @@ DECL_INTERFACE_SUB(IGetProp, IUnknown, 0x01, 0x20) STDMETHOD(UseExtractToStream)(Int32 *res) x; \ STDMETHOD(GetStream7)(const wchar_t *name, Int32 isDir, ISequentialOutStream **outStream, Int32 askExtractMode, IGetProp *getProp) x; \ STDMETHOD(PrepareOperation7)(Int32 askExtractMode) x; \ - STDMETHOD(SetOperationResult7)(Int32 resultEOperationResult, Int32 encrypted) x; \ + STDMETHOD(SetOperationResult8)(Int32 resultEOperationResult, Int32 encrypted, UInt64 size) x; \ -DECL_INTERFACE_SUB(IFolderExtractToStreamCallback, IUnknown, 0x01, 0x30) +DECL_INTERFACE_SUB(IFolderExtractToStreamCallback, IUnknown, 0x01, 0x31) { INTERFACE_IFolderExtractToStreamCallback(PURE) }; diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.cpp index 5435499baa8..b6a20732391 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.cpp @@ -33,12 +33,12 @@ EXPORT_CODECS #include "StdAfx.h" -#include "../../../../C/7zVersion.h" - #include "../../../Common/MyCom.h" #include "../../../Common/StringToInt.h" #include "../../../Common/StringConvert.h" +#include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/FileIO.h" #include "../../../Windows/PropVariant.h" #include "LoadCodecs.h" @@ -51,6 +51,7 @@ using namespace NWindows; #include "../../ICoder.h" #include "../../Common/RegisterArc.h" +#include "../../Common/RegisterCodec.h" #ifdef EXTERNAL_CODECS @@ -81,19 +82,19 @@ using namespace NFile; #define kFormatsFolderName FTEXT("Formats") -static CFSTR kMainDll = - // #ifdef _WIN32 +static CFSTR const kMainDll = + #ifdef _WIN32 FTEXT("7z.dll"); - // #else - // FTEXT("7z.so"); - // #endif + #else + FTEXT("7z.so"); + #endif #ifdef _WIN32 -static LPCTSTR kRegistryPath = TEXT("Software") TEXT(STRING_PATH_SEPARATOR) TEXT("7-zip"); -static LPCWSTR kProgramPathValue = L"Path"; -static LPCWSTR kProgramPath2Value = L"Path" +static LPCTSTR const kRegistryPath = TEXT("Software") TEXT(STRING_PATH_SEPARATOR) TEXT("7-zip"); +static LPCWSTR const kProgramPathValue = L"Path"; +static LPCWSTR const kProgramPath2Value = L"Path" #ifdef _WIN64 L"64"; #else @@ -110,7 +111,7 @@ static bool ReadPathFromRegistry(HKEY baseKey, LPCWSTR value, FString &path) { path = us2fs(pathU); NName::NormalizeDirPathPrefix(path); - return NFind::DoesFileExist(path + kMainDll); + return NFind::DoesFileExist_Raw(path + kMainDll); } } return false; @@ -163,7 +164,7 @@ int CArcInfoEx::FindExtension(const UString &ext) const { FOR_VECTOR (i, Exts) if (ext.IsEqualTo_NoCase(Exts[i].Ext)) - return i; + return (int)i; return -1; } @@ -191,9 +192,9 @@ void CArcInfoEx::AddExts(const UString &ext, const UString &addExt) static bool ParseSignatures(const Byte *data, unsigned size, CObjectVector &signatures) { signatures.Clear(); - while (size > 0) + while (size != 0) { - unsigned len = *data++; + const unsigned len = *data++; size--; if (len > size) return false; @@ -206,15 +207,18 @@ static bool ParseSignatures(const Byte *data, unsigned size, CObjectVector + #ifdef EXTERNAL_CODECS static FString GetBaseFolderPrefixFromRegistry() { FString moduleFolderPrefix = NDLL::GetModuleDirPrefix(); + #ifdef _WIN32 - if (!NFind::DoesFileExist(moduleFolderPrefix + kMainDll) && - !NFind::DoesDirExist(moduleFolderPrefix + kCodecsFolderName) && - !NFind::DoesDirExist(moduleFolderPrefix + kFormatsFolderName)) + if ( !NFind::DoesFileOrDirExist(moduleFolderPrefix + kMainDll) + && !NFind::DoesFileOrDirExist(moduleFolderPrefix + kCodecsFolderName) + && !NFind::DoesFileOrDirExist(moduleFolderPrefix + kFormatsFolderName)) { FString path; if (ReadPathFromRegistry(HKEY_CURRENT_USER, kProgramPath2Value, path)) return path; @@ -223,6 +227,8 @@ static FString GetBaseFolderPrefixFromRegistry() if (ReadPathFromRegistry(HKEY_LOCAL_MACHINE, kProgramPathValue, path)) return path; } #endif + + // printf("\nmoduleFolderPrefix = %s\n", (const char *)GetAnsiString(moduleFolderPrefix)); return moduleFolderPrefix; } @@ -238,25 +244,50 @@ static HRESULT GetCoderClass(Func_GetMethodProperty getMethodProperty, UInt32 in if (::SysStringByteLen(prop.bstrVal) != sizeof(GUID)) return E_FAIL; isAssigned = true; - clsId = *(const GUID *)prop.bstrVal; + clsId = *(const GUID *)(const void *)prop.bstrVal; } else if (prop.vt != VT_EMPTY) return E_FAIL; return S_OK; } + +static HRESULT GetMethodBoolProp(Func_GetMethodProperty getMethodProperty, UInt32 index, + PROPID propId, bool &resVal, bool &isAssigned) +{ + NCOM::CPropVariant prop; + resVal = false; + isAssigned = false; + RINOK(getMethodProperty(index, propId, &prop)); + if (prop.vt == VT_BOOL) + { + isAssigned = true; + resVal = VARIANT_BOOLToBool(prop.boolVal); + } + else if (prop.vt != VT_EMPTY) + return E_FAIL; + return S_OK; +} + + +#define MY_GET_FUNC(dest, type, func) *(void **)(&dest) = (func); +// #define MY_GET_FUNC(dest, type, func) dest = (type)(func); + +#define MY_GET_FUNC_LOC(dest, type, func) \ + type dest; MY_GET_FUNC(dest, type, func) + HRESULT CCodecs::LoadCodecs() { CCodecLib &lib = Libs.Back(); - lib.CreateDecoder = (Func_CreateDecoder)lib.Lib.GetProc("CreateDecoder"); - lib.CreateEncoder = (Func_CreateEncoder)lib.Lib.GetProc("CreateEncoder"); - lib.GetMethodProperty = (Func_GetMethodProperty)lib.Lib.GetProc("GetMethodProperty"); + MY_GET_FUNC (lib.CreateDecoder, Func_CreateDecoder, lib.Lib.GetProc("CreateDecoder")); + MY_GET_FUNC (lib.CreateEncoder, Func_CreateEncoder, lib.Lib.GetProc("CreateEncoder")); + MY_GET_FUNC (lib.GetMethodProperty, Func_GetMethodProperty, lib.Lib.GetProc("GetMethodProperty")); if (lib.GetMethodProperty) { UInt32 numMethods = 1; - Func_GetNumberOfMethods getNumberOfMethods = (Func_GetNumberOfMethods)lib.Lib.GetProc("GetNumberOfMethods"); + MY_GET_FUNC_LOC (getNumberOfMethods, Func_GetNumberOfMethods, lib.Lib.GetProc("GetNumberOfMethods")); if (getNumberOfMethods) { RINOK(getNumberOfMethods(&numMethods)); @@ -268,11 +299,12 @@ HRESULT CCodecs::LoadCodecs() info.CodecIndex = i; RINOK(GetCoderClass(lib.GetMethodProperty, i, NMethodPropID::kEncoder, info.Encoder, info.EncoderIsAssigned)); RINOK(GetCoderClass(lib.GetMethodProperty, i, NMethodPropID::kDecoder, info.Decoder, info.DecoderIsAssigned)); + RINOK(GetMethodBoolProp(lib.GetMethodProperty, i, NMethodPropID::kIsFilter, info.IsFilter, info.IsFilter_Assigned)); Codecs.Add(info); } } - Func_GetHashers getHashers = (Func_GetHashers)lib.Lib.GetProc("GetHashers"); + MY_GET_FUNC_LOC (getHashers, Func_GetHashers, lib.Lib.GetProc("GetHashers")); if (getHashers) { RINOK(getHashers(&lib.ComHashers)); @@ -381,14 +413,14 @@ HRESULT CCodecs::LoadFormats() const NDLL::CLibrary &lib = Libs.Back().Lib; Func_GetHandlerProperty getProp = NULL; - Func_GetHandlerProperty2 getProp2 = (Func_GetHandlerProperty2)lib.GetProc("GetHandlerProperty2"); - Func_GetIsArc getIsArc = (Func_GetIsArc)lib.GetProc("GetIsArc"); + MY_GET_FUNC_LOC (getProp2, Func_GetHandlerProperty2, lib.GetProc("GetHandlerProperty2")); + MY_GET_FUNC_LOC (getIsArc, Func_GetIsArc, lib.GetProc("GetIsArc")); UInt32 numFormats = 1; if (getProp2) { - Func_GetNumberOfFormats getNumberOfFormats = (Func_GetNumberOfFormats)lib.GetProc("GetNumberOfFormats"); + MY_GET_FUNC_LOC (getNumberOfFormats, Func_GetNumberOfFormats, lib.GetProc("GetNumberOfFormats")); if (getNumberOfFormats) { RINOK(getNumberOfFormats(&numFormats)); @@ -396,7 +428,7 @@ HRESULT CCodecs::LoadFormats() } else { - getProp = (Func_GetHandlerProperty)lib.GetProc("GetHandlerProperty"); + MY_GET_FUNC (getProp, Func_GetHandlerProperty, lib.GetProc("GetHandlerProperty")); if (!getProp) return S_OK; } @@ -404,7 +436,7 @@ HRESULT CCodecs::LoadFormats() for (UInt32 i = 0; i < numFormats; i++) { CArcInfoEx item; - item.LibIndex = Libs.Size() - 1; + item.LibIndex = (int)(Libs.Size() - 1); item.FormatIndex = i; RINOK(GetProp_String(getProp, getProp2, i, NArchive::NHandlerPropID::kName, item.Name)); @@ -417,7 +449,7 @@ HRESULT CCodecs::LoadFormats() continue; if (::SysStringByteLen(prop.bstrVal) != sizeof(GUID)) return E_FAIL; - item.ClassID = *(const GUID *)prop.bstrVal; + item.ClassID = *(const GUID *)(const void *)prop.bstrVal; prop.Clear(); } @@ -441,6 +473,11 @@ HRESULT CCodecs::LoadFormats() item.Flags |= kArcFlagsPars[j + 1]; } } + + { + bool defined = false; + RINOK(GetProp_UInt32(getProp, getProp2, i, NArchive::NHandlerPropID::kTimeFlags, item.TimeFlags, defined)); + } CByteBuffer sig; RINOK(GetProp_RawData(getProp, getProp2, i, NArchive::NHandlerPropID::kSignature, sig)); @@ -473,23 +510,53 @@ extern "C" } #endif + +void CCodecs::AddLastError(const FString &path) +{ + HRESULT res = GetLastError_noZero_HRESULT(); + CCodecError &error = Errors.AddNew(); + error.Path = path; + error.ErrorCode = res; +} + HRESULT CCodecs::LoadDll(const FString &dllPath, bool needCheckDll, bool *loadedOK) { if (loadedOK) *loadedOK = false; + // needCheckDll = 1; + + #ifdef _WIN32 if (needCheckDll) { NDLL::CLibrary lib; if (!lib.LoadEx(dllPath, LOAD_LIBRARY_AS_DATAFILE)) + { + /* if is not win32 + // %1 is not a valid Win32 application. + // #define ERROR_BAD_EXE_FORMAT 193L + */ + // return GetLastError_noZero_HRESULT(); + DWORD lastError = GetLastError(); + if (lastError != ERROR_BAD_EXE_FORMAT) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.Message = "cannot load file as datafile library"; + error.ErrorCode = HRESULT_FROM_WIN32(lastError); + } return S_OK; + } } + #else + UNUSED_VAR(needCheckDll) + #endif Libs.AddNew(); CCodecLib &lib = Libs.Back(); lib.Path = dllPath; bool used = false; - HRESULT res = S_OK; + // HRESULT res = S_OK; if (lib.Lib.Load(dllPath)) { @@ -499,52 +566,126 @@ HRESULT CCodecs::LoadDll(const FString &dllPath, bool needCheckDll, bool *loaded lib.LoadIcons(); #endif + /* + { + MY_GET_FUNC_LOC (_LibStartup, Func_LibStartup, lib.Lib.GetProc("LibStartup")); + if (_LibStartup) + { + HRESULT res = _LibStartup(); + if (res != 0) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.ErrorCode = res; + } + } + } + */ + #ifdef _7ZIP_LARGE_PAGES if (g_LargePageSize != 0) { - Func_SetLargePageMode setLargePageMode = (Func_SetLargePageMode)lib.Lib.GetProc("SetLargePageMode"); + MY_GET_FUNC_LOC (setLargePageMode, Func_SetLargePageMode, lib.Lib.GetProc("SetLargePageMode")); if (setLargePageMode) setLargePageMode(); } #endif - if (CaseSensitiveChange) + if (CaseSensitive_Change) { - Func_SetCaseSensitive setCaseSensitive = (Func_SetCaseSensitive)lib.Lib.GetProc("SetCaseSensitive"); + MY_GET_FUNC_LOC (setCaseSensitive, Func_SetCaseSensitive, lib.Lib.GetProc("SetCaseSensitive")); if (setCaseSensitive) setCaseSensitive(CaseSensitive ? 1 : 0); } - lib.CreateObject = (Func_CreateObject)lib.Lib.GetProc("CreateObject"); - if (lib.CreateObject) + /* + { + MY_GET_FUNC_LOC (setClientVersion, Func_SetClientVersion, lib.Lib.GetProc("SetClientVersion")); + if (setClientVersion) + { + // const UInt32 kVersion = (MY_VER_MAJOR << 16) | MY_VER_MINOR; + setClientVersion(g_ClientVersion); + } + } + */ + + + MY_GET_FUNC (lib.CreateObject, Func_CreateObject, lib.Lib.GetProc("CreateObject")); { unsigned startSize = Codecs.Size() + Hashers.Size(); - res = LoadCodecs(); - used = (startSize != Codecs.Size() + Hashers.Size()); - if (res == S_OK) + HRESULT res = LoadCodecs(); + if (startSize != Codecs.Size() + Hashers.Size()) + used = true; + if (res == S_OK && lib.CreateObject) { startSize = Formats.Size(); res = LoadFormats(); if (startSize != Formats.Size()) used = true; } + if (res != S_OK) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.ErrorCode = res; + } + } + // plugins can use non-7-zip dlls, so we silently ignore non7zip DLLs + /* + if (!used) + { + CCodecError &error = Errors.AddNew(); + error.Path = dllPath; + error.Message = "no 7-Zip code"; } + */ + } + else + { + AddLastError(dllPath); } if (!used) Libs.DeleteBack(); - return res; + return S_OK; } -HRESULT CCodecs::LoadDllsFromFolder(const FString &folderPrefix) +HRESULT CCodecs::LoadDllsFromFolder(const FString &folderPath) { - NFile::NFind::CEnumerator enumerator(folderPrefix + FCHAR_ANY_MASK); - NFile::NFind::CFileInfo fi; - while (enumerator.Next(fi)) + if (!NFile::NFind::DoesDirExist_FollowLink(folderPath)) + // if (!NFile::NFind::DoesDirExist(folderPath)) { + // AddLastError(folderPath); + return S_OK; + } + + FString folderPrefix = folderPath; + folderPrefix.Add_PathSepar(); + + NFile::NFind::CEnumerator enumerator; + enumerator.SetDirPrefix(folderPrefix); + NFile::NFind::CDirEntry fi; + for (;;) + { + bool found; + if (!enumerator.Next(fi, found)) + { + // it can be wrong Symbolic link to folder here + AddLastError(folderPath); + break; + // return GetLastError_noZero_HRESULT(); + } + if (!found) + break; + #ifdef _WIN32 if (fi.IsDir()) continue; + #else + if (enumerator.DirEntry_IsDir(fi, true)) // followLink + continue; + #endif + RINOK(LoadDll(folderPrefix + fi.Name, true)); } return S_OK; @@ -585,6 +726,7 @@ HRESULT CCodecs::Load() Formats.Clear(); #ifdef EXTERNAL_CODECS + Errors.Clear(); MainDll_ErrorPath.Empty(); Codecs.Clear(); Hashers.Clear(); @@ -595,7 +737,7 @@ HRESULT CCodecs::Load() const CArcInfo &arc = *g_Arcs[i]; CArcInfoEx item; - item.Name.SetFromAscii(arc.Name); + item.Name = arc.Name; item.CreateInArchive = arc.CreateInArchive; item.IsArcFunc = arc.IsArc; item.Flags = arc.Flags; @@ -603,9 +745,9 @@ HRESULT CCodecs::Load() { UString e, ae; if (arc.Ext) - e.SetFromAscii(arc.Ext); + e = arc.Ext; if (arc.AddExt) - ae.SetFromAscii(arc.AddExt); + ae = arc.AddExt; item.AddExts(e, ae); } @@ -620,13 +762,18 @@ HRESULT CCodecs::Load() if (arc.IsMultiSignature()) ParseSignatures(arc.Signature, arc.SignatureSize, item.Signatures); else - item.Signatures.AddNew().CopyFrom(arc.Signature, arc.SignatureSize); + { + if (arc.SignatureSize != 0) // 21.04 + item.Signatures.AddNew().CopyFrom(arc.Signature, arc.SignatureSize); + } #endif Formats.Add(item); } + // printf("\nLoad codecs \n"); + #ifdef EXTERNAL_CODECS const FString baseFolder = GetBaseFolderPrefixFromRegistry(); { @@ -635,8 +782,8 @@ HRESULT CCodecs::Load() if (!loadedOK) MainDll_ErrorPath = kMainDll; } - RINOK(LoadDllsFromFolder(baseFolder + kCodecsFolderName FSTRING_PATH_SEPARATOR)); - RINOK(LoadDllsFromFolder(baseFolder + kFormatsFolderName FSTRING_PATH_SEPARATOR)); + RINOK(LoadDllsFromFolder(baseFolder + kCodecsFolderName)); + RINOK(LoadDllsFromFolder(baseFolder + kFormatsFolderName)); NeedSetLibCodecs = true; @@ -659,7 +806,7 @@ HRESULT CCodecs::Load() FOR_VECTOR(i, Libs) { CCodecLib &lib = Libs[i]; - lib.SetCodecs = (Func_SetCodecs)lib.Lib.GetProc("SetCodecs"); + MY_GET_FUNC (lib.SetCodecs, Func_SetCodecs, lib.Lib.GetProc("SetCodecs")); if (lib.SetCodecs) { RINOK(lib.SetCodecs(this)); @@ -669,6 +816,8 @@ HRESULT CCodecs::Load() #endif + // we sort Formats to get fixed order of Formats after compilation. + Formats.Sort(); return S_OK; } @@ -679,7 +828,7 @@ int CCodecs::FindFormatForArchiveName(const UString &arcPath) const int dotPos = arcPath.ReverseFind_Dot(); if (dotPos <= arcPath.ReverseFind_PathSepar()) return -1; - const UString ext = arcPath.Ptr(dotPos + 1); + const UString ext = arcPath.Ptr((unsigned)(dotPos + 1)); if (ext.IsEmpty()) return -1; if (ext.IsEqualTo_Ascii_NoCase("exe")) @@ -692,7 +841,7 @@ int CCodecs::FindFormatForArchiveName(const UString &arcPath) const continue; */ if (arc.FindExtension(ext) >= 0) - return i; + return (int)i; } return -1; } @@ -703,7 +852,7 @@ int CCodecs::FindFormatForExtension(const UString &ext) const return -1; FOR_VECTOR (i, Formats) if (Formats[i].FindExtension(ext) >= 0) - return i; + return (int)i; return -1; } @@ -711,7 +860,7 @@ int CCodecs::FindFormatForArchiveType(const UString &arcType) const { FOR_VECTOR (i, Formats) if (Formats[i].Name.IsEqualTo_NoCase(arcType)) - return i; + return (int)i; return -1; } @@ -722,8 +871,8 @@ bool CCodecs::FindFormatForArchiveType(const UString &arcType, CIntVector &forma { int pos2 = arcType.Find(L'.', pos); if (pos2 < 0) - pos2 = arcType.Len(); - const UString name = arcType.Mid(pos, pos2 - pos); + pos2 = (int)arcType.Len(); + const UString name = arcType.Mid(pos, (unsigned)pos2 - pos); if (name.IsEmpty()) return false; int index = FindFormatForArchiveType(name); @@ -733,7 +882,7 @@ bool CCodecs::FindFormatForArchiveType(const UString &arcType, CIntVector &forma return false; } formatIndices.Add(index); - pos = pos2 + 1; + pos = (unsigned)pos2 + 1; } return true; } @@ -756,19 +905,19 @@ void CCodecIcons::LoadIcons(HMODULE m) CIconPair iconPair; iconPair.IconIndex = -1; if (pos < 0) - pos = s.Len(); + pos = (int)s.Len(); else { - UString num = s.Ptr(pos + 1); + const UString num = s.Ptr((unsigned)pos + 1); if (!num.IsEmpty()) { const wchar_t *end; - iconPair.IconIndex = ConvertStringToUInt32(num, &end); + iconPair.IconIndex = (int)ConvertStringToUInt32(num, &end); if (*end != 0) continue; } } - iconPair.Ext = s.Left(pos); + iconPair.Ext = s.Left((unsigned)pos); IconPairs.Add(iconPair); } } @@ -845,6 +994,15 @@ STDMETHODIMP CCodecs::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *valu prop.Detach(value); return S_OK; } + + if (propID == NMethodPropID::kIsFilter && ci.IsFilter_Assigned) + { + NCOM::CPropVariant prop; + prop = (bool)ci.IsFilter; + prop.Detach(value); + return S_OK; + } + const CCodecLib &lib = Libs[ci.LibIndex]; return lib.GetMethodProperty(ci.CodecIndex, propID, value); #else @@ -866,7 +1024,8 @@ STDMETHODIMP CCodecs::CreateDecoder(UInt32 index, const GUID *iid, void **coder) const CCodecLib &lib = Libs[ci.LibIndex]; if (lib.CreateDecoder) return lib.CreateDecoder(ci.CodecIndex, iid, (void **)coder); - return lib.CreateObject(&ci.Decoder, iid, (void **)coder); + if (lib.CreateObject) + return lib.CreateObject(&ci.Decoder, iid, (void **)coder); } return S_OK; #else @@ -888,7 +1047,8 @@ STDMETHODIMP CCodecs::CreateEncoder(UInt32 index, const GUID *iid, void **coder) const CCodecLib &lib = Libs[ci.LibIndex]; if (lib.CreateEncoder) return lib.CreateEncoder(ci.CodecIndex, iid, (void **)coder); - return lib.CreateObject(&ci.Encoder, iid, (void **)coder); + if (lib.CreateObject) + return lib.CreateObject(&ci.Encoder, iid, (void **)coder); } return S_OK; #else @@ -944,7 +1104,7 @@ int CCodecs::GetCodec_LibIndex(UInt32 index) const #ifdef EXTERNAL_CODECS const CDllCodecInfo &ci = Codecs[index - NUM_EXPORT_CODECS]; - return ci.LibIndex; + return (int)ci.LibIndex; #else return -1; #endif @@ -959,7 +1119,7 @@ int CCodecs::GetHasherLibIndex(UInt32 index) #ifdef EXTERNAL_CODECS const CDllHasherInfo &ci = Hashers[index - NUM_EXPORT_HASHERS]; - return ci.LibIndex; + return (int)ci.LibIndex; #else return -1; #endif @@ -987,6 +1147,7 @@ bool CCodecs::GetCodec_DecoderIsAssigned(UInt32 index) const #endif } + bool CCodecs::GetCodec_EncoderIsAssigned(UInt32 index) const { #ifdef EXPORT_CODECS @@ -1009,10 +1170,43 @@ bool CCodecs::GetCodec_EncoderIsAssigned(UInt32 index) const #endif } + +bool CCodecs::GetCodec_IsFilter(UInt32 index, bool &isAssigned) const +{ + isAssigned = false; + #ifdef EXPORT_CODECS + if (index < g_NumCodecs) + { + NCOM::CPropVariant prop; + if (GetProperty(index, NMethodPropID::kIsFilter, &prop) == S_OK) + { + if (prop.vt == VT_BOOL) + { + isAssigned = true; + return VARIANT_BOOLToBool(prop.boolVal); + } + } + return false; + } + #endif + + #ifdef EXTERNAL_CODECS + { + const CDllCodecInfo &c = Codecs[index - NUM_EXPORT_CODECS]; + isAssigned = c.IsFilter_Assigned; + return c.IsFilter; + } + #else + return false; + #endif +} + + UInt32 CCodecs::GetCodec_NumStreams(UInt32 index) { NCOM::CPropVariant prop; - RINOK(GetProperty(index, NMethodPropID::kPackStreams, &prop)); + if (GetProperty(index, NMethodPropID::kPackStreams, &prop) != S_OK) + return 0; if (prop.vt == VT_UI4) return (UInt32)prop.ulVal; if (prop.vt == VT_EMPTY) @@ -1063,10 +1257,75 @@ AString CCodecs::GetHasherName(UInt32 index) UInt32 CCodecs::GetHasherDigestSize(UInt32 index) { NCOM::CPropVariant prop; - RINOK(GetHasherProp(index, NMethodPropID::kDigestSize, &prop)); + if (GetHasherProp(index, NMethodPropID::kDigestSize, &prop) != S_OK) + return 0; if (prop.vt != VT_UI4) return 0; return prop.ulVal; } +void CCodecs::GetCodecsErrorMessage(UString &s) +{ + s.Empty(); + FOR_VECTOR (i, Errors) + { + const CCodecError &ce = Errors[i]; + s += "Codec Load Error: "; + s += fs2us(ce.Path); + if (ce.ErrorCode != 0) + { + s += " : "; + s += NWindows::NError::MyFormatMessage(ce.ErrorCode); + } + if (!ce.Message.IsEmpty()) + { + s += " : "; + s += ce.Message; + } + s.Add_LF(); + } +} + #endif // EXTERNAL_CODECS + +#ifndef _SFX + +extern unsigned g_NumCodecs; +extern const CCodecInfo *g_Codecs[]; + +void CCodecs::Get_CodecsInfoUser_Vector(CObjectVector &v) +{ + v.Clear(); + { + for (unsigned i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &cod = *g_Codecs[i]; + CCodecInfoUser &u = v.AddNew(); + u.EncoderIsAssigned = (cod.CreateEncoder != NULL); + u.DecoderIsAssigned = (cod.CreateDecoder != NULL); + u.IsFilter_Assigned = true; + u.IsFilter = cod.IsFilter; + u.NumStreams = cod.NumStreams; + u.Name = cod.Name; + } + } + + + #ifdef EXTERNAL_CODECS + { + UInt32 numMethods; + if (GetNumMethods(&numMethods) == S_OK) + for (UInt32 j = 0; j < numMethods; j++) + { + CCodecInfoUser &u = v.AddNew(); + u.EncoderIsAssigned = GetCodec_EncoderIsAssigned(j); + u.DecoderIsAssigned = GetCodec_DecoderIsAssigned(j); + u.IsFilter = GetCodec_IsFilter(j, u.IsFilter_Assigned); + u.NumStreams = GetCodec_NumStreams(j); + u.Name = GetCodec_Name(j); + } + } + #endif +} + +#endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.h b/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.h index ac9eeac71dd..50fb9f8f542 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/LoadCodecs.h @@ -68,6 +68,8 @@ struct CDllCodecInfo UInt32 CodecIndex; bool EncoderIsAssigned; bool DecoderIsAssigned; + bool IsFilter; + bool IsFilter_Assigned; CLSID Encoder; CLSID Decoder; }; @@ -94,6 +96,7 @@ struct CArcExtInfo struct CArcInfoEx { UInt32 Flags; + UInt32 TimeFlags; Func_CreateInArchive CreateInArchive; Func_IsArc IsArcFunc; @@ -119,11 +122,28 @@ struct CArcInfoEx CLSID ClassID; #endif + int Compare(const CArcInfoEx &a) const + { + int res = Name.Compare(a.Name); + if (res != 0) + return res; + #ifdef EXTERNAL_CODECS + return MyCompare(LibIndex, a.LibIndex); + #else + return 0; + #endif + /* + if (LibIndex < a.LibIndex) return -1; + if (LibIndex > a.LibIndex) return 1; + return 0; + */ + } + bool Flags_KeepName() const { return (Flags & NArcInfoFlags::kKeepName) != 0; } bool Flags_FindSignature() const { return (Flags & NArcInfoFlags::kFindSignature) != 0; } bool Flags_AltStreams() const { return (Flags & NArcInfoFlags::kAltStreams) != 0; } - bool Flags_NtSecure() const { return (Flags & NArcInfoFlags::kNtSecure) != 0; } + bool Flags_NtSecurity() const { return (Flags & NArcInfoFlags::kNtSecure) != 0; } bool Flags_SymLinks() const { return (Flags & NArcInfoFlags::kSymLinks) != 0; } bool Flags_HardLinks() const { return (Flags & NArcInfoFlags::kHardLinks) != 0; } @@ -132,7 +152,30 @@ struct CArcInfoEx bool Flags_BackwardOpen() const { return (Flags & NArcInfoFlags::kBackwardOpen) != 0; } bool Flags_PreArc() const { return (Flags & NArcInfoFlags::kPreArc) != 0; } bool Flags_PureStartOpen() const { return (Flags & NArcInfoFlags::kPureStartOpen) != 0; } - + bool Flags_ByExtOnlyOpen() const { return (Flags & NArcInfoFlags::kByExtOnlyOpen) != 0; } + bool Flags_HashHandler() const { return (Flags & NArcInfoFlags::kHashHandler) != 0; } + + bool Flags_CTime() const { return (Flags & NArcInfoFlags::kCTime) != 0; } + bool Flags_ATime() const { return (Flags & NArcInfoFlags::kATime) != 0; } + bool Flags_MTime() const { return (Flags & NArcInfoFlags::kMTime) != 0; } + + bool Flags_CTime_Default() const { return (Flags & NArcInfoFlags::kCTime_Default) != 0; } + bool Flags_ATime_Default() const { return (Flags & NArcInfoFlags::kATime_Default) != 0; } + bool Flags_MTime_Default() const { return (Flags & NArcInfoFlags::kMTime_Default) != 0; } + + UInt32 Get_TimePrecFlags() const + { + return (TimeFlags >> NArcInfoTimeFlags::kTime_Prec_Mask_bit_index) & + (((UInt32)1 << NArcInfoTimeFlags::kTime_Prec_Mask_num_bits) - 1); + } + + UInt32 Get_DefaultTimePrec() const + { + return (TimeFlags >> NArcInfoTimeFlags::kTime_Prec_Default_bit_index) & + (((UInt32)1 << NArcInfoTimeFlags::kTime_Prec_Default_num_bits) - 1); + } + + UString GetMainExt() const { if (Exts.IsEmpty()) @@ -141,6 +184,15 @@ struct CArcInfoEx } int FindExtension(const UString &ext) const; + bool Is_7z() const { return Name.IsEqualTo_Ascii_NoCase("7z"); } + bool Is_Split() const { return Name.IsEqualTo_Ascii_NoCase("Split"); } + bool Is_Xz() const { return Name.IsEqualTo_Ascii_NoCase("xz"); } + bool Is_BZip2() const { return Name.IsEqualTo_Ascii_NoCase("bzip2"); } + bool Is_GZip() const { return Name.IsEqualTo_Ascii_NoCase("gzip"); } + bool Is_Tar() const { return Name.IsEqualTo_Ascii_NoCase("tar"); } + bool Is_Zip() const { return Name.IsEqualTo_Ascii_NoCase("zip"); } + bool Is_Rar() const { return Name.IsEqualTo_Ascii_NoCase("rar"); } + /* UString GetAllExtensions() const { @@ -157,11 +209,10 @@ struct CArcInfoEx void AddExts(const UString &ext, const UString &addExt); - bool IsSplit() const { return StringsAreEqualNoCase_Ascii(Name, "Split"); } - // bool IsRar() const { return StringsAreEqualNoCase_Ascii(Name, "Rar"); } CArcInfoEx(): Flags(0), + TimeFlags(0), CreateInArchive(NULL), IsArcFunc(NULL) #ifndef _SFX @@ -227,6 +278,28 @@ struct CCodecLib #endif +struct CCodecError +{ + FString Path; + HRESULT ErrorCode; + AString Message; + CCodecError(): ErrorCode(0) {} +}; + + +struct CCodecInfoUser +{ + // unsigned LibIndex; + // UInt32 CodecIndex; + // UInt64 id; + bool EncoderIsAssigned; + bool DecoderIsAssigned; + bool IsFilter; + bool IsFilter_Assigned; + UInt32 NumStreams; + AString Name; +}; + class CCodecs: #ifdef EXTERNAL_CODECS @@ -243,7 +316,9 @@ public: CObjectVector Libs; FString MainDll_ErrorPath; - + CObjectVector Errors; + + void AddLastError(const FString &path); void CloseLibs(); class CReleaser @@ -272,7 +347,7 @@ public: HRESULT CreateArchiveHandler(const CArcInfoEx &ai, bool outHandler, void **archive) const { - return Libs[ai.LibIndex].CreateObject(&ai.ClassID, outHandler ? &IID_IOutArchive : &IID_IInArchive, (void **)archive); + return Libs[(unsigned)ai.LibIndex].CreateObject(&ai.ClassID, outHandler ? &IID_IOutArchive : &IID_IInArchive, (void **)archive); } #endif @@ -288,14 +363,14 @@ public: CRecordVector Hashers; #endif - bool CaseSensitiveChange; + bool CaseSensitive_Change; bool CaseSensitive; CCodecs(): #ifdef EXTERNAL_CODECS NeedSetLibCodecs(true), #endif - CaseSensitiveChange(false), + CaseSensitive_Change(false), CaseSensitive(false) {} @@ -306,11 +381,11 @@ public: const wchar_t *GetFormatNamePtr(int formatIndex) const { - return formatIndex < 0 ? L"#" : (const wchar_t *)Formats[formatIndex].Name; + return formatIndex < 0 ? L"#" : (const wchar_t *)Formats[(unsigned)formatIndex].Name; } HRESULT Load(); - + #ifndef _SFX int FindFormatForArchiveName(const UString &arcPath) const; int FindFormatForExtension(const UString &ext) const; @@ -343,6 +418,7 @@ public: int GetCodec_LibIndex(UInt32 index) const; bool GetCodec_DecoderIsAssigned(UInt32 index) const; bool GetCodec_EncoderIsAssigned(UInt32 index) const; + bool GetCodec_IsFilter(UInt32 index, bool &isAssigned) const; UInt32 GetCodec_NumStreams(UInt32 index); HRESULT GetCodec_Id(UInt32 index, UInt64 &id); AString GetCodec_Name(UInt32 index); @@ -352,6 +428,8 @@ public: AString GetHasherName(UInt32 index); UInt32 GetHasherDigestSize(UInt32 index); + void GetCodecsErrorMessage(UString &s); + #endif HRESULT CreateInArchive(unsigned formatIndex, CMyComPtr &archive) const @@ -399,11 +477,13 @@ public: if (!arc.UpdateEnabled) continue; if (arc.Name.IsEqualTo_NoCase(name)) - return i; + return (int)i; } return -1; } + void Get_CodecsInfoUser_Vector(CObjectVector &v); + #endif // _SFX }; @@ -420,5 +500,5 @@ public: CCodecs *codecs = new CCodecs; \ CMyComPtr __codecsRef = codecs; #endif - + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.cpp index bf3b982e44e..4a91a2686eb 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.cpp @@ -14,6 +14,7 @@ #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" +#include "../../../Common/UTFConvert.h" #include "../../../Common/Wildcard.h" #include "../../../Windows/FileDir.h" @@ -32,11 +33,13 @@ #include "SetProperties.h" #endif +#ifndef _SFX #ifdef SHOW_DEBUG_INFO #define PRF(x) x #else #define PRF(x) #endif +#endif // increase it, if you need to support larger SFX stubs static const UInt64 kMaxCheckStartPosition = 1 << 23; @@ -64,7 +67,7 @@ Open: - open FAIL: Try to open with all other types from offset 0 only. If some open type is OK and physical archive size is uequal or larger - than file size, then return that archive with warning that can not be open as [extension type]. + than file size, then return that archive with warning that cannot be open as [extension type]. If extension was EXE, it will try to open as unknown_extension case - file has unknown extension (like a.hhh) It tries to open via parser code. @@ -141,14 +144,14 @@ struct CParseItem bool LenIsUnknown; CParseItem(): - LenIsUnknown(false), + // OkSize(0), FileTime_Defined(false), UnpackSize_Defined(false), - NumSubFiles_Defined(false), NumSubDirs_Defined(false), + NumSubFiles_Defined(false), IsSelfExe(false), - IsNotArcType(false) - // OkSize(0) + IsNotArcType(false), + LenIsUnknown(false) {} /* @@ -206,23 +209,25 @@ int CHandler::FindInsertPos(const CParseItem &item) const unsigned left = 0, right = _items.Size(); while (left != right) { - unsigned mid = (left + right) / 2; - const CParseItem & midItem = _items[mid]; + const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const CParseItem &midItem = _items[mid]; if (item.Offset < midItem.Offset) right = mid; else if (item.Offset > midItem.Offset) left = mid + 1; else if (item.Size < midItem.Size) right = mid; + /* else if (item.Size > midItem.Size) left = mid + 1; + */ else { left = mid + 1; // return -1; } } - return left; + return (int)left; } void CHandler::AddUnknownItem(UInt64 next) @@ -257,10 +262,10 @@ void CHandler::AddUnknownItem(UInt64 next) void CHandler::AddItem(const CParseItem &item) { AddUnknownItem(item.Offset); - int pos = FindInsertPos(item); - if (pos >= 0) + const int pos = FindInsertPos(item); + if (pos != -1) { - _items.Insert(pos, item); + _items.Insert((unsigned)pos, item); UInt64 next = item.Offset + item.Size; if (_maxEndOffset < next) _maxEndOffset = next; @@ -330,17 +335,17 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val { case kpidPath: { - wchar_t sz[32]; + char sz[32]; ConvertUInt32ToString(index + 1, sz); - UString s = sz; + UString s(sz); if (!item.Name.IsEmpty()) { - s += L'.'; + s += '.'; s += item.Name; } if (!item.Extension.IsEmpty()) { - s += L'.'; + s += '.'; s += item.Extension; } prop = s; break; @@ -401,7 +406,7 @@ HRESULT CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; - Int32 index = allFilesMode ? i : indices[i]; + UInt32 index = allFilesMode ? i : indices[i]; const CParseItem &item = _items[index]; RINOK(extractCallback->GetStream(index, &realOutStream, askMode)); @@ -417,7 +422,7 @@ HRESULT CHandler::Extract(const UInt32 *indices, UInt32 numItems, outStreamSpec->Init(skipMode ? 0 : unpackSize, true); Int32 opRes = NExtract::NOperationResult::kOK; - RINOK(_stream->Seek(item.Offset, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek((Int64)item.Offset, STREAM_SEEK_SET, NULL)); streamSpec->Init(unpackSize); RINOK(copyCoder->Code(inStream, outStream, NULL, NULL, progress)); @@ -477,7 +482,7 @@ HRESULT Archive_IsItem_Deleted(IInArchive *arc, UInt32 index, bool &result) thro return Archive_GetItemBoolProp(arc, index, kpidIsDeleted, result); } -static HRESULT Archive_GetArcBoolProp(IInArchive *arc, PROPID propid, bool &result) throw() +static HRESULT Archive_GetArcProp_Bool(IInArchive *arc, PROPID propid, bool &result) throw() { NCOM::CPropVariant prop; result = false; @@ -496,13 +501,14 @@ static HRESULT Archive_GetArcProp_UInt(IInArchive *arc, PROPID propid, UInt64 &r RINOK(arc->GetArchiveProperty(propid, &prop)); switch (prop.vt) { - case VT_UI4: result = prop.ulVal; defined = true; break; - case VT_I4: result = (Int64)prop.lVal; defined = true; break; - case VT_UI8: result = (UInt64)prop.uhVal.QuadPart; defined = true; break; - case VT_I8: result = (UInt64)prop.hVal.QuadPart; defined = true; break; - case VT_EMPTY: break; + case VT_UI4: result = prop.ulVal; break; + case VT_I4: result = (UInt64)(Int64)prop.lVal; break; + case VT_UI8: result = (UInt64)prop.uhVal.QuadPart; break; + case VT_I8: result = (UInt64)prop.hVal.QuadPart; break; + case VT_EMPTY: return S_OK; default: return E_FAIL; } + defined = true; return S_OK; } @@ -513,19 +519,20 @@ static HRESULT Archive_GetArcProp_Int(IInArchive *arc, PROPID propid, Int64 &res RINOK(arc->GetArchiveProperty(propid, &prop)); switch (prop.vt) { - case VT_UI4: result = prop.ulVal; defined = true; break; - case VT_I4: result = prop.lVal; defined = true; break; - case VT_UI8: result = (Int64)prop.uhVal.QuadPart; defined = true; break; - case VT_I8: result = (Int64)prop.hVal.QuadPart; defined = true; break; - case VT_EMPTY: break; + case VT_UI4: result = prop.ulVal; break; + case VT_I4: result = prop.lVal; break; + case VT_UI8: result = (Int64)prop.uhVal.QuadPart; break; + case VT_I8: result = (Int64)prop.hVal.QuadPart; break; + case VT_EMPTY: return S_OK; default: return E_FAIL; } + defined = true; return S_OK; } #ifndef _SFX -HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &parts) const +HRESULT CArc::GetItem_PathToParent(UInt32 index, UInt32 parent, UStringVector &parts) const { if (!GetRawProps) return E_FAIL; @@ -563,6 +570,8 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa UInt32 parentType = 0; RINOK(GetRawProps->GetParent(curIndex, &curParent, &parentType)); + // 18.06: fixed : we don't want to split name to parts + /* if (parentType != NParentType::kAltStream) { for (;;) @@ -576,6 +585,7 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa s.DeleteFrom(pos); } } + */ parts.Insert(0, s); @@ -583,7 +593,7 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa { { UString &s2 = parts[parts.Size() - 2]; - s2 += L':'; + s2 += ':'; s2 += parts.Back(); } parts.DeleteBack(); @@ -604,7 +614,9 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa #endif -HRESULT CArc::GetItemPath(UInt32 index, UString &result) const + + +HRESULT CArc::GetItem_Path(UInt32 index, UString &result) const { #ifdef MY_CPU_LE if (GetRawProps) @@ -618,19 +630,42 @@ HRESULT CArc::GetItemPath(UInt32 index, UString &result) const propType == NPropDataType::kUtf16z) { unsigned len = size / 2 - 1; + // (len) doesn't include null terminator + + /* + #if WCHAR_MAX > 0xffff + len = (unsigned)Utf16LE__Get_Num_WCHARs(p, len); + + wchar_t *s = result.GetBuf(len); + wchar_t *sEnd = Utf16LE__To_WCHARs_Sep(p, len, s); + if (s + len != sEnd) return E_FAIL; + *sEnd = 0; + + #else + */ + wchar_t *s = result.GetBuf(len); for (unsigned i = 0; i < len; i++) { wchar_t c = GetUi16(p); p = (const void *)((const Byte *)p + 2); + #if WCHAR_PATH_SEPARATOR != L'/' if (c == L'/') c = WCHAR_PATH_SEPARATOR; + else if (c == L'\\') + c = WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT; // WSL scheme #endif + *s++ = c; } *s = 0; + + // #endif + result.ReleaseBuf_SetLen(len); + + Convert_UnicodeEsc16_To_UnicodeEscHigh(result); if (len != 0) return S_OK; } @@ -717,11 +752,13 @@ HRESULT CArc::GetItemPath(UInt32 index, UString &result) const } if (result.IsEmpty()) - return GetDefaultItemPath(index, result); + return GetItem_DefaultPath(index, result); + + Convert_UnicodeEsc16_To_UnicodeEscHigh(result); return S_OK; } -HRESULT CArc::GetDefaultItemPath(UInt32 index, UString &result) const +HRESULT CArc::GetItem_DefaultPath(UInt32 index, UString &result) const { result.Empty(); bool isDir; @@ -733,7 +770,7 @@ HRESULT CArc::GetDefaultItemPath(UInt32 index, UString &result) const RINOK(Archive->GetProperty(index, kpidExtension, &prop)); if (prop.vt == VT_BSTR) { - result += L'.'; + result += '.'; result += prop.bstrVal; } else if (prop.vt != VT_EMPTY) @@ -742,9 +779,9 @@ HRESULT CArc::GetDefaultItemPath(UInt32 index, UString &result) const return S_OK; } -HRESULT CArc::GetItemPath2(UInt32 index, UString &result) const +HRESULT CArc::GetItem_Path2(UInt32 index, UString &result) const { - RINOK(GetItemPath(index, result)); + RINOK(GetItem_Path(index, result)); if (Ask_Deleted) { bool isDeleted = false; @@ -769,7 +806,7 @@ int FindAltStreamColon_in_Path(const wchar_t *path) if (c == ':') { if (colonPos < 0) - colonPos = i; + colonPos = (int)i; continue; } if (c == WCHAR_PATH_SEPARATOR) @@ -796,7 +833,7 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const RINOK(Archive_IsItem_Dir(Archive, index, item.IsDir)); item.MainIsDir = item.IsDir; - RINOK(GetItemPath2(index, item.Path)); + RINOK(GetItem_Path2(index, item.Path)); #ifndef _SFX UInt32 mainIndex = index; @@ -848,7 +885,7 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const } else { - RINOK(GetItemPath2(parentIndex, item.MainPath)); + RINOK(GetItem_Path2(parentIndex, item.MainPath)); RINOK(Archive_IsItem_Dir(Archive, parentIndex, item.MainIsDir)); } } @@ -857,13 +894,13 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const if (item.WriteToAltStreamIfColon || needFindAltStream) { - /* Good handler must support GetRawProps::GetParent for alt streams./ + /* Good handler must support GetRawProps::GetParent for alt streams. So the following code currently is not used */ int colon = FindAltStreamColon_in_Path(item.Path); if (colon >= 0) { - item.MainPath.DeleteFrom(colon); - item.AltStreamName = item.Path.Ptr(colon + 1); + item.MainPath.DeleteFrom((unsigned)colon); + item.AltStreamName = item.Path.Ptr((unsigned)(colon + 1)); item.MainIsDir = (colon == 0 || IsPathSepar(item.Path[(unsigned)colon - 1])); item.IsAltStream = true; } @@ -874,7 +911,7 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const #ifndef _SFX if (item._use_baseParentFolder_mode) { - RINOK(GetItemPathToParent(mainIndex, item._baseParentFolder, item.PathParts)); + RINOK(GetItem_PathToParent(mainIndex, (unsigned)item._baseParentFolder, item.PathParts)); #ifdef SUPPORT_ALT_STREAMS if ((item.WriteToAltStreamIfColon || needFindAltStream) && !item.PathParts.IsEmpty()) @@ -885,10 +922,10 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const colon = FindAltStreamColon_in_Path(s); if (colon >= 0) { - item.AltStreamName = s.Ptr(colon + 1); + item.AltStreamName = s.Ptr((unsigned)(colon + 1)); item.MainIsDir = (colon == 0 || IsPathSepar(s[(unsigned)colon - 1])); item.IsAltStream = true; - s.DeleteFrom(colon); + s.DeleteFrom((unsigned)colon); } } if (colon == 0) @@ -933,7 +970,7 @@ static HRESULT Archive_GetItem_Size(IInArchive *archive, UInt32 index, UInt64 &s #endif -HRESULT CArc::GetItemSize(UInt32 index, UInt64 &size, bool &defined) const +HRESULT CArc::GetItem_Size(UInt32 index, UInt64 &size, bool &defined) const { NCOM::CPropVariant prop; defined = false; @@ -952,24 +989,52 @@ HRESULT CArc::GetItemSize(UInt32 index, UInt64 &size, bool &defined) const return S_OK; } -HRESULT CArc::GetItemMTime(UInt32 index, FILETIME &ft, bool &defined) const +HRESULT CArc::GetItem_MTime(UInt32 index, CArcTime &at) const { + at.Clear(); NCOM::CPropVariant prop; - defined = false; - ft.dwHighDateTime = ft.dwLowDateTime = 0; RINOK(Archive->GetProperty(index, kpidMTime, &prop)); + if (prop.vt == VT_FILETIME) { - ft = prop.filetime; - defined = true; + /* + // for debug + if (FILETIME_IsZero(prop.at) && MTime.Def) + { + at = MTime; + return S_OK; + } + */ + at.Set_From_Prop(prop); + if (at.Prec == 0) + { + // (at.Prec == 0) before version 22. + // so kpidTimeType is required for that code + prop.Clear(); + RINOK(Archive->GetProperty(index, kpidTimeType, &prop)); + if (prop.vt == VT_UI4) + { + UInt32 val = prop.ulVal; + if (val == NFileTimeType::kWindows) + val = k_PropVar_TimePrec_100ns; + /* + else if (val > k_PropVar_TimePrec_1ns) + { + val = k_PropVar_TimePrec_100ns; + // val = k_PropVar_TimePrec_1ns; + // return E_FAIL; // for debug + } + */ + at.Prec = (UInt16)val; + } + } + return S_OK; } - else if (prop.vt != VT_EMPTY) + + if (prop.vt != VT_EMPTY) return E_FAIL; - else if (MTimeDefined) - { - ft = MTime; - defined = true; - } + if (MTime.Def) + at = MTime; return S_OK; } @@ -983,52 +1048,52 @@ static inline bool TestSignature(const Byte *p1, const Byte *p2, size_t size) return true; } + static void MakeCheckOrder(CCodecs *codecs, CIntVector &orderIndices, unsigned numTypes, CIntVector &orderIndices2, const Byte *data, size_t dataSize) { for (unsigned i = 0; i < numTypes; i++) { - int index = orderIndices[i]; + const int index = orderIndices[i]; if (index < 0) continue; - const CArcInfoEx &ai = codecs->Formats[index]; - if (ai.SignatureOffset != 0) - { - orderIndices2.Add(index); - orderIndices[i] = -1; - continue; - } - - const CObjectVector &sigs = ai.Signatures; - FOR_VECTOR (k, sigs) + const CArcInfoEx &ai = codecs->Formats[(unsigned)index]; + if (ai.SignatureOffset == 0) { - const CByteBuffer &sig = sigs[k]; - if (sig.Size() == 0 && dataSize == 0 || - sig.Size() != 0 && sig.Size() <= dataSize && - TestSignature(data, sig, sig.Size())) + if (ai.Signatures.IsEmpty()) { - orderIndices2.Add(index); - orderIndices[i] = -1; - break; + if (dataSize != 0) // 21.04: no Signature means Empty Signature + continue; + } + else + { + unsigned k; + const CObjectVector &sigs = ai.Signatures; + for (k = 0; k < sigs.Size(); k++) + { + const CByteBuffer &sig = sigs[k]; + if (sig.Size() <= dataSize && TestSignature(data, sig, sig.Size())) + break; + } + if (k == sigs.Size()) + continue; } } + orderIndices2.Add(index); + orderIndices[i] = -1; } } -#endif - #ifdef UNDER_CE static const unsigned kNumHashBytes = 1; - #define HASH_VAL(buf, pos) ((buf)[pos]) + #define HASH_VAL(buf) ((buf)[0]) #else static const unsigned kNumHashBytes = 2; - #define HASH_VAL(buf, pos) ((buf)[pos] | ((UInt32)(buf)[pos + 1] << 8)) + // #define HASH_VAL(buf) ((buf)[0] | ((UInt32)(buf)[1] << 8)) + #define HASH_VAL(buf) GetUi16(buf) #endif - -#ifndef _SFX - static bool IsExeExt(const UString &ext) { return ext.IsEqualTo_Ascii_NoCase("exe"); @@ -1193,7 +1258,7 @@ void CArcErrorInfo::ClearErrors() HRESULT CArc::ReadBasicProps(IInArchive *archive, UInt64 startPos, HRESULT openRes) { // OkPhySize_Defined = false; - PhySizeDefined = false; + PhySize_Defined = false; PhySize = 0; Offset = 0; AvailPhySize = FileSize - startPos; @@ -1226,12 +1291,12 @@ HRESULT CArc::ReadBasicProps(IInArchive *archive, UInt64 startPos, HRESULT openR if (openRes == S_OK || ErrorInfo.IsArc_After_NonOpen()) { - RINOK(Archive_GetArcProp_UInt(archive, kpidPhySize, PhySize, PhySizeDefined)); + RINOK(Archive_GetArcProp_UInt(archive, kpidPhySize, PhySize, PhySize_Defined)); /* RINOK(Archive_GetArcProp_UInt(archive, kpidOkPhySize, OkPhySize, OkPhySize_Defined)); if (!OkPhySize_Defined) { - OkPhySize_Defined = PhySizeDefined; + OkPhySize_Defined = PhySize_Defined; OkPhySize = PhySize; } */ @@ -1239,11 +1304,11 @@ HRESULT CArc::ReadBasicProps(IInArchive *archive, UInt64 startPos, HRESULT openR bool offsetDefined; RINOK(Archive_GetArcProp_Int(archive, kpidOffset, Offset, offsetDefined)); - Int64 globalOffset = startPos + Offset; - AvailPhySize = FileSize - globalOffset; - if (PhySizeDefined) + Int64 globalOffset = (Int64)startPos + Offset; + AvailPhySize = (UInt64)((Int64)FileSize - globalOffset); + if (PhySize_Defined) { - UInt64 endPos = globalOffset + PhySize; + UInt64 endPos = (UInt64)(globalOffset + (Int64)PhySize); if (endPos < FileSize) { AvailPhySize = PhySize; @@ -1259,11 +1324,12 @@ HRESULT CArc::ReadBasicProps(IInArchive *archive, UInt64 startPos, HRESULT openR } /* -static PrintNumber(const char *s, int n) +static void PrintNumber(const char *s, int n) { char temp[100]; sprintf(temp, "%s %d", s, n); - OutputDebugStringA(temp); + // OutputDebugStringA(temp); + printf(temp); } */ @@ -1282,7 +1348,7 @@ HRESULT CArc::PrepareToOpen(const COpenOptions &op, unsigned formatIndex, CMyCom { const CArcInfoEx &ai = op.codecs->Formats[formatIndex]; if (ai.LibIndex >= 0 ? - !op.codecs->Libs[ai.LibIndex].SetCodecs : + !op.codecs->Libs[(unsigned)ai.LibIndex].SetCodecs : !op.codecs->Libs.IsEmpty()) { CMyComPtr setCompressCodecsInfo; @@ -1341,9 +1407,9 @@ static HRESULT ReadParseItemProps(IInArchive *archive, const CArcInfoEx &ai, NAr pi.FileTime_Defined = false; pi.ArcType = ai.Name; - RINOK(Archive_GetArcBoolProp(archive, kpidIsNotArcType, pi.IsNotArcType)); + RINOK(Archive_GetArcProp_Bool(archive, kpidIsNotArcType, pi.IsNotArcType)); - // RINOK(Archive_GetArcBoolProp(archive, kpidIsSelfExe, pi.IsSelfExe)); + // RINOK(Archive_GetArcProp_Bool(archive, kpidIsSelfExe, pi.IsSelfExe)); pi.IsSelfExe = ai.Flags_PreArc(); { @@ -1433,7 +1499,7 @@ HRESULT CArc::CheckZerosTail(const COpenOptions &op, UInt64 offset) { if (!op.stream) return S_OK; - RINOK(op.stream->Seek(offset, STREAM_SEEK_SET, NULL)); + RINOK(op.stream->Seek((Int64)offset, STREAM_SEEK_SET, NULL)); const UInt32 kBufSize = 1 << 11; Byte buf[kBufSize]; @@ -1459,6 +1525,8 @@ HRESULT CArc::CheckZerosTail(const COpenOptions &op, UInt64 offset) } } + + #ifndef _SFX class CExtractCallback_To_OpenCallback: @@ -1506,7 +1574,7 @@ STDMETHODIMP CExtractCallback_To_OpenCallback::SetRatioInfo(const UInt64 *inSize STDMETHODIMP CExtractCallback_To_OpenCallback::GetStream(UInt32 /* index */, ISequentialOutStream **outStream, Int32 /* askExtractMode */) { - *outStream = 0; + *outStream = NULL; return S_OK; } @@ -1520,6 +1588,7 @@ STDMETHODIMP CExtractCallback_To_OpenCallback::SetOperationResult(Int32 /* opera return S_OK; } + static HRESULT OpenArchiveSpec(IInArchive *archive, bool needPhySize, IInStream *stream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *openCallback, @@ -1543,22 +1612,32 @@ static HRESULT OpenArchiveSpec(IInArchive *archive, bool needPhySize, if (phySize_Defined) return S_OK; - bool phySizeCantBeDetected = false;; - RINOK(Archive_GetArcBoolProp(archive, kpidPhySizeCantBeDetected, phySizeCantBeDetected)); + bool phySizeCantBeDetected = false; + RINOK(Archive_GetArcProp_Bool(archive, kpidPhySizeCantBeDetected, phySizeCantBeDetected)); if (!phySizeCantBeDetected) { - RINOK(archive->Extract(0, (UInt32)(Int32)-1, BoolToInt(true), extractCallback)); + PRF(printf("\n-- !phySize_Defined after Open, call archive->Extract()")); + // It's for bzip2/gz and some xz archives, where Open operation doesn't know phySize. + // But the Handler will know phySize after full archive testing. + RINOK(archive->Extract(NULL, (UInt32)(Int32)-1, BoolToInt(true), extractCallback)); + PRF(printf("\n-- OK")); } } return S_OK; } + + static int FindFormatForArchiveType(CCodecs *codecs, CIntVector orderIndices, const char *name) { FOR_VECTOR (i, orderIndices) - if (StringsAreEqualNoCase_Ascii(codecs->Formats[orderIndices[i]].Name, name)) - return i; + { + int oi = orderIndices[i]; + if (oi >= 0) + if (StringsAreEqualNoCase_Ascii(codecs->Formats[(unsigned)oi].Name, name)) + return (int)i; + } return -1; } @@ -1586,7 +1665,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { int dotPos = fileName.ReverseFind_Dot(); if (dotPos >= 0) - extension = fileName.Ptr(dotPos + 1); + extension = fileName.Ptr((unsigned)(dotPos + 1)); } CIntVector orderIndices; @@ -1611,13 +1690,18 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool isUnknownExt = false; #endif + #ifndef _SFX bool isForced = false; + #endif + unsigned numMainTypes = 0; int formatIndex = op.openType.FormatIndex; if (formatIndex >= 0) { + #ifndef _SFX isForced = true; + #endif orderIndices.Add(formatIndex); numMainTypes = 1; isMainFormatArr[(unsigned)formatIndex] = true; @@ -1654,10 +1738,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) isNumber = true; } if (isNumber) + { if (c == 'z' || c == 'Z') isZip = true; else isRar = true; + } } #endif @@ -1667,9 +1753,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) const CArcInfoEx &ai = op.codecs->Formats[i]; if (IgnoreSplit || !op.openType.CanReturnArc) - if (ai.IsSplit()) + if (ai.Is_Split()) continue; - if (op.excludedFormats->FindInSorted(i) >= 0) + if (op.excludedFormats->FindInSorted((int)i) >= 0) continue; #ifndef _SFX @@ -1679,17 +1765,17 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (ai.FindExtension(extension) >= 0 #ifndef _SFX - || isZip && StringsAreEqualNoCase_Ascii(ai.Name, "zip") - || isRar && StringsAreEqualNoCase_Ascii(ai.Name, "rar") + || (isZip && ai.Is_Zip()) + || (isRar && ai.Is_Rar()) #endif ) { // PrintNumber("orderIndices.Insert", i); - orderIndices.Insert(numFinded++, i); + orderIndices.Insert(numFinded++, (int)i); isMainFormatArr[i] = true; } else - orderIndices.Add(i); + orderIndices.Add((int)i); } } @@ -1735,8 +1821,8 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) const Byte kRarHeader[] = { 0x52 , 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00 }; if (TestSignature(buf, kRarHeader, 7) && buf[9] == 0x73 && (buf[10] & 1) != 0) { - orderIndices2.Add(orderIndices[i]); - orderIndices[i] = -1; + orderIndices2.Add(orderIndices[(unsigned)i]); + orderIndices[(unsigned)i] = -1; if (i >= (int)numFinded) numFinded++; } @@ -1754,11 +1840,27 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) /* check type order: - 1) matched extension, no signuature - 2) matched extension, matched signuature + 0) matched_extension && Backward + 1) matched_extension && (no_signuature || SignatureOffset != 0) + 2) matched_extension && (matched_signature) // 3) no signuature // 4) matched signuature */ + // we move index from orderIndices to orderIndices2 for priority handlers. + + for (unsigned i = 0; i < numFinded; i++) + { + const int index = orderIndices[i]; + if (index < 0) + continue; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)index]; + if (ai.Flags_BackwardOpen()) + { + // backward doesn't need start signatures + orderIndices2.Add(index); + orderIndices[i] = -1; + } + } MakeCheckOrder(op.codecs, orderIndices, numFinded, orderIndices2, NULL, 0); MakeCheckOrder(op.codecs, orderIndices, numFinded, orderIndices2, byteBuffer, processedSize); @@ -1781,10 +1883,10 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) int iUdf = FindFormatForArchiveType(op.codecs, orderIndices, "udf"); if (iUdf > iIso && iIso >= 0) { - int isoIndex = orderIndices[iIso]; - int udfIndex = orderIndices[iUdf]; - orderIndices[iUdf] = isoIndex; - orderIndices[iIso] = udfIndex; + int isoIndex = orderIndices[(unsigned)iIso]; + int udfIndex = orderIndices[(unsigned)iUdf]; + orderIndices[(unsigned)iUdf] = isoIndex; + orderIndices[(unsigned)iIso] = udfIndex; } } @@ -1838,15 +1940,18 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) for (unsigned i = 0; i < numCheckTypes; i++) { FormatIndex = orderIndices[i]; + + // orderIndices[] item cannot be negative here bool exactOnly = false; #ifndef _SFX - const CArcInfoEx &ai = op.codecs->Formats[FormatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)FormatIndex]; // OutputDebugStringW(ai.Name); if (i >= numMainTypes) { + // here we allow mismatched extension only for backward handlers if (!ai.Flags_BackwardOpen() // && !ai.Flags_PureStartOpen() ) @@ -1867,7 +1972,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) CMyComPtr archive; - RINOK(PrepareToOpen(op, FormatIndex, archive)); + RINOK(PrepareToOpen(op, (unsigned)FormatIndex, archive)); if (!archive) continue; @@ -1944,7 +2049,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool thereIsTail = ErrorInfo.ThereIsTail; if (thereIsTail && mode.ZerosTailIsAllowed) { - RINOK(CheckZerosTail(op, Offset + PhySize)); + RINOK(CheckZerosTail(op, (UInt64)(Offset + (Int64)PhySize))); if (ErrorInfo.IgnoreTail) thereIsTail = false; } @@ -2012,7 +2117,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } else { - const CArcInfoEx &ai = op.codecs->Formats[formatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)formatIndex]; if (ai.FindExtension(extension) >= 0) { if (ai.Flags_FindSignature() && searchMarkerInHandler) @@ -2059,16 +2164,22 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) for (i = 0; i < orderIndices.Size(); i++) { - unsigned form = orderIndices[i]; + // orderIndices[] item cannot be negative here + unsigned form = (unsigned)orderIndices[i]; if (skipFrontalFormat[form]) continue; + const CArcInfoEx &ai = op.codecs->Formats[form]; - if (ai.IsSplit()) + + if (ai.Is_Split()) { - splitIndex = form; + splitIndex = (int)form; continue; } + if (ai.Flags_ByExtOnlyOpen()) + continue; + if (ai.IsArcFunc) { UInt32 isArcRes = ai.IsArcFunc(byteBuffer, processedSize); @@ -2081,7 +2192,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) continue; } - bool isNewStyleSignature = IsNewStyleSignature(ai); + const bool isNewStyleSignature = IsNewStyleSignature(ai); bool needCheck = !isNewStyleSignature || ai.Signatures.IsEmpty() || ai.Flags_PureStartOpen() @@ -2094,13 +2205,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) for (k = 0; k < ai.Signatures.Size(); k++) { const CByteBuffer &sig = ai.Signatures[k]; - UInt32 signatureEnd = ai.SignatureOffset + (UInt32)sig.Size(); - if (processedSize < signatureEnd) + if (processedSize < ai.SignatureOffset + sig.Size()) { if (!endOfFile) needCheck = true; } - else if (memcmp(sig, byteBuffer + ai.SignatureOffset, sig.Size()) == 0) + else if (TestSignature(sig, byteBuffer + ai.SignatureOffset, sig.Size())) break; } if (k != ai.Signatures.Size()) @@ -2114,12 +2224,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } if (splitIndex >= 0) - sortedFormats.Insert(0, splitIndex); + sortedFormats.Insert(0, (unsigned)splitIndex); for (i = 0; i < sortedFormats.Size(); i++) { - FormatIndex = sortedFormats[i]; - const CArcInfoEx &ai = op.codecs->Formats[FormatIndex]; + FormatIndex = (int)sortedFormats[i]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)FormatIndex]; if (op.callback) RINOK(op.callback->SetTotal(NULL, &fileSize)); @@ -2127,7 +2237,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) RINOK(op.stream->Seek(0, STREAM_SEEK_SET, NULL)); CMyComPtr archive; - RINOK(PrepareToOpen(op, FormatIndex, archive)); + RINOK(PrepareToOpen(op, (unsigned)FormatIndex, archive)); if (!archive) continue; @@ -2140,7 +2250,10 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) result = archive->Open(op.stream, &searchLimit, op.callback); else */ - result = OpenArchiveSpec(archive, !mode.CanReturnArc, op.stream, &searchLimit, op.callback, extractCallback_To_OpenCallback); + // if (!CanReturnArc), it's ParserMode, and we need phy size + result = OpenArchiveSpec(archive, + !mode.CanReturnArc, // needPhySize + op.stream, &searchLimit, op.callback, extractCallback_To_OpenCallback); } if (result == S_FALSE) @@ -2162,12 +2275,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } NArchive::NParser::CParseItem pi; - pi.Offset = Offset; + pi.Offset = (UInt64)Offset; pi.Size = AvailPhySize; // bool needScan = false; - if (!PhySizeDefined) + if (!PhySize_Defined) { // it's for Z format pi.LenIsUnknown = true; @@ -2199,7 +2312,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { if (mode.ZerosTailIsAllowed) { - RINOK(CheckZerosTail(op, Offset + PhySize)); + RINOK(CheckZerosTail(op, (UInt64)(Offset + (Int64)PhySize))); if (ErrorInfo.IgnoreTail) openCur = true; } @@ -2294,7 +2407,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) int index = orderIndices[i]; if (index < 0) continue; - const CArcInfoEx &ai = op.codecs->Formats[index]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)index]; + if (ai.Flags_ByExtOnlyOpen()) + continue; bool isDifficult = false; // if (ai.Version < 0x91F) // we don't use parser with old DLL (before 9.31) if (!ai.NewInterface) @@ -2317,7 +2432,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) continue; } thereAreHandlersForSearch = true; - UInt32 v = HASH_VAL(sig, 0); + UInt32 v = HASH_VAL(sig); unsigned sigIndex = arc2sig[(unsigned)index] + k; prevs[sigIndex] = hash[v]; hash[v] = (Byte)sigIndex; @@ -2325,8 +2440,8 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } if (isDifficult) { - difficultFormats.Add(index); - difficultBools[index] = true; + difficultFormats.Add((unsigned)index); + difficultBools[(unsigned)index] = true; } } @@ -2394,7 +2509,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) size_t processedSize = kBufSize - bytesInBuf; // printf("\nRead ask = %d", (unsigned)processedSize); UInt64 seekPos = bufPhyPos + bytesInBuf; - RINOK(op.stream->Seek(bufPhyPos + bytesInBuf, STREAM_SEEK_SET, NULL)); + RINOK(op.stream->Seek((Int64)(bufPhyPos + bytesInBuf), STREAM_SEEK_SET, NULL)); RINOK(ReadStream(op.stream, byteBuffer + bytesInBuf, &processedSize)); // printf(" processed = %d", (unsigned)processedSize); if (processedSize == 0) @@ -2440,6 +2555,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } } + if (bytesInBuf <= (size_t)posInBuf) + break; + bool useOffsetCallback = false; if (openCallback_Offset) { @@ -2464,7 +2582,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } } - size_t availSize = bytesInBuf - (size_t)posInBuf; + const size_t availSize = bytesInBuf - (size_t)posInBuf; if (availSize < kNumHashBytes) break; size_t scanSize = availSize - @@ -2489,17 +2607,19 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) scanSize++; const Byte *buf = byteBuffer + (size_t)posInBuf; + const Byte *bufLimit = buf + scanSize; size_t ppp = 0; if (!needCheckStartOpen) { - for (; ppp < scanSize && hash[HASH_VAL(buf, ppp)] == 0xFF; ppp++); + for (; buf < bufLimit && hash[HASH_VAL(buf)] == 0xFF; buf++); + ppp = (size_t)(buf - (byteBuffer + (size_t)posInBuf)); pos += ppp; - if (ppp == scanSize) + if (buf == bufLimit) continue; } - UInt32 v = HASH_VAL(buf, ppp); + UInt32 v = HASH_VAL(buf); bool nextNeedCheckStartOpen = true; unsigned i = hash[v]; unsigned indexOfDifficult = 0; @@ -2539,7 +2659,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) const CByteBuffer &sig = ai.Signatures[sigIndex]; if (ppp + sig.Size() > availSize - || !TestSignature(buf + ppp, sig, sig.Size())) + || !TestSignature(buf, sig, sig.Size())) continue; // printf("\nSignature OK: %10S %8x %5d", (const wchar_t *)ai.Name, (int)pos, (int)(pos - prevPos)); // prevPos = pos; @@ -2590,13 +2710,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) // printf("\nIsArc OK: %S", (const wchar_t *)ai.Name); } - /* - if (pos == 67109888) - pos = pos; - */ PRF(printf("\npos = %9I64d : %S", pos, (const wchar_t *)ai.Name)); - bool isMainFormat = isMainFormatArr[index]; + const bool isMainFormat = isMainFormatArr[index]; const COpenSpecFlags &specFlags = mode.GetSpec(isForced, isMainFormat, isUnknownExt); CMyComPtr archive; @@ -2606,14 +2722,14 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) // OutputDebugStringW(ai.Name); - UInt64 rem = fileSize - startArcPos; + const UInt64 rem = fileSize - startArcPos; UInt64 arcStreamOffset = 0; if (ai.Flags_UseGlobalOffset()) { limitedStreamSpec->InitAndSeek(0, fileSize); - limitedStream->Seek(startArcPos, STREAM_SEEK_SET, NULL); + limitedStream->Seek((Int64)startArcPos, STREAM_SEEK_SET, NULL); } else { @@ -2633,31 +2749,40 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) extractCallback_To_OpenCallback_Spec->Files = 0; extractCallback_To_OpenCallback_Spec->Offset = startArcPos; - HRESULT result = OpenArchiveSpec(archive, true, limitedStream, &maxCheckStartPosition, + HRESULT result = OpenArchiveSpec(archive, + true, // needPhySize + limitedStream, &maxCheckStartPosition, useOffsetCallback ? (IArchiveOpenCallback *)openCallback_Offset : (IArchiveOpenCallback *)op.callback, extractCallback_To_OpenCallback); RINOK(ReadBasicProps(archive, ai.Flags_UseGlobalOffset() ? 0 : startArcPos, result)); bool isOpen = false; + if (result == S_FALSE) { if (!mode.CanReturnParser) { if (formatIndex < 0 && ErrorInfo.IsArc_After_NonOpen()) { - ErrorInfo.ErrorFormatIndex = index; + ErrorInfo.ErrorFormatIndex = (int)index; NonOpen_ErrorInfo = ErrorInfo; // if archive was detected, we don't need additional open attempts return S_FALSE; } continue; } - if (!ErrorInfo.IsArc_After_NonOpen() || !PhySizeDefined || PhySize == 0) + if (!ErrorInfo.IsArc_After_NonOpen() || !PhySize_Defined || PhySize == 0) continue; } else { + if (PhySize_Defined && PhySize == 0) + { + PRF(printf(" phySize_Defined && PhySize == 0 ")); + // we skip that epmty archive case with unusual unexpected (PhySize == 0) from Code function. + continue; + } isOpen = true; RINOK(result); PRF(printf(" OK ")); @@ -2671,13 +2796,14 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) pi.Offset = startArcPos; if (ai.Flags_UseGlobalOffset()) - pi.Offset = Offset; + pi.Offset = (UInt64)Offset; else if (Offset != 0) return E_FAIL; - UInt64 arcRem = FileSize - pi.Offset; + + const UInt64 arcRem = FileSize - pi.Offset; UInt64 phySize = arcRem; - bool phySizeDefined = PhySizeDefined; - if (phySizeDefined) + const bool phySize_Defined = PhySize_Defined; + if (phySize_Defined) { if (pi.Offset + PhySize > FileSize) { @@ -2703,9 +2829,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool needScan = false; - if (isOpen && !phySizeDefined) + if (isOpen && !phySize_Defined) { - // it's for Z format + // it's for Z format, or bzip2,gz,xz with phySize that was not detected pi.LenIsUnknown = true; needScan = true; phySize = arcRem; @@ -2722,7 +2848,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) /* if (needSkipFullArc) - if (pi.Offset == 0 && phySizeDefined && pi.Size >= fileSize) + if (pi.Offset == 0 && phySize_Defined && pi.Size >= fileSize) continue; */ if (pi.Offset == 0 && !pi.LenIsUnknown && pi.Size >= FileSize) @@ -2750,7 +2876,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) RINOK(ReadParseItemProps(archive, ai, pi)); - if (pi.Offset < startArcPos && !mode.EachPos /* && phySizeDefined */) + if (pi.Offset < startArcPos && !mode.EachPos /* && phySize_Defined */) { /* It's for DMG format. This code deletes all previous items that are included to current item */ @@ -2769,7 +2895,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } - if (isOpen && mode.CanReturnArc && phySizeDefined) + if (isOpen && mode.CanReturnArc && phySize_Defined) { // if (pi.Offset + pi.Size >= fileSize) bool openCur = false; @@ -2777,7 +2903,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) bool thereIsTail = ErrorInfo.ThereIsTail; if (thereIsTail && mode.ZerosTailIsAllowed) { - RINOK(CheckZerosTail(op, arcStreamOffset + Offset + PhySize)); + RINOK(CheckZerosTail(op, (UInt64)((Int64)arcStreamOffset + Offset + (Int64)PhySize))); if (ErrorInfo.IgnoreTail) thereIsTail = false; } @@ -2785,10 +2911,12 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (pi.Offset != 0) { if (!pi.IsNotArcType) + { if (thereIsTail) openCur = specFlags.CanReturnMid; else openCur = specFlags.CanReturnTail; + } } else { @@ -2796,11 +2924,11 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) openCur = true; else openCur = specFlags.CanReturnFrontal; - if (formatIndex >= -2) openCur = true; } + if (formatIndex < 0 && pi.IsSelfExe /* && mode.SkipSfxStub */) openCur = false; @@ -2827,7 +2955,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { InStream = op.stream; Archive = archive; - FormatIndex = index; + FormatIndex = (int)index; ArcStreamOffset = arcStreamOffset; return S_OK; } @@ -2841,7 +2969,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } */ - pi.FormatIndex = index; + pi.FormatIndex = (int)index; // printf("\nAdd offset = %d", (int)pi.Offset); handlerSpec->AddItem(pi); @@ -2896,6 +3024,9 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) return S_OK; } + + + HRESULT CArc::OpenStream(const COpenOptions &op) { RINOK(OpenStream2(op)); @@ -2908,25 +3039,25 @@ HRESULT CArc::OpenStream(const COpenOptions &op) Archive->QueryInterface(IID_IArchiveGetRawProps, (void **)&GetRawProps); Archive->QueryInterface(IID_IArchiveGetRootProps, (void **)&GetRootProps); - RINOK(Archive_GetArcBoolProp(Archive, kpidIsTree, IsTree)); - RINOK(Archive_GetArcBoolProp(Archive, kpidIsDeleted, Ask_Deleted)); - RINOK(Archive_GetArcBoolProp(Archive, kpidIsAltStream, Ask_AltStream)); - RINOK(Archive_GetArcBoolProp(Archive, kpidIsAux, Ask_Aux)); - RINOK(Archive_GetArcBoolProp(Archive, kpidINode, Ask_INode)); - RINOK(Archive_GetArcBoolProp(Archive, kpidReadOnly, IsReadOnly)); + RINOK(Archive_GetArcProp_Bool(Archive, kpidIsTree, IsTree)); + RINOK(Archive_GetArcProp_Bool(Archive, kpidIsDeleted, Ask_Deleted)); + RINOK(Archive_GetArcProp_Bool(Archive, kpidIsAltStream, Ask_AltStream)); + RINOK(Archive_GetArcProp_Bool(Archive, kpidIsAux, Ask_Aux)); + RINOK(Archive_GetArcProp_Bool(Archive, kpidINode, Ask_INode)); + RINOK(Archive_GetArcProp_Bool(Archive, kpidReadOnly, IsReadOnly)); const UString fileName = ExtractFileNameFromPath(Path); UString extension; { int dotPos = fileName.ReverseFind_Dot(); if (dotPos >= 0) - extension = fileName.Ptr(dotPos + 1); + extension = fileName.Ptr((unsigned)(dotPos + 1)); } DefaultName.Empty(); if (FormatIndex >= 0) { - const CArcInfoEx &ai = op.codecs->Formats[FormatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)FormatIndex]; if (ai.Exts.Size() == 0) DefaultName = GetDefaultName2(fileName, UString(), UString()); else @@ -2934,7 +3065,7 @@ HRESULT CArc::OpenStream(const COpenOptions &op) int subExtIndex = ai.FindExtension(extension); if (subExtIndex < 0) subExtIndex = 0; - const CArcExtInfo &extInfo = ai.Exts[subExtIndex]; + const CArcExtInfo &extInfo = ai.Exts[(unsigned)subExtIndex]; DefaultName = GetDefaultName2(fileName, extInfo.Ext, extInfo.AddExt); } } @@ -2946,10 +3077,10 @@ HRESULT CArc::OpenStream(const COpenOptions &op) #ifdef _SFX #ifdef _WIN32 - static const char *k_ExeExt = ".exe"; + #define k_ExeExt ".exe" static const unsigned k_ExeExt_Len = 4; #else - static const char *k_ExeExt = ""; + #define k_ExeExt "" static const unsigned k_ExeExt_Len = 0; #endif @@ -2972,9 +3103,7 @@ HRESULT CArc::OpenStreamOrFile(COpenOptions &op) fileStream = fileStreamSpec; Path = filePath; if (!fileStreamSpec->Open(us2fs(Path))) - { - return GetLastError(); - } + return GetLastError_noZero_HRESULT(); op.stream = fileStream; #ifdef _SFX IgnoreSplit = true; @@ -3009,13 +3138,13 @@ HRESULT CArc::OpenStreamOrFile(COpenOptions &op) FOR_VECTOR (i, op.codecs->Formats) { const CArcInfoEx &ai = op.codecs->Formats[i]; - if (ai.IsSplit()) + if (ai.Is_Split()) continue; UString path3 = path2; - path3 += L'.'; + path3 += '.'; path3 += ai.GetMainExt(); // "7z" for SFX. Path = path3; - Path.AddAscii(".001"); + Path += ".001"; bool isOk = op.callbackSpec->SetSecondFileInfo(us2fs(Path)); if (!isOk) { @@ -3216,7 +3345,7 @@ HRESULT CArchiveLink::Open(COpenOptions &op) break; CArc arc2; - RINOK(arc.GetItemPath(mainSubfile, arc2.Path)); + RINOK(arc.GetItem_Path(mainSubfile, arc2.Path)); bool zerosTailIsAllowed; RINOK(Archive_GetItemBoolProp(arc.Archive, mainSubfile, kpidZerosTailIsAllowed, zerosTailIsAllowed)); @@ -3260,7 +3389,7 @@ HRESULT CArchiveLink::Open(COpenOptions &op) break; } RINOK(result); - RINOK(arc.GetItemMTime(mainSubfile, arc2.MTime, arc2.MTimeDefined)); + RINOK(arc.GetItem_MTime(mainSubfile, arc2.MTime)); Arcs.Add(arc2); } IsOpen = !Arcs.IsEmpty(); @@ -3279,7 +3408,7 @@ HRESULT CArchiveLink::Open2(COpenOptions &op, IOpenCallbackUI *callbackUI) if (!op.stream && !op.stdInMode) { NFile::NDir::GetFullPathAndSplit(us2fs(op.filePath), prefix, name); - openCallbackSpec->Init(prefix, name); + RINOK(openCallbackSpec->Init2(prefix, name)); } else { @@ -3309,7 +3438,7 @@ HRESULT CArchiveLink::Open2(COpenOptions &op, IOpenCallbackUI *callbackUI) return S_OK; } -HRESULT CArc::ReOpen(const COpenOptions &op) +HRESULT CArc::ReOpen(const COpenOptions &op, IArchiveOpenCallback *openCallback_Additional) { ErrorInfo.ClearErrors(); ErrorInfo.ErrorFormatIndex = -1; @@ -3331,7 +3460,7 @@ HRESULT CArc::ReOpen(const COpenOptions &op) CTailInStream *tailStreamSpec = new CTailInStream; stream2 = tailStreamSpec; tailStreamSpec->Stream = op.stream; - tailStreamSpec->Offset = globalOffset; + tailStreamSpec->Offset = (UInt64)globalOffset; tailStreamSpec->Init(); RINOK(tailStreamSpec->SeekToStart()); } @@ -3339,12 +3468,15 @@ HRESULT CArc::ReOpen(const COpenOptions &op) // There are archives with embedded STUBs (like ZIP), so we must support signature scanning // But for another archives we can use 0 here. So the code can be fixed !!! UInt64 maxStartPosition = kMaxCheckStartPosition; - HRESULT res = Archive->Open(stream2, &maxStartPosition, op.callback); + IArchiveOpenCallback *openCallback = openCallback_Additional; + if (!openCallback) + openCallback = op.callback; + HRESULT res = Archive->Open(stream2, &maxStartPosition, openCallback); if (res == S_OK) { - RINOK(ReadBasicProps(Archive, globalOffset, res)); - ArcStreamOffset = globalOffset; + RINOK(ReadBasicProps(Archive, (UInt64)globalOffset, res)); + ArcStreamOffset = (UInt64)globalOffset; if (ArcStreamOffset != 0) InStream = op.stream; } @@ -3384,18 +3516,18 @@ HRESULT CArchiveLink::ReOpen(COpenOptions &op) { FString dirPrefix, fileName; NFile::NDir::GetFullPathAndSplit(us2fs(op.filePath), dirPrefix, fileName); - openCallbackSpec->Init(dirPrefix, fileName); + RINOK(openCallbackSpec->Init2(dirPrefix, fileName)); } CInFileStream *fileStreamSpec = new CInFileStream; CMyComPtr stream(fileStreamSpec); if (!fileStreamSpec->Open(us2fs(op.filePath))) - return GetLastError(); + return GetLastError_noZero_HRESULT(); op.stream = stream; CArc &arc = Arcs[0]; - HRESULT res = arc.ReOpen(op); + HRESULT res = arc.ReOpen(op, openCallbackNew); PasswordWasAsked = openCallbackSpec->PasswordWasAsked; // Password = openCallbackSpec->Password; @@ -3406,6 +3538,7 @@ HRESULT CArchiveLink::ReOpen(COpenOptions &op) #ifndef _SFX +bool ParseComplexSize(const wchar_t *s, UInt64 &result); bool ParseComplexSize(const wchar_t *s, UInt64 &result) { result = 0; @@ -3463,7 +3596,7 @@ static bool ParseTypeParams(const UString &s, COpenType &type) return false; } -bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) +static bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) { int pos2 = s.Find(L':'); @@ -3472,11 +3605,11 @@ bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) if (pos2 < 0) { name = s; - pos2 = s.Len(); + pos2 = (int)s.Len(); } else { - name = s.Left(pos2); + name = s.Left((unsigned)pos2); pos2++; } @@ -3497,6 +3630,12 @@ bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) type.CanReturnArc = false; type.CanReturnParser = true; } + else if (name.IsEqualTo_Ascii_NoCase("hash")) + { + // type.CanReturnArc = false; + // type.CanReturnParser = false; + type.IsHashType = true; + } else return false; } @@ -3505,17 +3644,17 @@ bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) } - for (unsigned i = pos2; i < s.Len();) + for (unsigned i = (unsigned)pos2; i < s.Len();) { int next = s.Find(L':', i); if (next < 0) - next = s.Len(); - const UString name = s.Mid(i, next - i); + next = (int)s.Len(); + const UString name = s.Mid(i, (unsigned)next - i); if (name.IsEmpty()) return false; if (!ParseTypeParams(name, type)) return false; - i = next + 1; + i = (unsigned)next + 1; } return true; @@ -3524,21 +3663,36 @@ bool ParseType(CCodecs &codecs, const UString &s, COpenType &type) bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector &types) { types.Clear(); + bool isHashType = false; for (unsigned pos = 0; pos < s.Len();) { int pos2 = s.Find(L'.', pos); if (pos2 < 0) - pos2 = s.Len(); - UString name = s.Mid(pos, pos2 - pos); + pos2 = (int)s.Len(); + UString name = s.Mid(pos, (unsigned)pos2 - pos); if (name.IsEmpty()) return false; COpenType type; if (!ParseType(codecs, name, type)) return false; + if (isHashType) + return false; + if (type.IsHashType) + isHashType = true; types.Add(type); - pos = pos2 + 1; + pos = (unsigned)pos2 + 1; } return true; } +/* +bool IsHashType(const CObjectVector &types) +{ + if (types.Size() != 1) + return false; + return types[0].IsHashType; +} +*/ + + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.h b/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.h index d46d69e8a8d..e3220b94fd0 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/OpenArchive.h @@ -8,6 +8,7 @@ #include "ArchiveOpenCallback.h" #include "LoadCodecs.h" #include "Property.h" +#include "DirItem.h" #ifndef _SFX @@ -70,6 +71,7 @@ struct COpenType bool CanReturnArc; bool CanReturnParser; + bool IsHashType; bool EachPos; // bool SkipSfxStub; @@ -88,9 +90,10 @@ struct COpenType COpenType(): FormatIndex(-1), Recursive(true), - EachPos(false), CanReturnArc(true), CanReturnParser(false), + IsHashType(false), + EachPos(false), // SkipSfxStub(true), // ExeAsUnknown(true), ZerosTailIsAllowed(false), @@ -121,7 +124,7 @@ struct COpenOptions IInStream *stream; ISequentialInStream *seqStream; IArchiveOpenCallback *callback; - COpenCallbackImp *callbackSpec; + COpenCallbackImp *callbackSpec; // it's used for SFX only OPEN_PROPS_DECL // bool openOnlySpecifiedByExtension, @@ -258,6 +261,9 @@ struct CReadArcItem } }; + + + class CArc { HRESULT PrepareToOpen(const COpenOptions &op, unsigned formatIndex, CMyComPtr &archive); @@ -266,7 +272,7 @@ class CArc #ifndef _SFX // parts.Back() can contain alt stream name "nams:AltName" - HRESULT GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &parts) const; + HRESULT GetItem_PathToParent(UInt32 index, UInt32 parent, UStringVector &parts) const; #endif public: @@ -285,24 +291,26 @@ public: UString Path; UString filePath; UString DefaultName; - int FormatIndex; // - 1 means Parser. - int SubfileIndex; - FILETIME MTime; - bool MTimeDefined; + int FormatIndex; // -1 means Parser + UInt32 SubfileIndex; // (UInt32)(Int32)-1; means no subfile + + // CFiTime MTime; + // bool MTime_Defined; + CArcTime MTime; Int64 Offset; // it's offset of start of archive inside stream that is open by Archive Handler UInt64 PhySize; // UInt64 OkPhySize; - bool PhySizeDefined; + bool PhySize_Defined; // bool OkPhySize_Defined; UInt64 FileSize; UInt64 AvailPhySize; // PhySize, but it's reduced if exceed end of file // bool offsetDefined; - UInt64 GetEstmatedPhySize() const { return PhySizeDefined ? PhySize : FileSize; } + UInt64 GetEstmatedPhySize() const { return PhySize_Defined ? PhySize : FileSize; } UInt64 ArcStreamOffset; // offset of stream that is open by Archive Handler - Int64 GetGlobalOffset() const { return ArcStreamOffset + Offset; } // it's global offset of archive + Int64 GetGlobalOffset() const { return (Int64)ArcStreamOffset + Offset; } // it's global offset of archive // AString ErrorFlagsText; @@ -321,7 +329,7 @@ public: // void Set_ErrorFlagsText(); CArc(): - MTimeDefined(false), + // MTime_Defined(false), IsTree(false), IsReadOnly(false), Ask_Deleted(false), @@ -341,26 +349,45 @@ public: return Archive->Close(); } - HRESULT GetItemPath(UInt32 index, UString &result) const; - HRESULT GetDefaultItemPath(UInt32 index, UString &result) const; + HRESULT GetItem_Path(UInt32 index, UString &result) const; + HRESULT GetItem_DefaultPath(UInt32 index, UString &result) const; // GetItemPath2 adds [DELETED] dir prefix for deleted items. - HRESULT GetItemPath2(UInt32 index, UString &result) const; + HRESULT GetItem_Path2(UInt32 index, UString &result) const; HRESULT GetItem(UInt32 index, CReadArcItem &item) const; - HRESULT GetItemSize(UInt32 index, UInt64 &size, bool &defined) const; - HRESULT GetItemMTime(UInt32 index, FILETIME &ft, bool &defined) const; - HRESULT IsItemAnti(UInt32 index, bool &result) const + HRESULT GetItem_Size(UInt32 index, UInt64 &size, bool &defined) const; + + /* if (GetProperty() returns vt==VT_EMPTY), this function sets + timestamp from archive file timestamp (MTime). + So (at) will be set in most cases (at.Def == true) + if (at.Prec == 0) + { + it means that (Prec == 0) was returned for (kpidMTime), + and no value was returned for (kpidTimeType). + it can mean Windows precision or unknown precision. + } + */ + HRESULT GetItem_MTime(UInt32 index, CArcTime &at) const; + + HRESULT IsItem_Anti(UInt32 index, bool &result) const { return Archive_GetItemBoolProp(Archive, index, kpidIsAnti, result); } HRESULT OpenStream(const COpenOptions &options); HRESULT OpenStreamOrFile(COpenOptions &options); - HRESULT ReOpen(const COpenOptions &options); + HRESULT ReOpen(const COpenOptions &options, IArchiveOpenCallback *openCallback_Additional); HRESULT CreateNewTailStream(CMyComPtr &stream); + + bool IsHashHandler(const COpenOptions &options) const + { + if (FormatIndex < 0) + return false; + return options.codecs->Formats[(unsigned)FormatIndex].Flags_HashHandler(); + } }; struct CArchiveLink @@ -397,6 +424,13 @@ struct CArchiveLink IArchiveGetRawProps *GetArchiveGetRawProps() const { return Arcs.Back().GetRawProps; } IArchiveGetRootProps *GetArchiveGetRootProps() const { return Arcs.Back().GetRootProps; } + /* + Open() opens archive and COpenOptions::callback + Open2() uses COpenCallbackImp that implements Volumes and password callback + Open3() calls Open2() and callbackUI->Open_Finished(); + Open_Strict() returns S_FALSE also in case, if there is non-open expected nested archive. + */ + HRESULT Open(COpenOptions &options); HRESULT Open2(COpenOptions &options, IOpenCallbackUI *callbackUI); HRESULT Open3(COpenOptions &options, IOpenCallbackUI *callbackUI); @@ -414,4 +448,25 @@ struct CArchiveLink bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector &types); +// bool IsHashType(const CObjectVector &types); + + +struct CDirPathSortPair +{ + unsigned Len; + unsigned Index; + + void SetNumSlashes(const FChar *s); + + int Compare(const CDirPathSortPair &a) const + { + // We need sorting order where parent items will be after child items + if (Len < a.Len) return 1; + if (Len > a.Len) return -1; + if (Index < a.Index) return -1; + if (Index > a.Index) return 1; + return 0; + } +}; + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.cpp index dccc08b171e..72384b3effb 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.cpp @@ -14,17 +14,23 @@ #include "PropIDUtils.h" +#ifndef _SFX #define Get16(x) GetUi16(x) #define Get32(x) GetUi32(x) +#endif using namespace NWindows; -static const char g_WinAttribChars[16 + 1] = "RHS8DAdNTsLCOnE_"; +static const unsigned kNumWinAtrribFlags = 21; +static const char g_WinAttribChars[kNumWinAtrribFlags + 1] = "RHS8DAdNTsLCOIEV.X.PU"; + /* +FILE_ATTRIBUTE_ + 0 READONLY 1 HIDDEN 2 SYSTEM - +3 (Volume label - obsolete) 4 DIRECTORY 5 ARCHIVE 6 DEVICE @@ -34,12 +40,19 @@ static const char g_WinAttribChars[16 + 1] = "RHS8DAdNTsLCOnE_"; 10 REPARSE_POINT 11 COMPRESSED 12 OFFLINE -13 NOT_CONTENT_INDEXED +13 NOT_CONTENT_INDEXED (I - Win10 attrib/Explorer) 14 ENCRYPTED - -16 VIRTUAL +15 INTEGRITY_STREAM (V - ReFS Win8/Win2012) +16 VIRTUAL (reserved) +17 NO_SCRUB_DATA (X - ReFS Win8/Win2012 attrib) +18 RECALL_ON_OPEN or EA +19 PINNED +20 UNPINNED +21 STRICTLY_SEQUENTIAL +22 RECALL_ON_DATA_ACCESS */ + static const char kPosixTypes[16] = { '0', 'p', 'c', '3', 'd', '5', 'b', '7', '-', '9', 'l', 'B', 's', 'D', 'E', 'F' }; #define MY_ATTR_CHAR(a, n, c) ((a) & (1 << (n))) ? c : '-'; @@ -52,9 +65,9 @@ static void ConvertPosixAttribToString(char *s, UInt32 a) throw() s[8 - i] = MY_ATTR_CHAR(a, i + 1, 'w'); s[9 - i] = MY_ATTR_CHAR(a, i + 0, 'x'); } - if ((a & 0x800) != 0) s[3] = ((a & (1 << 6)) ? 's' : 'S'); - if ((a & 0x400) != 0) s[6] = ((a & (1 << 3)) ? 's' : 'S'); - if ((a & 0x200) != 0) s[9] = ((a & (1 << 0)) ? 't' : 'T'); + if ((a & 0x800) != 0) s[3] = ((a & (1 << 6)) ? 's' : 'S'); // S_ISUID + if ((a & 0x400) != 0) s[6] = ((a & (1 << 3)) ? 's' : 'S'); // S_ISGID + if ((a & 0x200) != 0) s[9] = ((a & (1 << 0)) ? 't' : 'T'); // S_ISVTX s[10] = 0; a &= ~(UInt32)0xFFFF; @@ -65,36 +78,95 @@ static void ConvertPosixAttribToString(char *s, UInt32 a) throw() } } + void ConvertWinAttribToString(char *s, UInt32 wa) throw() { - for (int i = 0; i < 16; i++) - if ((wa & (1 << i)) && i != 7) - *s++ = g_WinAttribChars[i]; - *s = 0; + /* + some programs store posix attributes in high 16 bits. + p7zip - stores additional 0x8000 flag marker. + macos - stores additional 0x4000 flag marker. + info-zip - no additional marker. + */ + + bool isPosix = ((wa & 0xF0000000) != 0); + + UInt32 posix = 0; + if (isPosix) + { + posix = wa >> 16; + wa &= (UInt32)0x3FFF; + } - // we support p7zip trick that stores posix attributes in high 16 bits, and 0x8000 flag - // we also support ZIP archives created in Unix, that store posix attributes in high 16 bits without 0x8000 flag + for (unsigned i = 0; i < kNumWinAtrribFlags; i++) + { + UInt32 flag = (1 << i); + if ((wa & flag) != 0) + { + char c = g_WinAttribChars[i]; + if (c != '.') + { + wa &= ~flag; + // if (i != 7) // we can disable N (NORMAL) printing + *s++ = c; + } + } + } - // if (wa & 0x8000) - if ((wa >> 16) != 0) + if (wa != 0) + { + *s++ = ' '; + ConvertUInt32ToHex8Digits(wa, s); + s += strlen(s); + } + + *s = 0; + + if (isPosix) { *s++ = ' '; - ConvertPosixAttribToString(s, wa >> 16); + ConvertPosixAttribToString(s, posix); } } -void ConvertPropertyToShortString(char *dest, const PROPVARIANT &prop, PROPID propID, bool full) throw() + +void ConvertPropertyToShortString2(char *dest, const PROPVARIANT &prop, PROPID propID, int level) throw() { *dest = 0; if (prop.vt == VT_FILETIME) { - FILETIME localFileTime; - if ((prop.filetime.dwHighDateTime == 0 && - prop.filetime.dwLowDateTime == 0) || - !::FileTimeToLocalFileTime(&prop.filetime, &localFileTime)) + const FILETIME &ft = prop.filetime; + unsigned ns100 = 0; + int numDigits = kTimestampPrintLevel_NTFS; + const unsigned prec = prop.wReserved1; + const unsigned ns100_Temp = prop.wReserved2; + if (prec != 0 + && prec <= k_PropVar_TimePrec_1ns + && ns100_Temp < 100 + && prop.wReserved3 == 0) + { + ns100 = ns100_Temp; + if (prec == k_PropVar_TimePrec_Unix || + prec == k_PropVar_TimePrec_DOS) + numDigits = 0; + else if (prec == k_PropVar_TimePrec_HighPrec) + numDigits = 9; + else + { + numDigits = (int)prec - (int)k_PropVar_TimePrec_Base; + if ( + // numDigits < kTimestampPrintLevel_DAY // for debuf + numDigits < kTimestampPrintLevel_SEC + ) + + numDigits = kTimestampPrintLevel_NTFS; + } + } + if (ft.dwHighDateTime == 0 && ft.dwLowDateTime == 0 && ns100 == 0) return; - ConvertFileTimeToString(localFileTime, dest, true, full); + if (level > numDigits) + level = numDigits; + ConvertUtcFileTimeToString2(ft, ns100, dest, level); return; } @@ -153,12 +225,30 @@ void ConvertPropertyToShortString(char *dest, const PROPVARIANT &prop, PROPID pr ConvertUInt64ToHex(v, dest + 2); return; } + + /* + case kpidDevice: + { + UInt64 v = 0; + if (prop.vt == VT_UI4) + v = prop.ulVal; + else if (prop.vt == VT_UI8) + v = (UInt64)prop.uhVal.QuadPart; + else + break; + ConvertUInt32ToString(MY_dev_major(v), dest); + dest += strlen(dest); + *dest++ = ','; + ConvertUInt32ToString(MY_dev_minor(v), dest); + return; + } + */ } ConvertPropVariantToShortString(prop, dest); } -void ConvertPropertyToString(UString &dest, const PROPVARIANT &prop, PROPID propID, bool full) +void ConvertPropertyToString2(UString &dest, const PROPVARIANT &prop, PROPID propID, int level) { if (prop.vt == VT_BSTR) { @@ -166,22 +256,21 @@ void ConvertPropertyToString(UString &dest, const PROPVARIANT &prop, PROPID prop return; } char temp[64]; - ConvertPropertyToShortString(temp, prop, propID, full); - dest.SetFromAscii(temp); + ConvertPropertyToShortString2(temp, prop, propID, level); + dest = temp; } +#ifndef _SFX + static inline unsigned GetHex(unsigned v) { return (v < 10) ? ('0' + v) : ('A' + (v - 10)); } -#ifndef _SFX - static inline void AddHexToString(AString &res, unsigned v) { res += (char)GetHex(v >> 4); res += (char)GetHex(v & 0xF); - res += ' '; } /* @@ -226,6 +315,14 @@ struct CSecID2Name const char *sz; }; +static int FindPairIndex(const CSecID2Name * pairs, unsigned num, UInt32 id) +{ + for (unsigned i = 0; i < num; i++) + if (pairs[i].n == id) + return (int)i; + return -1; +} + static const CSecID2Name sid_32_Names[] = { { 544, "Administrators" }, @@ -316,22 +413,22 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) if (v0 == 32 && num == 2) { UInt32 v1 = Get32(p + 12); - for (unsigned i = 0; i < ARRAY_SIZE(sid_32_Names); i++) - if (sid_32_Names[i].n == v1) - { - s += sid_32_Names[i].sz; - return; - } + int index = FindPairIndex(sid_32_Names, ARRAY_SIZE(sid_32_Names), v1); + if (index >= 0) + { + s += sid_32_Names[(unsigned)index].sz; + return; + } } if (v0 == 21 && num == 5) { UInt32 v4 = Get32(p + 8 + 4 * 4); - for (unsigned i = 0; i < ARRAY_SIZE(sid_21_Names); i++) - if (sid_21_Names[i].n == v4) - { - s += sid_21_Names[i].sz; - return; - } + int index = FindPairIndex(sid_21_Names, ARRAY_SIZE(sid_21_Names), v4); + if (index >= 0) + { + s += sid_21_Names[(unsigned)index].sz; + return; + } } if (v0 == 80 && num == 6) { @@ -349,13 +446,9 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) } } - char sz[16]; s += "S-1-"; if (p[2] == 0 && p[3] == 0) - { - ConvertUInt32ToString(authority, sz); - s += sz; - } + s.Add_UInt32(authority); else { s += "0x"; @@ -365,8 +458,7 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) for (UInt32 i = 0; i < num; i++) { s += '-'; - ConvertUInt32ToString(Get32(p + 8 + i * 4), sz); - s += sz; + s.Add_UInt32(Get32(p + 8 + i * 4)); } } @@ -381,20 +473,13 @@ static void ParseOwner(AString &s, const Byte *p, UInt32 size, UInt32 pos) ParseSid(s, p + pos, size - pos, sidSize); } -static void AddUInt32ToString(AString &s, UInt32 val) -{ - char sz[16]; - ConvertUInt32ToString(val, sz); - s += sz; -} - static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName, UInt32 flags, UInt32 offset) { UInt32 control = Get16(p + 2); if ((flags & control) == 0) return; UInt32 pos = Get32(p + offset); - s += ' '; + s.Add_Space(); s += strName; if (pos >= size) return; @@ -405,7 +490,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName if (Get16(p) != 2) // revision return; UInt32 num = Get32(p + 4); - AddUInt32ToString(s, num); + s.Add_UInt32(num); /* UInt32 aclSize = Get16(p + 2); @@ -428,7 +513,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName size -= 8; UInt32 sidSize = 0; - s += ' '; + s.Add_Space(); ParseSid(s, p, size, sidSize); if (sidSize == 0) return; @@ -441,11 +526,16 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName */ } +/* #define MY_SE_OWNER_DEFAULTED (0x0001) #define MY_SE_GROUP_DEFAULTED (0x0002) +*/ #define MY_SE_DACL_PRESENT (0x0004) +/* #define MY_SE_DACL_DEFAULTED (0x0008) +*/ #define MY_SE_SACL_PRESENT (0x0010) +/* #define MY_SE_SACL_DEFAULTED (0x0020) #define MY_SE_DACL_AUTO_INHERIT_REQ (0x0100) #define MY_SE_SACL_AUTO_INHERIT_REQ (0x0200) @@ -455,6 +545,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName #define MY_SE_SACL_PROTECTED (0x2000) #define MY_SE_RM_CONTROL_VALID (0x4000) #define MY_SE_SELF_RELATIVE (0x8000) +*/ void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s) { @@ -470,12 +561,12 @@ void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s) return; } ParseOwner(s, data, size, Get32(data + 4)); - s += ' '; + s.Add_Space(); ParseOwner(s, data, size, Get32(data + 8)); ParseAcl(s, data, size, "s:", MY_SE_SACL_PRESENT, 12); ParseAcl(s, data, size, "d:", MY_SE_DACL_PRESENT, 16); - s += ' '; - AddUInt32ToString(s, size); + s.Add_Space(); + s.Add_UInt32(size); // s += '\n'; // s += Data_To_Hex(data, size); } @@ -529,21 +620,68 @@ bool CheckNtSecure(const Byte *data, UInt32 size) throw() #endif + + +// IO_REPARSE_TAG_* + +static const CSecID2Name k_ReparseTags[] = +{ + { 0xA0000003, "MOUNT_POINT" }, + { 0xC0000004, "HSM" }, + { 0x80000005, "DRIVE_EXTENDER" }, + { 0x80000006, "HSM2" }, + { 0x80000007, "SIS" }, + { 0x80000008, "WIM" }, + { 0x80000009, "CSV" }, + { 0x8000000A, "DFS" }, + { 0x8000000B, "FILTER_MANAGER" }, + { 0xA000000C, "SYMLINK" }, + { 0xA0000010, "IIS_CACHE" }, + { 0x80000012, "DFSR" }, + { 0x80000013, "DEDUP" }, + { 0xC0000014, "APPXSTRM" }, + { 0x80000014, "NFS" }, + { 0x80000015, "FILE_PLACEHOLDER" }, + { 0x80000016, "DFM" }, + { 0x80000017, "WOF" }, + { 0x80000018, "WCI" }, + { 0x8000001B, "APPEXECLINK" }, + { 0xA000001D, "LX_SYMLINK" }, + { 0x80000023, "AF_UNIX" }, + { 0x80000024, "LX_FIFO" }, + { 0x80000025, "LX_CHR" }, + { 0x80000026, "LX_BLK" } +}; + bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) { s.Empty(); NFile::CReparseAttr attr; + if (attr.Parse(data, size)) { - if (!attr.IsSymLink()) - s.AddAscii("Junction: "); - s += attr.GetPath(); - if (!attr.IsOkNamePair()) + if (attr.IsSymLink_WSL()) { - s.AddAscii(" : "); - s += attr.PrintName; + s += "WSL: "; + s += attr.GetPath(); } + else + { + if (!attr.IsSymLink_Win()) + s += "Junction: "; + s += attr.GetPath(); + if (s.IsEmpty()) + s += "Link: "; + if (!attr.IsOkNamePair()) + { + s += " : "; + s += attr.PrintName; + } + } + if (attr.MinorError) + s += " : MINOR_ERROR"; return true; + // s += " "; // for debug } if (size < 8) @@ -555,19 +693,48 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) if (Get16(data + 6) != 0) // padding return false; - char hex[16]; - ConvertUInt32ToHex8Digits(tag, hex); - s.AddAscii(hex); - s.Add_Space(); + /* + #define _my_IO_REPARSE_TAG_DEDUP (0x80000013L) + if (tag == _my_IO_REPARSE_TAG_DEDUP) + { + } + */ + + { + int index = FindPairIndex(k_ReparseTags, ARRAY_SIZE(k_ReparseTags), tag); + if (index >= 0) + s += k_ReparseTags[(unsigned)index].sz; + else + { + s += "REPARSE:"; + char hex[16]; + ConvertUInt32ToHex8Digits(tag, hex); + s += hex; + } + } - data += 8; + s += ":"; + s.Add_UInt32(len); - for (UInt32 i = 0; i < len; i++) + if (len != 0) { - unsigned b = ((const Byte *)data)[i]; - s += (wchar_t)GetHex((b >> 4) & 0xF); - s += (wchar_t)GetHex(b & 0xF); + s.Add_Space(); + + data += 8; + + for (UInt32 i = 0; i < len; i++) + { + if (i >= 16) + { + s += "..."; + break; + } + unsigned b = data[i]; + s += (char)GetHex((b >> 4) & 0xF); + s += (char)GetHex(b & 0xF); + } } + return true; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.h b/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.h index 1dea321e9ac..915bfc28e03 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/PropIDUtils.h @@ -6,8 +6,8 @@ #include "../../../Common/MyString.h" // provide at least 64 bytes for buffer including zero-end -void ConvertPropertyToShortString(char *dest, const PROPVARIANT &propVariant, PROPID propID, bool full = true) throw(); -void ConvertPropertyToString(UString &dest, const PROPVARIANT &propVariant, PROPID propID, bool full = true); +void ConvertPropertyToShortString2(char *dest, const PROPVARIANT &propVariant, PROPID propID, int level = 0) throw(); +void ConvertPropertyToString2(UString &dest, const PROPVARIANT &propVariant, PROPID propID, int level = 0); bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s); void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s); diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/SetProperties.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/SetProperties.cpp index c3de5d5a323..4b3037af245 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/SetProperties.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/SetProperties.cpp @@ -27,6 +27,21 @@ static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) prop = result; } + +struct CPropPropetiesVector +{ + CPropVariant *values; + CPropPropetiesVector(unsigned num) + { + values = new CPropVariant[num]; + } + ~CPropPropetiesVector() + { + delete []values; + } +}; + + HRESULT SetProperties(IUnknown *unknown, const CObjectVector &properties) { if (properties.IsEmpty()) @@ -37,8 +52,7 @@ HRESULT SetProperties(IUnknown *unknown, const CObjectVector &propert return S_OK; UStringVector realNames; - CPropVariant *values = new CPropVariant[properties.Size()]; - try + CPropPropetiesVector values(properties.Size()); { unsigned i; for (i = 0; i < properties.Size(); i++) @@ -62,19 +76,12 @@ HRESULT SetProperties(IUnknown *unknown, const CObjectVector &propert else ParseNumberString(property.Value, propVariant); realNames.Add(name); - values[i] = propVariant; + values.values[i] = propVariant; } CRecordVector names; for (i = 0; i < realNames.Size(); i++) names.Add((const wchar_t *)realNames[i]); - RINOK(setProperties->SetProperties(&names.Front(), values, names.Size())); - } - catch(...) - { - delete []values; - throw; + return setProperties->SetProperties(&names.Front(), values.values, names.Size()); } - delete []values; - return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/Update.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/Update.cpp index 0db4a87f0d4..042991d196e 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/Update.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/Update.cpp @@ -2,9 +2,10 @@ #include "StdAfx.h" +// #include + #include "Update.h" -#include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" #include "../../../Windows/DLL.h" @@ -30,17 +31,21 @@ #include "TempFiles.h" #include "UpdateCallback.h" -static const char *kUpdateIsNotSupoorted = +static const char * const kUpdateIsNotSupoorted = "update operations are not supported for this archive"; +static const char * const kUpdateIsNotSupported_MultiVol = + "Updating for multivolume archives is not implemented"; + using namespace NWindows; using namespace NCOM; using namespace NFile; using namespace NDir; using namespace NName; -static CFSTR kTempFolderPrefix = FTEXT("7zE"); - +#ifdef _WIN32 +static CFSTR const kTempFolderPrefix = FTEXT("7zE"); +#endif void CUpdateErrorInfo::SetFromLastError(const char *message) { @@ -55,25 +60,12 @@ HRESULT CUpdateErrorInfo::SetFromLastError(const char *message, const FString &f return Get_HRESULT_Error(); } -static bool DeleteEmptyFolderAndEmptySubFolders(const FString &path) +HRESULT CUpdateErrorInfo::SetFromError_DWORD(const char *message, const FString &fileName, DWORD error) { - NFind::CFileInfo fileInfo; - FString pathPrefix = path + FCHAR_PATH_SEPARATOR; - { - NFind::CEnumerator enumerator(pathPrefix + FCHAR_ANY_MASK); - while (enumerator.Next(fileInfo)) - { - if (fileInfo.IsDir()) - if (!DeleteEmptyFolderAndEmptySubFolders(pathPrefix + fileInfo.Name)) - return false; - } - } - /* - // we don't need clear read-only for folders - if (!MySetFileAttributes(path, 0)) - return false; - */ - return RemoveDir(path); + Message = message; + FileNames.Add(fileName); + SystemError = error; + return Get_HRESULT_Error(); } @@ -111,7 +103,7 @@ public: _length = 0; } - bool SetMTime(const FILETIME *mTime); + bool SetMTime(const CFiTime *mTime); HRESULT Close(); UInt64 GetSize() const { return _length; } @@ -141,7 +133,7 @@ HRESULT COutMultiVolStream::Close() return res; } -bool COutMultiVolStream::SetMTime(const FILETIME *mTime) +bool COutMultiVolStream::SetMTime(const CFiTime *mTime) { bool res = true; FOR_VECTOR (i, Streams) @@ -156,7 +148,7 @@ bool COutMultiVolStream::SetMTime(const FILETIME *mTime) STDMETHODIMP COutMultiVolStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { - if (processedSize != NULL) + if (processedSize) *processedSize = 0; while (size > 0) { @@ -164,16 +156,15 @@ STDMETHODIMP COutMultiVolStream::Write(const void *data, UInt32 size, UInt32 *pr { CAltStreamInfo altStream; - FChar temp[16]; - ConvertUInt32ToString(_streamIndex + 1, temp); - FString name = temp; + FString name; + name.Add_UInt32(_streamIndex + 1); while (name.Len() < 3) name.InsertAtFront(FTEXT('0')); name.Insert(0, Prefix); altStream.StreamSpec = new COutFileStream; altStream.Stream = altStream.StreamSpec; if (!altStream.StreamSpec->Create(name, false)) - return ::GetLastError(); + return GetLastError_noZero_HRESULT(); { // NSynchronization::CCriticalSectionLock lock(g_TempPathsCS); TempFiles->Paths.Add(name); @@ -202,14 +193,14 @@ STDMETHODIMP COutMultiVolStream::Write(const void *data, UInt32 size, UInt32 *pr { // CMyComPtr outStream; // RINOK(altStream.Stream.QueryInterface(IID_IOutStream, &outStream)); - RINOK(altStream.Stream->Seek(_offsetPos, STREAM_SEEK_SET, NULL)); + RINOK(altStream.Stream->Seek((Int64)_offsetPos, STREAM_SEEK_SET, NULL)); altStream.Pos = _offsetPos; } UInt32 curSize = (UInt32)MyMin((UInt64)size, volSize - altStream.Pos); UInt32 realProcessed; RINOK(altStream.Stream->Write(data, curSize, &realProcessed)); - data = (void *)((Byte *)data + realProcessed); + data = (const void *)((const Byte *)data + realProcessed); size -= realProcessed; altStream.Pos += realProcessed; _offsetPos += realProcessed; @@ -218,7 +209,7 @@ STDMETHODIMP COutMultiVolStream::Write(const void *data, UInt32 size, UInt32 *pr _length = _absPos; if (_offsetPos > altStream.RealSize) altStream.RealSize = _offsetPos; - if (processedSize != NULL) + if (processedSize) *processedSize += realProcessed; if (altStream.Pos == volSize) { @@ -238,12 +229,12 @@ STDMETHODIMP COutMultiVolStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *n return STG_E_INVALIDFUNCTION; switch (seekOrigin) { - case STREAM_SEEK_SET: _absPos = offset; break; - case STREAM_SEEK_CUR: _absPos += offset; break; - case STREAM_SEEK_END: _absPos = _length + offset; break; + case STREAM_SEEK_SET: _absPos = (UInt64)offset; break; + case STREAM_SEEK_CUR: _absPos = (UInt64)((Int64)_absPos + offset); break; + case STREAM_SEEK_END: _absPos = (UInt64)((Int64)_length + offset); break; } _offsetPos = _absPos; - if (newPosition != NULL) + if (newPosition) *newPosition = _absPos; _streamIndex = 0; return S_OK; @@ -286,29 +277,27 @@ void CArchivePath::ParseFromPath(const UString &path, EArcNameMode mode) if (mode == k_ArcNameMode_Add) return; - if (mode == k_ArcNameMode_Exact) - { - BaseExtension.Empty(); - return; - } - int dotPos = Name.ReverseFind_Dot(); - if (dotPos < 0) - return; - if ((unsigned)dotPos == Name.Len() - 1) - { - Name.DeleteBack(); - BaseExtension.Empty(); - return; - } - const UString ext = Name.Ptr(dotPos + 1); - if (BaseExtension.IsEqualTo_NoCase(ext)) + if (mode != k_ArcNameMode_Exact) { - BaseExtension = ext; - Name.DeleteFrom(dotPos); + int dotPos = Name.ReverseFind_Dot(); + if (dotPos < 0) + return; + if ((unsigned)dotPos == Name.Len() - 1) + Name.DeleteBack(); + else + { + const UString ext = Name.Ptr((unsigned)(dotPos + 1)); + if (BaseExtension.IsEqualTo_NoCase(ext)) + { + BaseExtension = ext; + Name.DeleteFrom((unsigned)dotPos); + return; + } + } } - else - BaseExtension.Empty(); + + BaseExtension.Empty(); } UString CArchivePath::GetFinalPath() const @@ -316,7 +305,7 @@ UString CArchivePath::GetFinalPath() const UString path = GetPathWithoutExt(); if (!BaseExtension.IsEmpty()) { - path += L'.'; + path += '.'; path += BaseExtension; } return path; @@ -325,9 +314,10 @@ UString CArchivePath::GetFinalPath() const UString CArchivePath::GetFinalVolPath() const { UString path = GetPathWithoutExt(); + // if BaseExtension is empty, we must ignore VolExtension also. if (!BaseExtension.IsEmpty()) { - path += L'.'; + path += '.'; path += VolExtension; } return path; @@ -339,17 +329,17 @@ FString CArchivePath::GetTempPath() const path += us2fs(Name); if (!BaseExtension.IsEmpty()) { - path += FTEXT('.'); + path += '.'; path += us2fs(BaseExtension); } - path.AddAscii(".tmp"); + path += ".tmp"; path += TempPostfix; return path; } -static const wchar_t *kDefaultArcType = L"7z"; -static const wchar_t *kDefaultArcExt = L"7z"; -static const char *kSFXExtension = +static const char * const kDefaultArcType = "7z"; +static const char * const kDefaultArcExt = "7z"; +static const char * const kSFXExtension = #ifdef _WIN32 "exe"; #else @@ -391,14 +381,14 @@ bool CUpdateOptions::SetArcPath(const CCodecs *codecs, const UString &arcPath) } else { - const CArcInfoEx &arcInfo = codecs->Formats[formatIndex]; + const CArcInfoEx &arcInfo = codecs->Formats[(unsigned)formatIndex]; if (!arcInfo.UpdateEnabled) return false; typeExt = arcInfo.GetMainExt(); } UString ext = typeExt; if (SfxMode) - ext.SetFromAscii(kSFXExtension); + ext = kSFXExtension; ArchivePath.BaseExtension = ext; ArchivePath.VolExtension = typeExt; ArchivePath.ParseFromPath(arcPath, ArcNameMode); @@ -412,19 +402,43 @@ bool CUpdateOptions::SetArcPath(const CCodecs *codecs, const UString &arcPath) return true; } + struct CUpdateProduceCallbackImp: public IUpdateProduceCallback { const CObjectVector *_arcItems; + CDirItemsStat *_stat; IUpdateCallbackUI *_callback; - CUpdateProduceCallbackImp(const CObjectVector *a, - IUpdateCallbackUI *callback): _arcItems(a), _callback(callback) {} + CUpdateProduceCallbackImp( + const CObjectVector *a, + CDirItemsStat *stat, + IUpdateCallbackUI *callback): + _arcItems(a), + _stat(stat), + _callback(callback) {} + virtual HRESULT ShowDeleteFile(unsigned arcIndex); }; + HRESULT CUpdateProduceCallbackImp::ShowDeleteFile(unsigned arcIndex) { const CArcItem &ai = (*_arcItems)[arcIndex]; + { + CDirItemsStat &stat = *_stat; + if (ai.IsDir) + stat.NumDirs++; + else if (ai.IsAltStream) + { + stat.NumAltStreams++; + stat.AltStreamsSize += ai.Size; + } + else + { + stat.NumFiles++; + stat.FilesSize += ai.Size; + } + } return _callback->ShowDeleteFile(ai.Name, ai.IsDir); } @@ -516,7 +530,7 @@ static HRESULT Compress( } else { - RINOK(codecs->CreateOutArchive(formatIndex, outArchive)); + RINOK(codecs->CreateOutArchive((unsigned)formatIndex, outArchive)); #ifdef EXTERNAL_CODECS { @@ -530,14 +544,49 @@ static HRESULT Compress( #endif } - if (outArchive == 0) + if (!outArchive) throw kUpdateIsNotSupoorted; - + + // we need to set properties to get fileTimeType. + RINOK(SetProperties(outArchive, options.MethodMode.Properties)); + NFileTimeType::EEnum fileTimeType; { + /* + how we compare file_in_archive::MTime with dirItem.MTime + for GetUpdatePairInfoList(): + + if (kpidMTime is not defined), external MTime of archive is used. + + before 22.00: + if (kpidTimeType is defined) + { + kpidTimeType is used as precision. + (kpidTimeType > kDOS) is not allowed. + } + else GetFileTimeType() value is used as precision. + + 22.00: + if (kpidMTime is defined) + { + if (kpidMTime::precision != 0), then kpidMTime::precision is used as precision. + else + { + if (kpidTimeType is defined), kpidTimeType is used as precision. + else GetFileTimeType() value is used as precision. + } + } + else external MTime of archive is used as precision. + */ + UInt32 value; RINOK(outArchive->GetFileTimeType(&value)); + + // we support any future fileType here. + fileTimeType = (NFileTimeType::EEnum)value; + /* + old 21.07 code: switch (value) { case NFileTimeType::kWindows: @@ -548,13 +597,28 @@ static HRESULT Compress( default: return E_FAIL; } + */ } + // bool noTimestampExpected = false; { - const CArcInfoEx &arcInfo = codecs->Formats[formatIndex]; + const CArcInfoEx &arcInfo = codecs->Formats[(unsigned)formatIndex]; + + // if (arcInfo.Flags_KeepName()) noTimestampExpected = true; + if (arcInfo.Is_Xz() || + arcInfo.Is_BZip2()) + { + /* 7-zip before 22.00 returns NFileTimeType::kUnix for xz and bzip2, + but we want to set timestamp without reduction to unix. */ + // noTimestampExpected = true; + fileTimeType = NFileTimeType::kNotDefined; // it means not defined + } + if (options.AltStreams.Val && !arcInfo.Flags_AltStreams()) return E_NOTIMPL; - if (options.NtSecurity.Val && !arcInfo.Flags_NtSecure()) + if (options.NtSecurity.Val && !arcInfo.Flags_NtSecurity()) + return E_NOTIMPL; + if (options.DeleteAfterCompressing && arcInfo.Flags_HashHandler()) return E_NOTIMPL; } @@ -562,6 +626,8 @@ static HRESULT Compress( UStringVector newNames; + CArcToDoStat stat2; + if (options.RenamePairs.Size() != 0) { FOR_VECTOR (i, arcItems) @@ -587,7 +653,7 @@ static HRESULT Compress( int colonPos = FindAltStreamColon_in_Path(ai.Name); if (colonPos >= 0) { - UString mainName = ai.Name.Left(colonPos); + UString mainName = ai.Name.Left((unsigned)colonPos); /* actually we must improve that code to support cases with folder renaming like: rn arc dir1\ dir2\ @@ -595,8 +661,8 @@ static HRESULT Compress( if (rp.GetNewPath(false, mainName, dest)) { needRename = true; - dest += L':'; - dest += ai.Name.Ptr(colonPos + 1); + dest += ':'; + dest += ai.Name.Ptr((unsigned)(colonPos + 1)); break; } } @@ -610,8 +676,8 @@ static HRESULT Compress( if (needRename) { up2.NewProps = true; - RINOK(arc->IsItemAnti(i, up2.IsAnti)); - up2.NewNameIndex = newNames.Add(dest); + RINOK(arc->IsItem_Anti(i, up2.IsAnti)); + up2.NewNameIndex = (int)newNames.Add(dest); } updatePairs2.Add(up2); } @@ -620,23 +686,97 @@ static HRESULT Compress( { CRecordVector updatePairs; GetUpdatePairInfoList(dirItems, arcItems, fileTimeType, updatePairs); // must be done only once!!! - CUpdateProduceCallbackImp upCallback(&arcItems, callback); + CUpdateProduceCallbackImp upCallback(&arcItems, &stat2.DeleteData, callback); UpdateProduce(updatePairs, actionSet, updatePairs2, isUpdatingItself ? &upCallback : NULL); } { - UInt32 numItems = 0; FOR_VECTOR (i, updatePairs2) - if (updatePairs2[i].NewData) - numItems++; - RINOK(callback->SetNumItems(numItems)); + { + const CUpdatePair2 &up = updatePairs2[i]; + + // 17.01: anti-item is (up.NewData && (p.UseArcProps in most cases)) + + if (up.NewData && !up.UseArcProps) + { + if (up.ExistOnDisk()) + { + CDirItemsStat2 &stat = stat2.NewData; + const CDirItem &di = dirItems.Items[(unsigned)up.DirIndex]; + if (di.IsDir()) + { + if (up.IsAnti) + stat.Anti_NumDirs++; + else + stat.NumDirs++; + } + #ifdef _WIN32 + else if (di.IsAltStream) + { + if (up.IsAnti) + stat.Anti_NumAltStreams++; + else + { + stat.NumAltStreams++; + stat.AltStreamsSize += di.Size; + } + } + #endif + else + { + if (up.IsAnti) + stat.Anti_NumFiles++; + else + { + stat.NumFiles++; + stat.FilesSize += di.Size; + } + } + } + } + else if (up.ArcIndex >= 0) + { + CDirItemsStat2 &stat = *(up.NewData ? &stat2.NewData : &stat2.OldData); + const CArcItem &ai = arcItems[(unsigned)up.ArcIndex]; + if (ai.IsDir) + { + if (up.IsAnti) + stat.Anti_NumDirs++; + else + stat.NumDirs++; + } + else if (ai.IsAltStream) + { + if (up.IsAnti) + stat.Anti_NumAltStreams++; + else + { + stat.NumAltStreams++; + stat.AltStreamsSize += ai.Size; + } + } + else + { + if (up.IsAnti) + stat.Anti_NumFiles++; + else + { + stat.NumFiles++; + stat.FilesSize += ai.Size; + } + } + } + } + RINOK(callback->SetNumItems(stat2)); } CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; CMyComPtr updateCallback(updateCallbackSpec); + updateCallbackSpec->PreserveATime = options.PreserveATime; updateCallbackSpec->ShareForWrite = options.OpenShareForWrite; + updateCallbackSpec->StopAfterOpenError = options.StopAfterOpenError; updateCallbackSpec->StdInMode = options.StdInMode; updateCallbackSpec->Callback = callback; @@ -652,6 +792,8 @@ static HRESULT Compress( updateCallbackSpec->StoreNtSecurity = options.NtSecurity.Val; updateCallbackSpec->StoreHardLinks = options.HardLinks.Val; updateCallbackSpec->StoreSymLinks = options.SymLinks.Val; + updateCallbackSpec->StoreOwnerName = options.StoreOwnerName.Val; + updateCallbackSpec->StoreOwnerId = options.StoreOwnerId.Val; updateCallbackSpec->Arc = arc; updateCallbackSpec->ArcItems = &arcItems; @@ -659,9 +801,20 @@ static HRESULT Compress( updateCallbackSpec->ProcessedItemsStatuses = processedItemsStatuses; + { + const UString arcPath = archivePath.GetFinalPath(); + updateCallbackSpec->ArcFileName = ExtractFileNameFromPath(arcPath); + } + if (options.RenamePairs.Size() != 0) updateCallbackSpec->NewNames = &newNames; + if (options.SetArcMTime) + { + // updateCallbackSpec->Need_ArcMTime_Report = true; + updateCallbackSpec->Need_LatestMTime = true; + } + CMyComPtr outSeekStream; CMyComPtr outStream; @@ -698,9 +851,8 @@ static HRESULT Compress( { if (i > 0) { - FChar s[16]; - ConvertUInt32ToString(i, s); - archivePath.TempPostfix = s; + archivePath.TempPostfix.Empty(); + archivePath.TempPostfix.Add_UInt32(i); } realPath = archivePath.GetTempPath(); } @@ -734,7 +886,7 @@ static HRESULT Compress( outStream = outSeekStream; volStreamSpec->Sizes = options.VolumesSizes; volStreamSpec->Prefix = us2fs(archivePath.GetFinalVolPath()); - volStreamSpec->Prefix += FTEXT('.'); + volStreamSpec->Prefix += '.'; volStreamSpec->TempFiles = &tempFiles; volStreamSpec->Init(); @@ -742,12 +894,10 @@ static HRESULT Compress( updateCallbackSpec->VolumesSizes = volumesSizes; updateCallbackSpec->VolName = archivePath.Prefix + archivePath.Name; if (!archivePath.VolExtension.IsEmpty()) - updateCallbackSpec->VolExt = UString(L'.') + archivePath.VolExtension; + updateCallbackSpec->VolExt = UString('.') + archivePath.VolExtension; */ } - RINOK(SetProperties(outArchive, options.MethodMode.Properties)); - if (options.SfxMode) { CInFileStream *sfxStreamSpec = new CInFileStream; @@ -817,29 +967,76 @@ static HRESULT Compress( if (options.SetArcMTime) { - FILETIME ft; - ft.dwLowDateTime = 0; - ft.dwHighDateTime = 0; - FOR_VECTOR (i, updatePairs2) + CFiTime ft; + FiTime_Clear(ft); + bool isDefined = false; + + // bool needNormalizeAfterStream; + // needParse; + /* + if (updateCallbackSpec->ArcMTime_WasReported) { - CUpdatePair2 &pair2 = updatePairs2[i]; - const FILETIME *ft2 = NULL; - if (pair2.NewProps && pair2.DirIndex >= 0) - ft2 = &dirItems.Items[pair2.DirIndex].MTime; - else if (pair2.UseArcProps && pair2.ArcIndex >= 0) - ft2 = &arcItems[pair2.ArcIndex].MTime; - if (ft2) + isDefined = updateCallbackSpec->Reported_ArcMTime.Def; + if (isDefined) + updateCallbackSpec->Reported_ArcMTime.Write_To_FiTime(ft); + else + fileTimeType = NFileTimeType::kNotDefined; + } + if (!isDefined) + */ + { + if (updateCallbackSpec->LatestMTime_Defined) + { + // CArcTime at = StreamCallback_ArcMTime; + // updateCallbackSpec->StreamCallback_ArcMTime.Write_To_FiTime(ft); + // we must normalize with precision from archive; + ft = updateCallbackSpec->LatestMTime; + isDefined = true; + } + + FOR_VECTOR (i, updatePairs2) { - if (::CompareFileTime(&ft, ft2) < 0) - ft = *ft2; + const CUpdatePair2 &pair2 = updatePairs2[i]; + CFiTime ft2; + bool ft2_Defined = false; + /* we use full precision of dirItem, if dirItem is defined + and (dirItem will be used or dirItem is sameTime in dir and arc */ + if (pair2.DirIndex >= 0 && + (pair2.NewProps || pair2.IsSameTime)) + { + ft2 = dirItems.Items[(unsigned)pair2.DirIndex].MTime; + ft2_Defined = true; + } + else if (pair2.UseArcProps && pair2.ArcIndex >= 0) + { + const CArcItem &arcItem = arcItems[(unsigned)pair2.ArcIndex]; + if (arcItem.MTime.Def) + { + arcItem.MTime.Write_To_FiTime(ft2); + ft2_Defined = true; + } + } + if (ft2_Defined) + { + if (Compare_FiTime(&ft, &ft2) < 0) + { + ft = ft2; + isDefined = true; + } + } } + /* + if (fileTimeType != NFileTimeType::kNotDefined) + FiTime_Normalize_With_Prec(ft, fileTimeType); + */ } - if (ft.dwLowDateTime != 0 || ft.dwHighDateTime != 0) + // if (ft.dwLowDateTime != 0 || ft.dwHighDateTime != 0) + if (isDefined) { if (outStreamSpec) outStreamSpec->SetMTime(&ft); else if (volStreamSpec) - volStreamSpec->SetMTime(&ft);; + volStreamSpec->SetMTime(&ft); } } @@ -860,9 +1057,39 @@ static HRESULT Compress( result = outStreamSpec->Close(); else if (volStreamSpec) result = volStreamSpec->Close(); + + RINOK(result) + + if (processedItemsStatuses) + { + FOR_VECTOR (i, updatePairs2) + { + const CUpdatePair2 &up = updatePairs2[i]; + if (up.NewData && up.DirIndex >= 0) + { + const CDirItem &di = dirItems.Items[(unsigned)up.DirIndex]; + if (di.AreReparseData() || (!di.IsDir() && di.Size == 0)) + processedItemsStatuses[(unsigned)up.DirIndex] = 1; + } + } + } + return result; } + + +static bool Censor_AreAllAllowed(const NWildcard::CCensor &censor) +{ + if (censor.Pairs.Size() != 1) + return false; + const NWildcard::CPair &pair = censor.Pairs[0]; + /* Censor_CheckPath() ignores (CPair::Prefix). + So we also ignore (CPair::Prefix) here */ + // if (!pair.Prefix.IsEmpty()) return false; + return pair.Head.AreAllAllowed(); +} + bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include); static bool Censor_CheckPath(const NWildcard::CCensor &censor, const CReadArcItem &item) @@ -870,9 +1097,13 @@ static bool Censor_CheckPath(const NWildcard::CCensor &censor, const CReadArcIte bool finded = false; FOR_VECTOR (i, censor.Pairs) { + /* (CPair::Prefix) in not used for matching items in archive. + So we ignore (CPair::Prefix) here */ bool include; if (CensorNode_CheckPath2(censor.Pairs[i].Head, item, include)) { + // Check it and FIXME !!!! + // here we can exclude item via some Pair, that is still allowed by another Pair if (!include) return false; finded = true; @@ -895,6 +1126,8 @@ static HRESULT EnumerateInArchiveItems( CReadArcItem item; + const bool allFilesAreAllowed = Censor_AreAllAllowed(censor); + for (UInt32 i = 0; i < numItems; i++) { CArcItem ai; @@ -913,28 +1146,14 @@ static HRESULT EnumerateInArchiveItems( if (!storeStreamsMode && ai.IsAltStream) continue; */ - ai.Censored = Censor_CheckPath(censor, item); - - RINOK(arc.GetItemMTime(i, ai.MTime, ai.MTimeDefined)); - RINOK(arc.GetItemSize(i, ai.Size, ai.SizeDefined)); + if (allFilesAreAllowed) + ai.Censored = true; + else + ai.Censored = Censor_CheckPath(censor, item); - { - CPropVariant prop; - RINOK(archive->GetProperty(i, kpidTimeType, &prop)); - if (prop.vt == VT_UI4) - { - ai.TimeType = (int)(NFileTimeType::EEnum)prop.ulVal; - switch (ai.TimeType) - { - case NFileTimeType::kWindows: - case NFileTimeType::kUnix: - case NFileTimeType::kDOS: - break; - default: - return E_FAIL; - } - } - } + // ai.MTime will be set to archive MTime, if not present in archive item + RINOK(arc.GetItem_MTime(i, ai.MTime)); + RINOK(arc.GetItem_Size(i, ai.Size, ai.Size_Defined)); ai.IndexInServer = i; arcItems.AddInReserved(ai); @@ -944,38 +1163,8 @@ static HRESULT EnumerateInArchiveItems( #if defined(_WIN32) && !defined(UNDER_CE) -#include - -#endif - -struct CRefSortPair -{ - unsigned Len; - unsigned Index; -}; - -#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } - -static int CompareRefSortPair(const CRefSortPair *a1, const CRefSortPair *a2, void *) -{ - RINOZ(-MyCompare(a1->Len, a2->Len)); - return MyCompare(a1->Index, a2->Index); -} - -static unsigned GetNumSlashes(const FChar *s) -{ - for (unsigned numSlashes = 0;;) - { - FChar c = *s++; - if (c == 0) - return numSlashes; - if (IS_PATH_SEPAR(c)) - numSlashes++; - } -} +#include -#ifdef _WIN32 -void ConvertToLongNames(NWildcard::CCensor &censor); #endif HRESULT UpdateArchive( @@ -1007,7 +1196,7 @@ HRESULT UpdateArchive( if (options.Commands.Size() != 1) return E_NOTIMPL; const CActionSet &as = options.Commands[0].ActionSet; - for (int i = 2; i < NPairState::kNumValues; i++) + for (unsigned i = 2; i < NPairState::kNumValues; i++) if (as.StateActions[i] != NPairAction::kCompress) return E_NOTIMPL; } @@ -1025,7 +1214,7 @@ HRESULT UpdateArchive( if (options.SfxMode) { CProperty property; - property.Name.SetFromAscii("rsfx"); + property.Name = "rsfx"; options.MethodMode.Properties.Add(property); if (options.SfxModule.IsEmpty()) { @@ -1036,7 +1225,7 @@ HRESULT UpdateArchive( if (options.SfxModule.Find(FCHAR_PATH_SEPARATOR) < 0) { const FString fullName = NDLL::GetModuleDirPrefix() + options.SfxModule; - if (NFind::DoesFileExist(fullName)) + if (NFind::DoesFileExist_FollowLink(fullName)) { options.SfxModule = fullName; found = true; @@ -1044,7 +1233,7 @@ HRESULT UpdateArchive( } if (!found) { - if (!NFind::DoesFileExist(options.SfxModule)) + if (!NFind::DoesFileExist_FollowLink(options.SfxModule)) return errorInfo.SetFromLastError("cannot find specified SFX module", options.SfxModule); } } @@ -1058,7 +1247,15 @@ HRESULT UpdateArchive( !options.SetArcPath(codecs, cmdArcPath2)) return E_NOTIMPL; } - const UString arcPath = options.ArchivePath.GetFinalPath(); + + UString arcPath = options.ArchivePath.GetFinalPath(); + + if (!options.VolumesSizes.IsEmpty()) + { + arcPath = options.ArchivePath.GetFinalVolPath(); + arcPath += '.'; + arcPath += "001"; + } if (cmdArcPath2.IsEmpty()) { @@ -1068,7 +1265,7 @@ HRESULT UpdateArchive( else { NFind::CFileInfo fi; - if (!fi.Find(us2fs(arcPath))) + if (!fi.Find_FollowLink(us2fs(arcPath))) { if (renameMode) throw "can't find archive";; @@ -1081,11 +1278,39 @@ HRESULT UpdateArchive( else { if (fi.IsDir()) - throw "there is no such archive"; + return errorInfo.SetFromError_DWORD("There is a folder with the name of archive", + us2fs(arcPath), + #ifdef _WIN32 + ERROR_ACCESS_DENIED + #else + EISDIR + #endif + ); + #ifdef _WIN32 if (fi.IsDevice) return E_NOTIMPL; + #endif + + if (!options.StdOutMode && options.UpdateArchiveItself) + if (fi.IsReadOnly()) + { + return errorInfo.SetFromError_DWORD("The file is read-only", + us2fs(arcPath), + #ifdef _WIN32 + ERROR_ACCESS_DENIED + #else + EACCES + #endif + ); + } + if (options.VolumesSizes.Size() > 0) + { + errorInfo.FileNames.Add(us2fs(arcPath)); + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = kUpdateIsNotSupported_MultiVol; return E_NOTIMPL; + } CObjectVector types2; // change it. if (options.MethodMode.Type_Defined) @@ -1121,18 +1346,24 @@ HRESULT UpdateArchive( if (arcLink.VolumePaths.Size() > 1) { - errorInfo.SystemError = (DWORD)E_NOTIMPL; - errorInfo.Message = "Updating for multivolume archives is not implemented"; + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = kUpdateIsNotSupported_MultiVol; return E_NOTIMPL; } CArc &arc = arcLink.Arcs.Back(); - arc.MTimeDefined = !fi.IsDevice; - arc.MTime = fi.MTime; + arc.MTime.Def = + #ifdef _WIN32 + !fi.IsDevice; + #else + true; + #endif + if (arc.MTime.Def) + arc.MTime.Set_From_FiTime(fi.MTime); if (arc.ErrorInfo.ThereIsTail) { - errorInfo.SystemError = (DWORD)E_NOTIMPL; + // errorInfo.SystemError = (DWORD)E_NOTIMPL; errorInfo.Message = "There is some data block after the end of the archive"; return E_NOTIMPL; } @@ -1147,7 +1378,7 @@ HRESULT UpdateArchive( if (options.MethodMode.Type.FormatIndex < 0) { - options.MethodMode.Type.FormatIndex = codecs->FindFormatForArchiveType(kDefaultArcType); + options.MethodMode.Type.FormatIndex = codecs->FindFormatForArchiveType((UString)kDefaultArcType); if (options.MethodMode.Type.FormatIndex < 0) return E_NOTIMPL; } @@ -1172,10 +1403,11 @@ HRESULT UpdateArchive( if (options.StdInMode) { CDirItem di; + di.ClearBase(); di.Name = options.StdInFileName; di.Size = (UInt64)(Int64)-1; - di.Attrib = 0; - NTime::GetCurUtcFileTime(di.MTime); + di.SetAsFile(); + NTime::GetCurUtc_FiTime(di.MTime); di.CTime = di.ATime = di.MTime; dirItems.Items.Add(di); } @@ -1199,10 +1431,18 @@ HRESULT UpdateArchive( #endif dirItems.ScanAltStreams = options.AltStreams.Val; + dirItems.ExcludeDirItems = censor.ExcludeDirItems; + dirItems.ExcludeFileItems = censor.ExcludeFileItems; + + dirItems.ShareForWrite = options.OpenShareForWrite; + + #ifndef _WIN32 + dirItems.StoreOwnerName = options.StoreOwnerName.Val; + #endif - HRESULT res = EnumerateItems(censor, + const HRESULT res = EnumerateItems(censor, options.PathMode, - options.AddPathPrefix, + UString(), // options.AddPathPrefix, dirItems); if (res != S_OK) @@ -1214,11 +1454,13 @@ HRESULT UpdateArchive( RINOK(callback->FinishScanning(dirItems.Stat)); + // 22.00: we don't need parent folder, if absolute path mode + if (options.PathMode != NWildcard::k_AbsPath) if (censor.Pairs.Size() == 1) { NFind::CFileInfo fi; FString prefix = us2fs(censor.Pairs[0].Prefix); - prefix += FTEXT('.'); + prefix += '.'; // UString prefix = censor.Pairs[0].Prefix; /* if (prefix.Back() == WCHAR_PATH_SEPARATOR) @@ -1229,11 +1471,7 @@ HRESULT UpdateArchive( if (fi.Find(prefix)) if (fi.IsDir()) { - parentDirItem.Size = fi.Size; - parentDirItem.CTime = fi.CTime; - parentDirItem.ATime = fi.ATime; - parentDirItem.MTime = fi.MTime; - parentDirItem.Attrib = fi.Attrib; + parentDirItem.Copy_From_FileInfoBase(fi); parentDirItem_Ptr = &parentDirItem; int secureIndex = -1; @@ -1242,8 +1480,6 @@ HRESULT UpdateArchive( dirItems.AddSecurityItem(prefix, secureIndex); #endif parentDirItem.SecureIndex = secureIndex; - - parentDirItem_Ptr = &parentDirItem; } } } @@ -1286,6 +1522,31 @@ HRESULT UpdateArchive( unsigned ci; + + // self including protection + if (options.DeleteAfterCompressing) + { + for (ci = 0; ci < options.Commands.Size(); ci++) + { + CArchivePath &ap = options.Commands[ci].ArchivePath; + const FString path = us2fs(ap.GetFinalPath()); + // maybe we must compare absolute paths path here + FOR_VECTOR (i, dirItems.Items) + { + const FString phyPath = dirItems.GetPhyPath(i); + if (phyPath == path) + { + UString s; + s = "It is not allowed to include archive to itself"; + s.Add_LF(); + s += fs2us(path); + throw s; + } + } + } + } + + for (ci = 0; ci < options.Commands.Size(); ci++) { CArchivePath &ap = options.Commands[ci].ArchivePath; @@ -1328,7 +1589,7 @@ HRESULT UpdateArchive( CByteBuffer processedItems; if (options.DeleteAfterCompressing) { - unsigned num = dirItems.Items.Size(); + const unsigned num = dirItems.Items.Size(); processedItems.Alloc(num); for (unsigned i = 0; i < num; i++) processedItems[i] = 0; @@ -1353,7 +1614,7 @@ HRESULT UpdateArchive( if (options.StdOutMode) { - name.SetFromAscii("stdout"); + name = "stdout"; isUpdating = thereIsInArchive; } else @@ -1399,9 +1660,13 @@ HRESULT UpdateArchive( CArchivePath &ap = options.Commands[0].ArchivePath; const FString &tempPath = ap.GetTempPath(); + // DWORD attrib = 0; if (thereIsInArchive) + { + // attrib = NFind::GetFileAttrib(us2fs(arcPath)); if (!DeleteFileAlways(us2fs(arcPath))) return errorInfo.SetFromLastError("cannot delete the file", us2fs(arcPath)); + } if (!MyMoveFile(tempPath, us2fs(arcPath))) { @@ -1409,6 +1674,15 @@ HRESULT UpdateArchive( errorInfo.FileNames.Add(us2fs(arcPath)); return errorInfo.Get_HRESULT_Error(); } + + /* + if (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_READONLY)) + { + DWORD attrib2 = NFind::GetFileAttrib(us2fs(arcPath)); + if (attrib2 != INVALID_FILE_ATTRIBUTES) + NDir::SetFileAttrib(us2fs(arcPath), attrib2 | FILE_ATTRIBUTE_READONLY); + } + */ } catch(...) { @@ -1437,7 +1711,7 @@ HRESULT UpdateArchive( } */ - LPMAPISENDMAIL sendMail = (LPMAPISENDMAIL)mapiLib.GetProc("MAPISendMail"); + LPMAPISENDMAIL sendMail = (LPMAPISENDMAIL)(void *)mapiLib.GetProc("MAPISendMail"); if (sendMail == 0) { errorInfo.SetFromLastError("7-Zip cannot find MAPISendMail function"); @@ -1458,34 +1732,47 @@ HRESULT UpdateArchive( } CCurrentDirRestorer curDirRestorer; + + AStringVector paths; + AStringVector names; for (i = 0; i < fullPaths.Size(); i++) { - UString arcPath2 = fs2us(fullPaths[i]); - UString fileName = ExtractFileNameFromPath(arcPath2); - AString path = GetAnsiString(arcPath2); - AString name = GetAnsiString(fileName); + const UString arcPath2 = fs2us(fullPaths[i]); + const UString fileName = ExtractFileNameFromPath(arcPath2); + paths.Add(GetAnsiString(arcPath2)); + names.Add(GetAnsiString(fileName)); + // const AString path (GetAnsiString(arcPath2)); + // const AString name (GetAnsiString(fileName)); // Warning!!! MAPISendDocuments function changes Current directory // fnSend(0, ";", (LPSTR)(LPCSTR)path, (LPSTR)(LPCSTR)name, 0); + } - MapiFileDesc f; + CRecordVector files; + files.ClearAndSetSize(paths.Size()); + + for (i = 0; i < paths.Size(); i++) + { + MapiFileDesc &f = files[i]; memset(&f, 0, sizeof(f)); f.nPosition = 0xFFFFFFFF; - f.lpszPathName = (char *)(const char *)path; - f.lpszFileName = (char *)(const char *)name; - + f.lpszPathName = paths[i].Ptr_non_const(); + f.lpszFileName = names[i].Ptr_non_const(); + } + + { MapiMessage m; memset(&m, 0, sizeof(m)); - m.nFileCount = 1; - m.lpFiles = &f; + m.nFileCount = files.Size(); + m.lpFiles = &files.Front(); - const AString addr = GetAnsiString(options.EMailAddress); + const AString addr (GetAnsiString(options.EMailAddress)); MapiRecipDesc rec; if (!addr.IsEmpty()) { memset(&rec, 0, sizeof(rec)); rec.ulRecipClass = MAPI_TO; - rec.lpszAddress = (char *)(const char *)addr; + rec.lpszAddress = addr.Ptr_non_const(); m.nRecipCount = 1; m.lpRecips = &rec; } @@ -1498,7 +1785,7 @@ HRESULT UpdateArchive( if (options.DeleteAfterCompressing) { - CRecordVector pairs; + CRecordVector pairs; FStringVector foldersNames; unsigned i; @@ -1506,39 +1793,63 @@ HRESULT UpdateArchive( for (i = 0; i < dirItems.Items.Size(); i++) { const CDirItem &dirItem = dirItems.Items[i]; - FString phyPath = dirItems.GetPhyPath(i); + const FString phyPath = dirItems.GetPhyPath(i); if (dirItem.IsDir()) { - CRefSortPair pair; + CDirPathSortPair pair; pair.Index = i; - pair.Len = GetNumSlashes(phyPath); + pair.SetNumSlashes(phyPath); pairs.Add(pair); } else { - if (processedItems[i] != 0 || dirItem.Size == 0) + // 21.04: we have set processedItems[*] before for all required items + if (processedItems[i] != 0 + // || dirItem.Size == 0 + // || dirItem.AreReparseData() + ) { - RINOK(callback->DeletingAfterArchiving(phyPath, false)); - DeleteFileAlways(phyPath); + NFind::CFileInfo fileInfo; + /* if (!SymLinks), we follow link here, similar to (dirItem) filling */ + if (fileInfo.Find(phyPath, !options.SymLinks.Val)) + { + bool is_SameSize = false; + if (options.SymLinks.Val && dirItem.AreReparseData()) + { + /* (dirItem.Size = dirItem.ReparseData.Size()) was set before. + So we don't compare sizes for that case here */ + is_SameSize = fileInfo.IsOsSymLink(); + } + else + is_SameSize = (fileInfo.Size == dirItem.Size); + + if (is_SameSize + && Compare_FiTime(&fileInfo.MTime, &dirItem.MTime) == 0 + && Compare_FiTime(&fileInfo.CTime, &dirItem.CTime) == 0) + { + RINOK(callback->DeletingAfterArchiving(phyPath, false)); + DeleteFileAlways(phyPath); + } + } } else { - // file was skipped + // file was skipped by some reason. We can throw error for debug: /* errorInfo.SystemError = 0; errorInfo.Message = "file was not processed"; - errorInfo.FileName = phyPath; + errorInfo.FileNames.Add(phyPath); return E_FAIL; */ } } } - pairs.Sort(CompareRefSortPair, NULL); + pairs.Sort2(); for (i = 0; i < pairs.Size(); i++) { - FString phyPath = dirItems.GetPhyPath(pairs[i].Index); + const FString phyPath = dirItems.GetPhyPath(pairs[i].Index); if (NFind::DoesDirExist(phyPath)) { RINOK(callback->DeletingAfterArchiving(phyPath, true)); diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/Update.h b/3rdparty/lzma/CPP/7zip/UI/Common/Update.h index 0301ae7660e..01fc43e23e0 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/Update.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/Update.h @@ -18,7 +18,7 @@ enum EArcNameMode { k_ArcNameMode_Smart, k_ArcNameMode_Exact, - k_ArcNameMode_Add, + k_ArcNameMode_Add }; struct CArchivePath @@ -91,7 +91,9 @@ struct CUpdateOptions bool SfxMode; FString SfxModule; + bool PreserveATime; bool OpenShareForWrite; + bool StopAfterOpenError; bool StdInMode; UString StdInFileName; @@ -103,13 +105,16 @@ struct CUpdateOptions FString WorkingDir; NWildcard::ECensorPathMode PathMode; - UString AddPathPrefix; + // UString AddPathPrefix; CBoolPair NtSecurity; CBoolPair AltStreams; CBoolPair HardLinks; CBoolPair SymLinks; + CBoolPair StoreOwnerId; + CBoolPair StoreOwnerName; + bool DeleteAfterCompressing; bool SetArcMTime; @@ -121,19 +126,26 @@ struct CUpdateOptions CUpdateOptions(): UpdateArchiveItself(true), + ArcNameMode(k_ArcNameMode_Smart), + SfxMode(false), + + PreserveATime(false), + OpenShareForWrite(false), + StopAfterOpenError(false), + StdInMode(false), StdOutMode(false), + EMailMode(false), EMailRemoveAfter(false), - OpenShareForWrite(false), - ArcNameMode(k_ArcNameMode_Smart), + PathMode(NWildcard::k_RelatPath), DeleteAfterCompressing(false), SetArcMTime(false) - {}; + {}; void SetActionCommand_Add() { @@ -148,7 +160,7 @@ struct CUpdateOptions struct CUpdateErrorInfo { - DWORD SystemError; + DWORD SystemError; // it's DWORD (WRes) only; AString Message; FStringVector FileNames; @@ -156,6 +168,7 @@ struct CUpdateErrorInfo HRESULT Get_HRESULT_Error() const { return SystemError == 0 ? E_FAIL : HRESULT_FROM_WIN32(SystemError); } void SetFromLastError(const char *message); HRESULT SetFromLastError(const char *message, const FString &fileName); + HRESULT SetFromError_DWORD(const char *message, const FString &fileName, DWORD error); CUpdateErrorInfo(): SystemError(0) {}; }; diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.cpp index 5499f2a1a9f..c93bfc7e02b 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.cpp @@ -2,6 +2,21 @@ #include "StdAfx.h" +// #include + +#ifndef _WIN32 +// #include +// #include + +// for major()/minor(): +#if defined(__FreeBSD__) || defined(BSD) +#include +#else +#include +#endif + +#endif + #ifndef _7ZIP_ST #include "../../../Windows/Synchronization.h" #endif @@ -10,6 +25,7 @@ #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" #include "../../../Common/Wildcard.h" +#include "../../../Common/UTFConvert.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" @@ -51,14 +67,30 @@ CArchiveUpdateCallback::CArchiveUpdateCallback(): ArcItems(NULL), UpdatePairs(NULL), NewNames(NULL), + CommentIndex(-1), + Comment(NULL), + PreserveATime(false), ShareForWrite(false), + StopAfterOpenError(false), StdInMode(false), KeepOriginalItemNames(false), StoreNtSecurity(false), StoreHardLinks(false), StoreSymLinks(false), + + #ifndef _WIN32 + StoreOwnerId(false), + StoreOwnerName(false), + #endif + + /* + , Need_ArcMTime_Report(false), + , ArcMTime_WasReported(false), + */ + Need_LatestMTime(false), + LatestMTime_Defined(false), ProcessedItemsStatuses(NULL) { @@ -121,22 +153,24 @@ STDMETHODIMP CArchiveUpdateCallback::GetUpdateItemInfo(UInt32 index, { *indexInArchive = (UInt32)(Int32)-1; if (up.ExistInArchive()) - *indexInArchive = (ArcItems == 0) ? up.ArcIndex : (*ArcItems)[up.ArcIndex].IndexInServer; + *indexInArchive = ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex; } return S_OK; COM_TRY_END } + STDMETHODIMP CArchiveUpdateCallback::GetRootProp(PROPID propID, PROPVARIANT *value) { NCOM::CPropVariant prop; switch (propID) { case kpidIsDir: prop = true; break; - case kpidAttrib: if (ParentDirItem) prop = ParentDirItem->Attrib; break; - case kpidCTime: if (ParentDirItem) prop = ParentDirItem->CTime; break; - case kpidATime: if (ParentDirItem) prop = ParentDirItem->ATime; break; - case kpidMTime: if (ParentDirItem) prop = ParentDirItem->MTime; break; + case kpidAttrib: if (ParentDirItem) prop = ParentDirItem->GetWinAttrib(); break; + case kpidCTime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->CTime); break; + case kpidATime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->ATime); break; + case kpidMTime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->MTime); break; + case kpidArcFileName: if (!ArcFileName.IsEmpty()) prop = ArcFileName; break; } prop.Detach(value); return S_OK; @@ -185,7 +219,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRootRawProp(PROPID { if (ParentDirItem->SecureIndex < 0) return S_OK; - const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[ParentDirItem->SecureIndex]; + const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[(unsigned)ParentDirItem->SecureIndex]; *data = buf; *dataSize = (UInt32)buf.Size(); *propType = NPropDataType::kRaw; @@ -217,7 +251,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con const CUpdatePair2 &up = (*UpdatePairs)[index]; if (up.UseArcProps && up.ExistInArchive() && Arc->GetRawProps) return Arc->GetRawProps->GetRawProp( - ArcItems ? (*ArcItems)[up.ArcIndex].IndexInServer : up.ArcIndex, + ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex, propID, data, dataSize, propType); { /* @@ -227,8 +261,8 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con if (up.IsAnti) return S_OK; - #ifndef UNDER_CE - const CDirItem &di = DirItems->Items[up.DirIndex]; + #if defined(_WIN32) && !defined(UNDER_CE) + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; #endif #ifdef _USE_SECURITY_CODE @@ -238,18 +272,19 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con return S_OK; if (di.SecureIndex < 0) return S_OK; - const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[di.SecureIndex]; + const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[(unsigned)di.SecureIndex]; *data = buf; *dataSize = (UInt32)buf.Size(); *propType = NPropDataType::kRaw; } else #endif + if (propID == kpidNtReparse) { - // propID == kpidNtReparse if (!StoreSymLinks) return S_OK; - #ifndef UNDER_CE + #if defined(_WIN32) && !defined(UNDER_CE) + // we use ReparseData2 instead of ReparseData for WIM format const CByteBuffer *buf = &di.ReparseData2; if (buf->Size() == 0) buf = &di.ReparseData; @@ -269,7 +304,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, con return S_OK; } -#ifndef UNDER_CE +#if defined(_WIN32) && !defined(UNDER_CE) static UString GetRelativePath(const UString &to, const UString &from) { @@ -300,7 +335,7 @@ static UString GetRelativePath(const UString &to, const UString &from) unsigned k; for (k = i + 1; k < partsFrom.Size(); k++) - s += L".." WSTRING_PATH_SEPARATOR; + s += ".." STRING_PATH_SEPARATOR; for (k = i; k < partsTo.Size(); k++) { @@ -337,21 +372,25 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR prop.Detach(value); return S_OK; } + + #if !defined(UNDER_CE) + if (up.DirIndex >= 0) { - #ifndef UNDER_CE - const CDirItem &di = DirItems->Items[up.DirIndex]; + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + + #ifdef _WIN32 // if (di.IsDir()) { CReparseAttr attr; if (attr.Parse(di.ReparseData, di.ReparseData.Size())) { UString simpleName = attr.GetPath(); - if (attr.IsRelative()) + if (!attr.IsSymLink_WSL() && attr.IsRelative_Win()) prop = simpleName; else { - const FString phyPath = DirItems->GetPhyPath(up.DirIndex); + const FString phyPath = DirItems->GetPhyPath((unsigned)up.DirIndex); FString fullPath; if (NDir::MyGetFullPathName(phyPath, fullPath)) { @@ -362,8 +401,26 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR return S_OK; } } - #endif + + #else // _WIN32 + + if (di.ReparseData.Size() != 0) + { + AString utf; + utf.SetFrom_CalcLen((const char *)(const Byte *)di.ReparseData, (unsigned)di.ReparseData.Size()); + + UString us; + if (ConvertUTF8ToUnicode(utf, us)) + { + prop = us; + prop.Detach(value); + return S_OK; + } + } + + #endif // _WIN32 } + #endif // !defined(UNDER_CE) } else if (propID == kpidHardLink) { @@ -371,7 +428,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR { const CKeyKeyValPair &pair = _map[_hardIndex_To]; const CUpdatePair2 &up2 = (*UpdatePairs)[pair.Value]; - prop = DirItems->GetLogPath(up2.DirIndex); + prop = DirItems->GetLogPath((unsigned)up2.DirIndex); prop.Detach(value); return S_OK; } @@ -395,30 +452,66 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR } } else if (propID == kpidPath && up.NewNameIndex >= 0) - prop = (*NewNames)[up.NewNameIndex]; + prop = (*NewNames)[(unsigned)up.NewNameIndex]; + else if (propID == kpidComment + && CommentIndex >= 0 + && (unsigned)CommentIndex == index + && Comment) + prop = *Comment; else if (propID == kpidShortName && up.NewNameIndex >= 0 && up.IsMainRenameItem) { // we can generate new ShortName here; } else if ((up.UseArcProps || (KeepOriginalItemNames && (propID == kpidPath || propID == kpidIsAltStream))) && up.ExistInArchive() && Archive) - return Archive->GetProperty(ArcItems ? (*ArcItems)[up.ArcIndex].IndexInServer : up.ArcIndex, propID, value); + return Archive->GetProperty(ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex, propID, value); else if (up.ExistOnDisk()) { - const CDirItem &di = DirItems->Items[up.DirIndex]; + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; switch (propID) { - case kpidPath: prop = DirItems->GetLogPath(up.DirIndex); break; + case kpidPath: prop = DirItems->GetLogPath((unsigned)up.DirIndex); break; case kpidIsDir: prop = di.IsDir(); break; - case kpidSize: prop = di.IsDir() ? (UInt64)0 : di.Size; break; - case kpidAttrib: prop = di.Attrib; break; - case kpidCTime: prop = di.CTime; break; - case kpidATime: prop = di.ATime; break; - case kpidMTime: prop = di.MTime; break; + case kpidSize: prop = (UInt64)(di.IsDir() ? (UInt64)0 : di.Size); break; + case kpidCTime: PropVariant_SetFrom_FiTime(prop, di.CTime); break; + case kpidATime: PropVariant_SetFrom_FiTime(prop, di.ATime); break; + case kpidMTime: PropVariant_SetFrom_FiTime(prop, di.MTime); break; + case kpidAttrib: prop = (UInt32)di.GetWinAttrib(); break; + case kpidPosixAttrib: prop = (UInt32)di.GetPosixAttrib(); break; + + #if defined(_WIN32) case kpidIsAltStream: prop = di.IsAltStream; break; - #if defined(_WIN32) && !defined(UNDER_CE) // case kpidShortName: prop = di.ShortName; break; - #endif + #else + + case kpidDeviceMajor: + /* + printf("\ndi.mode = %o\n", di.mode); + printf("\nst.st_rdev major = %d\n", (unsigned)major(di.rdev)); + printf("\nst.st_rdev minor = %d\n", (unsigned)minor(di.rdev)); + */ + if (S_ISCHR(di.mode) || S_ISBLK(di.mode)) + prop = (UInt32)major(di.rdev); + break; + + case kpidDeviceMinor: + if (S_ISCHR(di.mode) || S_ISBLK(di.mode)) + prop = (UInt32)minor(di.rdev); + break; + + // case kpidDevice: if (S_ISCHR(di.mode) || S_ISBLK(di.mode)) prop = (UInt64)(di.rdev); break; + + case kpidUserId: if (StoreOwnerId) prop = (UInt32)di.uid; break; + case kpidGroupId: if (StoreOwnerId) prop = (UInt32)di.gid; break; + case kpidUser: + if (di.OwnerNameIndex >= 0) + prop = DirItems->OwnerNameMap.Strings[(unsigned)di.OwnerNameIndex]; + break; + case kpidGroup: + if (di.OwnerGroupIndex >= 0) + prop = DirItems->OwnerGroupMap.Strings[(unsigned)di.OwnerGroupIndex]; + break; + #endif } } prop.Detach(value); @@ -430,6 +523,17 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR static NSynchronization::CCriticalSection CS; #endif +void CArchiveUpdateCallback::UpdateProcessedItemStatus(unsigned dirIndex) +{ + if (ProcessedItemsStatuses) + { + #ifndef _7ZIP_ST + NSynchronization::CCriticalSectionLock lock(CS); + #endif + ProcessedItemsStatuses[dirIndex] = 1; + } +} + STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStream **inStream, UInt32 mode) { COM_TRY_BEGIN @@ -447,9 +551,9 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea { UString name; if (up.ArcIndex >= 0) - name = (*ArcItems)[up.ArcIndex].Name; + name = (*ArcItems)[(unsigned)up.ArcIndex].Name; else if (up.DirIndex >= 0) - name = DirItems->GetLogPath(up.DirIndex); + name = DirItems->GetLogPath((unsigned)up.DirIndex); RINOK(Callback->GetStream(name, isDir, true, mode)); /* 9.33: fixed. Handlers expect real stream object for files, even for anti-file. @@ -465,7 +569,7 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea return S_OK; } - RINOK(Callback->GetStream(DirItems->GetLogPath(up.DirIndex), isDir, false, mode)); + RINOK(Callback->GetStream(DirItems->GetLogPath((unsigned)up.DirIndex), isDir, false, mode)); if (isDir) return S_OK; @@ -482,32 +586,100 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea } else { + #if !defined(UNDER_CE) + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + if (di.AreReparseData()) + { + /* + // we still need DeviceIoControlOut() instead of Read + if (!inStreamSpec->File.OpenReparse(path)) + { + return Callback->OpenFileError(path, ::GetLastError()); + } + */ + // 20.03: we use Reparse Data instead of real data + + CBufInStream *inStreamSpec = new CBufInStream(); + CMyComPtr inStreamLoc = inStreamSpec; + inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); + *inStream = inStreamLoc.Detach(); + + UpdateProcessedItemStatus((unsigned)up.DirIndex); + return S_OK; + } + #endif // !defined(UNDER_CE) + CInFileStream *inStreamSpec = new CInFileStream; CMyComPtr inStreamLoc(inStreamSpec); + /* + // for debug: + #ifdef _WIN32 + inStreamSpec->StoreOwnerName = true; + inStreamSpec->OwnerName = "user_name"; + inStreamSpec->OwnerName += di.Name; + inStreamSpec->OwnerName += "11111111112222222222222333333333333"; + inStreamSpec->OwnerGroup = "gname_"; + inStreamSpec->OwnerGroup += inStreamSpec->OwnerName; + #endif + */ + + #ifndef _WIN32 + inStreamSpec->StoreOwnerId = StoreOwnerId; + inStreamSpec->StoreOwnerName = StoreOwnerName; + + // if (StoreOwner) + { + inStreamSpec->_uid = di.uid; + inStreamSpec->_gid = di.gid; + if (di.OwnerNameIndex >= 0) + inStreamSpec->OwnerName = DirItems->OwnerNameMap.Strings[(unsigned)di.OwnerNameIndex]; + if (di.OwnerGroupIndex >= 0) + inStreamSpec->OwnerGroup = DirItems->OwnerGroupMap.Strings[(unsigned)di.OwnerGroupIndex]; + } + #endif + inStreamSpec->SupportHardLinks = StoreHardLinks; - inStreamSpec->Callback = this; - inStreamSpec->CallbackRef = index; + inStreamSpec->Set_PreserveATime(PreserveATime + || mode == NUpdateNotifyOp::kAnalyze); // 22.00 : we don't change access time in Analyze pass. - const FString path = DirItems->GetPhyPath(up.DirIndex); + const FString path = DirItems->GetPhyPath((unsigned)up.DirIndex); _openFiles_Indexes.Add(index); _openFiles_Paths.Add(path); + // _openFiles_Streams.Add(inStreamSpec); + + /* 21.02 : we set Callback/CallbackRef after _openFiles_Indexes adding + for correct working if exception was raised in GetPhyPath */ + inStreamSpec->Callback = this; + inStreamSpec->CallbackRef = index; - #if defined(_WIN32) && !defined(UNDER_CE) - if (DirItems->Items[up.DirIndex].AreReparseData()) + if (!inStreamSpec->OpenShared(path, ShareForWrite)) { - if (!inStreamSpec->File.OpenReparse(path)) - { - return Callback->OpenFileError(path, ::GetLastError()); - } + const DWORD error = ::GetLastError(); + const HRESULT hres = Callback->OpenFileError(path, error); + if (StopAfterOpenError) + if (hres == S_OK || hres == S_FALSE) + return HRESULT_FROM_WIN32(error); + return hres; } - else - #endif - if (!inStreamSpec->OpenShared(path, ShareForWrite)) + + /* + { + // for debug: + Byte b = 0; + UInt32 processedSize = 0; + if (inStreamSpec->Read(&b, 1, &processedSize) != S_OK || + processedSize != 1) + return E_FAIL; + } + */ + + if (Need_LatestMTime) { - return Callback->OpenFileError(path, ::GetLastError()); + inStreamSpec->ReloadProps(); } + // #if defined(USE_WIN_FILE) || !defined(_WIN32) if (StoreHardLinks) { CStreamFileProps props; @@ -532,14 +704,9 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea } } } + // #endif - if (ProcessedItemsStatuses) - { - #ifndef _7ZIP_ST - NSynchronization::CCriticalSectionLock lock(CS); - #endif - ProcessedItemsStatuses[(unsigned)up.DirIndex] = 1; - } + UpdateProcessedItemStatus((unsigned)up.DirIndex); *inStream = inStreamLoc.Detach(); } @@ -568,6 +735,8 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in { COM_TRY_BEGIN + // if (op == NUpdateNotifyOp::kOpFinished) return Callback->ReportFinished(indexType, index); + bool isDir = false; if (indexType == NArchive::NEventIndexType::kOutArcIndex) @@ -578,11 +747,11 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in const CUpdatePair2 &up = (*UpdatePairs)[index]; if (up.ExistOnDisk()) { - name = DirItems->GetLogPath(up.DirIndex); - isDir = DirItems->Items[up.DirIndex].IsDir(); + name = DirItems->GetLogPath((unsigned)up.DirIndex); + isDir = DirItems->Items[(unsigned)up.DirIndex].IsDir(); } } - return Callback->ReportUpdateOpeartion(op, name.IsEmpty() ? NULL : name.Ptr(), isDir); + return Callback->ReportUpdateOperation(op, name.IsEmpty() ? NULL : name.Ptr(), isDir); } wchar_t temp[16]; @@ -601,7 +770,7 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in } else if (Arc) { - RINOK(Arc->GetItemPath(index, s2)); + RINOK(Arc->GetItem_Path(index, s2)); s = s2; RINOK(Archive_IsItem_Dir(Arc->Archive, index, isDir)); } @@ -617,7 +786,7 @@ STDMETHODIMP CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 in if (!s) s = L""; - return Callback->ReportUpdateOpeartion(op, s, isDir); + return Callback->ReportUpdateOperation(op, s, isDir); COM_TRY_END } @@ -656,7 +825,7 @@ STDMETHODIMP CArchiveUpdateCallback::ReportExtractResult(UInt32 indexType, UInt3 s = (*ArcItems)[index].Name; else if (Arc) { - RINOK(Arc->GetItemPath(index, s2)); + RINOK(Arc->GetItem_Path(index, s2)); s = s2; } if (Archive) @@ -677,6 +846,51 @@ STDMETHODIMP CArchiveUpdateCallback::ReportExtractResult(UInt32 indexType, UInt3 COM_TRY_END } + +/* +STDMETHODIMP CArchiveUpdateCallback::DoNeedArcProp(PROPID propID, Int32 *answer) +{ + *answer = 0; + if (Need_ArcMTime_Report && propID == kpidComboMTime) + *answer = 1; + return S_OK; +} + +STDMETHODIMP CArchiveUpdateCallback::ReportProp(UInt32 indexType, UInt32 index, PROPID propID, const PROPVARIANT *value) +{ + if (indexType == NArchive::NEventIndexType::kArcProp) + { + if (propID == kpidComboMTime) + { + ArcMTime_WasReported = true; + if (value->vt == VT_FILETIME) + { + Reported_ArcMTime.Set_From_Prop(*value); + Reported_ArcMTime.Def = true; + } + else + { + Reported_ArcMTime.Clear(); + if (value->vt != VT_EMPTY) + return E_FAIL; // for debug + } + } + } + return Callback->ReportProp(indexType, index, propID, value); +} + +STDMETHODIMP CArchiveUpdateCallback::ReportRawProp(UInt32 indexType, UInt32 index, + PROPID propID, const void *data, UInt32 dataSize, UInt32 propType) +{ + return Callback->ReportRawProp(indexType, index, propID, data, dataSize, propType); +} + +STDMETHODIMP CArchiveUpdateCallback::ReportFinished(UInt32 indexType, UInt32 index, Int32 opRes) +{ + return Callback->ReportFinished(indexType, index, opRes); +} +*/ + STDMETHODIMP CArchiveUpdateCallback::GetVolumeSize(UInt32 index, UInt64 *size) { if (VolumesSizes.Size() == 0) @@ -690,19 +904,19 @@ STDMETHODIMP CArchiveUpdateCallback::GetVolumeSize(UInt32 index, UInt64 *size) STDMETHODIMP CArchiveUpdateCallback::GetVolumeStream(UInt32 index, ISequentialOutStream **volumeStream) { COM_TRY_BEGIN - FChar temp[16]; + char temp[16]; ConvertUInt32ToString(index + 1, temp); - FString res = temp; + FString res (temp); while (res.Len() < 2) res.InsertAtFront(FTEXT('0')); FString fileName = VolName; - fileName += FTEXT('.'); + fileName += '.'; fileName += res; fileName += VolExt; COutFileStream *streamSpec = new COutFileStream; CMyComPtr streamLoc(streamSpec); if (!streamSpec->Create(fileName, false)) - return ::GetLastError(); + return GetLastError_noZero_HRESULT(); *volumeStream = streamLoc.Detach(); return S_OK; COM_TRY_END @@ -724,10 +938,13 @@ STDMETHODIMP CArchiveUpdateCallback::CryptoGetTextPassword(BSTR *password) HRESULT CArchiveUpdateCallback::InFileStream_On_Error(UINT_PTR val, DWORD error) { - if (error == ERROR_LOCK_VIOLATION) + #ifdef _WIN32 // FIX IT !!! + // why did we check only for ERROR_LOCK_VIOLATION ? + // if (error == ERROR_LOCK_VIOLATION) + #endif { MT_LOCK - UInt32 index = (UInt32)val; + const UInt32 index = (UInt32)val; FOR_VECTOR(i, _openFiles_Indexes) { if (_openFiles_Indexes[i] == index) @@ -740,18 +957,35 @@ HRESULT CArchiveUpdateCallback::InFileStream_On_Error(UINT_PTR val, DWORD error) return HRESULT_FROM_WIN32(error); } -void CArchiveUpdateCallback::InFileStream_On_Destroy(UINT_PTR val) +void CArchiveUpdateCallback::InFileStream_On_Destroy(CInFileStream *stream, UINT_PTR val) { MT_LOCK - UInt32 index = (UInt32)val; + if (Need_LatestMTime) + { + if (stream->_info_WasLoaded) + { + const CFiTime &ft = ST_MTIME(stream->_info); + if (!LatestMTime_Defined + || Compare_FiTime(&LatestMTime, &ft) < 0) + LatestMTime = ft; + LatestMTime_Defined = true; + } + } + const UInt32 index = (UInt32)val; FOR_VECTOR(i, _openFiles_Indexes) { if (_openFiles_Indexes[i] == index) { _openFiles_Indexes.Delete(i); _openFiles_Paths.Delete(i); + // _openFiles_Streams.Delete(i); return; } } - throw 20141125; + /* 21.02 : this function can be called in destructor. + And destructor can be called after some exception. + If we don't want to throw exception in desctructors or after another exceptions, + we must disable the code below that raises new exception. + */ + // throw 20141125; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.h b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.h index 4ff4a7ebe1d..3719c1ea67a 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateCallback.h @@ -15,6 +15,18 @@ #include "OpenArchive.h" +struct CArcToDoStat +{ + CDirItemsStat2 NewData; + CDirItemsStat2 OldData; + CDirItemsStat2 DeleteData; + + UInt64 Get_NumDataItems_Total() const + { + return NewData.Get_NumDataItems2() + OldData.Get_NumDataItems2(); + } +}; + #define INTERFACE_IUpdateCallbackUI(x) \ virtual HRESULT WriteSfx(const wchar_t *name, UInt64 size) x; \ virtual HRESULT SetTotal(UInt64 size) x; \ @@ -22,17 +34,24 @@ virtual HRESULT SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) x; \ virtual HRESULT CheckBreak() x; \ /* virtual HRESULT Finalize() x; */ \ - virtual HRESULT SetNumItems(UInt64 numItems) x; \ + virtual HRESULT SetNumItems(const CArcToDoStat &stat) x; \ virtual HRESULT GetStream(const wchar_t *name, bool isDir, bool isAnti, UInt32 mode) x; \ virtual HRESULT OpenFileError(const FString &path, DWORD systemError) x; \ virtual HRESULT ReadingFileError(const FString &path, DWORD systemError) x; \ virtual HRESULT SetOperationResult(Int32 opRes) x; \ virtual HRESULT ReportExtractResult(Int32 opRes, Int32 isEncrypted, const wchar_t *name) x; \ - virtual HRESULT ReportUpdateOpeartion(UInt32 op, const wchar_t *name, bool isDir) x; \ + virtual HRESULT ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) x; \ /* virtual HRESULT SetPassword(const UString &password) x; */ \ virtual HRESULT CryptoGetTextPassword2(Int32 *passwordIsDefined, BSTR *password) x; \ virtual HRESULT CryptoGetTextPassword(BSTR *password) x; \ virtual HRESULT ShowDeleteFile(const wchar_t *name, bool isDir) x; \ + + /* + virtual HRESULT ReportProp(UInt32 indexType, UInt32 index, PROPID propID, const PROPVARIANT *value) x; \ + virtual HRESULT ReportRawProp(UInt32 indexType, UInt32 index, PROPID propID, const void *data, UInt32 dataSize, UInt32 propType) x; \ + virtual HRESULT ReportFinished(UInt32 indexType, UInt32 index, Int32 opRes) x; \ + */ + /* virtual HRESULT CloseProgress() { return S_OK; } */ struct IUpdateCallbackUI @@ -58,6 +77,7 @@ struct CKeyKeyValPair class CArchiveUpdateCallback: public IArchiveUpdateCallback2, public IArchiveUpdateCallbackFile, + // public IArchiveUpdateCallbackArcProp, public IArchiveExtractCallbackMessage, public IArchiveGetRawProps, public IArchiveGetRootProps, @@ -75,9 +95,12 @@ class CArchiveUpdateCallback: UInt32 _hardIndex_From; UInt32 _hardIndex_To; + void UpdateProcessedItemStatus(unsigned dirIndex); + public: MY_QUERYINTERFACE_BEGIN2(IArchiveUpdateCallback2) MY_QUERYINTERFACE_ENTRY(IArchiveUpdateCallbackFile) + // MY_QUERYINTERFACE_ENTRY(IArchiveUpdateCallbackArcProp) MY_QUERYINTERFACE_ENTRY(IArchiveExtractCallbackMessage) MY_QUERYINTERFACE_ENTRY(IArchiveGetRawProps) MY_QUERYINTERFACE_ENTRY(IArchiveGetRootProps) @@ -92,6 +115,7 @@ public: INTERFACE_IArchiveUpdateCallback2(;) INTERFACE_IArchiveUpdateCallbackFile(;) + // INTERFACE_IArchiveUpdateCallbackArcProp(;) INTERFACE_IArchiveExtractCallbackMessage(;) INTERFACE_IArchiveGetRawProps(;) INTERFACE_IArchiveGetRootProps(;) @@ -101,14 +125,16 @@ public: CRecordVector _openFiles_Indexes; FStringVector _openFiles_Paths; + // CRecordVector< CInFileStream* > _openFiles_Streams; bool AreAllFilesClosed() const { return _openFiles_Indexes.IsEmpty(); } virtual HRESULT InFileStream_On_Error(UINT_PTR val, DWORD error); - virtual void InFileStream_On_Destroy(UINT_PTR val); + virtual void InFileStream_On_Destroy(CInFileStream *stream, UINT_PTR val); CRecordVector VolumesSizes; FString VolName; FString VolExt; + UString ArcFileName; // without path prefix IUpdateCallbackUI *Callback; @@ -120,8 +146,12 @@ public: const CObjectVector *ArcItems; const CRecordVector *UpdatePairs; const UStringVector *NewNames; + int CommentIndex; + const UString *Comment; + bool PreserveATime; bool ShareForWrite; + bool StopAfterOpenError; bool StdInMode; bool KeepOriginalItemNames; @@ -129,17 +159,30 @@ public: bool StoreHardLinks; bool StoreSymLinks; + bool StoreOwnerId; + bool StoreOwnerName; + + /* + bool Need_ArcMTime_Report; + bool ArcMTime_WasReported; + CArcTime Reported_ArcMTime; + */ + bool Need_LatestMTime; + bool LatestMTime_Defined; + CFiTime LatestMTime; + Byte *ProcessedItemsStatuses; + CArchiveUpdateCallback(); bool IsDir(const CUpdatePair2 &up) const { if (up.DirIndex >= 0) - return DirItems->Items[up.DirIndex].IsDir(); + return DirItems->Items[(unsigned)up.DirIndex].IsDir(); else if (up.ArcIndex >= 0) - return (*ArcItems)[up.ArcIndex].IsDir; + return (*ArcItems)[(unsigned)up.ArcIndex].IsDir; return false; } }; diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/UpdatePair.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/UpdatePair.cpp index e21151974be..e9a16444bad 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/UpdatePair.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/UpdatePair.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include +// #include #include "../../../Common/Wildcard.h" @@ -14,38 +15,99 @@ using namespace NWindows; using namespace NTime; -static int MyCompareTime(NFileTimeType::EEnum fileTimeType, const FILETIME &time1, const FILETIME &time2) + +/* + a2.Prec = + { + 0 (k_PropVar_TimePrec_0): + if GetProperty(kpidMTime) returned 0 and + GetProperty(kpidTimeType) did not returned VT_UI4. + 7z, wim, tar in 7-Zip before v21) + in that case we use + (prec) that is set by IOutArchive::GetFileTimeType() + } +*/ + +static int MyCompareTime(unsigned prec, const CFiTime &f1, const CArcTime &a2) { - switch (fileTimeType) + // except of precision, we also have limitation, when timestamp is out of range + + /* if (Prec) in archive item is defined, then use global (prec) */ + if (a2.Prec != k_PropVar_TimePrec_0) + prec = a2.Prec; + + CArcTime a1; + a1.Set_From_FiTime(f1); + /* Set_From_FiTime() must set full form precision: + k_PropVar_TimePrec_Base + numDigits + windows: 7 digits, non-windows: 9 digits */ + + if (prec == k_PropVar_TimePrec_DOS) { - case NFileTimeType::kWindows: - return ::CompareFileTime(&time1, &time2); - case NFileTimeType::kUnix: - { - UInt32 unixTime1, unixTime2; - FileTimeToUnixTime(time1, unixTime1); - FileTimeToUnixTime(time2, unixTime2); - return MyCompare(unixTime1, unixTime2); - } - case NFileTimeType::kDOS: - { - UInt32 dosTime1, dosTime2; - FileTimeToDosTime(time1, dosTime1); - FileTimeToDosTime(time2, dosTime2); - return MyCompare(dosTime1, dosTime2); - } + const UInt32 dosTime1 = a1.Get_DosTime(); + const UInt32 dosTime2 = a2.Get_DosTime(); + return MyCompare(dosTime1, dosTime2); } - throw 4191618; + + if (prec == k_PropVar_TimePrec_Unix) + { + const Int64 u2 = FileTime_To_UnixTime64(a2.FT); + if (u2 == 0 || u2 == (UInt32)0xFFFFFFFF) + { + // timestamp probably was saturated in archive to 32-bit + // so we use saturated 32-bit value for disk file too. + UInt32 u1; + FileTime_To_UnixTime(a1.FT, u1); + const UInt32 u2_32 = (UInt32)u2; + return MyCompare(u1, u2_32); + } + + const Int64 u1 = FileTime_To_UnixTime64(a1.FT); + return MyCompare(u1, u2); + // prec = k_PropVar_TimePrec_Base; // for debug + } + + if (prec == k_PropVar_TimePrec_0) + prec = k_PropVar_TimePrec_Base + 7; + else if (prec == k_PropVar_TimePrec_HighPrec) + prec = k_PropVar_TimePrec_Base + 9; + else if (prec < k_PropVar_TimePrec_Base) + prec = k_PropVar_TimePrec_Base; + else if (prec > k_PropVar_TimePrec_Base + 9) + prec = k_PropVar_TimePrec_Base + 7; + + // prec now is full form: k_PropVar_TimePrec_Base + numDigits; + if (prec > a1.Prec && a1.Prec >= k_PropVar_TimePrec_Base) + prec = a1.Prec; + + const unsigned numDigits = prec - k_PropVar_TimePrec_Base; + if (numDigits >= 7) + { + const int comp = CompareFileTime(&a1.FT, &a2.FT); + if (comp != 0 || numDigits == 7) + return comp; + return MyCompare(a1.Ns100, a2.Ns100); + } + UInt32 d = 1; + for (unsigned k = numDigits; k < 7; k++) + d *= 10; + const UInt64 v1 = a1.Get_FILETIME_as_UInt64() / d * d; + const UInt64 v2 = a2.Get_FILETIME_as_UInt64() / d * d; + // printf("\ndelta=%d numDigits=%d\n", (unsigned)(v1- v2), numDigits); + return MyCompare(v1, v2); } -static const char *k_Duplicate_inArc_Message = "Duplicate filename in archive:"; -static const char *k_Duplicate_inDir_Message = "Duplicate filename on disk:"; -static const char *k_NotCensoredCollision_Message = "Internal file name collision (file on disk, file in archive):"; -static void ThrowError(const char *message, const UString &s1, const UString &s2) + +static const char * const k_Duplicate_inArc_Message = "Duplicate filename in archive:"; +static const char * const k_Duplicate_inDir_Message = "Duplicate filename on disk:"; +static const char * const k_NotCensoredCollision_Message = "Internal file name collision (file on disk, file in archive):"; + +MY_ATTR_NORETURN +static +void ThrowError(const char *message, const UString &s1, const UString &s2) { - UString m; - m.SetFromAscii(message); + UString m (message); m.Add_LF(); m += s1; m.Add_LF(); m += s2; throw m; @@ -63,8 +125,8 @@ static int CompareArcItemsBase(const CArcItem &ai1, const CArcItem &ai2) static int CompareArcItems(const unsigned *p1, const unsigned *p2, void *param) { - unsigned i1 = *p1; - unsigned i2 = *p2; + const unsigned i1 = *p1; + const unsigned i2 = *p2; const CObjectVector &arcItems = *(const CObjectVector *)param; int res = CompareArcItemsBase(arcItems[i1], arcItems[i2]); if (res != 0) @@ -80,8 +142,8 @@ void GetUpdatePairInfoList( { CUIntVector dirIndices, arcIndices; - unsigned numDirItems = dirItems.Items.Size(); - unsigned numArcItems = arcItems.Size(); + const unsigned numDirItems = dirItems.Items.Size(); + const unsigned numArcItems = arcItems.Size(); CIntArr duplicatedArcItem(numArcItems); { @@ -145,18 +207,18 @@ void GetUpdatePairInfoList( if (dirIndex < numDirItems) { - dirIndex2 = dirIndices[dirIndex]; - di = &dirItems.Items[dirIndex2]; + dirIndex2 = (int)dirIndices[dirIndex]; + di = &dirItems.Items[(unsigned)dirIndex2]; } if (arcIndex < numArcItems) { - arcIndex2 = arcIndices[arcIndex]; - ai = &arcItems[arcIndex2]; + arcIndex2 = (int)arcIndices[arcIndex]; + ai = &arcItems[(unsigned)arcIndex2]; compareResult = 1; if (dirIndex < numDirItems) { - compareResult = CompareFileNames(dirNames[dirIndex2], ai->Name); + compareResult = CompareFileNames(dirNames[(unsigned)dirIndex2], ai->Name); if (compareResult == 0) { if (di->IsDir() != ai->IsDir) @@ -167,7 +229,7 @@ void GetUpdatePairInfoList( if (compareResult < 0) { - name = &dirNames[dirIndex2]; + name = &dirNames[(unsigned)dirIndex2]; pair.State = NUpdateArchive::NPairState::kOnlyOnDisk; pair.DirIndex = dirIndex2; dirIndex++; @@ -183,25 +245,28 @@ void GetUpdatePairInfoList( } else { - int dupl = duplicatedArcItem[arcIndex]; + const int dupl = duplicatedArcItem[arcIndex]; if (dupl != 0) - ThrowError(k_Duplicate_inArc_Message, ai->Name, arcItems[arcIndices[arcIndex + dupl]].Name); + ThrowError(k_Duplicate_inArc_Message, ai->Name, arcItems[arcIndices[(unsigned)((int)arcIndex + dupl)]].Name); - name = &dirNames[dirIndex2]; + name = &dirNames[(unsigned)dirIndex2]; if (!ai->Censored) ThrowError(k_NotCensoredCollision_Message, *name, ai->Name); pair.DirIndex = dirIndex2; pair.ArcIndex = arcIndex2; - switch (ai->MTimeDefined ? MyCompareTime( - ai->TimeType != - 1 ? (NFileTimeType::EEnum)ai->TimeType : fileTimeType, - di->MTime, ai->MTime): 0) + int compResult = 0; + if (ai->MTime.Def) + { + compResult = MyCompareTime(fileTimeType, di->MTime, ai->MTime); + } + switch (compResult) { case -1: pair.State = NUpdateArchive::NPairState::kNewInArchive; break; case 1: pair.State = NUpdateArchive::NPairState::kOldInArchive; break; default: - pair.State = (ai->SizeDefined && di->Size == ai->Size) ? + pair.State = (ai->Size_Defined && di->Size == ai->Size) ? NUpdateArchive::NPairState::kSameFiles : NUpdateArchive::NPairState::kUnknowNewerFiles; } @@ -210,7 +275,10 @@ void GetUpdatePairInfoList( arcIndex++; } - if ((di && di->IsAltStream) || + if ( + #ifdef _WIN32 + (di && di->IsAltStream) || + #endif (ai && ai->IsAltStream)) { if (prevHostName) @@ -223,7 +291,7 @@ void GetUpdatePairInfoList( } else { - prevHostFile = updatePairs.Size(); + prevHostFile = (int)updatePairs.Size(); prevHostName = name; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.cpp index e6eabcf148e..e921dc326b6 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.cpp @@ -6,7 +6,7 @@ using namespace NUpdateArchive; -static const char *kUpdateActionSetCollision = "Internal collision in update action set"; +static const char * const kUpdateActionSetCollision = "Internal collision in update action set"; void UpdateProduce( const CRecordVector &updatePairs, @@ -28,7 +28,7 @@ void UpdateProduce( { case NPairAction::kIgnore: if (pair.ArcIndex >= 0 && callback) - callback->ShowDeleteFile(pair.ArcIndex); + callback->ShowDeleteFile((unsigned)pair.ArcIndex); continue; case NPairAction::kCopy: @@ -43,7 +43,7 @@ void UpdateProduce( 1) no such alt stream in Disk 2) there is Host file in disk */ - if (updatePairs[pair.HostIndex].DirIndex >= 0) + if (updatePairs[(unsigned)pair.HostIndex].DirIndex >= 0) continue; } } @@ -63,6 +63,8 @@ void UpdateProduce( break; } + up2.IsSameTime = ((unsigned)pair.State == NUpdateArchive::NPairState::kSameFiles); + operationChain.Add(up2); } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.h b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.h index 64c58cc5772..24bb32ec06f 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/UpdateProduce.h @@ -17,13 +17,14 @@ struct CUpdatePair2 int NewNameIndex; bool IsMainRenameItem; + bool IsSameTime; - void SetAs_NoChangeArcItem(int arcIndex) + void SetAs_NoChangeArcItem(unsigned arcIndex) // int { NewData = NewProps = false; UseArcProps = true; IsAnti = false; - ArcIndex = arcIndex; + ArcIndex = (int)arcIndex; } bool ExistOnDisk() const { return DirIndex != -1; } @@ -37,7 +38,8 @@ struct CUpdatePair2 DirIndex(-1), ArcIndex(-1), NewNameIndex(-1), - IsMainRenameItem(false) + IsMainRenameItem(false), + IsSameTime(false) {} }; diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/WorkDir.cpp b/3rdparty/lzma/CPP/7zip/UI/Common/WorkDir.cpp index 284eaa16ac3..1307ceebb9b 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/WorkDir.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Common/WorkDir.cpp @@ -5,6 +5,7 @@ #include "../../../Common/StringConvert.h" #include "../../../Common/Wildcard.h" +#include "../../../Windows/FileFind.h" #include "../../../Windows/FileName.h" #include "WorkDir.h" @@ -39,13 +40,13 @@ FString GetWorkDir(const NWorkDir::CInfo &workDirInfo, const FString &path, FStr #endif int pos = path.ReverseFind_PathSepar() + 1; - fileName = path.Ptr(pos); + fileName = path.Ptr((unsigned)pos); switch (mode) { case NWorkDir::NMode::kCurrent: { - return path.Left(pos); + return path.Left((unsigned)pos); } case NWorkDir::NMode::kSpecified: { @@ -75,8 +76,7 @@ HRESULT CWorkDirTempFile::CreateTempFile(const FString &originalPath) OutStream = _outStreamSpec; if (!_tempFile.Create(workDir + namePart, &_outStreamSpec->File)) { - DWORD error = GetLastError(); - return error ? error : E_FAIL; + return GetLastError_noZero_HRESULT(); } _originalPath = originalPath; return S_OK; @@ -87,8 +87,7 @@ HRESULT CWorkDirTempFile::MoveToOriginal(bool deleteOriginal) OutStream.Release(); if (!_tempFile.MoveTo(_originalPath, deleteOriginal)) { - DWORD error = GetLastError(); - return error ? error : E_FAIL; + return GetLastError_noZero_HRESULT(); } return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Common/ZipRegistry.h b/3rdparty/lzma/CPP/7zip/UI/Common/ZipRegistry.h index 935f97bbaa4..b7075e67780 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Common/ZipRegistry.h +++ b/3rdparty/lzma/CPP/7zip/UI/Common/ZipRegistry.h @@ -6,8 +6,20 @@ #include "../../../Common/MyTypes.h" #include "../../../Common/MyString.h" +#include "../../Common/MethodProps.h" + #include "ExtractMode.h" +/* +CBoolPair::Def in writing functions means: + if ( CBoolPair::Def ), we write CBoolPair::Val + if ( !CBoolPair::Def ) + { + in NCompression functions we delete registry value + in another functions we do nothing + } +*/ + namespace NExtract { struct CInfo @@ -35,6 +47,36 @@ namespace NExtract namespace NCompression { + struct CMemUse + { + // UString Str; + bool IsDefined; + bool IsPercent; + UInt64 Val; + + CMemUse(): + IsDefined(false), + IsPercent(false), + Val(0) + {} + + void Clear() + { + // Str.Empty(); + IsDefined = false; + IsPercent = false; + Val = 0; + } + + UInt64 GetBytes(UInt64 ramSize) const + { + if (!IsPercent) + return Val; + return Calc_From_Val_Percents(ramSize, Val); + } + void Parse(const UString &s); + }; + struct CFormatOptions { UInt32 Level; @@ -43,19 +85,47 @@ namespace NCompression UInt32 BlockLogSize; UInt32 NumThreads; + UInt32 TimePrec; + CBoolPair MTime; + CBoolPair ATime; + CBoolPair CTime; + CBoolPair SetArcMTime; + CSysString FormatID; UString Method; UString Options; UString EncryptionMethod; + UString MemUse; + + void Reset_TimePrec() + { + TimePrec = (UInt32)(Int32)-1; + } + + bool IsSet_TimePrec() const + { + return TimePrec != (UInt32)(Int32)-1; + } + + + void Reset_BlockLogSize() + { + BlockLogSize = (UInt32)(Int32)-1; + } void ResetForLevelChange() { - BlockLogSize = NumThreads = Level = Dictionary = Order = UInt32(-1); + BlockLogSize = NumThreads = Level = Dictionary = Order = (UInt32)(Int32)-1; Method.Empty(); // Options.Empty(); // EncryptionMethod.Empty(); } - CFormatOptions() { ResetForLevelChange(); } + CFormatOptions() + { + // TimePrec = 0; + Reset_TimePrec(); + ResetForLevelChange(); + } }; struct CInfo @@ -73,6 +143,8 @@ namespace NCompression CBoolPair HardLinks; CBoolPair SymLinks; + CBoolPair PreserveATime; + void Save() const; void Load(); }; @@ -114,9 +186,18 @@ struct CContextMenuInfo CBoolPair Cascaded; CBoolPair MenuIcons; CBoolPair ElimDup; - + bool Flags_Def; UInt32 Flags; + UInt32 WriteZone; + + /* + CContextMenuInfo(): + Flags_Def(0), + WriteZone((UInt32)(Int32)-1), + Flags((UInt32)(Int32)-1) + {} + */ void Save() const; void Load(); diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/Console.mak b/3rdparty/lzma/CPP/7zip/UI/Console/Console.mak index 05a07e5ad8a..bd4c1da4f3f 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/Console.mak +++ b/3rdparty/lzma/CPP/7zip/UI/Console/Console.mak @@ -1,3 +1,9 @@ +MY_CONSOLE = 1 + +!IFNDEF UNDER_CE +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE +!ENDIF + CONSOLE_OBJS = \ $O\BenchCon.obj \ $O\ConsoleClose.obj \ @@ -33,4 +39,5 @@ UI_COMMON_OBJS = \ $O\UpdatePair.obj \ $O\UpdateProduce.obj \ -# +C_OBJS = $(C_OBJS) \ + $O\DllSecur.obj \ diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/Console.manifest b/3rdparty/lzma/CPP/7zip/UI/Console/Console.manifest new file mode 100644 index 00000000000..58b68ced854 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/UI/Console/Console.manifest @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.cpp index 703f821395c..9e4c040d3f8 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.cpp @@ -4,8 +4,13 @@ #include "ConsoleClose.h" -#if !defined(UNDER_CE) && defined(_WIN32) +#ifndef UNDER_CE + +#ifdef _WIN32 #include "../../../Common/MyWindows.h" +#else +#include +#include #endif namespace NConsoleClose { @@ -13,7 +18,8 @@ namespace NConsoleClose { unsigned g_BreakCounter = 0; static const unsigned kBreakAbortThreshold = 2; -#if !defined(UNDER_CE) && defined(_WIN32) +#ifdef _WIN32 + static BOOL WINAPI HandlerRoutine(DWORD ctrlType) { if (ctrlType == CTRL_LOGOFF_EVENT) @@ -37,33 +43,58 @@ static BOOL WINAPI HandlerRoutine(DWORD ctrlType) return FALSE; */ } -#endif - -/* -void CheckCtrlBreak() -{ - if (TestBreakSignal()) - throw CCtrlBreakException(); -} -*/ CCtrlHandlerSetter::CCtrlHandlerSetter() { - #if !defined(UNDER_CE) && defined(_WIN32) if (!SetConsoleCtrlHandler(HandlerRoutine, TRUE)) throw "SetConsoleCtrlHandler fails"; - #endif } CCtrlHandlerSetter::~CCtrlHandlerSetter() { - #if !defined(UNDER_CE) && defined(_WIN32) if (!SetConsoleCtrlHandler(HandlerRoutine, FALSE)) { // warning for throw in destructor. // throw "SetConsoleCtrlHandler fails"; } - #endif } +#else // _WIN32 + +static void HandlerRoutine(int) +{ + g_BreakCounter++; + if (g_BreakCounter < kBreakAbortThreshold) + return; + exit(EXIT_FAILURE); +} + +CCtrlHandlerSetter::CCtrlHandlerSetter() +{ + memo_sig_int = signal(SIGINT, HandlerRoutine); // CTRL-C + if (memo_sig_int == SIG_ERR) + throw "SetConsoleCtrlHandler fails (SIGINT)"; + memo_sig_term = signal(SIGTERM, HandlerRoutine); // for kill -15 (before "kill -9") + if (memo_sig_term == SIG_ERR) + throw "SetConsoleCtrlHandler fails (SIGTERM)"; +} + +CCtrlHandlerSetter::~CCtrlHandlerSetter() +{ + signal(SIGINT, memo_sig_int); // CTRL-C + signal(SIGTERM, memo_sig_term); // kill {pid} +} + +#endif // _WIN32 + +/* +void CheckCtrlBreak() +{ + if (TestBreakSignal()) + throw CCtrlBreakException(); +} +*/ + } + +#endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.h b/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.h index 11c1631ca71..9c9e035c6f3 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/ConsoleClose.h @@ -5,28 +5,34 @@ namespace NConsoleClose { +class CCtrlBreakException {}; + +#ifdef UNDER_CE + +inline bool TestBreakSignal() { return false; } +struct CCtrlHandlerSetter {}; + +#else + extern unsigned g_BreakCounter; inline bool TestBreakSignal() { - #ifdef UNDER_CE - return false; - #else return (g_BreakCounter != 0); - #endif } class CCtrlHandlerSetter { + #ifndef _WIN32 + void (*memo_sig_int)(int); + void (*memo_sig_term)(int); + #endif public: CCtrlHandlerSetter(); virtual ~CCtrlHandlerSetter(); }; -class CCtrlBreakException -{}; - -// void CheckCtrlBreak(); +#endif } diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index 8dac7fbc3e9..7f791b00bfa 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -32,7 +32,7 @@ static HRESULT CheckBreak2() return NConsoleClose::TestBreakSignal() ? E_ABORT : S_OK; } -static const char *kError = "ERROR: "; +static const char * const kError = "ERROR: "; void CExtractScanConsole::StartScanning() @@ -56,18 +56,27 @@ HRESULT CExtractScanConsole::ScanProgress(const CDirItemsStat &st, const FString HRESULT CExtractScanConsole::ScanError(const FString &path, DWORD systemError) { + // 22.00: + // ScanErrors.AddError(path, systemError); + ClosePercentsAndFlush(); if (_se) { - *_se << endl << kError << NError::MyFormatMessage(systemError) << endl << - fs2us(path) << endl << endl; + *_se << endl << kError << NError::MyFormatMessage(systemError) << endl; + _se->NormalizePrint_UString(fs2us(path)); + *_se << endl << endl; _se->Flush(); } return HRESULT_FROM_WIN32(systemError); + + // 22.00: commented + // CommonError(path, systemError, true); + // return S_OK; } +void Print_UInt64_and_String(AString &s, UInt64 val, const char *name); void Print_UInt64_and_String(AString &s, UInt64 val, const char *name) { char temp[32]; @@ -77,6 +86,7 @@ void Print_UInt64_and_String(AString &s, UInt64 val, const char *name) s += name; } +void PrintSize_bytes_Smart(AString &s, UInt64 val); void PrintSize_bytes_Smart(AString &s, UInt64 val) { Print_UInt64_and_String(s, val, "bytes"); @@ -95,6 +105,17 @@ void PrintSize_bytes_Smart(AString &s, UInt64 val) s += ')'; } +static void PrintSize_bytes_Smart_comma(AString &s, UInt64 val) +{ + if (val == (UInt64)(Int64)-1) + return; + s += ", "; + PrintSize_bytes_Smart(s, val); +} + + + +void Print_DirItemsStat(AString &s, const CDirItemsStat &st); void Print_DirItemsStat(AString &s, const CDirItemsStat &st) { if (st.NumDirs != 0) @@ -103,17 +124,49 @@ void Print_DirItemsStat(AString &s, const CDirItemsStat &st) s += ", "; } Print_UInt64_and_String(s, st.NumFiles, st.NumFiles == 1 ? "file" : "files"); - s += ", "; - PrintSize_bytes_Smart(s, st.FilesSize); + PrintSize_bytes_Smart_comma(s, st.FilesSize); if (st.NumAltStreams != 0) { s.Add_LF(); Print_UInt64_and_String(s, st.NumAltStreams, "alternate streams"); - s += ", "; - PrintSize_bytes_Smart(s, st.AltStreamsSize); + PrintSize_bytes_Smart_comma(s, st.AltStreamsSize); } } + +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st); +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st) +{ + Print_DirItemsStat(s, (CDirItemsStat &)st); + bool needLF = true; + if (st.Anti_NumDirs != 0) + { + if (needLF) + s.Add_LF(); + needLF = false; + Print_UInt64_and_String(s, st.Anti_NumDirs, st.Anti_NumDirs == 1 ? "anti-folder" : "anti-folders"); + } + if (st.Anti_NumFiles != 0) + { + if (needLF) + s.Add_LF(); + else + s += ", "; + needLF = false; + Print_UInt64_and_String(s, st.Anti_NumFiles, st.Anti_NumFiles == 1 ? "anti-file" : "anti-files"); + } + if (st.Anti_NumAltStreams != 0) + { + if (needLF) + s.Add_LF(); + else + s += ", "; + needLF = false; + Print_UInt64_and_String(s, st.Anti_NumAltStreams, "anti-alternate-streams"); + } +} + + void CExtractScanConsole::PrintStat(const CDirItemsStat &st) { if (_so) @@ -138,33 +191,34 @@ static NSynchronization::CCriticalSection g_CriticalSection; #endif -static const char *kTestString = "T"; -static const char *kExtractString = "-"; -static const char *kSkipString = "."; +static const char * const kTestString = "T"; +static const char * const kExtractString = "-"; +static const char * const kSkipString = "."; +static const char * const kReadString = "H"; -// static const char *kCantAutoRename = "can not create file with auto name\n"; -// static const char *kCantRenameFile = "can not rename existing file\n"; -// static const char *kCantDeleteOutputFile = "can not delete output file "; +// static const char * const kCantAutoRename = "cannot create file with auto name\n"; +// static const char * const kCantRenameFile = "cannot rename existing file\n"; +// static const char * const kCantDeleteOutputFile = "cannot delete output file "; -static const char *kMemoryExceptionMessage = "Can't allocate required memory!"; +static const char * const kMemoryExceptionMessage = "Can't allocate required memory!"; -static const char *kExtracting = "Extracting archive: "; -static const char *kTesting = "Testing archive: "; +static const char * const kExtracting = "Extracting archive: "; +static const char * const kTesting = "Testing archive: "; -static const char *kEverythingIsOk = "Everything is Ok"; -static const char *kNoFiles = "No files to process"; +static const char * const kEverythingIsOk = "Everything is Ok"; +static const char * const kNoFiles = "No files to process"; -static const char *kUnsupportedMethod = "Unsupported Method"; -static const char *kCrcFailed = "CRC Failed"; -static const char *kCrcFailedEncrypted = "CRC Failed in encrypted file. Wrong password?"; -static const char *kDataError = "Data Error"; -static const char *kDataErrorEncrypted = "Data Error in encrypted file. Wrong password?"; -static const char *kUnavailableData = "Unavailable data"; -static const char *kUnexpectedEnd = "Unexpected end of data"; -static const char *kDataAfterEnd = "There are some data after the end of the payload data"; -static const char *kIsNotArc = "Is not archive"; -static const char *kHeadersError = "Headers Error"; -static const char *kWrongPassword = "Wrong password"; +static const char * const kUnsupportedMethod = "Unsupported Method"; +static const char * const kCrcFailed = "CRC Failed"; +static const char * const kCrcFailedEncrypted = "CRC Failed in encrypted file. Wrong password?"; +static const char * const kDataError = "Data Error"; +static const char * const kDataErrorEncrypted = "Data Error in encrypted file. Wrong password?"; +static const char * const kUnavailableData = "Unavailable data"; +static const char * const kUnexpectedEnd = "Unexpected end of data"; +static const char * const kDataAfterEnd = "There are some data after the end of the payload data"; +static const char * const kIsNotArc = "Is not archive"; +static const char * const kHeadersError = "Headers Error"; +static const char * const kWrongPassword = "Wrong password"; static const char * const k_ErrorFlagsMessages[] = { @@ -206,12 +260,14 @@ STDMETHODIMP CExtractCallbackConsole::SetCompleted(const UInt64 *completeValue) return CheckBreak2(); } -static const char *kTab = " "; +static const char * const kTab = " "; static void PrintFileInfo(CStdOutStream *_so, const wchar_t *path, const FILETIME *ft, const UInt64 *size) { - *_so << kTab << "Path: " << path << endl; - if (size) + *_so << kTab << "Path: "; + _so->NormalizePrint_wstr(path); + *_so << endl; + if (size && *size != (UInt64)(Int64)-1) { AString s; PrintSize_bytes_Smart(s, *size); @@ -220,10 +276,8 @@ static void PrintFileInfo(CStdOutStream *_so, const wchar_t *path, const FILETIM if (ft) { char temp[64]; - FILETIME locTime; - if (FileTimeToLocalFileTime(ft, &locTime)) - if (ConvertFileTimeToString(locTime, temp, true, true)) - *_so << kTab << "Modified: " << temp << endl; + if (ConvertUtcFileTimeToString(*ft, temp, kTimestampPrintLevel_SEC)) + *_so << kTab << "Modified: " << temp << endl; } } @@ -256,6 +310,8 @@ STDMETHODIMP CExtractCallbackConsole::AskOverwrite( case NUserAnswerMode::kYesAll: *answer = NOverwriteAnswer::kYesToAll; break; case NUserAnswerMode::kYes: *answer = NOverwriteAnswer::kYes; break; case NUserAnswerMode::kAutoRenameAll: *answer = NOverwriteAnswer::kAutoRename; break; + case NUserAnswerMode::kEof: return E_ABORT; + case NUserAnswerMode::kError: return E_FAIL; default: return E_FAIL; } @@ -269,7 +325,7 @@ STDMETHODIMP CExtractCallbackConsole::AskOverwrite( return CheckBreak2(); } -STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int32 /* isFolder */, Int32 askExtractMode, const UInt64 *position) +STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int32 isFolder, Int32 askExtractMode, const UInt64 *position) { MT_LOCK @@ -283,6 +339,7 @@ STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 case NArchive::NExtract::NAskMode::kExtract: s = kExtractString; break; case NArchive::NExtract::NAskMode::kTest: s = kTestString; break; case NArchive::NExtract::NAskMode::kSkip: s = kSkipString; requiredLevel = 2; break; + case NArchive::NExtract::NAskMode::kReadExternal: s = kReadString; requiredLevel = 0; break; default: s = "???"; requiredLevel = 2; }; @@ -299,7 +356,16 @@ STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 _tempU.Empty(); if (name) + { _tempU = name; + _so->Normalize_UString(_tempU); + // 21.04 + if (isFolder) + { + if (!_tempU.IsEmpty() && _tempU.Back() != WCHAR_PATH_SEPARATOR) + _tempU.Add_PathSepar(); + } + } _so->PrintUString(_tempU, _tempA); if (position) *_so << " <" << *position << ">"; @@ -347,6 +413,7 @@ STDMETHODIMP CExtractCallbackConsole::MessageError(const wchar_t *message) return CheckBreak2(); } +void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest); void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest) { dest.Empty(); @@ -388,10 +455,8 @@ void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest) dest += s; else { - char temp[16]; - ConvertUInt32ToString(opRes, temp); dest += "Error #"; - dest += temp; + dest.Add_UInt32((UInt32)opRes); } } @@ -422,7 +487,10 @@ STDMETHODIMP CExtractCallbackConsole::SetOperationResult(Int32 opRes, Int32 encr *_se << s; if (!_currentName.IsEmpty()) - *_se << " : " << _currentName; + { + *_se << " : "; + _se->NormalizePrint_UString(_currentName); + } *_se << endl; _se->Flush(); } @@ -474,7 +542,11 @@ HRESULT CExtractCallbackConsole::BeforeOpen(const wchar_t *name, bool testMode) ClosePercents_for_so(); if (_so) - *_so << endl << (testMode ? kTesting : kExtracting) << name << endl; + { + *_so << endl << (testMode ? kTesting : kExtracting); + _so->NormalizePrint_wstr(name); + *_so << endl; + } if (NeedPercents()) _percent.Command = "Open"; @@ -514,6 +586,7 @@ static AString GetOpenArcErrorMessage(UInt32 errorFlags) return s; } +void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags); void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags) { if (errorFlags == 0) @@ -521,29 +594,31 @@ void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags) so << s << endl << GetOpenArcErrorMessage(errorFlags) << endl; } -void Add_Messsage_Pre_ArcType(UString &s, const char *pre, const wchar_t *arcType) +static void Add_Messsage_Pre_ArcType(UString &s, const char *pre, const wchar_t *arcType) { s.Add_LF(); - s.AddAscii(pre); - s.AddAscii(" as ["); + s += pre; + s += " as ["; s += arcType; - s.AddAscii("] archive"); + s += "] archive"; } +void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, const CArc &arc); void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, const CArc &arc) { const CArcErrorInfo &er = arc.ErrorInfo; - UString s = L"WARNING:\n"; - s += arc.Path; + *_so << "WARNING:\n"; + _so->NormalizePrint_UString(arc.Path); + UString s; if (arc.FormatIndex == er.ErrorFormatIndex) { s.Add_LF(); - s.AddAscii("The archive is open with offset"); + s += "The archive is open with offset"; } else { - Add_Messsage_Pre_ArcType(s, "Can not open the file", codecs->GetFormatNamePtr(er.ErrorFormatIndex)); + Add_Messsage_Pre_ArcType(s, "Cannot open the file", codecs->GetFormatNamePtr(er.ErrorFormatIndex)); Add_Messsage_Pre_ArcType(s, "The file is open", codecs->GetFormatNamePtr(arc.FormatIndex)); } @@ -580,7 +655,10 @@ HRESULT CExtractCallbackConsole::OpenResult( { *_se << endl; if (level != 0) - *_se << arc.Path << endl; + { + _se->NormalizePrint_UString(arc.Path); + *_se << endl; + } } if (errorFlags != 0) @@ -614,7 +692,10 @@ HRESULT CExtractCallbackConsole::OpenResult( { *_so << endl; if (level != 0) - *_so << arc.Path << endl; + { + _so->NormalizePrint_UString(arc.Path); + *_so << endl; + } } if (warningFlags != 0) @@ -669,7 +750,9 @@ HRESULT CExtractCallbackConsole::OpenResult( _so->Flush(); if (_se) { - *_se << kError << name << endl; + *_se << kError; + _se->NormalizePrint_wstr(name); + *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res); if (result == S_FALSE) @@ -745,7 +828,9 @@ HRESULT CExtractCallbackConsole::ExtractResult(HRESULT result) else { NumArcsWithError++; - if (result == E_ABORT || result == ERROR_DISK_FULL) + if (result == E_ABORT + || result == HRESULT_FROM_WIN32(ERROR_DISK_FULL) + ) return result; if (_se) diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.h b/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.h index dc659521e03..7964813d60f 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/ExtractCallbackConsole.h @@ -15,12 +15,33 @@ #include "OpenCallbackConsole.h" +/* +struct CErrorPathCodes2 +{ + FStringVector Paths; + CRecordVector Codes; + + void AddError(const FString &path, DWORD systemError) + { + Paths.Add(path); + Codes.Add(systemError); + } + void Clear() + { + Paths.Clear(); + Codes.Clear(); + } +}; +*/ + class CExtractScanConsole: public IDirItemsCallback { CStdOutStream *_so; CStdOutStream *_se; CPercentPrinter _percent; + // CErrorPathCodes2 ScanErrors; + bool NeedPercents() const { return _percent._so != NULL; } void ClosePercentsAndFlush() @@ -32,6 +53,9 @@ class CExtractScanConsole: public IDirItemsCallback } public: + + virtual ~CExtractScanConsole() {} + void Init(CStdOutStream *outStream, CStdOutStream *errorStream, CStdOutStream *percentStream) { _so = outStream; diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.cpp index 6a82daf2eb2..5a34976502b 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.cpp @@ -4,12 +4,14 @@ #include "../../../Common/IntToString.h" +#include "../../../Windows/FileName.h" + #include "ConsoleClose.h" #include "HashCon.h" -static const wchar_t *kEmptyFileAlias = L"[Content]"; +static const char * const kEmptyFileAlias = "[Content]"; -static const char *kScanningMessage = "Scanning"; +static const char * const kScanningMessage = "Scanning"; static HRESULT CheckBreak2() { @@ -33,13 +35,15 @@ HRESULT CHashCallbackConsole::StartScanning() return CheckBreak2(); } -HRESULT CHashCallbackConsole::ScanProgress(const CDirItemsStat &st, const FString &path, bool /* isDir */) +HRESULT CHashCallbackConsole::ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) { if (NeedPercents()) { _percent.Files = st.NumDirs + st.NumFiles + st.NumAltStreams; _percent.Completed = st.GetTotalBytes(); _percent.FileName = fs2us(path); + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_percent.FileName); _percent.Print(); } return CheckBreak2(); @@ -111,6 +115,15 @@ static void SetSpacesAndNul(char *s, unsigned num) s[num] = 0; } +static void SetSpacesAndNul_if_Positive(char *s, int num) +{ + if (num < 0) + return; + for (int i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + static const unsigned kSizeField_Len = 13; static const unsigned kNameField_Len = 12; @@ -122,61 +135,83 @@ static unsigned GetColumnWidth(unsigned digestSize) return width < kHashColumnWidth_Min ? kHashColumnWidth_Min: width; } -void CHashCallbackConsole::PrintSeparatorLine(const CObjectVector &hashers) + +AString CHashCallbackConsole::GetFields() const { - _s.Empty(); - - for (unsigned i = 0; i < hashers.Size(); i++) - { - if (i != 0) - _s.Add_Space(); - const CHasherState &h = hashers[i]; - AddMinuses(_s, GetColumnWidth(h.DigestSize)); - } + AString s (PrintFields); + if (s.IsEmpty()) + s = "hsn"; + s.MakeLower_Ascii(); + return s; +} - if (PrintSize) - { - _s.Add_Space(); - AddMinuses(_s, kSizeField_Len); - } - if (PrintName) +void CHashCallbackConsole::PrintSeparatorLine(const CObjectVector &hashers) +{ + _s.Empty(); + const AString fields = GetFields(); + for (unsigned pos = 0; pos < fields.Len(); pos++) { - AddSpacesBeforeName(); - AddMinuses(_s, kNameField_Len); + const char c = fields[pos]; + if (c == 'h') + { + for (unsigned i = 0; i < hashers.Size(); i++) + { + AddSpace(); + const CHasherState &h = hashers[i]; + AddMinuses(_s, GetColumnWidth(h.DigestSize)); + } + } + else if (c == 's') + { + AddSpace(); + AddMinuses(_s, kSizeField_Len); + } + else if (c == 'n') + { + AddSpacesBeforeName(); + AddMinuses(_s, kNameField_Len); + } } *_so << _s << endl; } + HRESULT CHashCallbackConsole::BeforeFirstFile(const CHashBundle &hb) { if (PrintHeaders && _so) { _s.Empty(); ClosePercents_for_so(); - - FOR_VECTOR (i, hb.Hashers) - { - if (i != 0) - _s.Add_Space(); - const CHasherState &h = hb.Hashers[i]; - _s += h.Name; - AddSpaces_if_Positive(_s, (int)GetColumnWidth(h.DigestSize) - (int)h.Name.Len()); - } - - if (PrintSize) - { - _s.Add_Space(); - const AString s2 = "Size"; - AddSpaces_if_Positive(_s, (int)kSizeField_Len - (int)s2.Len()); - _s += s2; - } - - if (PrintName) + + const AString fields = GetFields(); + for (unsigned pos = 0; pos < fields.Len(); pos++) { - AddSpacesBeforeName(); - _s += "Name"; + const char c = fields[pos]; + if (c == 'h') + { + FOR_VECTOR (i, hb.Hashers) + { + AddSpace(); + const CHasherState &h = hb.Hashers[i]; + _s += h.Name; + AddSpaces_if_Positive(_s, (int)GetColumnWidth(h.DigestSize) - (int)h.Name.Len()); + } + } + + else if (c == 's') + { + AddSpace(); + const AString s2 ("Size"); + AddSpaces_if_Positive(_s, (int)kSizeField_Len - (int)s2.Len()); + _s += s2; + } + else if (c == 'n') + { + AddSpacesBeforeName(); + _s += "Name"; + } } *_so << _s << endl; @@ -191,9 +226,11 @@ HRESULT CHashCallbackConsole::OpenFileError(const FString &path, DWORD systemErr return OpenFileError_Base(path, systemError); } -HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool /* isFolder */) +HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool isDir) { _fileName = name; + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_fileName); if (NeedPercents()) { @@ -208,70 +245,92 @@ HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool /* isFolder */ return CheckBreak2(); } + +static const unsigned k_DigestStringSize = k_HashCalc_DigestSize_Max * 2 + k_HashCalc_ExtraSize * 2 + 16; + + + void CHashCallbackConsole::PrintResultLine(UInt64 fileSize, - const CObjectVector &hashers, unsigned digestIndex, bool showHash) + const CObjectVector &hashers, unsigned digestIndex, bool showHash, + const AString &path) { ClosePercents_for_so(); _s.Empty(); - - FOR_VECTOR (i, hashers) - { - const CHasherState &h = hashers[i]; - char s[k_HashCalc_DigestSize_Max * 2 + 64]; - s[0] = 0; - if (showHash) - AddHashHexToString(s, h.Digests[digestIndex], h.DigestSize); - SetSpacesAndNul(s + strlen(s), (int)GetColumnWidth(h.DigestSize) - (int)strlen(s)); - if (i != 0) - _s.Add_Space(); - _s += s; - } + const AString fields = GetFields(); - if (PrintSize) + for (unsigned pos = 0; pos < fields.Len(); pos++) { - _s.Add_Space(); - - char s[kSizeField_Len + 32]; - char *p = s; - - if (showHash) + const char c = fields[pos]; + if (c == 'h') { - p = s + kSizeField_Len; - ConvertUInt64ToString(fileSize, p); - int numSpaces = kSizeField_Len - (int)strlen(p); - if (numSpaces > 0) + FOR_VECTOR (i, hashers) { - p -= (unsigned)numSpaces; - for (unsigned i = 0; i < (unsigned)numSpaces; i++) - p[i] = ' '; + AddSpace(); + const CHasherState &h = hashers[i]; + char s[k_DigestStringSize]; + s[0] = 0; + if (showHash) + h.WriteToString(digestIndex, s); + const unsigned len = (unsigned)strlen(s); + SetSpacesAndNul_if_Positive(s + len, (int)GetColumnWidth(h.DigestSize) - (int)len); + _s += s; } } - else + else if (c == 's') + { + AddSpace(); + char s[kSizeField_Len + 32]; + char *p = s; SetSpacesAndNul(s, kSizeField_Len); - - _s += p; + if (showHash) + { + p = s + kSizeField_Len; + ConvertUInt64ToString(fileSize, p); + const int numSpaces = (int)kSizeField_Len - (int)strlen(p); + if (numSpaces > 0) + p -= (unsigned)numSpaces; + } + _s += p; + } + else if (c == 'n') + { + AddSpacesBeforeName(); + _s += path; + } } - - if (PrintName) - AddSpacesBeforeName(); *_so << _s; } + HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBundle &hb, bool showHash) { if (_so) { + AString s; + if (_fileName.IsEmpty()) + s = kEmptyFileAlias; + else + { + UString temp (_fileName); + _so->Normalize_UString(temp); + _so->Convert_UString_to_AString(temp, s); + } + PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash, s); + + /* PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash); if (PrintName) { if (_fileName.IsEmpty()) *_so << kEmptyFileAlias; else - *_so << _fileName; + _so->NormalizePrint_UString(_fileName); } - *_so << endl; + */ + // if (PrintNewLine) + *_so << endl; } if (NeedPercents()) @@ -303,9 +362,9 @@ static void PrintSum(CStdOutStream &so, const CHasherState &h, unsigned digestIn so << k_DigestTitles[digestIndex]; - char s[k_HashCalc_DigestSize_Max * 2 + 64]; - s[0] = 0; - AddHashHexToString(s, h.Digests[digestIndex], h.DigestSize); + char s[k_DigestStringSize]; + // s[0] = 0; + h.WriteToString(digestIndex, s); so << s << endl; } @@ -332,7 +391,7 @@ void CHashCallbackConsole::PrintProperty(const char *name, UInt64 value) *_so << name << s << endl; } -HRESULT CHashCallbackConsole::AfterLastFile(const CHashBundle &hb) +HRESULT CHashCallbackConsole::AfterLastFile(CHashBundle &hb) { ClosePercents2(); @@ -340,7 +399,7 @@ HRESULT CHashCallbackConsole::AfterLastFile(const CHashBundle &hb) { PrintSeparatorLine(hb.Hashers); - PrintResultLine(hb.FilesSize, hb.Hashers, k_HashCalc_Index_DataSum, true); + PrintResultLine(hb.FilesSize, hb.Hashers, k_HashCalc_Index_DataSum, true, AString()); *_so << endl << endl; diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.h b/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.h index 5b30b69a058..f926d4d33e0 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/HashCon.h @@ -12,15 +12,23 @@ class CHashCallbackConsole: public IHashCallbackUI, public CCallbackConsoleBase UString _fileName; AString _s; + void AddSpace() + { + _s.Add_Space_if_NotEmpty(); + } + void AddSpacesBeforeName() { - _s.Add_Space(); - _s.Add_Space(); + if (!_s.IsEmpty()) + { + _s.Add_Space(); + _s.Add_Space(); + } } void PrintSeparatorLine(const CObjectVector &hashers); void PrintResultLine(UInt64 fileSize, - const CObjectVector &hashers, unsigned digestIndex, bool showHash); + const CObjectVector &hashers, unsigned digestIndex, bool showHash, const AString &path); void PrintProperty(const char *name, UInt64 value); public: @@ -28,17 +36,20 @@ public: bool PrintHeaders; - bool PrintSize; - bool PrintName; + // bool PrintSize; + // bool PrintNewLine; // set it too (false), if you need only hash for single file without LF char. + AString PrintFields; + + AString GetFields() const; CHashCallbackConsole(): PrintNameInPercents(true), - PrintHeaders(false), - PrintSize(true), - PrintName(true) + PrintHeaders(false) + // , PrintSize(true), + // , PrintNewLine(true) {} - ~CHashCallbackConsole() { } + virtual ~CHashCallbackConsole() {} INTERFACE_IHashCallbackUI(;) }; diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/List.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/List.cpp index 62e813b2834..f764f07e428 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/List.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/List.cpp @@ -124,18 +124,25 @@ static const char * const kPropIdToName[] = , "Read-only" , "Out Name" , "Copy Link" + , "ArcFileName" + , "IsHash" + , "Metadata Changed" + , "User ID" + , "Group ID" + , "Device Major" + , "Device Minor" }; static const char kEmptyAttribChar = '.'; -static const char *kListing = "Listing archive: "; +static const char * const kListing = "Listing archive: "; -static const char *kString_Files = "files"; -static const char *kString_Dirs = "folders"; -static const char *kString_AltStreams = "alternate streams"; -static const char *kString_Streams = "streams"; +static const char * const kString_Files = "files"; +static const char * const kString_Dirs = "folders"; +static const char * const kString_AltStreams = "alternate streams"; +static const char * const kString_Streams = "streams"; -static const char *kError = "ERROR: "; +static const char * const kError = "ERROR: "; static void GetAttribString(UInt32 wa, bool isDir, bool allAttribs, char *s) { @@ -303,22 +310,18 @@ struct CListUInt64Def void Add(const CListUInt64Def &v) { if (v.Def) Add(v.Val); } }; -struct CListFileTimeDef -{ - FILETIME Val; - bool Def; - CListFileTimeDef(): Def(false) { Val.dwLowDateTime = 0; Val.dwHighDateTime = 0; } +struct CListFileTimeDef: public CArcTime +{ void Update(const CListFileTimeDef &t) { - if (t.Def && (!Def || CompareFileTime(&Val, &t.Val) < 0)) - { - Val = t.Val; - Def = true; - } + if (t.Def && (!Def || CompareWith(t) < 0)) + (*this) = t; } }; + + struct CListStat { CListUInt64Def Size; @@ -351,7 +354,7 @@ struct CListStat2 AltStreams.Update(st.AltStreams); NumDirs += st.NumDirs; } - const UInt64 GetNumStreams() const { return MainFiles.NumFiles + AltStreams.NumFiles; } + UInt64 GetNumStreams() const { return MainFiles.NumFiles + AltStreams.NumFiles; } CListStat &GetStat(bool altStreamsMode) { return altStreamsMode ? AltStreams : MainFiles; } }; @@ -417,9 +420,8 @@ static void GetPropName(PROPID propID, const wchar_t *name, AString &nameA, UStr nameU = name; else { - char s[16]; - ConvertUInt32ToString(propID, s); - nameA = s; + nameA.Empty(); + nameA.Add_UInt32(propID); } } @@ -429,7 +431,7 @@ void CFieldPrinter::AddProp(const wchar_t *name, PROPID propID, bool isRawProp) f.PropID = propID; f.IsRawProp = isRawProp; GetPropName(propID, name, f.NameA, f.NameU); - f.NameU.AddAscii(" = "); + f.NameU += " = "; if (!f.NameA.IsEmpty()) f.NameA += " = "; else @@ -494,22 +496,31 @@ void CFieldPrinter::PrintTitleLines() g_StdOut << LinesString; } -static void PrintTime(char *dest, const FILETIME *ft) +static void PrintTime(char *dest, const CListFileTimeDef &t, bool showNS) { *dest = 0; - if (ft->dwLowDateTime == 0 && ft->dwHighDateTime == 0) + if (t.IsZero()) return; - FILETIME locTime; - if (!FileTimeToLocalFileTime(ft, &locTime)) - throw 20121211; - ConvertFileTimeToString(locTime, dest, true, true); + int prec = kTimestampPrintLevel_SEC; + if (showNS) + { + prec = kTimestampPrintLevel_NTFS; + if (t.Prec != 0) + { + prec = t.GetNumDigits(); + if (prec < kTimestampPrintLevel_DAY) + prec = kTimestampPrintLevel_NTFS; + } + } + + ConvertUtcFileTimeToString2(t.FT, t.Ns100, dest, prec); } #ifndef _SFX static inline char GetHex(Byte value) { - return (char)((value < 10) ? ('0' + value) : ('A' + (value - 10))); + return (char)((value < 10) ? ('0' + value) : ('a' + (value - 10))); } static void HexToString(char *dest, const Byte *data, UInt32 size) @@ -564,7 +575,7 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { if (!techMode) g_StdOut << temp; - g_StdOut.PrintUString(FilePath, TempAString); + g_StdOut.NormalizePrint_UString(FilePath, TempWString, TempAString); if (techMode) g_StdOut << MY_ENDL; continue; @@ -635,7 +646,13 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { case kpidSize: if (st.Size.Def) prop = st.Size.Val; break; case kpidPackSize: if (st.PackSize.Def) prop = st.PackSize.Val; break; - case kpidMTime: if (st.MTime.Def) prop = st.MTime.Val; break; + case kpidMTime: + { + const CListFileTimeDef &mtime = st.MTime; + if (mtime.Def) + prop.SetAsTimeFrom_FT_Prec_Ns100(mtime.FT, mtime.Prec, mtime.Ns100); + break; + } default: RINOK(Arc->Archive->GetProperty(index, f.PropID, &prop)); } @@ -657,7 +674,9 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) } else if (prop.vt == VT_FILETIME) { - PrintTime(temp + tempPos, &prop.filetime); + CListFileTimeDef t; + t.Set_From_Prop(prop); + PrintTime(temp + tempPos, t, techMode); if (techMode) g_StdOut << temp + tempPos; else @@ -675,9 +694,10 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) else if (prop.vt == VT_BSTR) { TempWString.SetFromBstr(prop.bstrVal); + // do we need multi-line support here ? + g_StdOut.Normalize_UString(TempWString); if (techMode) { - // replace CR/LF here. g_StdOut.PrintUString(TempWString, TempAString); } else @@ -686,7 +706,7 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) else { char s[64]; - ConvertPropertyToShortString(s, prop, f.PropID); + ConvertPropertyToShortString2(s, prop, f.PropID); if (techMode) g_StdOut << s; else @@ -729,7 +749,7 @@ void CFieldPrinter::PrintSum(const CListStat &st, UInt64 numDirs, const char *st char s[64]; s[0] = 0; if (st.MTime.Def) - PrintTime(s, &st.MTime.Val); + PrintTime(s, st.MTime, false); // showNS PrintString(f.TextAdjustment, f.Width, s); } else if (f.PropID == kpidPath) @@ -754,7 +774,7 @@ void CFieldPrinter::PrintSum(const CListStat2 &stat2) PrintSum(stat2.MainFiles, stat2.NumDirs, kString_Files); if (stat2.AltStreams.NumFiles != 0) { - PrintSum(stat2.AltStreams, 0, kString_AltStreams);; + PrintSum(stat2.AltStreams, 0, kString_AltStreams); CListStat st = stat2.MainFiles; st.Update(stat2.AltStreams); PrintSum(st, 0, kString_Streams); @@ -773,16 +793,14 @@ static HRESULT GetUInt64Value(IInArchive *archive, UInt32 index, PROPID propID, static HRESULT GetItemMTime(IInArchive *archive, UInt32 index, CListFileTimeDef &t) { - t.Val.dwLowDateTime = 0; - t.Val.dwHighDateTime = 0; - t.Def = false; + /* maybe we could call CArc::GetItemMTime(UInt32 index, CArcTime &ft, bool &defined) here + that can set default timestamp, if not defined */ + t.Clear(); + // t.Def = false; CPropVariant prop; RINOK(archive->GetProperty(index, kpidMTime, &prop)); if (prop.vt == VT_FILETIME) - { - t.Val = prop.filetime; - t.Def = true; - } + t.Set_From_Prop(prop); else if (prop.vt != VT_EMPTY) return E_FAIL; return S_OK; @@ -819,25 +837,82 @@ static void PrintPropNameAndNumber_Signed(CStdOutStream &so, PROPID propID, Int6 so << val << endl; } -static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val) + +static void UString_Replace_CRLF_to_LF(UString &s) { - so << name << " = " << val << endl; + // s.Replace(L"\r\n", L"\n"); + wchar_t *src = s.GetBuf(); + wchar_t *dest = src; + for (;;) + { + wchar_t c = *src++; + if (c == 0) + break; + if (c == '\r' && *src == '\n') + { + src++; + c = '\n'; + } + *dest++ = c; + } + s.ReleaseBuf_SetEnd((unsigned)(dest - s.GetBuf())); +} + + +static void PrintPropVal_MultiLine(CStdOutStream &so, const wchar_t *val) +{ + UString s (val); + if (s.Find(L'\n') >= 0) + { + so << endl; + so << "{"; + so << endl; + UString_Replace_CRLF_to_LF(s); + so.Normalize_UString__LF_Allowed(s); + so << s; + so << endl; + so << "}"; + } + else + { + so.Normalize_UString(s); + so << s; + } + so << endl; +} + + +static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val, bool multiLine) +{ + so << name << " = "; + if (multiLine) + { + PrintPropVal_MultiLine(so, val); + return; + } + UString s (val); + so.Normalize_UString(s); + so << s; + so << endl; } static void PrintPropertyPair2(CStdOutStream &so, PROPID propID, const wchar_t *name, const CPropVariant &prop) { UString s; - ConvertPropertyToString(s, prop, propID); + const int levelTopLimit = 9; // 1ns level + ConvertPropertyToString2(s, prop, propID, levelTopLimit); if (!s.IsEmpty()) { AString nameA; UString nameU; GetPropName(propID, name, nameA, nameU); if (!nameA.IsEmpty()) - PrintPropPair(so, nameA, s); + so << nameA; else - so << nameU << " = " << s << endl; + so << nameU; + so << " = "; + PrintPropVal_MultiLine(so, s); } } @@ -852,7 +927,7 @@ static HRESULT PrintArcProp(CStdOutStream &so, IInArchive *archive, PROPID propI static void PrintArcTypeError(CStdOutStream &so, const UString &type, bool isWarning) { so << "Open " << (isWarning ? "WARNING" : "ERROR") - << ": Can not open the file as [" + << ": Cannot open the file as [" << type << "] archive" << endl; @@ -866,13 +941,14 @@ static void ErrorInfo_Print(CStdOutStream &so, const CArcErrorInfo &er) { PrintErrorFlags(so, "ERRORS:", er.GetErrorFlags()); if (!er.ErrorMessage.IsEmpty()) - PrintPropPair(so, "ERROR", er.ErrorMessage); + PrintPropPair(so, "ERROR", er.ErrorMessage, true); PrintErrorFlags(so, "WARNINGS:", er.GetWarningFlags()); if (!er.WarningMessage.IsEmpty()) - PrintPropPair(so, "WARNING", er.WarningMessage); + PrintPropPair(so, "WARNING", er.WarningMessage, true); } +HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink); HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink) { FOR_VECTOR (r, arcLink.Arcs) @@ -881,7 +957,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const const CArcErrorInfo &er = arc.ErrorInfo; so << "--\n"; - PrintPropPair(so, "Path", arc.Path); + PrintPropPair(so, "Path", arc.Path, false); if (er.ErrorFormatIndex >= 0) { if (er.ErrorFormatIndex == arc.FormatIndex) @@ -889,7 +965,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const else PrintArcTypeError(so, codecs->GetFormatNamePtr(er.ErrorFormatIndex), true); } - PrintPropPair(so, "Type", codecs->GetFormatNamePtr(arc.FormatIndex)); + PrintPropPair(so, "Type", codecs->GetFormatNamePtr(arc.FormatIndex), false); ErrorInfo_Print(so, er); @@ -937,21 +1013,23 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const return S_OK; } +HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink); HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink) { #ifndef _NO_CRYPTO if (arcLink.PasswordWasAsked) - so << "Can not open encrypted archive. Wrong password?"; + so << "Cannot open encrypted archive. Wrong password?"; else #endif { if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { - so << arcLink.NonOpen_ArcPath << endl; - PrintArcTypeError(so, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); + so.NormalizePrint_UString(arcLink.NonOpen_ArcPath); + so << endl; + PrintArcTypeError(so, codecs->Formats[(unsigned)arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } else - so << "Can not open the file as archive"; + so << "Cannot open the file as archive"; } so << endl; @@ -963,7 +1041,9 @@ HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item); -HRESULT ListArchives(CCodecs *codecs, +HRESULT ListArchives( + const CListOptions &listOptions, + CCodecs *codecs, const CObjectVector &types, const CIntVector &excludedFormats, bool stdInMode, @@ -1011,22 +1091,31 @@ HRESULT ListArchives(CCodecs *codecs, if (!stdInMode) { NFile::NFind::CFileInfo fi; - if (!fi.Find(us2fs(arcPath))) + if (!fi.Find_FollowLink(us2fs(arcPath))) { DWORD errorCode = GetLastError(); if (errorCode == 0) errorCode = ERROR_FILE_NOT_FOUND; - lastError = HRESULT_FROM_WIN32(lastError);; + lastError = HRESULT_FROM_WIN32(errorCode); g_StdOut.Flush(); - *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << - endl << arcPath << endl << endl; + if (g_ErrStream) + { + *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << endl; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << endl << endl; + } numErrors++; continue; } if (fi.IsDir()) { g_StdOut.Flush(); - *g_ErrStream << endl << kError << arcPath << " is not a file" << endl << endl; + if (g_ErrStream) + { + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " is not a file" << endl << endl; + } numErrors++; continue; } @@ -1065,7 +1154,9 @@ HRESULT ListArchives(CCodecs *codecs, if (enableHeaders) { - g_StdOut << endl << kListing << arcPath << endl << endl; + g_StdOut << endl << kListing; + g_StdOut.NormalizePrint_UString(arcPath); + g_StdOut << endl << endl; } HRESULT result = arcLink.Open_Strict(options, &openCallback); @@ -1074,22 +1165,28 @@ HRESULT ListArchives(CCodecs *codecs, { if (result == E_ABORT) return result; + if (result != S_FALSE) + lastError = result; g_StdOut.Flush(); - *g_ErrStream << endl << kError << arcPath << " : "; - if (result == S_FALSE) + if (g_ErrStream) { - Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); - } - else - { - lastError = result; - *g_ErrStream << "opening : "; - if (result == E_OUTOFMEMORY) - *g_ErrStream << "Can't allocate required memory"; + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " : "; + if (result == S_FALSE) + { + Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); + } else - *g_ErrStream << NError::MyFormatMessage(result); + { + *g_ErrStream << "opening : "; + if (result == E_OUTOFMEMORY) + *g_ErrStream << "Can't allocate required memory"; + else + *g_ErrStream << NError::MyFormatMessage(result); + } + *g_ErrStream << endl; } - *g_ErrStream << endl; numErrors++; continue; } @@ -1174,7 +1271,7 @@ HRESULT ListArchives(CCodecs *codecs, if (NConsoleClose::TestBreakSignal()) return E_ABORT; - HRESULT res = arc.GetItemPath2(i, fp.FilePath); + HRESULT res = arc.GetItem_Path2(i, fp.FilePath); if (stdInMode && res == E_INVALIDARG) break; @@ -1198,6 +1295,9 @@ HRESULT ListArchives(CCodecs *codecs, RINOK(Archive_IsItem_Dir(archive, i, fp.IsDir)); + if (fp.IsDir ? listOptions.ExcludeDirItems : listOptions.ExcludeFileItems) + continue; + if (!allFilesAreAllowed) { if (isAltStream) @@ -1208,7 +1308,7 @@ HRESULT ListArchives(CCodecs *codecs, } else { - SplitPathToParts(fp.FilePath, pathParts);; + SplitPathToParts(fp.FilePath, pathParts); bool include; if (!wildcardCensor.CheckPathVect(pathParts, !fp.IsDir, include)) continue; @@ -1259,8 +1359,8 @@ HRESULT ListArchives(CCodecs *codecs, if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { g_StdOut << "----------\n"; - PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath); - PrintArcTypeError(g_StdOut, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); + PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath, false); + PrintArcTypeError(g_StdOut, codecs->Formats[(unsigned)arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } } diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/List.h b/3rdparty/lzma/CPP/7zip/UI/Console/List.h index 462c4710375..79d2ed482d5 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/List.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/List.h @@ -7,7 +7,20 @@ #include "../Common/LoadCodecs.h" -HRESULT ListArchives(CCodecs *codecs, +struct CListOptions +{ + bool ExcludeDirItems; + bool ExcludeFileItems; + + CListOptions(): + ExcludeDirItems(false), + ExcludeFileItems(false) + {} +}; + +HRESULT ListArchives( + const CListOptions &listOptions, + CCodecs *codecs, const CObjectVector &types, const CIntVector &excludedFormats, bool stdInMode, @@ -15,12 +28,12 @@ HRESULT ListArchives(CCodecs *codecs, bool processAltStreams, bool showAltStreams, const NWildcard::CCensorNode &wildcardCensor, bool enableHeaders, bool techMode, - #ifndef _NO_CRYPTO + #ifndef _NO_CRYPTO bool &passwordEnabled, UString &password, - #endif - #ifndef _SFX + #endif + #ifndef _SFX const CObjectVector *props, - #endif + #endif UInt64 &errors, UInt64 &numWarnings); diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/Main.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/Main.cpp index 50652555de7..d01aa4d0523 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/Main.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/Main.cpp @@ -6,32 +6,31 @@ #ifdef _WIN32 #include +#else +#include +#include +#include +#include #endif #include "../../../../C/CpuArch.h" -#if defined( _7ZIP_LARGE_PAGES) -#include "../../../../C/Alloc.h" -#endif - #include "../../../Common/MyInitGuid.h" #include "../../../Common/CommandLineParser.h" #include "../../../Common/IntToString.h" #include "../../../Common/MyException.h" +#include "../../../Common/StdInStream.h" +#include "../../../Common/StdOutStream.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" #include "../../../Common/UTFConvert.h" #include "../../../Windows/ErrorMsg.h" - -#ifdef _WIN32 -#include "../../../Windows/MemoryLock.h" -#endif - #include "../../../Windows/TimeUtils.h" #include "../Common/ArchiveCommandLine.h" +#include "../Common/Bench.h" #include "../Common/ExitCode.h" #include "../Common/Extract.h" @@ -44,12 +43,11 @@ #include "BenchCon.h" #include "ConsoleClose.h" #include "ExtractCallbackConsole.h" +#include "HashCon.h" #include "List.h" #include "OpenCallbackConsole.h" #include "UpdateCallbackConsole.h" -#include "HashCon.h" - #ifdef PROG_VARIANT_R #include "../../../../C/7zVersion.h" #else @@ -73,34 +71,36 @@ extern const CCodecInfo *g_Codecs[]; extern unsigned g_NumHashers; extern const CHasherInfo *g_Hashers[]; -static const char *kCopyrightString = "\n7-Zip" -#ifndef EXTERNAL_CODECS -#ifdef PROG_VARIANT_R -" (r)" -#else -" (a)" -#endif +#ifdef EXTERNAL_CODECS +const CExternalCodecs *g_ExternalCodecs_Ptr; #endif -#ifdef MY_CPU_64BIT -" [64]" -#elif defined MY_CPU_32BIT -" [32]" +DECLARE_AND_SET_CLIENT_VERSION_VAR + +#if defined(PROG_VARIANT_Z) + #define PROG_POSTFIX "z" + #define PROG_POSTFIX_2 " (z)" +#elif defined(PROG_VARIANT_R) + #define PROG_POSTFIX "r" + #define PROG_POSTFIX_2 " (r)" +#elif !defined(EXTERNAL_CODECS) + #define PROG_POSTFIX "a" + #define PROG_POSTFIX_2 " (a)" +#else + #define PROG_POSTFIX "" + #define PROG_POSTFIX_2 "" #endif -" " MY_VERSION_COPYRIGHT_DATE "\n\n"; -static const char *kHelpString = +static const char * const kCopyrightString = "\n7-Zip" + PROG_POSTFIX_2 + " " MY_VERSION_CPU + " : " MY_COPYRIGHT_DATE "\n"; + +static const char * const kHelpString = "Usage: 7z" -#ifndef EXTERNAL_CODECS -#ifdef PROG_VARIANT_R - "r" -#else - "a" -#endif -#endif - " [...] [...]\n" - " [<@listfiles...>]\n" + PROG_POSTFIX + " [...] [...] [@listfile]\n" "\n" "\n" " a : Add files to archive\n" @@ -116,7 +116,7 @@ static const char *kHelpString = " x : eXtract files with full paths\n" "\n" "\n" - " -- : Stop switches parsing\n" + " -- : Stop switches and @listfile parsing\n" " -ai[r[-|0]]{@listfile|!wildcard} : Include archives\n" " -ax[r[-|0]]{@listfile|!wildcard} : eXclude archives\n" " -ao{a|s|t|u} : set Overwrite mode\n" @@ -128,11 +128,12 @@ static const char *kHelpString = " -i[r[-|0]]{@listfile|!wildcard} : Include filenames\n" " -m{Parameters} : set compression Method\n" " -mmt[N] : set number of CPU threads\n" + " -mx[N] : set compression level: -mx1 (fastest) ... -mx9 (ultra)\n" " -o{Directory} : set Output directory\n" #ifndef _NO_CRYPTO " -p{Password} : set Password\n" #endif - " -r[-|0] : Recurse subdirectories\n" + " -r[-|0] : Recurse subdirectories for name search\n" " -sa{a|e|s} : set Archive name mode\n" " -scc{UTF-8|WIN|DOS} : set charset for for console input/output\n" " -scs{UTF-8|UTF-16LE|UTF-16BE|WIN|DOS|{id}} : set charset for list files\n" @@ -152,6 +153,8 @@ static const char *kHelpString = " -spe : eliminate duplication of root folder for extract command\n" " -spf : use fully qualified file paths\n" " -ssc[-] : set sensitive case mode\n" + " -sse : stop archive creating, if it can't open some input file\n" + " -ssp : do not change Last Access Time of source files while archiving\n" " -ssw : compress shared files\n" " -stl : set archive timestamp from the most recently modified file\n" " -stm{HexMask} : set CPU thread affinity mask (hexadecimal number)\n" @@ -166,14 +169,15 @@ static const char *kHelpString = // --------------------------- // exception messages -static const char *kEverythingIsOk = "Everything is Ok"; -static const char *kUserErrorMessage = "Incorrect command line"; -static const char *kNoFormats = "7-Zip cannot find the code that works with archives."; -static const char *kUnsupportedArcTypeMessage = "Unsupported archive type"; -// static const char *kUnsupportedUpdateArcType = "Can't create archive for that type"; +static const char * const kEverythingIsOk = "Everything is Ok"; +static const char * const kUserErrorMessage = "Incorrect command line"; +static const char * const kNoFormats = "7-Zip cannot find the code that works with archives."; +static const char * const kUnsupportedArcTypeMessage = "Unsupported archive type"; +// static const char * const kUnsupportedUpdateArcType = "Can't create archive for that type"; -static CFSTR kDefaultSfxModule = FTEXT("7zCon.sfx"); +#define kDefaultSfxModule "7zCon.sfx" +MY_ATTR_NORETURN static void ShowMessageAndThrowException(LPCSTR message, NExitCode::EEnum code) { if (g_ErrStream) @@ -181,15 +185,103 @@ static void ShowMessageAndThrowException(LPCSTR message, NExitCode::EEnum code) throw code; } -#ifndef _WIN32 -static void GetArguments(int numArgs, const char *args[], UStringVector &parts) + +#ifdef _WIN32 +#define ShowProgInfo(so) +#else +static void ShowProgInfo(CStdOutStream *so) { - parts.Clear(); - for (int i = 0; i < numArgs; i++) + if (!so) + return; + + *so + + /* + #ifdef __DATE__ + << " " << __DATE__ + #endif + #ifdef __TIME__ + << " " << __TIME__ + #endif + */ + + << " " << (unsigned)(sizeof(void *)) * 8 << "-bit" + + #ifdef __ILP32__ + << " ILP32" + #endif + + #ifdef __ARM_ARCH + << " arm_v:" << __ARM_ARCH + #ifdef __ARM_ARCH_ISA_THUMB + << " thumb:" << __ARM_ARCH_ISA_THUMB + #endif + #endif + ; + + + + #ifdef ENV_HAVE_LOCALE + *so << " locale=" << GetLocale(); + #endif + #ifndef _WIN32 + { + const bool is_IsNativeUTF8 = IsNativeUTF8(); + if (!is_IsNativeUTF8) + *so << " UTF8=" << (is_IsNativeUTF8 ? "+" : "-"); + } + if (!g_ForceToUTF8) + *so << " use-UTF8=" << (g_ForceToUTF8 ? "+" : "-"); + { + const unsigned wchar_t_size = (unsigned)sizeof(wchar_t); + if (wchar_t_size != 4) + *so << " wchar_t=" << wchar_t_size * 8 << "-bit"; + } + { + const unsigned off_t_size = (unsigned)sizeof(off_t); + if (off_t_size != 8) + *so << " Files=" << off_t_size * 8 << "-bit"; + } + #endif + + { + const UInt32 numCpus = NWindows::NSystem::GetNumberOfProcessors(); + *so << " Threads:" << numCpus; + } + + #ifdef _7ZIP_ASM + *so << ", ASM"; + #endif + + /* { - UString s = MultiByteToUnicodeString(args[i]); - parts.Add(s); + AString s; + GetCpuName(s); + s.Trim(); + *so << ", " << s; } + + #ifdef __ARM_FEATURE_CRC32 + << " CRC32" + #endif + + + #if (defined MY_CPU_X86_OR_AMD64 || defined(MY_CPU_ARM_OR_ARM64)) + if (CPU_IsSupported_AES()) *so << ",AES"; + #endif + + #ifdef MY_CPU_ARM_OR_ARM64 + if (CPU_IsSupported_CRC32()) *so << ",CRC32"; + #if defined(_WIN32) + if (CPU_IsSupported_CRYPTO()) *so << ",CRYPTO"; + #else + if (CPU_IsSupported_SHA1()) *so << ",SHA1"; + if (CPU_IsSupported_SHA2()) *so << ",SHA2"; + #endif + #endif + */ + + *so << endl; } #endif @@ -199,14 +291,16 @@ static void ShowCopyrightAndHelp(CStdOutStream *so, bool needHelp) return; *so << kCopyrightString; // *so << "# CPUs: " << (UInt64)NWindows::NSystem::GetNumberOfProcessors() << endl; + ShowProgInfo(so); + *so << endl; if (needHelp) *so << kHelpString; } -static void PrintStringRight(CStdOutStream &so, const AString &s, unsigned size) +static void PrintStringRight(CStdOutStream &so, const char *s, unsigned size) { - unsigned len = s.Len(); + unsigned len = MyStringLen(s); for (unsigned i = len; i < size; i++) so << ' '; so << s; @@ -222,7 +316,7 @@ static void PrintUInt32(CStdOutStream &so, UInt32 val, unsigned size) static void PrintLibIndex(CStdOutStream &so, int libIndex) { if (libIndex >= 0) - PrintUInt32(so, libIndex, 2); + PrintUInt32(so, (UInt32)libIndex, 2); else so << " "; so << ' '; @@ -245,7 +339,8 @@ static void PrintWarningsPaths(const CErrorPathCodes &pc, CStdOutStream &so) { FOR_VECTOR(i, pc.Paths) { - so << pc.Paths[i] << " : "; + so.NormalizePrint_UString(fs2us(pc.Paths[i])); + so << " : "; so << NError::MyFormatMessage(pc.Codes[i]) << endl; } so << "----------------" << endl; @@ -279,7 +374,7 @@ static int WarningsCheck(HRESULT result, const CCallbackConsoleBase &callback, UString message; if (!errorInfo.Message.IsEmpty()) { - message.AddAscii(errorInfo.Message); + message += errorInfo.Message.Ptr(); message.Add_LF(); } { @@ -339,7 +434,6 @@ static void ThrowException_if_Error(HRESULT res) throw CSystemException(res); } - static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') { char temp[64]; @@ -351,6 +445,8 @@ static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') *g_StdStream << p; } +#ifdef _WIN32 + static void PrintTime(const char *s, UInt64 val, UInt64 total) { *g_StdStream << endl << s << " Time ="; @@ -384,11 +480,19 @@ static void PrintMemUsage(const char *s, UInt64 val) *g_StdStream << " " << s << " Memory ="; PrintNum(SHIFT_SIZE_VALUE(val, 20), 7); *g_StdStream << " MB"; + + #ifdef _7ZIP_LARGE_PAGES + AString lp; + Add_LargePages_String(lp); + if (!lp.IsEmpty()) + *g_StdStream << lp; + #endif } EXTERN_C_BEGIN typedef BOOL (WINAPI *Func_GetProcessMemoryInfo)(HANDLE Process, PPROCESS_MEMORY_COUNTERS ppsmemCounters, DWORD cb); +typedef BOOL (WINAPI *Func_QueryProcessCycleTime)(HANDLE Process, PULONG64 CycleTime); EXTERN_C_END #endif @@ -408,13 +512,15 @@ static void PrintStat() , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT)) return; FILETIME curTimeFT; - NTime::GetCurUtcFileTime(curTimeFT); + NTime::GetCurUtc_FiTime(curTimeFT); #ifndef UNDER_CE PROCESS_MEMORY_COUNTERS m; memset(&m, 0, sizeof(m)); BOOL memDefined = FALSE; + BOOL cycleDefined = FALSE; + ULONG64 cycleTime = 0; { /* NT 4.0: GetProcessMemoryInfo() in Psapi.dll Win7: new function K32GetProcessMemoryInfo() in kernel32.dll @@ -425,17 +531,23 @@ static void PrintStat() // memDefined = GetProcessMemoryInfo(GetCurrentProcess(), &m, sizeof(m)); */ + HMODULE kern = ::GetModuleHandleW(L"kernel32.dll"); Func_GetProcessMemoryInfo my_GetProcessMemoryInfo = (Func_GetProcessMemoryInfo) - ::GetProcAddress(::GetModuleHandleW(L"kernel32.dll"), "K32GetProcessMemoryInfo"); + (void *)::GetProcAddress(kern, "K32GetProcessMemoryInfo"); if (!my_GetProcessMemoryInfo) { HMODULE lib = LoadLibraryW(L"Psapi.dll"); if (lib) - my_GetProcessMemoryInfo = (Func_GetProcessMemoryInfo)::GetProcAddress(lib, "GetProcessMemoryInfo"); + my_GetProcessMemoryInfo = (Func_GetProcessMemoryInfo)(void *)::GetProcAddress(lib, "GetProcessMemoryInfo"); } if (my_GetProcessMemoryInfo) memDefined = my_GetProcessMemoryInfo(GetCurrentProcess(), &m, sizeof(m)); // FreeLibrary(lib); + + Func_QueryProcessCycleTime my_QueryProcessCycleTime = (Func_QueryProcessCycleTime) + (void *)::GetProcAddress(kern, "QueryProcessCycleTime"); + if (my_QueryProcessCycleTime) + cycleDefined = my_QueryProcessCycleTime(GetCurrentProcess(), &cycleTime); } #endif @@ -448,9 +560,33 @@ static void PrintStat() UInt64 totalTime = curTime - creationTime; PrintTime("Kernel ", kernelTime, totalTime); + + const UInt64 processTime = kernelTime + userTime; + + #ifndef UNDER_CE + if (cycleDefined) + { + *g_StdStream << " Cnt:"; + PrintNum(cycleTime / 1000000, 15); + *g_StdStream << " MCycles"; + } + #endif + PrintTime("User ", userTime, totalTime); + + #ifndef UNDER_CE + if (cycleDefined) + { + *g_StdStream << " Freq (cnt/ptime):"; + UInt64 us = processTime / 10; + if (us == 0) + us = 1; + PrintNum(cycleTime / us, 6); + *g_StdStream << " MHz"; + } + #endif - PrintTime("Process", kernelTime + userTime, totalTime); + PrintTime("Process", processTime, totalTime); #ifndef UNDER_CE if (memDefined) PrintMemUsage("Virtual ", m.PeakPagefileUsage); #endif @@ -459,10 +595,96 @@ static void PrintStat() #ifndef UNDER_CE if (memDefined) PrintMemUsage("Physical", m.PeakWorkingSetSize); #endif +} + + +#else // ! _WIN32 + +static UInt64 Get_timeofday_us() +{ + struct timeval now; + if (gettimeofday(&now, 0 ) == 0) + return (UInt64)now.tv_sec * 1000000 + (UInt64)now.tv_usec; + return 0; +} + +static void PrintTime(const char *s, UInt64 val, UInt64 total_us, UInt64 kFreq) +{ + *g_StdStream << endl << s << " Time ="; + + { + UInt64 sec, ms; + + if (kFreq == 0) + { + sec = val / 1000000; + ms = val % 1000000 / 1000; + } + else + { + sec = val / kFreq; + ms = (UInt32)((val - (sec * kFreq)) * 1000 / kFreq); + } + + PrintNum(sec, 6); + *g_StdStream << '.'; + PrintNum(ms, 3, '0'); + } + if (total_us == 0) + return; + + UInt64 percent = 0; + if (kFreq == 0) + percent = val * 100 / total_us; + else + { + const UInt64 kMaxVal = (UInt64)(Int64)-1; + UInt32 m = 100000000; + for (;;) + { + if (m == 0 || kFreq == 0) + break; + if (kMaxVal / m > val && + kMaxVal / kFreq > total_us) + break; + if (val > m) + val >>= 1; + else + m >>= 1; + if (kFreq > total_us) + kFreq >>= 1; + else + total_us >>= 1; + } + const UInt64 total = kFreq * total_us; + if (total != 0) + percent = val * m / total; + } + *g_StdStream << " ="; + PrintNum(percent, 5); + *g_StdStream << '%'; +} + +static void PrintStat(UInt64 startTime) +{ + tms t; + /* clock_t res = */ times(&t); + const UInt64 totalTime = Get_timeofday_us() - startTime; + const UInt64 kFreq = (UInt64)sysconf(_SC_CLK_TCK); + PrintTime("Kernel ", (UInt64)t.tms_stime, totalTime, kFreq); + PrintTime("User ", (UInt64)t.tms_utime, totalTime, kFreq); + PrintTime("Process", (UInt64)t.tms_utime + (UInt64)t.tms_stime, totalTime, kFreq); + PrintTime("Global ", totalTime, totalTime, 0); *g_StdStream << endl; } +#endif // ! _WIN32 + + + + + static void PrintHexId(CStdOutStream &so, UInt64 id) { char s[32]; @@ -470,38 +692,83 @@ static void PrintHexId(CStdOutStream &so, UInt64 id) PrintStringRight(so, s, 8); } +#ifndef _WIN32 +void Set_ModuleDirPrefix_From_ProgArg0(const char *s); +#endif + +int Main2( + #ifndef _WIN32 + int numArgs, char *args[] + #endif +); int Main2( #ifndef _WIN32 int numArgs, char *args[] #endif ) { + #if defined(MY_CPU_SIZEOF_POINTER) + { unsigned k = sizeof(void *); if (k != MY_CPU_SIZEOF_POINTER) throw "incorrect MY_CPU_PTR_SIZE"; } + #endif + #if defined(_WIN32) && !defined(UNDER_CE) SetFileApisToOEM(); #endif + #ifdef ENV_HAVE_LOCALE + // printf("\nBefore SetLocale() : %s\n", IsNativeUtf8() ? "NATIVE UTF-8" : "IS NOT NATIVE UTF-8"); + MY_SetLocale(); + // printf("\nAfter SetLocale() : %s\n", IsNativeUtf8() ? "NATIVE UTF-8" : "IS NOT NATIVE UTF-8"); + #endif + + #ifndef _WIN32 + UInt64 startTime = Get_timeofday_us(); + #endif + UStringVector commandStrings; #ifdef _WIN32 NCommandLineParser::SplitCommandLine(GetCommandLineW(), commandStrings); #else - GetArguments(numArgs, args, commandStrings); + { + if (numArgs > 0) + Set_ModuleDirPrefix_From_ProgArg0(args[0]); + + for (int i = 0; i < numArgs; i++) + { + AString a (args[i]); + /* + printf("\n%d %s :", i, a.Ptr()); + for (unsigned k = 0; k < a.Len(); k++) + printf(" %2x", (unsigned)(Byte)a[k]); + */ + const UString s = MultiByteToUnicodeString(a); + commandStrings.Add(s); + } + // printf("\n"); + } + + #endif + + #ifndef UNDER_CE + if (commandStrings.Size() > 0) + commandStrings.Delete(0); #endif - if (commandStrings.Size() == 1) + if (commandStrings.Size() == 0) { ShowCopyrightAndHelp(g_StdStream, true); return 0; } - commandStrings.Delete(0); - CArcCmdLineOptions options; CArcCmdLineParser parser; parser.Parse1(commandStrings, options); + g_StdOut.IsTerminalMode = options.IsStdOutTerminal; + g_StdErr.IsTerminalMode = options.IsStdErrTerminal; if (options.Number_for_Out != k_OutStream_stdout) g_StdStream = (options.Number_for_Out == k_OutStream_stderr ? &g_StdErr : NULL); @@ -519,25 +786,54 @@ int Main2( return 0; } - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_SymLink(); - #endif - - #ifdef _7ZIP_LARGE_PAGES - if (options.LargePages) - { - SetLargePageSize(); - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_LockMemory(); - #endif - } - #endif - if (options.EnableHeaders) + { ShowCopyrightAndHelp(g_StdStream, false); + if (!parser.Parse1Log.IsEmpty()) + *g_StdStream << parser.Parse1Log; + } parser.Parse2(options); + { + int cp = options.ConsoleCodePage; + + int stdout_cp = cp; + int stderr_cp = cp; + int stdin_cp = cp; + + /* + // these cases are complicated. + // maybe we must use CRT functions instead of console WIN32. + // different Windows/CRT versions also can work different ways. + // so the following code was not enabled: + if (cp == -1) + { + // we set CodePage only if stream is attached to terminal + // maybe we should set CodePage even if is not terminal? + #ifdef _WIN32 + { + UINT ccp = GetConsoleOutputCP(); + if (ccp != 0) + { + if (options.IsStdOutTerminal) stdout_cp = ccp; + if (options.IsStdErrTerminal) stderr_cp = ccp; + } + } + if (options.IsInTerminal) + { + UINT ccp = GetConsoleCP(); + if (ccp != 0) stdin_cp = ccp; + } + #endif + } + */ + + if (stdout_cp != -1) g_StdOut.CodePage = stdout_cp; + if (stderr_cp != -1) g_StdErr.CodePage = stderr_cp; + if (stdin_cp != -1) g_StdIn.CodePage = stdin_cp; + } + unsigned percentsNameLevel = 1; if (options.LogLevel == 0 || options.Number_for_Percents != options.Number_for_Out) percentsNameLevel = 2; @@ -551,13 +847,13 @@ int Main2( #if !defined(UNDER_CE) CONSOLE_SCREEN_BUFFER_INFO consoleInfo; if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &consoleInfo)) - consoleWidth = consoleInfo.dwSize.X; + consoleWidth = (unsigned)(unsigned short)consoleInfo.dwSize.X; #endif #else struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) == ) + if (ioctl(0, TIOCGWINSZ, &w) == 0) consoleWidth = w.ws_col; #endif @@ -565,11 +861,25 @@ int Main2( CREATE_CODECS_OBJECT - codecs->CaseSensitiveChange = options.CaseSensitiveChange; + codecs->CaseSensitive_Change = options.CaseSensitive_Change; codecs->CaseSensitive = options.CaseSensitive; ThrowException_if_Error(codecs->Load()); + Codecs_AddHashArcHandler(codecs); + + #ifdef EXTERNAL_CODECS + { + g_ExternalCodecs_Ptr = &__externalCodecs; + UString s; + codecs->GetCodecsErrorMessage(s); + if (!s.IsEmpty()) + { + CStdOutStream &so = (g_StdStream ? *g_StdStream : g_StdOut); + so << endl << s << endl; + } + } + #endif - bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); + const bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); if (codecs->Formats.Size() == 0 && (isExtractGroupCommand @@ -579,18 +889,20 @@ int Main2( #ifdef EXTERNAL_CODECS if (!codecs->MainDll_ErrorPath.IsEmpty()) { - UString s = L"Can't load module "; + UString s ("Can't load module: "); s += fs2us(codecs->MainDll_ErrorPath); throw s; } #endif - throw kNoFormats; } CObjectVector types; if (!ParseOpenTypes(*codecs, options.ArcType, types)) + { throw kUnsupportedArcTypeMessage; + } + CIntVector excludedFormats; FOR_VECTOR (k, options.ExcludedArcTypes) @@ -599,13 +911,16 @@ int Main2( if (!codecs->FindFormatForArchiveType(options.ExcludedArcTypes[k], tempIndices) || tempIndices.Size() != 1) throw kUnsupportedArcTypeMessage; + + + excludedFormats.AddToUniqueSorted(tempIndices[0]); // excludedFormats.Sort(); } - #ifdef EXTERNAL_CODECS if (isExtractGroupCommand + || options.Command.IsFromUpdateGroup() || options.Command.CommandType == NCommandType::kHash || options.Command.CommandType == NCommandType::kBenchmark) ThrowException_if_Error(__externalCodecs.Load()); @@ -632,16 +947,18 @@ int Main2( so << endl << "Libs:" << endl; for (i = 0; i < codecs->Libs.Size(); i++) { - PrintLibIndex(so, i); + PrintLibIndex(so, (int)i); so << ' ' << codecs->Libs[i].Path << endl; } #endif so << endl << "Formats:" << endl; - const char *kArcFlags = "KSNFMGOPBELH"; + const char * const kArcFlags = "KSNFMGOPBELHXCc+a+m+r+"; + const char * const kArcTimeFlags = "wudn"; const unsigned kNumArcFlags = (unsigned)strlen(kArcFlags); - + const unsigned kNumArcTimeFlags = (unsigned)strlen(kArcTimeFlags); + for (i = 0; i < codecs->Formats.Size(); i++) { const CArcInfoEx &arc = codecs->Formats[i]; @@ -649,17 +966,27 @@ int Main2( #ifdef EXTERNAL_CODECS PrintLibIndex(so, arc.LibIndex); #else - so << " "; + so << " "; #endif so << (char)(arc.UpdateEnabled ? 'C' : ' '); - for (unsigned b = 0; b < kNumArcFlags; b++) { - so << (char) - ((arc.Flags & ((UInt32)1 << b)) != 0 ? kArcFlags[b] : ' '); + unsigned b; + for (b = 0; b < kNumArcFlags; b++) + so << (char)((arc.Flags & ((UInt32)1 << b)) != 0 ? kArcFlags[b] : '.'); + so << ' '; } - + + if (arc.TimeFlags != 0) + { + unsigned b; + for (b = 0; b < kNumArcTimeFlags; b++) + so << (char)((arc.TimeFlags & ((UInt32)1 << b)) != 0 ? kArcTimeFlags[b] : '.'); + so << arc.Get_DefaultTimePrec(); + so << ' '; + } + so << ' '; PrintString(so, arc.Name, 8); so << ' '; @@ -673,9 +1000,9 @@ int Main2( s += ext.Ext; if (!ext.AddExt.IsEmpty()) { - s += L" ("; + s += " ("; s += ext.AddExt; - s += L')'; + s += ')'; } } @@ -685,6 +1012,8 @@ int Main2( if (arc.SignatureOffset != 0) so << "offset=" << arc.SignatureOffset << ' '; + // so << "numSignatures = " << arc.Signatures.Size() << " "; + FOR_VECTOR(si, arc.Signatures) { if (si != 0) @@ -726,6 +1055,7 @@ int Main2( so << (char)(cod.CreateEncoder ? 'E' : ' '); so << (char)(cod.CreateDecoder ? 'D' : ' '); + so << (char)(cod.IsFilter ? 'F' : ' '); so << ' '; PrintHexId(so, cod.Id); @@ -749,6 +1079,12 @@ int Main2( so << (char)(codecs->GetCodec_EncoderIsAssigned(j) ? 'E' : ' '); so << (char)(codecs->GetCodec_DecoderIsAssigned(j) ? 'D' : ' '); + { + bool isFilter_Assigned; + const bool isFilter = codecs->GetCodec_IsFilter(j, isFilter_Assigned); + so << (char)(isFilter ? 'F' : isFilter_Assigned ? ' ' : '*'); + } + so << ' '; UInt64 id; @@ -796,6 +1132,7 @@ int Main2( options.Properties, options.NumIterations, (FILE *)so); if (hresultMain == S_FALSE) { + so << endl; if (g_ErrStream) *g_ErrStream << "\nDecoding ERROR\n"; retCode = NExitCode::kFatalError; @@ -855,7 +1192,7 @@ int Main2( } } - if (hresultMain == S_OK) + if (hresultMain == S_OK) { if (isExtractGroupCommand) { CExtractCallbackConsole *ecs = new CExtractCallbackConsole; @@ -906,10 +1243,11 @@ int Main2( { hashCalc = &hb; ThrowException_if_Error(hb.SetMethods(EXTERNAL_CODECS_VARS_L options.HashMethods)); - hb.Init(); + // hb.Init(); } hresultMain = Extract( + // EXTERNAL_CODECS_VARS_L codecs, types, excludedFormats, @@ -984,7 +1322,7 @@ int Main2( if (isError) retCode = NExitCode::kFatalError; - if (so) + if (so) { if (ecs->NumArcsWithError != 0 || ecs->NumFileErrors != 0) { // if (ecs->NumArchives > 1) @@ -1015,15 +1353,21 @@ int Main2( PrintHashStat(*so, hb); } } + } // if (so) } - else + else // if_(!isExtractGroupCommand) { UInt64 numErrors = 0; UInt64 numWarnings = 0; // options.ExtractNtOptions.StoreAltStreams = true, if -sns[-] is not definmed + CListOptions lo; + lo.ExcludeDirItems = options.Censor.ExcludeDirItems; + lo.ExcludeFileItems = options.Censor.ExcludeFileItems; + hresultMain = ListArchives( + lo, codecs, types, excludedFormats, @@ -1052,7 +1396,8 @@ int Main2( g_StdOut << endl << "Errors: " << numErrors << endl; retCode = NExitCode::kFatalError; } - } + } // if_(isExtractGroupCommand) + } // if_(hresultMain == S_OK) } else if (options.Command.IsFromUpdateGroup()) { @@ -1122,6 +1467,7 @@ int Main2( callback.Init(g_StdStream, g_ErrStream, percentsStream); callback.PrintHeaders = options.EnableHeaders; + callback.PrintFields = options.ListFields; AString errorInfoString; hresultMain = HashCalc(EXTERNAL_CODECS_VARS_L @@ -1138,7 +1484,11 @@ int Main2( ShowMessageAndThrowException(kUserErrorMessage, NExitCode::kUserError); if (options.ShowTime && g_StdStream) - PrintStat(); + PrintStat( + #ifndef _WIN32 + startTime + #endif + ); ThrowException_if_Error(hresultMain); diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/MainAr.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/MainAr.cpp index d30a96e4be2..0e45312c2e6 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/MainAr.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/MainAr.cpp @@ -2,6 +2,10 @@ #include "StdAfx.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif + #include "../../../Common/MyException.h" #include "../../../Common/StdOutStream.h" @@ -15,7 +19,11 @@ using namespace NWindows; +extern +CStdOutStream *g_StdStream; CStdOutStream *g_StdStream = NULL; +extern +CStdOutStream *g_ErrStream; CStdOutStream *g_ErrStream = NULL; extern int Main2( @@ -24,12 +32,12 @@ extern int Main2( #endif ); -static const char *kException_CmdLine_Error_Message = "Command Line Error:"; -static const char *kExceptionErrorMessage = "ERROR:"; -static const char *kUserBreakMessage = "Break signaled"; -static const char *kMemoryExceptionMessage = "ERROR: Can't allocate required memory!"; -static const char *kUnknownExceptionMessage = "Unknown Error"; -static const char *kInternalExceptionMessage = "\n\nInternal Error #"; +static const char * const kException_CmdLine_Error_Message = "Command Line Error:"; +static const char * const kExceptionErrorMessage = "ERROR:"; +static const char * const kUserBreakMessage = "Break signaled"; +static const char * const kMemoryExceptionMessage = "ERROR: Can't allocate required memory!"; +static const char * const kUnknownExceptionMessage = "Unknown Error"; +static const char * const kInternalExceptionMessage = "\n\nInternal Error #"; static void FlushStreams() { @@ -44,7 +52,9 @@ static void PrintError(const char *message) *g_ErrStream << "\n\n" << message << endl; } +#if defined(_WIN32) && defined(_UNICODE) && !defined(_WIN64) && !defined(UNDER_CE) #define NT_CHECK_FAIL_ACTION *g_StdStream << "Unsupported Windows version"; return NExitCode::kFatalError; +#endif int MY_CDECL main ( @@ -63,6 +73,10 @@ int MY_CDECL main try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif + res = Main2( #ifndef _WIN32 numArgs, args @@ -79,7 +93,7 @@ int MY_CDECL main PrintError(kUserBreakMessage); return (NExitCode::kUserBreak); } - catch(const CArcCmdLineException &e) + catch(const CMessagePathException &e) { PrintError(kException_CmdLine_Error_Message); if (g_ErrStream) diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.cpp index 9d25a729ca0..a074fa1fb64 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.cpp @@ -32,13 +32,17 @@ HRESULT COpenCallbackConsole::Open_SetTotal(const UInt64 *files, const UInt64 *b if (bytes) { - _totalBytesDefined = true; - // _totalBytes = *bytes; + // _totalBytesDefined = true; + _totalBytes = *bytes; if (!files) _percent.Total = *bytes; } else - _totalBytesDefined = false; + { + // _totalBytesDefined = false; + if (!files) + _percent.Total = _totalBytes; + } } return CheckBreak2(); @@ -83,7 +87,7 @@ HRESULT COpenCallbackConsole::Open_CryptoGetTextPassword(BSTR *password) if (!PasswordIsDefined) { ClosePercents(); - Password = GetPassword(_so); + RINOK(GetPassword_HRESULT(_so, Password)); PasswordIsDefined = true; } return StringToBstr(Password, password); diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.h b/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.h index 0dd4e1d1c82..075d37413d6 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/OpenCallbackConsole.h @@ -18,9 +18,9 @@ protected: CStdOutStream *_se; bool _totalFilesDefined; - bool _totalBytesDefined; + // bool _totalBytesDefined; // UInt64 _totalFiles; - // UInt64 _totalBytes; + UInt64 _totalBytes; bool NeedPercents() const { return _percent._so != NULL; } @@ -36,7 +36,8 @@ public: COpenCallbackConsole(): _totalFilesDefined(false), - _totalBytesDefined(false), + // _totalBytesDefined(false), + _totalBytes(0), MultiArcMode(false) #ifndef _NO_CRYPTO @@ -45,6 +46,8 @@ public: #endif {} + + virtual ~COpenCallbackConsole() {} void Init(CStdOutStream *outStream, CStdOutStream *errorStream, CStdOutStream *percentStream) { diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/PercentPrinter.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/PercentPrinter.cpp index e3e8d599811..49d039329ba 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/PercentPrinter.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/PercentPrinter.cpp @@ -63,7 +63,8 @@ void CPercentPrinter::GetPercents() { char c = '%'; UInt64 val = 0; - if (Total == (UInt64)(Int64)-1) + if (Total == (UInt64)(Int64)-1 || + (Total == 0 && Completed != 0)) { val = Completed >> 20; c = 'M'; @@ -141,8 +142,9 @@ void CPercentPrinter::Print() { _s += ' '; - StdOut_Convert_UString_to_AString(FileName, _temp); - _temp.Replace('\n', ' '); + _tempU = FileName; + _so->Normalize_UString(_tempU); + _so->Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() > MaxLen) { unsigned len = FileName.Len(); @@ -153,9 +155,10 @@ void CPercentPrinter::Print() delta = 1; len -= delta; _tempU = FileName; - _tempU.Delete(len / 2, FileName.Len() - len); + _tempU.Delete(len / 2, _tempU.Len() - len); _tempU.Insert(len / 2, L" . "); - StdOut_Convert_UString_to_AString(_tempU, _temp); + _so->Normalize_UString(_tempU); + _so->Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() <= MaxLen) break; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index f4f2d10210d..1a6b820c431 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -5,11 +5,14 @@ #include "../../../Common/IntToString.h" #include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/FileName.h" #ifndef _7ZIP_ST #include "../../../Windows/Synchronization.h" #endif +// #include "../Common/PropIDUtils.h" + #include "ConsoleClose.h" #include "UserInputUtils.h" #include "UpdateCallbackConsole.h" @@ -23,15 +26,15 @@ static NSynchronization::CCriticalSection g_CriticalSection; #define MT_LOCK #endif -static const wchar_t *kEmptyFileAlias = L"[Content]"; +static const wchar_t * const kEmptyFileAlias = L"[Content]"; -static const char *kOpenArchiveMessage = "Open archive: "; -static const char *kCreatingArchiveMessage = "Creating archive: "; -static const char *kUpdatingArchiveMessage = "Updating archive: "; -static const char *kScanningMessage = "Scanning the drive:"; +static const char * const kOpenArchiveMessage = "Open archive: "; +static const char * const kCreatingArchiveMessage = "Creating archive: "; +static const char * const kUpdatingArchiveMessage = "Updating archive: "; +static const char * const kScanningMessage = "Scanning the drive:"; -static const char *kError = "ERROR: "; -static const char *kWarning = "WARNING: "; +static const char * const kError = "ERROR: "; +static const char * const kWarning = "WARNING: "; static HRESULT CheckBreak2() { @@ -143,7 +146,9 @@ HRESULT CUpdateCallbackConsole::OpenResult( _so->Flush(); if (_se) { - *_se << kError << name << endl; + *_se << kError; + _se->NormalizePrint_wstr(name); + *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res); _se->Flush(); @@ -185,11 +190,29 @@ void CCallbackConsoleBase::CommonError(const FString &path, DWORD systemError, b *_se << endl << (isWarning ? kWarning : kError) << NError::MyFormatMessage(systemError) - << endl << fs2us(path) << endl << endl; + << endl; + _se->NormalizePrint_UString(fs2us(path)); + *_se << endl << endl; _se->Flush(); } } +/* +void CCallbackConsoleBase::CommonError(const char *message) +{ + ClosePercents2(); + + if (_se) + { + if (_so) + _so->Flush(); + + *_se << endl << kError << message << endl; + _se->Flush(); + } +} +*/ + HRESULT CCallbackConsoleBase::ScanError_Base(const FString &path, DWORD systemError) { @@ -205,6 +228,7 @@ HRESULT CCallbackConsoleBase::OpenFileError_Base(const FString &path, DWORD syst { MT_LOCK FailedFiles.AddError(path, systemError); + NumNonOpenFiles++; /* if (systemError == ERROR_SHARING_VIOLATION) { @@ -241,6 +265,7 @@ static void PrintPropPair(AString &s, const char *name, UInt64 val) void PrintSize_bytes_Smart(AString &s, UInt64 val); void Print_DirItemsStat(AString &s, const CDirItemsStat &st); +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st); HRESULT CUpdateCallbackConsole::FinishScanning(const CDirItemsStat &st) { @@ -259,7 +284,7 @@ HRESULT CUpdateCallbackConsole::FinishScanning(const CDirItemsStat &st) return S_OK; } -static const char *k_StdOut_ArcName = "StdOut"; +static const char * const k_StdOut_ArcName = "StdOut"; HRESULT CUpdateCallbackConsole::StartOpenArchive(const wchar_t *name) { @@ -277,11 +302,17 @@ HRESULT CUpdateCallbackConsole::StartOpenArchive(const wchar_t *name) HRESULT CUpdateCallbackConsole::StartArchive(const wchar_t *name, bool updating) { + if (NeedPercents()) + _percent.ClosePrint(true); + + _percent.ClearCurState(); + NumNonOpenFiles = 0; + if (_so) { *_so << (updating ? kUpdatingArchiveMessage : kCreatingArchiveMessage); - if (name != 0) - *_so << name; + if (name) + _so->NormalizePrint_wstr(name); else *_so << k_StdOut_ArcName; *_so << endl << endl; @@ -297,7 +328,7 @@ HRESULT CUpdateCallbackConsole::FinishArchive(const CFinishArchiveStat &st) { AString s; // Print_UInt64_and_String(s, _percent.Files == 1 ? "file" : "files", _percent.Files); - PrintPropPair(s, "Files read from disk", _percent.Files); + PrintPropPair(s, "Files read from disk", _percent.Files - NumNonOpenFiles); s.Add_LF(); s += "Archive size: "; PrintSize_bytes_Smart(s, st.OutArcFileSize); @@ -316,7 +347,7 @@ HRESULT CUpdateCallbackConsole::WriteSfx(const wchar_t *name, UInt64 size) { *_so << "Write SFX: "; *_so << name; - AString s = " : "; + AString s (" : "); PrintSize_bytes_Smart(s, size); *_so << s << endl; } @@ -342,6 +373,7 @@ HRESULT CUpdateCallbackConsole::DeletingAfterArchiving(const FString &path, bool _tempA.Add_Space(); *_so << _tempA; _tempU = fs2us(path); + _so->Normalize_UString(_tempU); _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -398,14 +430,31 @@ HRESULT CUpdateCallbackConsole::Finalize() } */ -HRESULT CUpdateCallbackConsole::SetNumItems(UInt64 numItems) + +void static PrintToDoStat(CStdOutStream *_so, const CDirItemsStat2 &stat, const char *name) +{ + AString s; + Print_DirItemsStat2(s, stat); + *_so << name << ": " << s << endl; +} + +HRESULT CUpdateCallbackConsole::SetNumItems(const CArcToDoStat &stat) { if (_so) { ClosePercents_for_so(); - AString s; - PrintPropPair(s, "Items to compress", numItems); - *_so << s << endl << endl; + if (!stat.DeleteData.IsEmpty()) + { + *_so << endl; + PrintToDoStat(_so, stat.DeleteData, "Delete data from archive"); + } + if (!stat.OldData.IsEmpty()) + PrintToDoStat(_so, stat.OldData, "Keep old data in archive"); + // if (!stat.NewData.IsEmpty()) + { + PrintToDoStat(_so, stat.NewData, "Add new data to archive"); + } + *_so << endl; } return S_OK; } @@ -440,7 +489,7 @@ HRESULT CUpdateCallbackConsole::SetRatioInfo(const UInt64 * /* inSize */, const return CheckBreak2(); } -HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *command, bool showInLog) +HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, bool isDir, const char *command, bool showInLog) { MT_LOCK @@ -457,7 +506,12 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *com _tempU.Empty(); if (name) + { _tempU = name; + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_tempU); + _so->Normalize_UString(_tempU); + } _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -483,7 +537,29 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *com return CheckBreak2(); } -HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool /* isDir */, bool isAnti, UInt32 mode) + +/* +void CCallbackConsoleBase::PrintInfoLine(const UString &s) +{ + if (LogLevel < 1000) + return; + + MT_LOCK + + const bool show2 = (_so != NULL); + + if (show2) + { + ClosePercents_for_so(); + _so->PrintUString(s, _tempA); + *_so << endl; + if (NeedFlush) + _so->Flush(); + } +} +*/ + +HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool isDir, bool isAnti, UInt32 mode) { if (StdOutMode) return S_OK; @@ -513,7 +589,7 @@ HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool /* isDir */, s = "Reading"; } - return PrintProgress(name, s, LogLevel >= requiredLevel); + return PrintProgress(name, isDir, s, LogLevel >= requiredLevel); } HRESULT CUpdateCallbackConsole::OpenFileError(const FString &path, DWORD systemError) @@ -526,10 +602,19 @@ HRESULT CUpdateCallbackConsole::ReadingFileError(const FString &path, DWORD syst return ReadingFileError_Base(path, systemError); } -HRESULT CUpdateCallbackConsole::SetOperationResult(Int32) +HRESULT CUpdateCallbackConsole::SetOperationResult(Int32 /* opRes */) { MT_LOCK _percent.Files++; + /* + if (opRes != NArchive::NUpdate::NOperationResult::kOK) + { + if (opRes == NArchive::NUpdate::NOperationResult::kError_FileChanged) + { + CommonError("Input file changed"); + } + } + */ return S_OK; } @@ -550,7 +635,9 @@ HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypt AString s; SetExtractErrorMessage(opRes, isEncrypted, s); - *_se << s << " : " << endl << name << endl << endl; + *_se << s << " : " << endl; + _se->NormalizePrint_wstr(name); + *_se << endl << endl; _se->Flush(); } return S_OK; @@ -559,7 +646,7 @@ HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypt } -HRESULT CUpdateCallbackConsole::ReportUpdateOpeartion(UInt32 op, const wchar_t *name, bool /* isDir */) +HRESULT CUpdateCallbackConsole::ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) { // if (StdOutMode) return S_OK; @@ -578,6 +665,8 @@ HRESULT CUpdateCallbackConsole::ReportUpdateOpeartion(UInt32 op, const wchar_t * case NUpdateNotifyOp::kSkip: s = "."; requiredLevel = 2; break; case NUpdateNotifyOp::kDelete: s = "D"; requiredLevel = 3; break; case NUpdateNotifyOp::kHeader: s = "Header creation"; requiredLevel = 100; break; + case NUpdateNotifyOp::kInFileChanged: s = "Size of input file was changed:"; requiredLevel = 10; break; + // case NUpdateNotifyOp::kOpFinished: s = "Finished"; requiredLevel = 100; break; default: { temp[0] = 'o'; @@ -587,7 +676,7 @@ HRESULT CUpdateCallbackConsole::ReportUpdateOpeartion(UInt32 op, const wchar_t * } } - return PrintProgress(name, s, LogLevel >= requiredLevel); + return PrintProgress(name, isDir, s, LogLevel >= requiredLevel); } /* @@ -622,7 +711,7 @@ HRESULT CUpdateCallbackConsole::CryptoGetTextPassword2(Int32 *passwordIsDefined, { if (AskPassword) { - Password = GetPassword(_so); + RINOK(GetPassword_HRESULT(_so, Password)); PasswordIsDefined = true; } } @@ -649,7 +738,7 @@ HRESULT CUpdateCallbackConsole::CryptoGetTextPassword(BSTR *password) if (!PasswordIsDefined) { { - Password = GetPassword(_so); + RINOK(GetPassword_HRESULT(_so, Password)) PasswordIsDefined = true; } } @@ -659,7 +748,7 @@ HRESULT CUpdateCallbackConsole::CryptoGetTextPassword(BSTR *password) COM_TRY_END } -HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool /* isDir */) +HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool isDir) { if (StdOutMode) return S_OK; @@ -668,7 +757,123 @@ HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool /* isDi { if (!name || name[0] == 0) name = kEmptyFileAlias; - return PrintProgress(name, "D", true); + return PrintProgress(name, isDir, "D", true); } return S_OK; } + +/* +void GetPropName(PROPID propID, const wchar_t *name, AString &nameA, UString &nameU); + +static void GetPropName(PROPID propID, UString &nameU) +{ + AString nameA; + GetPropName(propID, NULL, nameA, nameU); + // if (!nameA.IsEmpty()) + nameU = nameA; +} + + +static void AddPropNamePrefix(UString &s, PROPID propID) +{ + UString name; + GetPropName(propID, name); + s += name; + s += " = "; +} + +void CCallbackConsoleBase::PrintPropInfo(UString &s, PROPID propID, const PROPVARIANT *value) +{ + AddPropNamePrefix(s, propID); + { + UString dest; + const int level = 9; // we show up to ns precision level + ConvertPropertyToString2(dest, *value, propID, level); + s += dest; + } + PrintInfoLine(s); +} + +static void Add_IndexType_Index(UString &s, UInt32 indexType, UInt32 index) +{ + if (indexType == NArchive::NEventIndexType::kArcProp) + { + } + else + { + if (indexType == NArchive::NEventIndexType::kBlockIndex) + { + s += "#"; + } + else if (indexType == NArchive::NEventIndexType::kOutArcIndex) + { + } + else + { + s += "indexType_"; + s.Add_UInt32(indexType); + s.Add_Space(); + } + s.Add_UInt32(index); + } + s += ": "; +} + +HRESULT CUpdateCallbackConsole::ReportProp(UInt32 indexType, UInt32 index, PROPID propID, const PROPVARIANT *value) +{ + UString s; + Add_IndexType_Index(s, indexType, index); + PrintPropInfo(s, propID, value); + return S_OK; +} + +static inline char GetHex(Byte value) +{ + return (char)((value < 10) ? ('0' + value) : ('a' + (value - 10))); +} + +static void AddHexToString(UString &dest, const Byte *data, UInt32 size) +{ + for (UInt32 i = 0; i < size; i++) + { + Byte b = data[i]; + dest += GetHex((Byte)((b >> 4) & 0xF)); + dest += GetHex((Byte)(b & 0xF)); + } +} + +void HashHexToString(char *dest, const Byte *data, UInt32 size); + +HRESULT CUpdateCallbackConsole::ReportRawProp(UInt32 indexType, UInt32 index, + PROPID propID, const void *data, UInt32 dataSize, UInt32 propType) +{ + UString s; + propType = propType; + Add_IndexType_Index(s, indexType, index); + AddPropNamePrefix(s, propID); + if (propID == kpidChecksum) + { + char temp[k_HashCalc_DigestSize_Max + 8]; + HashHexToString(temp, (const Byte *)data, dataSize); + s += temp; + } + else + AddHexToString(s, (const Byte *)data, dataSize); + PrintInfoLine(s); + return S_OK; +} + +HRESULT CUpdateCallbackConsole::ReportFinished(UInt32 indexType, UInt32 index, Int32 opRes) +{ + UString s; + Add_IndexType_Index(s, indexType, index); + s += "finished"; + if (opRes != NArchive::NUpdate::NOperationResult::kOK) + { + s += ": "; + s.Add_UInt32(opRes); + } + PrintInfoLine(s); + return S_OK; +} +*/ diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.h b/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.h index ba8614ebaa3..b7ffef036ae 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/UpdateCallbackConsole.h @@ -35,7 +35,8 @@ protected: CStdOutStream *_se; void CommonError(const FString &path, DWORD systemError, bool isWarning); - + // void CommonError(const char *message); + HRESULT ScanError_Base(const FString &path, DWORD systemError); HRESULT OpenFileError_Base(const FString &name, DWORD systemError); HRESULT ReadingFileError_Base(const FString &name, DWORD systemError); @@ -56,7 +57,8 @@ public: StdOutMode(false), NeedFlush(false), PercentsNameLevel(1), - LogLevel(0) + LogLevel(0), + NumNonOpenFiles(0) {} void SetWindowWidth(unsigned width) { _percent.MaxLen = width - 1; } @@ -82,12 +84,14 @@ public: _percent.ClosePrint(false); } - CErrorPathCodes FailedFiles; CErrorPathCodes ScanErrors; + UInt64 NumNonOpenFiles; - HRESULT PrintProgress(const wchar_t *name, const char *command, bool showInLog); + HRESULT PrintProgress(const wchar_t *name, bool isDir, const char *command, bool showInLog); + // void PrintInfoLine(const UString &s); + // void PrintPropInfo(UString &s, PROPID propID, const PROPVARIANT *value); }; class CUpdateCallbackConsole: public IUpdateCallbackUI2, public CCallbackConsoleBase @@ -95,21 +99,23 @@ class CUpdateCallbackConsole: public IUpdateCallbackUI2, public CCallbackConsole // void PrintPropPair(const char *name, const wchar_t *val); public: + bool DeleteMessageWasShown; + #ifndef _NO_CRYPTO bool PasswordIsDefined; UString Password; bool AskPassword; #endif - bool DeleteMessageWasShown; - - CUpdateCallbackConsole() - : DeleteMessageWasShown(false) + CUpdateCallbackConsole(): + DeleteMessageWasShown(false) #ifndef _NO_CRYPTO , PasswordIsDefined(false) , AskPassword(false) #endif {} + + virtual ~CUpdateCallbackConsole() {} /* void Init(CStdOutStream *outStream) diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.cpp b/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.cpp index c7f830ff999..93f60eb291a 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.cpp @@ -14,8 +14,8 @@ static const char kNoAll = 's'; static const char kAutoRenameAll = 'u'; static const char kQuit = 'q'; -static const char *kFirstQuestionMessage = "? "; -static const char *kHelpQuestionMessage = +static const char * const kFirstQuestionMessage = "? "; +static const char * const kHelpQuestionMessage = "(Y)es / (N)o / (A)lways / (S)kip all / A(u)to rename all / (Q)uit? "; // return true if pressed Quite; @@ -31,9 +31,16 @@ NUserAnswerMode::EEnum ScanUserYesNoAllQuit(CStdOutStream *outStream) *outStream << kHelpQuestionMessage; outStream->Flush(); } - AString scannedString = g_StdIn.ScanStringUntilNewLine(); + AString scannedString; + if (!g_StdIn.ScanAStringUntilNewLine(scannedString)) + return NUserAnswerMode::kError; + if (g_StdIn.Error()) + return NUserAnswerMode::kError; scannedString.Trim(); - if (!scannedString.IsEmpty()) + if (scannedString.IsEmpty() && g_StdIn.Eof()) + return NUserAnswerMode::kEof; + + if (scannedString.Len() == 1) switch (::MyCharLower_Ascii(scannedString[0])) { case kYes: return NUserAnswerMode::kYes; @@ -52,7 +59,7 @@ NUserAnswerMode::EEnum ScanUserYesNoAllQuit(CStdOutStream *outStream) #endif #endif -UString GetPassword(CStdOutStream *outStream) +static bool GetPassword(CStdOutStream *outStream, UString &psw) { if (outStream) { @@ -71,20 +78,33 @@ UString GetPassword(CStdOutStream *outStream) DWORD mode = 0; if (console != INVALID_HANDLE_VALUE && console != 0) if (GetConsoleMode(console, &mode)) - wasChanged = (SetConsoleMode(console, mode & ~ENABLE_ECHO_INPUT) != 0); - UString res = g_StdIn.ScanUStringUntilNewLine(); + wasChanged = (SetConsoleMode(console, mode & ~(DWORD)ENABLE_ECHO_INPUT) != 0); + bool res = g_StdIn.ScanUStringUntilNewLine(psw); if (wasChanged) SetConsoleMode(console, mode); + + #else + + bool res = g_StdIn.ScanUStringUntilNewLine(psw); + + #endif + if (outStream) { *outStream << endl; outStream->Flush(); } + return res; - - #else - - return g_StdIn.ScanUStringUntilNewLine(); - - #endif +} + +HRESULT GetPassword_HRESULT(CStdOutStream *outStream, UString &psw) +{ + if (!GetPassword(outStream, psw)) + return E_INVALIDARG; + if (g_StdIn.Error()) + return E_FAIL; + if (g_StdIn.Eof() && psw.IsEmpty()) + return E_ABORT; + return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.h b/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.h index 16f045803e4..256feafef75 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.h +++ b/3rdparty/lzma/CPP/7zip/UI/Console/UserInputUtils.h @@ -14,11 +14,14 @@ enum EEnum kYesAll, kNoAll, kAutoRenameAll, - kQuit + kQuit, + kEof, + kError }; } NUserAnswerMode::EEnum ScanUserYesNoAllQuit(CStdOutStream *outStream); -UString GetPassword(CStdOutStream *outStream); +// bool GetPassword(CStdOutStream *outStream, UString &psw); +HRESULT GetPassword_HRESULT(CStdOutStream *outStream, UString &psw); #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/makefile b/3rdparty/lzma/CPP/7zip/UI/Console/makefile index 98ad91c2440..acc3f107cb5 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/makefile +++ b/3rdparty/lzma/CPP/7zip/UI/Console/makefile @@ -1,15 +1,11 @@ PROG = 7z.exe -MY_CONSOLE = 1 CFLAGS = $(CFLAGS) \ -DEXTERNAL_CODECS \ -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -!ENDIF - COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ + $O\DynLimBuf.obj \ $O\IntToString.obj \ $O\ListFileUtils.obj \ $O\NewHandler.obj \ @@ -36,6 +32,7 @@ WIN_OBJS = \ $O\PropVariantConv.obj \ $O\Registry.obj \ $O\System.obj \ + $O\SystemInfo.obj \ $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ @@ -52,12 +49,13 @@ WIN_OBJS = \ $O\UniqBlocks.obj \ AR_COMMON_OBJS = \ + $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ COMPRESS_OBJS = \ $O\CopyCoder.obj \ -C_OBJS = \ +C_OBJS = $(C_OBJS) \ $O\Alloc.obj \ $O\CpuArch.obj \ $O\Sort.obj \ diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/makefile.gcc b/3rdparty/lzma/CPP/7zip/UI/Console/makefile.gcc new file mode 100644 index 00000000000..d7e444780ac --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/UI/Console/makefile.gcc @@ -0,0 +1,185 @@ +PROG = 7z +IS_NOT_STANDALONE = 1 + +# IS_X64 = 1 +# USE_ASM = 1 +# ST_MODE = 1 + + +LOCAL_FLAGS_ST = +MT_OBJS = + +ifdef SystemDrive +IS_MINGW = 1 +else +ifdef SYSTEMDRIVE +# ifdef OS +IS_MINGW = 1 +endif +endif + +ifdef ST_MODE + +LOCAL_FLAGS_ST = -D_7ZIP_ST + +ifdef IS_MINGW +MT_OBJS = \ + $O/Threads.o \ + +endif + +else + +MT_OBJS = \ + $O/Synchronization.o \ + $O/Threads.o \ + +endif + + + +LOCAL_FLAGS_WIN= + +ifdef IS_MINGW + +LOCAL_FLAGS_WIN = \ + -D_7ZIP_LARGE_PAGES \ + -DWIN_LONG_PATH \ + -DSUPPORT_DEVICE_FILE \ + +SYS_OBJS = \ + $O/FileSystem.o \ + $O/Registry.o \ + $O/MemoryLock.o \ + $O/DllSecur.o \ + $O/resource.o \ + +else + +SYS_OBJS = \ + $O/MyWindows.o \ + +endif + + + +LOCAL_FLAGS = \ + $(LOCAL_FLAGS_WIN) \ + $(LOCAL_FLAGS_ST) \ + -DEXTERNAL_CODECS \ + + + +CONSOLE_OBJS = \ + $O/BenchCon.o \ + $O/ConsoleClose.o \ + $O/ExtractCallbackConsole.o \ + $O/HashCon.o \ + $O/List.o \ + $O/Main.o \ + $O/MainAr.o \ + $O/OpenCallbackConsole.o \ + $O/PercentPrinter.o \ + $O/UpdateCallbackConsole.o \ + $O/UserInputUtils.o \ + +UI_COMMON_OBJS = \ + $O/ArchiveCommandLine.o \ + $O/ArchiveExtractCallback.o \ + $O/ArchiveOpenCallback.o \ + $O/Bench.o \ + $O/DefaultName.o \ + $O/EnumDirItems.o \ + $O/Extract.o \ + $O/ExtractingFilePath.o \ + $O/HashCalc.o \ + $O/LoadCodecs.o \ + $O/OpenArchive.o \ + $O/PropIDUtils.o \ + $O/SetProperties.o \ + $O/SortUtils.o \ + $O/TempFiles.o \ + $O/Update.o \ + $O/UpdateAction.o \ + $O/UpdateCallback.o \ + $O/UpdatePair.o \ + $O/UpdateProduce.o \ + +COMMON_OBJS = \ + $O/CommandLineParser.o \ + $O/CRC.o \ + $O/CrcReg.o \ + $O/DynLimBuf.o \ + $O/IntToString.o \ + $O/ListFileUtils.o \ + $O/NewHandler.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ + $O/MyString.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ + $O/UTFConvert.o \ + $O/MyVector.o \ + $O/Wildcard.o \ + +WIN_OBJS = \ + $O/DLL.o \ + $O/ErrorMsg.o \ + $O/FileDir.o \ + $O/FileFind.o \ + $O/FileIO.o \ + $O/FileLink.o \ + $O/FileName.o \ + $O/PropVariant.o \ + $O/PropVariantConv.o \ + $O/System.o \ + $O/SystemInfo.o \ + $O/TimeUtils.o \ + +7ZIP_COMMON_OBJS = \ + $O/CreateCoder.o \ + $O/CWrappers.o \ + $O/FilePathAutoRename.o \ + $O/FileStreams.o \ + $O/InBuffer.o \ + $O/InOutTempBuffer.o \ + $O/FilterCoder.o \ + $O/LimitedStreams.o \ + $O/MethodId.o \ + $O/MethodProps.o \ + $O/OffsetStream.o \ + $O/OutBuffer.o \ + $O/ProgressUtils.o \ + $O/PropId.o \ + $O/StreamObjects.o \ + $O/StreamUtils.o \ + $O/UniqBlocks.o \ + +COMPRESS_OBJS = \ + $O/CopyCoder.o \ + +AR_COMMON_OBJS = \ + $O/ItemNameUtils.o \ + +C_OBJS = \ + $O/Alloc.o \ + $O/CpuArch.o \ + $O/Sort.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ + + +OBJS = \ + $(C_OBJS) \ + $(MT_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ + $(SYS_OBJS) \ + $(COMPRESS_OBJS) \ + $(AR_COMMON_OBJS) \ + $(7ZIP_COMMON_OBJS) \ + $(UI_COMMON_OBJS) \ + $(CONSOLE_OBJS) \ + + +include ../../7zip_gcc.mak diff --git a/3rdparty/lzma/CPP/7zip/UI/Console/resource.rc b/3rdparty/lzma/CPP/7zip/UI/Console/resource.rc index 6e09bb968be..414427fb2d1 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Console/resource.rc +++ b/3rdparty/lzma/CPP/7zip/UI/Console/resource.rc @@ -1,3 +1,7 @@ #include "../../MyVersionInfo.rc" -MY_VERSION_INFO_APP("7-Zip Console", "7z") +MY_VERSION_INFO_APP("7-Zip Console" , "7z") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "Console.manifest" +#endif diff --git a/3rdparty/lzma/CPP/7zip/UI/Explorer/MyMessages.cpp b/3rdparty/lzma/CPP/7zip/UI/Explorer/MyMessages.cpp index 70c2a4608d3..c912504ca2a 100644 --- a/3rdparty/lzma/CPP/7zip/UI/Explorer/MyMessages.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/Explorer/MyMessages.cpp @@ -26,7 +26,7 @@ void ShowErrorMessageRes(UINT resID) ShowErrorMessageHwndRes(0, resID); } -void ShowErrorMessageDWORD(HWND window, DWORD errorCode) +static void ShowErrorMessageDWORD(HWND window, DWORD errorCode) { ShowErrorMessage(window, NError::MyFormatMessage(errorCode)); } diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/BrowseDialog.cpp b/3rdparty/lzma/CPP/7zip/UI/FileManager/BrowseDialog.cpp index 066ac219190..6d2b6b55e24 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/BrowseDialog.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/BrowseDialog.cpp @@ -4,7 +4,7 @@ #include "../../../Common/MyWindows.h" -#include +#include #ifndef UNDER_CE #include "../../../Windows/CommonDialog.h" @@ -139,7 +139,7 @@ public: UStringVector Filters; UString FilterDescription; - CBrowseDialog(): FolderMode(false), _showDots(false), ShowAllFiles(true) {} + CBrowseDialog(): _showDots(false), FolderMode(false), ShowAllFiles(true) {} void SetFilter(const UString &s); INT_PTR Create(HWND parent = 0) { return CModalDialog::Create(IDD_BROWSE, parent); } int CompareItems(LPARAM lParam1, LPARAM lParam2); @@ -209,7 +209,7 @@ bool CBrowseDialog::OnInit() if (!FilterDescription.IsEmpty()) s = FilterDescription; else if (ShowAllFiles) - s = L"*.*"; + s = "*.*"; else { FOR_VECTOR (i, Filters) @@ -235,7 +235,7 @@ bool CBrowseDialog::OnInit() column.fmt = LVCFMT_RIGHT; column.cx = 100; const UString s = LangString(IDS_PROP_SIZE); - column.pszText = (wchar_t *)(const wchar_t *)s; + column.pszText = s.Ptr_non_const(); _list.InsertColumn(2, &column); } @@ -468,15 +468,15 @@ bool CBrowseDialog::GetParentPath(const UString &path, UString &parentPrefix, US if (_topDirPrefix == path) return false; UString s = path; - if (s.Back() == WCHAR_PATH_SEPARATOR) + if (IS_PATH_SEPAR(s.Back())) s.DeleteBack(); if (s.IsEmpty()) return false; - if (s.Back() == WCHAR_PATH_SEPARATOR) + if (IS_PATH_SEPAR(s.Back())) return false; int pos = s.ReverseFind_PathSepar(); parentPrefix.SetFrom(s, pos + 1); - name = s.Ptr(pos + 1); + name = s.Ptr((unsigned)(pos + 1)); return true; } @@ -531,7 +531,7 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected #ifndef UNDER_CE bool isDrive = false; - if (pathPrefix.IsEmpty() || pathPrefix == kSuperPathPrefix) + if (pathPrefix.IsEmpty() || pathPrefix.IsEqualTo(kSuperPathPrefix)) { isDrive = true; FStringVector drives; @@ -551,7 +551,8 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected else #endif { - CEnumerator enumerator(us2fs(pathPrefix + L'*')); + CEnumerator enumerator; + enumerator.SetDirPrefix(us2fs(pathPrefix)); for (;;) { bool found; @@ -596,14 +597,14 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected if (_showDots && _topDirPrefix != DirPrefix) { item.iItem = index; - const UString itemName = L".."; + const UString itemName (".."); if (selectedName.IsEmpty()) cursorIndex = index; item.mask = LVIF_TEXT | LVIF_PARAM | LVIF_IMAGE; int subItem = 0; item.iSubItem = subItem++; item.lParam = kParentIndex; - item.pszText = (wchar_t *)(const wchar_t *)itemName; + item.pszText = itemName.Ptr_non_const(); item.iImage = _extToIconMap.GetIconIndex(FILE_ATTRIBUTE_DIRECTORY, DirPrefix); if (item.iImage < 0) item.iImage = 0; @@ -625,7 +626,7 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected int subItem = 0; item.iSubItem = subItem++; item.lParam = i; - item.pszText = (wchar_t *)(const wchar_t *)name; + item.pszText = name.Ptr_non_const(); const UString fullPath = DirPrefix + name; #ifndef UNDER_CE @@ -642,16 +643,14 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected _list.InsertItem(&item); wchar_t s[32]; { - FILETIME ft; s[0] = 0; - if (FileTimeToLocalFileTime(&fi.MTime, &ft)) - ConvertFileTimeToString(ft, s, + ConvertUtcFileTimeToString(fi.MTime, s, #ifndef UNDER_CE - true + kTimestampPrintLevel_MIN #else - false + kTimestampPrintLevel_DAY #endif - , false); + ); _list.SetSubItem(index, subItem++, s); } { @@ -780,8 +779,10 @@ void CBrowseDialog::OnItemEnter() */ return; } - UString s = DirPrefix + fs2us(file.Name) + WCHAR_PATH_SEPARATOR; - HRESULT res = Reload(s, L""); + UString s = DirPrefix; + s += fs2us(file.Name); + s.Add_PathSepar(); + HRESULT res = Reload(s, UString()); if (res != S_OK) MessageBox_HResError(*this, res, s); SetPathEditText(); @@ -853,17 +854,17 @@ bool MyBrowseForFile(HWND owner, LPCWSTR title, LPCWSTR path, #else // maybe we must use GetLastError in WinCE. DWORD errorCode = CommDlgExtendedError(); - const wchar_t *errorMessage = NULL; + const char *errorMessage = NULL; switch (errorCode) { case 0: return false; // cancel or close obn dialog - case FNERR_INVALIDFILENAME: errorMessage = L"Invalid File Name"; break; - default: errorMessage = L"Open Dialog Error"; + case FNERR_INVALIDFILENAME: errorMessage = "Invalid File Name"; break; + default: errorMessage = "Open Dialog Error"; } if (!errorMessage) return false; { - UString s = errorMessage; + UString s (errorMessage); s.Add_LF(); s += path; MessageBox_Error_Global(owner, s); @@ -952,7 +953,7 @@ bool CorrectFsPath(const UString &relBase, const UString &path2, UString &result if (path.Back() == WCHAR_PATH_SEPARATOR) { path.DeleteBack(); - result.Insert(0, WCHAR_PATH_SEPARATOR);; + result.Insert(0, WCHAR_PATH_SEPARATOR); } int pos = path.ReverseFind(WCHAR_PATH_SEPARATOR) + 1; UString cur = path.Ptr(pos); diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.cpp index 7d61d14d9d9..232717f89f4 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -44,6 +44,7 @@ void CExtractCallbackImp::Init() _lang_Extracting = LangString(IDS_PROGRESS_EXTRACTING); _lang_Testing = LangString(IDS_PROGRESS_TESTING); _lang_Skipping = LangString(IDS_PROGRESS_SKIPPING); + _lang_Reading = "Reading"; NumArchiveErrors = 0; ThereAreMessageErrors = false; @@ -233,6 +234,7 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation(const wchar_t *name, Int32 is case NArchive::NExtract::NAskMode::kExtract: msg = &_lang_Extracting; break; case NArchive::NExtract::NAskMode::kTest: msg = &_lang_Testing; break; case NArchive::NExtract::NAskMode::kSkip: msg = &_lang_Skipping; break; + case NArchive::NExtract::NAskMode::kReadExternal: msg = &_lang_Reading; break; // default: s = "Unknown operation"; } @@ -262,6 +264,7 @@ STDMETHODIMP CExtractCallbackImp::ShowMessage(const wchar_t *s) #endif +void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, const wchar_t *fileName, UString &s); void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, const wchar_t *fileName, UString &s) { s.Empty(); @@ -335,20 +338,18 @@ void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, const wchar_t *fileNam s += msg; else { - char temp[16]; - ConvertUInt32ToString(opRes, temp); - s.AddAscii("Error #"); - s.AddAscii(temp); + s += "Error #"; + s.Add_UInt32(opRes); } if (encrypted && opRes != NArchive::NExtract::NOperationResult::kWrongPassword) { - // s.AddAscii(" : "); + // s += " : "; // AddLangString(s, IDS_EXTRACT_MSG_ENCRYPTED); - s.AddAscii(" : "); + s += " : "; AddLangString(s, IDS_EXTRACT_MSG_WRONG_PSW_GUESS); } - s.AddAscii(" : "); + s += " : "; s += fileName; } } @@ -451,6 +452,7 @@ static void AddNewLineString(UString &s, const UString &m) s.Add_LF(); } +UString GetOpenArcErrorMessage(UInt32 errorFlags); UString GetOpenArcErrorMessage(UInt32 errorFlags) { UString s; @@ -466,7 +468,7 @@ UString GetOpenArcErrorMessage(UInt32 errorFlags) continue; if (f == kpv_ErrorFlags_EncryptedHeadersError) { - m.AddAscii(" : "); + m += " : "; AddLangString(m, IDS_EXTRACT_MSG_WRONG_PSW_GUESS); } if (!s.IsEmpty()) @@ -483,7 +485,7 @@ UString GetOpenArcErrorMessage(UInt32 errorFlags) ConvertUInt32ToHex(errorFlags, sz + 2); if (!s.IsEmpty()) s.Add_LF(); - s.AddAscii(sz); + s += sz; } return s; @@ -503,7 +505,7 @@ static void ErrorInfo_Print(UString &s, const CArcErrorInfo &er) if (warningFlags != 0) { s += GetNameOfProperty(kpidWarningFlags, L"Warnings"); - s.AddAscii(":"); + s += ":"; s.Add_LF(); AddNewLineString(s, GetOpenArcErrorMessage(warningFlags)); } @@ -511,7 +513,7 @@ static void ErrorInfo_Print(UString &s, const CArcErrorInfo &er) if (!er.WarningMessage.IsEmpty()) { s += GetNameOfProperty(kpidWarning, L"Warning"); - s.AddAscii(": "); + s += ": "; s += er.WarningMessage; s.Add_LF(); } @@ -519,12 +521,13 @@ static void ErrorInfo_Print(UString &s, const CArcErrorInfo &er) static UString GetBracedType(const wchar_t *type) { - UString s = L'['; + UString s ('['); s += type; - s += L']'; + s += ']'; return s; } +void OpenResult_GUI(UString &s, const CCodecs *codecs, const CArchiveLink &arcLink, const wchar_t *name, HRESULT result); void OpenResult_GUI(UString &s, const CCodecs *codecs, const CArchiveLink &arcLink, const wchar_t *name, HRESULT result) { FOR_VECTOR (level, arcLink.Arcs) @@ -634,7 +637,9 @@ HRESULT CExtractCallbackImp::ExtractResult(HRESULT result) if (result == S_OK) return result; NumArchiveErrors++; - if (result == E_ABORT || result == ERROR_DISK_FULL) + if (result == E_ABORT + || result == HRESULT_FROM_WIN32(ERROR_DISK_FULL) + ) return result; Add_ArchiveName_Error(); @@ -704,7 +709,7 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( { if (!destFileInfo.IsDir()) { - RINOK(MessageError("can not replace file with folder with same name", destPathSys)); + RINOK(MessageError("Cannot replace file with folder with same name", destPathSys)); return E_ABORT; } *writeAnswer = BoolToInt(false); @@ -713,7 +718,7 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( if (destFileInfo.IsDir()) { - RINOK(MessageError("can not replace folder with file with same name", destPathSys)); + RINOK(MessageError("Cannot replace folder with file with same name", destPathSys)); *writeAnswer = BoolToInt(false); return S_OK; } @@ -727,7 +732,7 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( Int32 overwriteResult; UString destPathSpec = destPath; int slashPos = destPathSpec.ReverseFind_PathSepar(); - destPathSpec.DeleteFrom(slashPos + 1); + destPathSpec.DeleteFrom((unsigned)(slashPos + 1)); destPathSpec += fs2us(destFileInfo.Name); RINOK(AskOverwrite( @@ -748,24 +753,31 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( default: return E_FAIL; } + break; } + default: + break; } if (OverwriteMode == NExtract::NOverwriteMode::kRename) { if (!AutoRenamePath(destPathSys)) { - RINOK(MessageError("can not create name for file", destPathSys)); + RINOK(MessageError("Cannot create name for file", destPathSys)); return E_ABORT; } destPathResultTemp = fs2us(destPathSys); } else + { + if (NFind::DoesFileExist_Raw(destPathSys)) if (!NDir::DeleteFileAlways(destPathSys)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) { - RINOK(MessageError("can not delete output file", destPathSys)); + RINOK(MessageError("Cannot delete output file", destPathSys)); return E_ABORT; } + } } *writeAnswer = BoolToInt(true); return StringToBstr(destPathResultTemp, destPathResult); @@ -901,8 +913,10 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation7(Int32 askExtractMode) { COM_TRY_BEGIN _needUpdateStat = ( - askExtractMode == NArchive::NExtract::NAskMode::kExtract || - askExtractMode == NArchive::NExtract::NAskMode::kTest); + askExtractMode == NArchive::NExtract::NAskMode::kExtract + || askExtractMode == NArchive::NExtract::NAskMode::kTest + || askExtractMode == NArchive::NExtract::NAskMode::kReadExternal + ); /* _extractMode = false; @@ -920,7 +934,7 @@ STDMETHODIMP CExtractCallbackImp::PrepareOperation7(Int32 askExtractMode) COM_TRY_END } -STDMETHODIMP CExtractCallbackImp::SetOperationResult7(Int32 opRes, Int32 encrypted) +STDMETHODIMP CExtractCallbackImp::SetOperationResult8(Int32 opRes, Int32 encrypted, UInt64 size) { COM_TRY_BEGIN if (VirtFileSystem && _newVirtFileWasAdded) @@ -940,7 +954,7 @@ STDMETHODIMP CExtractCallbackImp::SetOperationResult7(Int32 opRes, Int32 encrypt } else if (_hashCalc && _needUpdateStat) { - _hashCalc->SetSize(_curSize); + _hashCalc->SetSize(size); // (_curSize) before 21.04 _hashCalc->Final(_isFolder, _isAltStream, _filePath); } return SetOperationResult(opRes, encrypted); @@ -948,9 +962,8 @@ STDMETHODIMP CExtractCallbackImp::SetOperationResult7(Int32 opRes, Int32 encrypt } -static const size_t k_SizeT_MAX = (size_t)((size_t)0 - 1); -static const UInt32 kBlockSize = ((UInt32)1 << 31); +// static const UInt32 kBlockSize = ((UInt32)1 << 31); STDMETHODIMP CVirtFileSystem::Write(const void *data, UInt32 size, UInt32 *processedSize) { @@ -975,8 +988,9 @@ STDMETHODIMP CVirtFileSystem::Write(const void *data, UInt32 size, UInt32 *proce if (b < a) b = a; useMem = false; - if (b <= k_SizeT_MAX && b <= MaxTotalAllocSize) - useMem = file.Data.ReAlloc_KeepData((size_t)b, (size_t)file.Size); + const size_t b_sizet = (size_t)b; + if (b == b_sizet && b <= MaxTotalAllocSize) + useMem = file.Data.ReAlloc_KeepData(b_sizet, (size_t)file.Size); } if (useMem) { @@ -1009,7 +1023,7 @@ HRESULT CVirtFileSystem::FlushToDisk(bool closeLast) { _outFileStream.Release(); return E_FAIL; - // MessageBoxMyError(UString(L"Can't create file ") + fs2us(tempFilePath)); + // MessageBoxMyError(UString("Can't create file ") + fs2us(tempFilePath)); } _fileIsOpen = true; RINOK(WriteStream(_outFileStream, file.Data, (size_t)file.Size)); @@ -1027,7 +1041,7 @@ HRESULT CVirtFileSystem::FlushToDisk(bool closeLast) _numFlushed++; _fileIsOpen = false; if (file.AttribDefined) - NDir::SetFileAttrib(path, file.Attrib); + NDir::SetFileAttrib_PosixHighDetect(path, file.Attrib); } return S_OK; } diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.h index 6cd8d0aa83d..02578bb421c 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ExtractCallback.h @@ -282,6 +282,7 @@ public: UString _lang_Extracting; UString _lang_Testing; UString _lang_Skipping; + UString _lang_Reading; UString _lang_Empty; bool _totalFilesDefined; @@ -289,21 +290,19 @@ public: bool MultiArcMode; CExtractCallbackImp(): + #ifndef _SFX + _hashCalc(NULL), + #endif + ProcessAltStreams(true), + StreamMode(false), + OverwriteMode(NExtract::NOverwriteMode::kAsk), #ifndef _NO_CRYPTO PasswordIsDefined(false), PasswordWasAsked(false), #endif - OverwriteMode(NExtract::NOverwriteMode::kAsk), - StreamMode(false), - ProcessAltStreams(true), - _totalFilesDefined(false), _totalBytesDefined(false), MultiArcMode(false) - - #ifndef _SFX - , _hashCalc(NULL) - #endif {} ~CExtractCallbackImp(); diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.cpp b/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.cpp index c6feb9021bb..b455b14a3ca 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.cpp @@ -40,6 +40,12 @@ void COverwriteDialog::ReduceString(UString &s) s.Delete(size / 2, s.Len() - size); s.Insert(size / 2, L" ... "); } + if (!s.IsEmpty() && s.Back() == ' ') + { + // s += (wchar_t)(0x2423); + s.InsertAtFront(L'\"'); + s += L'\"'; + } } void COverwriteDialog::SetFileInfoControl(int textID, int iconID, @@ -51,8 +57,8 @@ void COverwriteDialog::SetFileInfoControl(int textID, int iconID, const UString &fileName = fileInfo.Name; int slashPos = fileName.ReverseFind_PathSepar(); - UString s1 = fileName.Left(slashPos + 1); - UString s2 = fileName.Ptr(slashPos + 1); + UString s1 = fileName.Left((unsigned)(slashPos + 1)); + UString s2 = fileName.Ptr((unsigned)(slashPos + 1)); ReduceString(s1); ReduceString(s2); @@ -66,13 +72,10 @@ void COverwriteDialog::SetFileInfoControl(int textID, int iconID, if (fileInfo.TimeIsDefined) { - FILETIME localFileTime; - if (!FileTimeToLocalFileTime(&fileInfo.Time, &localFileTime)) - throw 4190402; AddLangString(s, IDS_PROP_MTIME); - s += L": "; - wchar_t t[32]; - ConvertFileTimeToString(localFileTime, t); + s += ": "; + char t[32]; + ConvertUtcFileTimeToString(fileInfo.Time, t); s += t; } @@ -100,6 +103,22 @@ bool COverwriteDialog::OnInit() SetFileInfoControl(IDT_OVERWRITE_OLD_FILE_SIZE_TIME, IDI_OVERWRITE_OLD_FILE, OldFileInfo); SetFileInfoControl(IDT_OVERWRITE_NEW_FILE_SIZE_TIME, IDI_OVERWRITE_NEW_FILE, NewFileInfo); NormalizePosition(); + + if (!ShowExtraButtons) + { + HideItem(IDB_YES_TO_ALL); + HideItem(IDB_NO_TO_ALL); + HideItem(IDB_AUTO_RENAME); + } + + if (DefaultButton_is_NO) + { + PostMsg(DM_SETDEFID, IDNO); + HWND h = GetItem(IDNO); + PostMsg(WM_NEXTDLGCTL, (WPARAM)h, TRUE); + // ::SetFocus(h); + } + return CModalDialog::OnInit(); } diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.h index da7fa55ff68..24e56cac987 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/OverwriteDialog.h @@ -20,7 +20,7 @@ namespace NOverwriteDialog void SetTime(const FILETIME *t) { - if (t == 0) + if (!t) TimeIsDefined = false; else { @@ -28,15 +28,19 @@ namespace NOverwriteDialog Time = *t; } } + + void SetSize(UInt64 size) + { + SizeIsDefined = true; + Size = size; + } + void SetSize(const UInt64 *size) { - if (size == 0) + if (!size) SizeIsDefined = false; else - { - SizeIsDefined = true; - Size = *size; - } + SetSize(*size); } }; } @@ -51,6 +55,12 @@ class COverwriteDialog: public NWindows::NControl::CModalDialog void ReduceString(UString &s); public: + bool ShowExtraButtons; + bool DefaultButton_is_NO; + + + COverwriteDialog(): ShowExtraButtons(true), DefaultButton_is_NO(false) {} + INT_PTR Create(HWND parent = 0) { BIG_DIALOG_SIZE(280, 200); diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.cpp b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.cpp index 65201a9ed16..b688a901506 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.cpp @@ -120,7 +120,7 @@ bool CProgressDialog::OnTimer(WPARAM /* timerID */, LPARAM /* callback */) wchar_t s[64]; ConvertUInt64ToString(percentValue, s); UString title = s; - title += L"% "; + title += "% "; SetText(title + _title); #ifndef _SFX AddToTitle(title + MainAddTitle); @@ -136,8 +136,11 @@ bool CProgressDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) { case kCloseMessage: { - KillTimer(_timer); - _timer = 0; + if (_timer) + { + KillTimer(kTimerID); + _timer = 0; + } if (_inCancelMessageBox) { _externalCloseMessageWasReceived = true; diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.h index 0f41b57aed6..35c182a8121 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog.h @@ -137,7 +137,7 @@ public: { _title = title; INT_PTR res = CModalDialog::Create(IDD_PROGRESS, wndParent); - thread.Wait(); + thread.Wait_Close(); return res; } diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.cpp b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.cpp index 8c38b81aa0a..54273d0ce12 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.cpp @@ -5,7 +5,7 @@ #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" -#include "../../../Windows/Control/Static.h" +#include "../../../Windows/Clipboard.h" #include "../../../Windows/ErrorMsg.h" #include "../GUI/ExtractRes.h" @@ -217,7 +217,7 @@ void CProgressSync::AddError_Message_Name(const wchar_t *message, const wchar_t UString s; if (name && *name != 0) s += name; - if (message && *message != 0 ) + if (message && *message != 0) { if (!s.IsEmpty()) s.Add_LF(); @@ -232,11 +232,14 @@ void CProgressSync::AddError_Code_Name(DWORD systemError, const wchar_t *name) { UString s = NError::MyFormatMessage(systemError); if (systemError == 0) - s = L"Error"; + s = "Error"; AddError_Message_Name(s, name); } -CProgressDialog::CProgressDialog(): _timer(0), CompressingMode(true), MainWindow(0) +CProgressDialog::CProgressDialog(): + _timer(0), + CompressingMode(true), + MainWindow(NULL) { _isDir = false; @@ -277,7 +280,7 @@ CProgressDialog::~CProgressDialog() } void CProgressDialog::AddToTitle(LPCWSTR s) { - if (MainWindow != 0) + if (MainWindow) { CWindow window(MainWindow); window.SetText((UString)s + MainTitle); @@ -345,13 +348,16 @@ bool CProgressDialog::OnInit() INIT_AS_UNDEFINED(_processed_Prev); INIT_AS_UNDEFINED(_packed_Prev); INIT_AS_UNDEFINED(_ratio_Prev); + _filesStr_Prev.Empty(); + _filesTotStr_Prev.Empty(); _foreground = true; m_ProgressBar.Attach(GetItem(IDC_PROGRESS1)); _messageList.Attach(GetItem(IDL_PROGRESS_MESSAGES)); _messageList.SetUnicodeFormat(); + _messageList.SetExtendedListViewStyle(LVS_EX_FULLROWSELECT); _wasCreated = true; _dialogCreatedEvent.Set(); @@ -420,13 +426,14 @@ static const UINT kIDs[] = IDT_PROGRESS_ELAPSED, IDT_PROGRESS_ELAPSED_VAL, IDT_PROGRESS_REMAINING, IDT_PROGRESS_REMAINING_VAL, IDT_PROGRESS_FILES, IDT_PROGRESS_FILES_VAL, - IDT_PROGRESS_RATIO, IDT_PROGRESS_RATIO_VAL, + 0, IDT_PROGRESS_FILES_TOTAL, IDT_PROGRESS_ERRORS, IDT_PROGRESS_ERRORS_VAL, IDT_PROGRESS_TOTAL, IDT_PROGRESS_TOTAL_VAL, IDT_PROGRESS_SPEED, IDT_PROGRESS_SPEED_VAL, IDT_PROGRESS_PROCESSED, IDT_PROGRESS_PROCESSED_VAL, - IDT_PROGRESS_PACKED, IDT_PROGRESS_PACKED_VAL + IDT_PROGRESS_PACKED, IDT_PROGRESS_PACKED_VAL, + IDT_PROGRESS_RATIO, IDT_PROGRESS_RATIO_VAL }; bool CProgressDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) @@ -533,16 +540,17 @@ bool CProgressDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) labelSize = gSize - valueSize; yPos = my; - for (int i = 0; i < ARRAY_SIZE(kIDs); i += 2) + for (unsigned i = 0; i < ARRAY_SIZE(kIDs); i += 2) { int x = mx; - const int kNumColumn1Items = 5 * 2; + const unsigned kNumColumn1Items = 5 * 2; if (i >= kNumColumn1Items) { if (i == kNumColumn1Items) yPos = my; x = mx + gSize + padSize; } + if (kIDs[i] != 0) MoveItem(kIDs[i], x, yPos, labelSize, sY); MoveItem(kIDs[i + 1], x + labelSize, yPos, valueSize, sY); yPos += sStep; @@ -580,6 +588,7 @@ void CProgressDialog::SetProgressPos(UInt64 pos) #define UINT_TO_STR_2(val) { s[0] = (wchar_t)('0' + (val) / 10); s[1] = (wchar_t)('0' + (val) % 10); s += 2; } +void GetTimeString(UInt64 timeValue, wchar_t *s); void GetTimeString(UInt64 timeValue, wchar_t *s) { UInt64 hours = timeValue / 3600; @@ -613,6 +622,7 @@ static void ConvertSizeToString(UInt64 v, wchar_t *s) s += MyStringLen(s); *s++ = ' '; *s++ = c; + *s++ = 'B'; *s++ = 0; } } @@ -825,17 +835,25 @@ void CProgressDialog::UpdateStatInfo(bool showAll) { wchar_t s[64]; + ConvertUInt64ToString(completedFiles, s); - if (IS_DEFINED_VAL(totalFiles)) - { - wcscat(s, L" / "); - ConvertUInt64ToString(totalFiles, s + wcslen(s)); - } if (_filesStr_Prev != s) { _filesStr_Prev = s; SetItemText(IDT_PROGRESS_FILES_VAL, s); } + + s[0] = 0; + if (IS_DEFINED_VAL(totalFiles)) + { + MyStringCopy(s, L" / "); + ConvertUInt64ToString(totalFiles, s + MyStringLen(s)); + } + if (_filesTotStr_Prev != s) + { + _filesTotStr_Prev = s; + SetItemText(IDT_PROGRESS_FILES_TOTAL, s); + } } const UInt64 packSize = CompressingMode ? outSize : inSize; @@ -862,7 +880,7 @@ void CProgressDialog::UpdateStatInfo(bool showAll) { _ratio_Prev = ratio; ConvertUInt64ToString(ratio, s); - wcscat(s, L"%"); + MyStringCat(s, L"%"); SetItemText(IDT_PROGRESS_RATIO_VAL, s); } } @@ -889,8 +907,8 @@ void CProgressDialog::UpdateStatInfo(bool showAll) int slashPos = _filePath.ReverseFind_PathSepar(); if (slashPos >= 0) { - s1.SetFrom(_filePath, slashPos + 1); - s2 = _filePath.Ptr(slashPos + 1); + s1.SetFrom(_filePath, (unsigned)(slashPos + 1)); + s2 = _filePath.Ptr((unsigned)(slashPos + 1)); } else s2 = _filePath; @@ -939,7 +957,7 @@ INT_PTR CProgressDialog::Create(const UString &title, NWindows::CThread &thread, CWaitCursor waitCursor; HANDLE h[] = { thread, _createDialogEvent }; - WRes res2 = WaitForMultipleObjects(ARRAY_SIZE(h), h, FALSE, kCreateDelay); + DWORD res2 = WaitForMultipleObjects(ARRAY_SIZE(h), h, FALSE, kCreateDelay); if (res2 == WAIT_OBJECT_0 && !Sync.ThereIsMessage()) return 0; } @@ -951,9 +969,8 @@ INT_PTR CProgressDialog::Create(const UString &title, NWindows::CThread &thread, { _wasCreated = true; _dialogCreatedEvent.Set(); - res = res; } - thread.Wait(); + thread.Wait_Close(); if (!MessagesDisplayed) MessageBoxW(wndParent, L"Progress Error", L"7-Zip", MB_ICONERROR); return res; @@ -974,7 +991,9 @@ bool CProgressDialog::OnExternalCloseMessage() ::SendMessage(GetItem(IDCANCEL), BM_SETSTYLE, BS_DEFPUSHBUTTON, MAKELPARAM(TRUE, 0)); HideItem(IDB_PROGRESS_BACKGROUND); HideItem(IDB_PAUSE); - + + ProcessWasFinished_GuiVirt(); + bool thereAreMessages; CProgressFinalMessage fm; { @@ -982,20 +1001,22 @@ bool CProgressDialog::OnExternalCloseMessage() thereAreMessages = !Sync.Messages.IsEmpty(); fm = Sync.FinalMessage; } + if (!fm.ErrorMessage.Message.IsEmpty()) { MessagesDisplayed = true; if (fm.ErrorMessage.Title.IsEmpty()) - fm.ErrorMessage.Title = L"7-Zip"; + fm.ErrorMessage.Title = "7-Zip"; MessageBoxW(*this, fm.ErrorMessage.Message, fm.ErrorMessage.Title, MB_ICONERROR); } else if (!thereAreMessages) { MessagesDisplayed = true; + if (!fm.OkMessage.Message.IsEmpty()) { if (fm.OkMessage.Title.IsEmpty()) - fm.OkMessage.Title = L"7-Zip"; + fm.OkMessage.Title = "7-Zip"; MessageBoxW(*this, fm.OkMessage.Message, fm.OkMessage.Title, MB_OK); } } @@ -1017,10 +1038,17 @@ bool CProgressDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) { case kCloseMessage: { - KillTimer(_timer); - _timer = 0; + if (_timer) + { + /* 21.03 : KillTimer(kTimerID) instead of KillTimer(_timer). + But (_timer == kTimerID) in Win10. So it worked too */ + KillTimer(kTimerID); + _timer = 0; + } if (_inCancelMessageBox) { + /* if user is in MessageBox(), we will call OnExternalCloseMessage() + later, when MessageBox() will be closed */ _externalCloseMessageWasReceived = true; break; } @@ -1050,8 +1078,8 @@ void CProgressDialog::SetTitleText() { char temp[32]; ConvertUInt64ToString(_prevPercentValue, temp); - s.AddAscii(temp); - s += L'%'; + s += temp; + s += '%'; } if (!_foreground) { @@ -1117,13 +1145,16 @@ void CProgressDialog::OnPriorityButton() void CProgressDialog::AddMessageDirect(LPCWSTR message, bool needNumber) { - int itemIndex = _messageList.GetItemCount(); wchar_t sz[16]; sz[0] = 0; if (needNumber) ConvertUInt32ToString(_numMessages + 1, sz); - _messageList.InsertItem(itemIndex, sz); - _messageList.SetSubItem(itemIndex, 1, message); + const unsigned itemIndex = _messageStrings.Size(); // _messageList.GetItemCount(); + if (_messageList.InsertItem((int)itemIndex, sz) == (int)itemIndex) + { + _messageList.SetSubItem((int)itemIndex, 1, message); + _messageStrings.Add(message); + } } void CProgressDialog::AddMessage(LPCWSTR message) @@ -1192,24 +1223,42 @@ bool CProgressDialog::OnButtonClicked(int buttonID, HWND buttonHWND) End(IDCLOSE); break; } + + if (_cancelWasPressed) + return true; - bool paused = Sync.Get_Paused(); + const bool paused = Sync.Get_Paused(); + if (!paused) + { OnPauseButton(); + } + _inCancelMessageBox = true; - int res = ::MessageBoxW(*this, LangString(IDS_PROGRESS_ASK_CANCEL), _title, MB_YESNOCANCEL); + const int res = ::MessageBoxW(*this, LangString(IDS_PROGRESS_ASK_CANCEL), _title, MB_YESNOCANCEL); _inCancelMessageBox = false; + if (res == IDYES) + _cancelWasPressed = true; + if (!paused) + { OnPauseButton(); - if (res == IDCANCEL || res == IDNO) + } + + if (_externalCloseMessageWasReceived) { - if (_externalCloseMessageWasReceived) - OnExternalCloseMessage(); + /* we have received kCloseMessage while we were in MessageBoxW(). + so we call OnExternalCloseMessage() here. + it can show MessageBox and it can close dialog */ + OnExternalCloseMessage(); return true; } - _cancelWasPressed = true; + if (!_cancelWasPressed) + return true; + MessagesDisplayed = true; + // we will call Sync.Set_Stopped(true) in OnButtonClicked() : OnCancel() break; } @@ -1245,11 +1294,105 @@ void CProgressDialog::ProcessWasFinished() } +bool CProgressDialog::OnNotify(UINT /* controlID */, LPNMHDR header) +{ + if (header->hwndFrom != _messageList) + return false; + switch (header->code) + { + case LVN_KEYDOWN: + { + LPNMLVKEYDOWN keyDownInfo = LPNMLVKEYDOWN(header); + switch (keyDownInfo->wVKey) + { + case 'A': + { + if (IsKeyDown(VK_CONTROL)) + { + _messageList.SelectAll(); + return true; + } + break; + } + case VK_INSERT: + case 'C': + { + if (IsKeyDown(VK_CONTROL)) + { + CopyToClipboard(); + return true; + } + break; + } + } + } + } + return false; +} + + +static void ListView_GetSelected(NControl::CListView &listView, CUIntVector &vector) +{ + vector.Clear(); + int index = -1; + for (;;) + { + index = listView.GetNextSelectedItem(index); + if (index < 0) + break; + vector.Add(index); + } +} + + +void CProgressDialog::CopyToClipboard() +{ + CUIntVector indexes; + ListView_GetSelected(_messageList, indexes); + UString s; + unsigned numIndexes = indexes.Size(); + if (numIndexes == 0) + numIndexes = _messageList.GetItemCount(); + + for (unsigned i = 0; i < numIndexes; i++) + { + const unsigned index = (i < indexes.Size() ? indexes[i] : i); + // s.Add_UInt32(index); + // s += ": "; + s += _messageStrings[index]; + { + s += + #ifdef _WIN32 + "\r\n" + #else + "\n" + #endif + ; + } + } + + ClipboardSetText(*this, s); +} + + +static THREAD_FUNC_DECL MyThreadFunction(void *param) +{ + CProgressThreadVirt *p = (CProgressThreadVirt *)param; + try + { + p->Process(); + p->ThreadFinishedOK = true; + } + catch (...) { p->Result = E_FAIL; } + return 0; +} + + HRESULT CProgressThreadVirt::Create(const UString &title, HWND parentWindow) { NWindows::CThread thread; RINOK(thread.Create(MyThreadFunction, this)); - ProgressDialog.Create(title, thread, parentWindow); + CProgressDialog::Create(title, thread, parentWindow); return S_OK; } @@ -1265,7 +1408,7 @@ static void AddMessageToString(UString &dest, const UString &src) void CProgressThreadVirt::Process() { - CProgressCloser closer(ProgressDialog); + CProgressCloser closer(*this); UString m; try { Result = ProcessVirt(); } catch(const wchar_t *s) { m = s; } @@ -1273,12 +1416,10 @@ void CProgressThreadVirt::Process() catch(const char *s) { m = GetUnicodeString(s); } catch(int v) { - wchar_t s[16]; - ConvertUInt32ToString(v, s); - m = L"Error #"; - m += s; + m = "Error #"; + m.Add_UInt32(v); } - catch(...) { m = L"Error"; } + catch(...) { m = "Error"; } if (Result != E_ABORT) { if (m.IsEmpty() && Result != S_OK) @@ -1295,7 +1436,7 @@ void CProgressThreadVirt::Process() } } - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; NSynchronization::CCriticalSectionLock lock(sync._cs); if (m.IsEmpty()) { diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.h index 03f836da219..d8259d6aff1 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2.h @@ -152,6 +152,7 @@ class CProgressDialog: public NWindows::NControl::CModalDialog NWindows::NControl::CListView _messageList; int _numMessages; + UStringVector _messageStrings; #ifdef __ITaskbarList3_INTERFACE_DEFINED__ CMyComPtr _taskbarList; @@ -169,7 +170,9 @@ class CProgressDialog: public NWindows::NControl::CModalDialog UInt64 _processed_Prev; UInt64 _packed_Prev; UInt64 _ratio_Prev; + UString _filesStr_Prev; + UString _filesTotStr_Prev; unsigned _prevSpeed_MoveBits; UInt64 _prevSpeed; @@ -210,6 +213,9 @@ class CProgressDialog: public NWindows::NControl::CModalDialog virtual bool OnSize(WPARAM wParam, int xSize, int ySize); virtual void OnCancel(); virtual void OnOK(); + virtual bool OnNotify(UINT /* controlID */, LPNMHDR header); + void CopyToClipboard(); + NWindows::NSynchronization::CManualResetEvent _createDialogEvent; NWindows::NSynchronization::CManualResetEvent _dialogCreatedEvent; #ifndef _SFX @@ -261,7 +267,18 @@ public: INT_PTR Create(const UString &title, NWindows::CThread &thread, HWND wndParent = 0); + + /* how it works: + 1) the working thread calls ProcessWasFinished() + that sends kCloseMessage message to CProgressDialog (GUI) thread + 2) CProgressDialog (GUI) thread receives kCloseMessage message and + calls ProcessWasFinished_GuiVirt(); + So we can implement ProcessWasFinished_GuiVirt() and show special + results window in GUI thread with CProgressDialog as parent window + */ + void ProcessWasFinished(); + virtual void ProcessWasFinished_GuiVirt() {} }; @@ -273,7 +290,8 @@ public: ~CProgressCloser() { _p->ProcessWasFinished(); } }; -class CProgressThreadVirt + +class CProgressThreadVirt: public CProgressDialog { protected: FStringVector ErrorPaths; @@ -281,33 +299,59 @@ protected: // error if any of HRESULT, ErrorMessage, ErrorPath virtual HRESULT ProcessVirt() = 0; - void Process(); public: HRESULT Result; bool ThreadFinishedOK; // if there is no fatal exception - CProgressDialog ProgressDialog; - - static THREAD_FUNC_DECL MyThreadFunction(void *param) - { - CProgressThreadVirt *p = (CProgressThreadVirt *)param; - try - { - p->Process(); - p->ThreadFinishedOK = true; - } - catch (...) { p->Result = E_FAIL; } - return 0; - } + void Process(); void AddErrorPath(const FString &path) { ErrorPaths.Add(path); } HRESULT Create(const UString &title, HWND parentWindow = 0); CProgressThreadVirt(): Result(E_FAIL), ThreadFinishedOK(false) {} CProgressMessageBoxPair &GetMessagePair(bool isError) { return isError ? FinalMessage.ErrorMessage : FinalMessage.OkMessage; } - }; UString HResultToMessage(HRESULT errorCode); +/* +how it works: + +client code inherits CProgressThreadVirt and calls +CProgressThreadVirt::Create() +{ + it creates new thread that calls CProgressThreadVirt::Process(); + it creates modal progress dialog window with ProgressDialog.Create() +} + +CProgressThreadVirt::Process() +{ + { + Result = ProcessVirt(); // virtual function that must implement real work + } + if (exceptions) or FinalMessage.ErrorMessage.Message + { + set message to ProgressDialog.Sync.FinalMessage.ErrorMessage.Message + } + else if (FinalMessage.OkMessage.Message) + { + set message to ProgressDialog.Sync.FinalMessage.OkMessage + } + + PostMsg(kCloseMessage); +} + + +CProgressDialog::OnExternalCloseMessage() +{ + if (ProgressDialog.Sync.FinalMessage) + { + WorkWasFinishedVirt(); + Show (ProgressDialog.Sync.FinalMessage) + MessagesDisplayed = true; + } +} + +*/ + #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2Res.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2Res.h index b45d7b49506..736c7179a18 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2Res.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2Res.h @@ -28,6 +28,7 @@ #define IDT_PROGRESS_PACKED_VAL 110 #define IDT_PROGRESS_FILES_VAL 111 +#define IDT_PROGRESS_FILES_TOTAL 112 #define IDT_PROGRESS_ELAPSED_VAL 120 #define IDT_PROGRESS_REMAINING_VAL 121 @@ -41,7 +42,7 @@ #ifdef UNDER_CE #define MY_PROGRESS_VAL_UNITS 44 #else -#define MY_PROGRESS_VAL_UNITS 76 +#define MY_PROGRESS_VAL_UNITS 72 #endif #define MY_PROGRESS_LABEL_UNITS_MIN 60 #define MY_PROGRESS_LABEL_UNITS_START 90 diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2a.rc b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2a.rc index e9713930be1..dc7d797ff72 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2a.rc +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/ProgressDialog2a.rc @@ -44,30 +44,35 @@ DIALOG_ID DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT CAPTION "Progress" { DEFPUSHBUTTON "&Background", IDB_PROGRESS_BACKGROUND, bx3, by, bxs, bys - PUSHBUTTON "&Pause", IDB_PAUSE bx2, by, bxs, bys + PUSHBUTTON "&Pause", IDB_PAUSE, bx2, by, bxs, bys PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys + LTEXT "Elapsed time:", IDT_PROGRESS_ELAPSED, m, y0, x0s, 8 LTEXT "Remaining time:", IDT_PROGRESS_REMAINING, m, y1, x0s, 8 LTEXT "Files:", IDT_PROGRESS_FILES, m, y2, x0s, 8 - LTEXT "Compression ratio:", IDT_PROGRESS_RATIO, m, y3, x0s, 8 + LTEXT "Errors:", IDT_PROGRESS_ERRORS, m, y4, x0s, 8 + LTEXT "Total size:", IDT_PROGRESS_TOTAL, x2, y0, x2s, 8 LTEXT "Speed:", IDT_PROGRESS_SPEED, x2, y1, x2s, 8 LTEXT "Processed:", IDT_PROGRESS_PROCESSED,x2, y2, x2s, 8 LTEXT "Compressed size:" , IDT_PROGRESS_PACKED, x2, y3, x2s, 8 + LTEXT "Compression ratio:", IDT_PROGRESS_RATIO, x2, y4, x2s, 8 + RTEXT "", IDT_PROGRESS_ELAPSED_VAL, x1, y0, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_REMAINING_VAL, x1, y1, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_FILES_VAL, x1, y2, x1s, MY_TEXT_NOPREFIX - RTEXT "", IDT_PROGRESS_RATIO_VAL, x1, y3, x1s, MY_TEXT_NOPREFIX + RTEXT "", IDT_PROGRESS_FILES_TOTAL, x1, y3, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_ERRORS_VAL, x1, y4, x1s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_TOTAL_VAL, x3, y0, x3s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_SPEED_VAL, x3, y1, x3s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_PROCESSED_VAL, x3, y2, x3s, MY_TEXT_NOPREFIX RTEXT "", IDT_PROGRESS_PACKED_VAL, x3, y3, x3s, MY_TEXT_NOPREFIX + RTEXT "", IDT_PROGRESS_RATIO_VAL, x3, y4, x3s, MY_TEXT_NOPREFIX LTEXT "", IDT_PROGRESS_STATUS, m, z3, xc, MY_TEXT_NOPREFIX CONTROL "", IDT_PROGRESS_FILE_NAME, "Static", SS_NOPREFIX | SS_LEFTNOWORDWRAP, m, z2, xc, z2s diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/PropertyNameRes.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/PropertyNameRes.h index 67f339028fe..e3d08db97cc 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/PropertyNameRes.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/PropertyNameRes.h @@ -93,3 +93,10 @@ #define IDS_PROP_READ_ONLY 1093 #define IDS_PROP_OUT_NAME 1094 #define IDS_PROP_COPY_LINK 1095 +#define IDS_PROP_ARC_FILE_NAME 1096 +#define IDS_PROP_IS_HASH 1097 +#define IDS_PROP_CHANGE_TIME 1098 +#define IDS_PROP_USER_ID 1099 +#define IDS_PROP_GROUP_ID 1100 +#define IDS_PROP_DEVICE_MAJOR 1101 +#define IDS_PROP_DEVICE_MINOR 1102 diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.cpp b/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.cpp index c8ea8b95aef..d8e0f8b1f84 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.cpp @@ -12,6 +12,9 @@ #include +#define MY_CAST_FUNC (void(*)()) +// #define MY_CAST_FUNC + #ifndef _UNICODE extern bool g_IsNT; #endif @@ -23,7 +26,7 @@ int GetIconIndexForCSIDL(int csidl) if (pidl) { SHFILEINFO shellInfo; - SHGetFileInfo(LPCTSTR(pidl), FILE_ATTRIBUTE_NORMAL, + SHGetFileInfo((LPCTSTR)(const void *)(pidl), FILE_ATTRIBUTE_NORMAL, &shellInfo, sizeof(shellInfo), SHGFI_PIDL | SHGFI_SYSICONINDEX); IMalloc *pMalloc; @@ -39,15 +42,16 @@ int GetIconIndexForCSIDL(int csidl) } #ifndef _UNICODE -typedef int (WINAPI * SHGetFileInfoWP)(LPCWSTR pszPath, DWORD attrib, SHFILEINFOW *psfi, UINT cbFileInfo, UINT uFlags); +typedef int (WINAPI * Func_SHGetFileInfoW)(LPCWSTR pszPath, DWORD attrib, SHFILEINFOW *psfi, UINT cbFileInfo, UINT uFlags); -struct CSHGetFileInfoInit +static struct CSHGetFileInfoInit { - SHGetFileInfoWP shGetFileInfoW; + Func_SHGetFileInfoW shGetFileInfoW; CSHGetFileInfoInit() { - shGetFileInfoW = (SHGetFileInfoWP) - ::GetProcAddress(::GetModuleHandleW(L"shell32.dll"), "SHGetFileInfoW"); + shGetFileInfoW = (Func_SHGetFileInfoW) + MY_CAST_FUNC + ::GetProcAddress(::GetModuleHandleW(L"shell32.dll"), "SHGetFileInfoW"); } } g_SHGetFileInfoInit; #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.h index f1b27fa1541..fde16e46ebc 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/SysIconUtils.h @@ -5,7 +5,7 @@ #include "../../../Common/MyWindows.h" -#include +#include #include "../../../Common/MyString.h" diff --git a/3rdparty/lzma/CPP/7zip/UI/FileManager/resource.h b/3rdparty/lzma/CPP/7zip/UI/FileManager/resource.h index b7ca17f1b22..98cb4fd44e8 100644 --- a/3rdparty/lzma/CPP/7zip/UI/FileManager/resource.h +++ b/3rdparty/lzma/CPP/7zip/UI/FileManager/resource.h @@ -47,6 +47,11 @@ #define IDM_LINK 558 #define IDM_ALT_STREAMS 559 +#define IDM_VER_EDIT 580 +#define IDM_VER_COMMIT 581 +#define IDM_VER_REVERT 582 +#define IDM_VER_DIFF 583 + #define IDM_OPEN_INSIDE_ONE 590 #define IDM_OPEN_INSIDE_PARSER 591 @@ -85,6 +90,8 @@ #define IDM_VIEW_TOOLBARS_LARGE_BUTTONS 752 #define IDM_VIEW_TOOLBARS_SHOW_BUTTONS_TEXT 753 +#define IDM_VIEW_TIME 761 + #define IDS_BOOKMARK 801 #define IDM_OPTIONS 900 diff --git a/3rdparty/lzma/CPP/7zip/UI/GUI/Extract.rc b/3rdparty/lzma/CPP/7zip/UI/GUI/Extract.rc index f75b2e6fcb3..36bfb00945e 100644 --- a/3rdparty/lzma/CPP/7zip/UI/GUI/Extract.rc +++ b/3rdparty/lzma/CPP/7zip/UI/GUI/Extract.rc @@ -7,11 +7,11 @@ BEGIN IDS_MEM_ERROR "The system cannot allocate the required amount of memory" IDS_CANNOT_CREATE_FOLDER "Cannot create folder '{0}'" IDS_UPDATE_NOT_SUPPORTED "Update operations are not supported for this archive." - IDS_CANT_OPEN_ARCHIVE "Can not open file '{0}' as archive" - IDS_CANT_OPEN_ENCRYPTED_ARCHIVE "Can not open encrypted archive '{0}'. Wrong password?" + IDS_CANT_OPEN_ARCHIVE "Cannot open file '{0}' as archive" + IDS_CANT_OPEN_ENCRYPTED_ARCHIVE "Cannot open encrypted archive '{0}'. Wrong password?" IDS_UNSUPPORTED_ARCHIVE_TYPE "Unsupported archive type" - IDS_CANT_OPEN_AS_TYPE "Can not open the file as {0} archive" + IDS_CANT_OPEN_AS_TYPE "Cannot open the file as {0} archive" IDS_IS_OPEN_AS_TYPE "The file is open as {0} archive" IDS_IS_OPEN_WITH_OFFSET "The archive is open with offset" @@ -45,7 +45,7 @@ BEGIN IDS_EXTRACT_MSG_DATA_ERROR "Data error" IDS_EXTRACT_MSG_CRC_ERROR "CRC failed" IDS_EXTRACT_MSG_UNAVAILABLE_DATA "Unavailable data" - IDS_EXTRACT_MSG_UEXPECTED_END "Unexpected end of data"; + IDS_EXTRACT_MSG_UEXPECTED_END "Unexpected end of data" IDS_EXTRACT_MSG_DATA_AFTER_END "There are some data after the end of the payload data" IDS_EXTRACT_MSG_IS_NOT_ARC "Is not archive" IDS_EXTRACT_MSG_HEADERS_ERROR "Headers Error" diff --git a/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractDialog.cpp b/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractDialog.cpp index af47fc2d205..5132084d6e4 100644 --- a/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractDialog.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractDialog.cpp @@ -28,6 +28,8 @@ using namespace NName; extern HINSTANCE g_hInstance; +#ifndef _SFX + static const UInt32 kPathMode_IDs[] = { IDS_EXTRACT_PATHS_FULL, @@ -44,8 +46,6 @@ static const UInt32 kOverwriteMode_IDs[] = IDS_EXTRACT_OVERWRITE_RENAME_EXISTING }; -#ifndef _SFX - static const // NExtract::NPathMode::EEnum int @@ -94,6 +94,7 @@ static const unsigned kHistorySize = 16; #ifndef _SFX // it's used in CompressDialog also +void AddComboItems(NControl::CComboBox &combo, const UInt32 *langIDs, unsigned numItems, const int *values, int curVal); void AddComboItems(NControl::CComboBox &combo, const UInt32 *langIDs, unsigned numItems, const int *values, int curVal) { int curSel = 0; @@ -110,6 +111,7 @@ void AddComboItems(NControl::CComboBox &combo, const UInt32 *langIDs, unsigned n } // it's used in CompressDialog also +bool GetBoolsVal(const CBoolPair &b1, const CBoolPair &b2); bool GetBoolsVal(const CBoolPair &b1, const CBoolPair &b2) { if (b1.Def) return b1.Val; @@ -143,7 +145,7 @@ bool CExtractDialog::OnInit() GetText(s); if (!ArcPath.IsEmpty()) { - s.AddAscii(" : "); + s += " : "; s += ArcPath; } SetText(s); @@ -286,6 +288,7 @@ void CExtractDialog::OnButtonSetPath() _path.SetText(resultPath); } +void AddUniqueString(UStringVector &list, const UString &s); void AddUniqueString(UStringVector &list, const UString &s) { FOR_VECTOR (i, list) @@ -357,7 +360,7 @@ void CExtractDialog::OnOK() if (currentItem == CB_ERR) { _path.GetText(s); - if (_path.GetCount() >= kHistorySize) + if (_path.GetCount() >= (int)kHistorySize) currentItem = _path.GetCount() - 1; } else @@ -409,10 +412,10 @@ void CExtractDialog::OnOK() } #ifndef NO_REGISTRY -static LPCWSTR kHelpTopic = L"fm/plugins/7-zip/extract.htm"; +#define kHelpTopic "fm/plugins/7-zip/extract.htm" void CExtractDialog::OnHelp() { - ShowHelpWindow(NULL, kHelpTopic); + ShowHelpWindow(kHelpTopic); CModalDialog::OnHelp(); } #endif diff --git a/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.cpp b/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.cpp index 7fb843a15fc..a9191a8be09 100644 --- a/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.cpp +++ b/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.cpp @@ -34,7 +34,7 @@ using namespace NWindows; using namespace NFile; using namespace NDir; -static const wchar_t *kIncorrectOutDir = L"Incorrect output directory path"; +static const wchar_t * const kIncorrectOutDir = L"Incorrect output directory path"; #ifndef _SFX @@ -42,29 +42,17 @@ static void AddValuePair(UString &s, UINT resourceID, UInt64 value, bool addColo { AddLangString(s, resourceID); if (addColon) - s += L':'; + s += ':'; s.Add_Space(); - char sz[32]; - ConvertUInt64ToString(value, sz); - s.AddAscii(sz); + s.Add_UInt64(value); s.Add_LF(); } static void AddSizePair(UString &s, UINT resourceID, UInt64 value) { - wchar_t sz[32]; AddLangString(s, resourceID); - s += L": "; - ConvertUInt64ToString(value, sz); - s += MyFormatNew(IDS_FILE_SIZE, sz); - // s += sz; - if (value >= (1 << 20)) - { - ConvertUInt64ToString(value >> 20, sz); - s += L" ("; - s += sz; - s += L" MB)"; - } + s += ": "; + AddSizeValue(s, value); s.Add_LF(); } @@ -74,6 +62,12 @@ class CThreadExtracting: public CProgressThreadVirt { HRESULT ProcessVirt(); public: + /* + #ifdef EXTERNAL_CODECS + const CExternalCodecs *externalCodecs; + #endif + */ + CCodecs *codecs; CExtractCallbackImp *ExtractCallbackSpec; const CObjectVector *FormatIndices; @@ -83,22 +77,45 @@ public: UStringVector *ArchivePathsFull; const NWildcard::CCensorNode *WildcardCensor; const CExtractOptions *Options; + #ifndef _SFX CHashBundle *HashBundle; + virtual void ProcessWasFinished_GuiVirt(); #endif + CMyComPtr ExtractCallback; UString Title; + + CPropNameValPairs Pairs; }; + +#ifndef _SFX +void CThreadExtracting::ProcessWasFinished_GuiVirt() +{ + if (HashBundle && !Pairs.IsEmpty()) + ShowHashResults(Pairs, *this); +} +#endif + HRESULT CThreadExtracting::ProcessVirt() { CDecompressStat Stat; + #ifndef _SFX + /* if (HashBundle) HashBundle->Init(); + */ #endif - HRESULT res = Extract(codecs, + HRESULT res = Extract( + /* + #ifdef EXTERNAL_CODECS + externalCodecs, + #endif + */ + codecs, *FormatIndices, *ExcludedFormatIndices, *ArchivePaths, *ArchivePathsFull, *WildcardCensor, *Options, ExtractCallbackSpec, ExtractCallback, @@ -106,16 +123,23 @@ HRESULT CThreadExtracting::ProcessVirt() HashBundle, #endif FinalMessage.ErrorMessage.Message, Stat); + #ifndef _SFX - if (res == S_OK && Options->TestMode && ExtractCallbackSpec->IsOK()) + if (res == S_OK && ExtractCallbackSpec->IsOK()) { - UString s; + if (HashBundle) + { + AddValuePair(Pairs, IDS_ARCHIVES_COLON, Stat.NumArchives); + AddSizeValuePair(Pairs, IDS_PROP_PACKED_SIZE, Stat.PackSize); + AddHashBundleRes(Pairs, *HashBundle); + } + else if (Options->TestMode) + { + UString s; - AddValuePair(s, IDS_ARCHIVES_COLON, Stat.NumArchives, false); - AddSizePair(s, IDS_PROP_PACKED_SIZE, Stat.PackSize); + AddValuePair(s, IDS_ARCHIVES_COLON, Stat.NumArchives, false); + AddSizePair(s, IDS_PROP_PACKED_SIZE, Stat.PackSize); - if (!HashBundle) - { if (Stat.NumFolders != 0) AddValuePair(s, IDS_PROP_FOLDERS, Stat.NumFolders); AddValuePair(s, IDS_PROP_FILES, Stat.NumFiles); @@ -126,25 +150,21 @@ HRESULT CThreadExtracting::ProcessVirt() AddValuePair(s, IDS_PROP_NUM_ALT_STREAMS, Stat.NumAltStreams); AddSizePair(s, IDS_PROP_ALT_STREAMS_SIZE, Stat.AltStreams_UnpackSize); } - } - - if (HashBundle) - { s.Add_LF(); - AddHashBundleRes(s, *HashBundle, UString()); + AddLangString(s, IDS_MESSAGE_NO_ERRORS); + FinalMessage.OkMessage.Title = Title; + FinalMessage.OkMessage.Message = s; } - - s.Add_LF(); - AddLangString(s, IDS_MESSAGE_NO_ERRORS); - - FinalMessage.OkMessage.Title = Title; - FinalMessage.OkMessage.Message = s; } #endif + return res; } + + HRESULT ExtractGUI( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector &formatIndices, const CIntVector &excludedFormatIndices, @@ -163,6 +183,11 @@ HRESULT ExtractGUI( messageWasDisplayed = false; CThreadExtracting extracter; + /* + #ifdef EXTERNAL_CODECS + extracter.externalCodecs = __externalCodecs; + #endif + */ extracter.codecs = codecs; extracter.FormatIndices = &formatIndices; extracter.ExcludedFormatIndices = &excludedFormatIndices; @@ -249,11 +274,11 @@ HRESULT ExtractGUI( extracter.Title = title; extracter.ExtractCallbackSpec = extractCallback; - extracter.ExtractCallbackSpec->ProgressDialog = &extracter.ProgressDialog; + extracter.ExtractCallbackSpec->ProgressDialog = &extracter; extracter.ExtractCallback = extractCallback; extracter.ExtractCallbackSpec->Init(); - extracter.ProgressDialog.CompressingMode = false; + extracter.CompressingMode = false; extracter.ArchivePaths = &archivePaths; extracter.ArchivePathsFull = &archivePathsFull; @@ -263,10 +288,9 @@ HRESULT ExtractGUI( extracter.HashBundle = hb; #endif - extracter.ProgressDialog.IconID = IDI_ICON; + extracter.IconID = IDI_ICON; RINOK(extracter.Create(title, hwndParent)); - messageWasDisplayed = extracter.ThreadFinishedOK & - extracter.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = extracter.ThreadFinishedOK && extracter.MessagesDisplayed; return extracter.Result; } diff --git a/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.h b/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.h index d55b30de15c..3b412590977 100644 --- a/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.h +++ b/3rdparty/lzma/CPP/7zip/UI/GUI/ExtractGUI.h @@ -20,6 +20,7 @@ */ HRESULT ExtractGUI( + // DECL_EXTERNAL_CODECS_LOC_VARS CCodecs *codecs, const CObjectVector &formatIndices, const CIntVector &excludedFormatIndices, diff --git a/3rdparty/lzma/CPP/7zip/UI/GUI/HashGUI.h b/3rdparty/lzma/CPP/7zip/UI/GUI/HashGUI.h index 40340365abe..8264453522c 100644 --- a/3rdparty/lzma/CPP/7zip/UI/GUI/HashGUI.h +++ b/3rdparty/lzma/CPP/7zip/UI/GUI/HashGUI.h @@ -4,6 +4,7 @@ #define __HASH_GUI_H #include "../Common/HashCalc.h" +#include "../Common/Property.h" HRESULT HashCalcGUI( DECL_EXTERNAL_CODECS_LOC_VARS @@ -11,6 +12,16 @@ HRESULT HashCalcGUI( const CHashOptions &options, bool &messageWasDisplayed); -void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName); +typedef CObjectVector CPropNameValPairs; + +void AddValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); +void AddSizeValue(UString &s, UInt64 value); +void AddSizeValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); + +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb); +void AddHashBundleRes(UString &s, const CHashBundle &hb); + +void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd); +void ShowHashResults(const CHashBundle &hb, HWND hwnd); #endif diff --git a/3rdparty/lzma/CPP/7zip/cmpl_clang.mak b/3rdparty/lzma/CPP/7zip/cmpl_clang.mak new file mode 100644 index 00000000000..e62e1e62faa --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_clang.mak @@ -0,0 +1,3 @@ +include ../../var_clang.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_clang_arm64.mak b/3rdparty/lzma/CPP/7zip/cmpl_clang_arm64.mak new file mode 100644 index 00000000000..3f6b02bfcf5 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_clang_arm64.mak @@ -0,0 +1,3 @@ +include ../../var_clang_arm64.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_clang_x64.mak b/3rdparty/lzma/CPP/7zip/cmpl_clang_x64.mak new file mode 100644 index 00000000000..b61e2af6608 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_clang_x64.mak @@ -0,0 +1,3 @@ +include ../../var_clang_x64.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_clang_x86.mak b/3rdparty/lzma/CPP/7zip/cmpl_clang_x86.mak new file mode 100644 index 00000000000..0e5cb76c0b6 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_clang_x86.mak @@ -0,0 +1,3 @@ +include ../../var_clang_x86.mak +include ../../warn_clang.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_gcc.mak b/3rdparty/lzma/CPP/7zip/cmpl_gcc.mak new file mode 100644 index 00000000000..7a1aef2e3f6 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_gcc.mak @@ -0,0 +1,3 @@ +include ../../var_gcc.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_gcc_arm64.mak b/3rdparty/lzma/CPP/7zip/cmpl_gcc_arm64.mak new file mode 100644 index 00000000000..53a85844728 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_gcc_arm64.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_arm64.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_gcc_x64.mak b/3rdparty/lzma/CPP/7zip/cmpl_gcc_x64.mak new file mode 100644 index 00000000000..500c30e47b9 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_gcc_x64.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_x64.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_gcc_x86.mak b/3rdparty/lzma/CPP/7zip/cmpl_gcc_x86.mak new file mode 100644 index 00000000000..e7687070791 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_gcc_x86.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_x86.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_mac_arm64.mak b/3rdparty/lzma/CPP/7zip/cmpl_mac_arm64.mak new file mode 100644 index 00000000000..941028e9584 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_mac_arm64.mak @@ -0,0 +1,3 @@ +include ../../var_mac_arm64.mak +include ../../warn_clang_mac.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/cmpl_mac_x64.mak b/3rdparty/lzma/CPP/7zip/cmpl_mac_x64.mak new file mode 100644 index 00000000000..d3aa03967d9 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/cmpl_mac_x64.mak @@ -0,0 +1,3 @@ +include ../../var_mac_x64.mak +include ../../warn_clang_mac.mak +include makefile.gcc diff --git a/3rdparty/lzma/CPP/7zip/var_clang.mak b/3rdparty/lzma/CPP/7zip/var_clang.mak new file mode 100644 index 00000000000..a6df26e77c8 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_clang.mak @@ -0,0 +1,11 @@ +PLATFORM= +O=b/c +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/CPP/7zip/var_clang_arm64.mak b/3rdparty/lzma/CPP/7zip/var_clang_arm64.mak new file mode 100644 index 00000000000..4b354096956 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_clang_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/CPP/7zip/var_clang_x64.mak b/3rdparty/lzma/CPP/7zip/var_clang_x64.mak new file mode 100644 index 00000000000..fefed51c5bd --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_clang_x64.mak @@ -0,0 +1,12 @@ +PLATFORM=x64 +O=b/c_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 + diff --git a/3rdparty/lzma/CPP/7zip/var_clang_x86.mak b/3rdparty/lzma/CPP/7zip/var_clang_x86.mak new file mode 100644 index 00000000000..5f3c2d9cb15 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_clang_x86.mak @@ -0,0 +1,12 @@ +PLATFORM=x86 +O=b/c_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 + diff --git a/3rdparty/lzma/CPP/7zip/var_gcc.mak b/3rdparty/lzma/CPP/7zip/var_gcc.mak new file mode 100644 index 00000000000..664491cf2fc --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_gcc.mak @@ -0,0 +1,12 @@ +PLATFORM= +O=b/g +IS_X64= +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM= +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/3rdparty/lzma/CPP/7zip/var_gcc_arm64.mak b/3rdparty/lzma/CPP/7zip/var_gcc_arm64.mak new file mode 100644 index 00000000000..4bbb687dbb3 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_gcc_arm64.mak @@ -0,0 +1,12 @@ +PLATFORM=arm64 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-mtune=cortex-a53 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + +# -march=armv8-a+crc+crypto diff --git a/3rdparty/lzma/CPP/7zip/var_gcc_x64.mak b/3rdparty/lzma/CPP/7zip/var_gcc_x64.mak new file mode 100644 index 00000000000..1acf604f5f2 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_gcc_x64.mak @@ -0,0 +1,10 @@ +PLATFORM=x64 +O=b/g_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH= +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ diff --git a/3rdparty/lzma/CPP/7zip/var_gcc_x86.mak b/3rdparty/lzma/CPP/7zip/var_gcc_x86.mak new file mode 100644 index 00000000000..288bf94bead --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_gcc_x86.mak @@ -0,0 +1,11 @@ +PLATFORM=x86 +O=b/g_$(PLATFORM) +IS_X64= +IS_X86=1 +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-m32 +USE_ASM=1 +CC=$(CROSS_COMPILE)gcc +CXX=$(CROSS_COMPILE)g++ + diff --git a/3rdparty/lzma/CPP/7zip/var_mac_arm64.mak b/3rdparty/lzma/CPP/7zip/var_mac_arm64.mak new file mode 100644 index 00000000000..adf5fa1d703 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_mac_arm64.mak @@ -0,0 +1,11 @@ +PLATFORM=arm64 +O=b/m_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64=1 +CROSS_COMPILE= +MY_ARCH=-arch arm64 +USE_ASM=1 +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/CPP/7zip/var_mac_x64.mak b/3rdparty/lzma/CPP/7zip/var_mac_x64.mak new file mode 100644 index 00000000000..13d7aa7fedf --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/var_mac_x64.mak @@ -0,0 +1,11 @@ +PLATFORM=x64 +O=b/m_$(PLATFORM) +IS_X64=1 +IS_X86= +IS_ARM64= +CROSS_COMPILE= +MY_ARCH=-arch x86_64 +USE_ASM= +CC=$(CROSS_COMPILE)clang +CXX=$(CROSS_COMPILE)clang++ +USE_CLANG=1 diff --git a/3rdparty/lzma/CPP/7zip/warn_clang.mak b/3rdparty/lzma/CPP/7zip/warn_clang.mak new file mode 100644 index 00000000000..ed4f908f607 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/warn_clang.mak @@ -0,0 +1,37 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + +CFLAGS_WARN_1 = \ + -Wno-deprecated-copy-dtor \ + + + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_1) diff --git a/3rdparty/lzma/CPP/7zip/warn_clang_mac.mak b/3rdparty/lzma/CPP/7zip/warn_clang_mac.mak new file mode 100644 index 00000000000..aadf14f77c1 --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/warn_clang_mac.mak @@ -0,0 +1,39 @@ +CFLAGS_WARN_CLANG_3_8_UNIQ = \ + -Wno-reserved-id-macro \ + -Wno-old-style-cast \ + -Wno-c++11-long-long \ + -Wno-unused-macros \ + +CFLAGS_WARN_CLANG_3_8 = \ + $(CFLAGS_WARN_CLANG_3_8_UNIQ) \ + -Weverything \ + -Wno-extra-semi \ + -Wno-sign-conversion \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-cast-qual \ + -Wno-padded \ + -Wno-exit-time-destructors \ + -Wno-weak-vtables \ + +CFLAGS_WARN_CLANG_12= $(CFLAGS_WARN_CLANG_3_8) \ + -Wno-extra-semi-stmt \ + -Wno-zero-as-null-pointer-constant \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-c++98-compat-pedantic \ + -Wno-atomic-implicit-seq-cst \ + -Wconversion \ + -Wno-sign-conversion \ + -Wno-suggest-override \ + -Wno-suggest-destructor-override \ + +CFLAGS_WARN_MAC = \ + -Wno-poison-system-directories \ + -Wno-c++11-long-long \ + -Wno-atomic-implicit-seq-cst \ + + +CFLAGS_WARN = $(CFLAGS_WARN_CLANG_12) $(CFLAGS_WARN_MAC) diff --git a/3rdparty/lzma/CPP/7zip/warn_gcc.mak b/3rdparty/lzma/CPP/7zip/warn_gcc.mak new file mode 100644 index 00000000000..3185326a3af --- /dev/null +++ b/3rdparty/lzma/CPP/7zip/warn_gcc.mak @@ -0,0 +1,57 @@ +CFLAGS_WARN_GCC_4_5 = \ + +CFLAGS_WARN_GCC_6 = \ + -Waddress \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wcast-align \ + -Wcomment \ + -Wdiv-by-zero \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Winit-self \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + +# -Wno-strict-aliasing + +CFLAGS_WARN_GCC_9 = \ + -Waddress \ + -Waddress-of-packed-member \ + -Waggressive-loop-optimizations \ + -Wattributes \ + -Wbool-compare \ + -Wbool-operation \ + -Wcast-align \ + -Wcast-align=strict \ + -Wcomment \ + -Wdangling-else \ + -Wdiv-by-zero \ + -Wduplicated-branches \ + -Wduplicated-cond \ + -Wformat-contains-nul \ + -Wimplicit-fallthrough=5 \ + -Winit-self \ + -Wint-in-bool-context \ + -Wint-to-pointer-cast \ + -Wunused \ + -Wunused-macros \ + -Wconversion \ + +# -Wno-sign-conversion \ + +CFLAGS_WARN_GCC_10 = $(CFLAGS_WARN_GCC_9) \ + -Wmaybe-uninitialized \ + -Wmisleading-indentation \ + +CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ + -Wno-strict-aliasing \ + + +CFLAGS_WARN = $(CFLAGS_WARN_GCC_9) \ + +# $(CFLAGS_WARN_GCC_PPMD_UNALIGNED) + + \ No newline at end of file diff --git a/3rdparty/lzma/CPP/Build.mak b/3rdparty/lzma/CPP/Build.mak index ef0ed0c4112..1ef676e063f 100644 --- a/3rdparty/lzma/CPP/Build.mak +++ b/3rdparty/lzma/CPP/Build.mak @@ -4,24 +4,31 @@ LIBS = $(LIBS) oleaut32.lib ole32.lib CFLAGS = $(CFLAGS) -DUNICODE -D_UNICODE !ENDIF -# CFLAGS = $(CFLAGS) -FAsc -Fa$O/Asm/ - !IFNDEF O -!IFDEF CPU -O=$(CPU) +!IFDEF PLATFORM +O=$(PLATFORM) !ELSE -O=O +O=o +!ENDIF !ENDIF + +!IF "$(CC)" != "clang-cl" +# CFLAGS = $(CFLAGS) -FAsc -Fa$O/asm/ !ENDIF -!IF "$(CPU)" == "AMD64" -MY_ML = ml64 -Dx64 -!ELSEIF "$(CPU)" == "ARM" -MY_ML = armasm + +!IF "$(PLATFORM)" == "x64" +MY_ML = ml64 -WX +#-Dx64 +!ELSEIF "$(PLATFORM)" == "arm" +MY_ML = armasm -WX !ELSE -MY_ML = ml +MY_ML = ml -WX +# -DABI_CDECL !ENDIF +# MY_ML = "$(MY_ML) -Fl$O\asm\ + !IFDEF UNDER_CE RFLAGS = $(RFLAGS) -dUNDER_CE @@ -29,14 +36,16 @@ RFLAGS = $(RFLAGS) -dUNDER_CE LFLAGS = $(LFLAGS) /ENTRY:mainACRTStartup !ENDIF !ELSE -!IFNDEF NEW_COMPILER +!IFDEF OLD_COMPILER LFLAGS = $(LFLAGS) -OPT:NOWIN98 !ENDIF +!IF "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" CFLAGS = $(CFLAGS) -Gr +!ENDIF LIBS = $(LIBS) user32.lib advapi32.lib shell32.lib !ENDIF -!IF "$(CPU)" == "ARM" +!IF "$(PLATFORM)" == "arm" COMPL_ASM = $(MY_ML) $** $O/$(*B).obj !ELSE COMPL_ASM = $(MY_ML) -c -Fo$O/ $** @@ -44,24 +53,74 @@ COMPL_ASM = $(MY_ML) -c -Fo$O/ $** CFLAGS = $(CFLAGS) -nologo -c -Fo$O/ -W4 -WX -EHsc -Gy -GR- -GF -!IFDEF MY_STATIC_LINK +!IF "$(CC)" == "clang-cl" + +CFLAGS = $(CFLAGS) \ + -Werror \ + -Wextra \ + -Wall \ + -Weverything \ + -Wno-extra-semi-stmt \ + -Wno-extra-semi \ + -Wno-zero-as-null-pointer-constant \ + -Wno-sign-conversion \ + -Wno-old-style-cast \ + -Wno-reserved-id-macro \ + -Wno-deprecated-dynamic-exception-spec \ + -Wno-language-extension-token \ + -Wno-global-constructors \ + -Wno-non-virtual-dtor \ + -Wno-deprecated-copy-dtor \ + -Wno-exit-time-destructors \ + -Wno-switch-enum \ + -Wno-covered-switch-default \ + -Wno-nonportable-system-include-path \ + -Wno-c++98-compat-pedantic \ + -Wno-cast-qual \ + -Wc++11-extensions \ + +!ENDIF + +!IFDEF MY_DYNAMIC_LINK +CFLAGS = $(CFLAGS) -MD +!ELSE !IFNDEF MY_SINGLE_THREAD CFLAGS = $(CFLAGS) -MT !ENDIF -!ELSE -CFLAGS = $(CFLAGS) -MD !ENDIF -!IFDEF NEW_COMPILER -CFLAGS = $(CFLAGS) -GS- -Zc:forScope + +CFLAGS = $(CFLAGS_COMMON) $(CFLAGS) + +!IFNDEF OLD_COMPILER +CFLAGS = $(CFLAGS) -GS- -Zc:forScope -Zc:wchar_t !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -MP2 +!IF "$(CC)" != "clang-cl" +CFLAGS = $(CFLAGS) -MP4 +!ENDIF +!IFNDEF PLATFORM +# CFLAGS = $(CFLAGS) -arch:IA32 +!ENDIF !ENDIF !ELSE CFLAGS = $(CFLAGS) !ENDIF +!IFDEF MY_CONSOLE +CFLAGS = $(CFLAGS) -D_CONSOLE +!ENDIF + +!IFNDEF UNDER_CE +!IF "$(PLATFORM)" == "arm" +CFLAGS = $(CFLAGS) -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +!ENDIF +!ENDIF + +!IF "$(PLATFORM)" == "x64" +CFLAGS_O1 = $(CFLAGS) -O1 +!ELSE CFLAGS_O1 = $(CFLAGS) -O1 +!ENDIF CFLAGS_O2 = $(CFLAGS) -O2 LFLAGS = $(LFLAGS) -nologo -OPT:REF -OPT:ICF @@ -72,15 +131,31 @@ LFLAGS = $(LFLAGS) /LARGEADDRESSAWARE !IFDEF DEF_FILE LFLAGS = $(LFLAGS) -DLL -DEF:$(DEF_FILE) +!ELSE +!IF defined(MY_FIXED) && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" +LFLAGS = $(LFLAGS) /FIXED +!ELSE +LFLAGS = $(LFLAGS) /FIXED:NO +!ENDIF +# /BASE:0x400000 !ENDIF -MY_SUB_SYS_VER=6.0 + +# !IF "$(PLATFORM)" == "x64" + +!IFDEF SUB_SYS_VER + +MY_SUB_SYS_VER=5.02 + !IFDEF MY_CONSOLE -# LFLAGS = $(LFLAGS) /SUBSYSTEM:console,$(MY_SUB_SYS_VER) +LFLAGS = $(LFLAGS) /SUBSYSTEM:console,$(MY_SUB_SYS_VER) !ELSE -# LFLAGS = $(LFLAGS) /SUBSYSTEM:windows,$(MY_SUB_SYS_VER) +LFLAGS = $(LFLAGS) /SUBSYSTEM:windows,$(MY_SUB_SYS_VER) !ENDIF +!ENDIF + + PROGPATH = $O\$(PROG) COMPL_O1 = $(CC) $(CFLAGS_O1) $** @@ -92,11 +167,17 @@ COMPLB = $(CC) $(CFLAGS_O1) -Yu"StdAfx.h" -Fp$O/a.pch $< # COMPLB_O2 = $(CC) $(CFLAGS_O2) -Yu"StdAfx.h" -Fp$O/a.pch $< COMPLB_O2 = $(CC) $(CFLAGS_O2) $< -CCOMPL_PCH = $(CC) $(CFLAGS_O2) -Yc"Precomp.h" -Fp$O/a.pch $** -CCOMPL_USE = $(CC) $(CFLAGS_O2) -Yu"Precomp.h" -Fp$O/a.pch $** -CCOMPL = $(CC) $(CFLAGS_O2) $** -CCOMPLB = $(CC) $(CFLAGS_O2) $< +CFLAGS_C_ALL = $(CFLAGS_O2) $(CFLAGS_C_SPEC) +CCOMPL_PCH = $(CC) $(CFLAGS_C_ALL) -Yc"Precomp.h" -Fp$O/a.pch $** +CCOMPL_USE = $(CC) $(CFLAGS_C_ALL) -Yu"Precomp.h" -Fp$O/a.pch $** +CCOMPL = $(CC) $(CFLAGS_C_ALL) $** +CCOMPLB = $(CC) $(CFLAGS_C_ALL) $< +!IF "$(CC)" == "clang-cl" +COMPL = $(COMPL) -FI StdAfx.h +COMPLB = $(COMPLB) -FI StdAfx.h +CCOMPL_USE = $(CCOMPL_USE) -FI Precomp.h +!ENDIF all: $(PROGPATH) @@ -105,10 +186,10 @@ clean: $O: if not exist "$O" mkdir "$O" -$O/Asm: - if not exist "$O/Asm" mkdir "$O/Asm" +$O/asm: + if not exist "$O/asm" mkdir "$O/asm" -$(PROGPATH): $O $O/Asm $(OBJS) $(DEF_FILE) +$(PROGPATH): $O $O/asm $(OBJS) $(DEF_FILE) link $(LFLAGS) -out:$(PROGPATH) $(OBJS) $(LIBS) !IFNDEF NO_DEFAULT_RES diff --git a/3rdparty/lzma/CPP/Common/CRC.cpp b/3rdparty/lzma/CPP/Common/CRC.cpp index 9a9f81fb713..c6b7d5e4a7e 100644 --- a/3rdparty/lzma/CPP/Common/CRC.cpp +++ b/3rdparty/lzma/CPP/Common/CRC.cpp @@ -4,4 +4,4 @@ #include "../../C/7zCrc.h" -struct CCRCTableInit { CCRCTableInit() { CrcGenerateTable(); } } g_CRCTableInit; +static struct CCRCTableInit { CCRCTableInit() { CrcGenerateTable(); } } g_CRCTableInit; diff --git a/3rdparty/lzma/CPP/Common/C_FileIO.cpp b/3rdparty/lzma/CPP/Common/C_FileIO.cpp index 7c629390232..4bd3fadc448 100644 --- a/3rdparty/lzma/CPP/Common/C_FileIO.cpp +++ b/3rdparty/lzma/CPP/Common/C_FileIO.cpp @@ -1,92 +1,3 @@ // Common/C_FileIO.cpp -#include "C_FileIO.h" - -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -namespace NC { -namespace NFile { -namespace NIO { - -bool CFileBase::OpenBinary(const char *name, int flags) -{ - #ifdef O_BINARY - flags |= O_BINARY; - #endif - Close(); - _handle = ::open(name, flags, 0666); - return _handle != -1; -} - -bool CFileBase::Close() -{ - if (_handle == -1) - return true; - if (close(_handle) != 0) - return false; - _handle = -1; - return true; -} - -bool CFileBase::GetLength(UInt64 &length) const -{ - off_t curPos = Seek(0, SEEK_CUR); - off_t lengthTemp = Seek(0, SEEK_END); - Seek(curPos, SEEK_SET); - length = (UInt64)lengthTemp; - return true; -} - -off_t CFileBase::Seek(off_t distanceToMove, int moveMethod) const -{ - return ::lseek(_handle, distanceToMove, moveMethod); -} - -///////////////////////// -// CInFile - -bool CInFile::Open(const char *name) -{ - return CFileBase::OpenBinary(name, O_RDONLY); -} - -bool CInFile::OpenShared(const char *name, bool) -{ - return Open(name); -} - -ssize_t CInFile::Read(void *data, size_t size) -{ - return read(_handle, data, size); -} - -///////////////////////// -// COutFile - -bool COutFile::Create(const char *name, bool createAlways) -{ - if (createAlways) - { - Close(); - _handle = ::creat(name, 0666); - return _handle != -1; - } - return OpenBinary(name, O_CREAT | O_EXCL | O_WRONLY); -} - -bool COutFile::Open(const char *name, DWORD creationDisposition) -{ - return Create(name, false); -} - -ssize_t COutFile::Write(const void *data, size_t size) -{ - return write(_handle, data, size); -} - -}}} +#include "StdAfx.h" diff --git a/3rdparty/lzma/CPP/Common/C_FileIO.h b/3rdparty/lzma/CPP/Common/C_FileIO.h index ff4ec162c59..6818558b253 100644 --- a/3rdparty/lzma/CPP/Common/C_FileIO.h +++ b/3rdparty/lzma/CPP/Common/C_FileIO.h @@ -3,51 +3,4 @@ #ifndef __COMMON_C_FILEIO_H #define __COMMON_C_FILEIO_H -#include -#include - -#include "MyTypes.h" -#include "MyWindows.h" - -#ifdef _WIN32 -#ifdef _MSC_VER -typedef size_t ssize_t; -#endif -#endif - -namespace NC { -namespace NFile { -namespace NIO { - -class CFileBase -{ -protected: - int _handle; - bool OpenBinary(const char *name, int flags); -public: - CFileBase(): _handle(-1) {}; - ~CFileBase() { Close(); } - bool Close(); - bool GetLength(UInt64 &length) const; - off_t Seek(off_t distanceToMove, int moveMethod) const; -}; - -class CInFile: public CFileBase -{ -public: - bool Open(const char *name); - bool OpenShared(const char *name, bool shareForWrite); - ssize_t Read(void *data, size_t size); -}; - -class COutFile: public CFileBase -{ -public: - bool Create(const char *name, bool createAlways); - bool Open(const char *name, DWORD creationDisposition); - ssize_t Write(const void *data, size_t size); -}; - -}}} - #endif diff --git a/3rdparty/lzma/CPP/Common/ComTry.h b/3rdparty/lzma/CPP/Common/ComTry.h index fb4ef04594a..297c407ba46 100644 --- a/3rdparty/lzma/CPP/Common/ComTry.h +++ b/3rdparty/lzma/CPP/Common/ComTry.h @@ -10,7 +10,11 @@ #define COM_TRY_BEGIN try { #define COM_TRY_END } catch(...) { return E_OUTOFMEMORY; } - // catch(const CNewException &) { return E_OUTOFMEMORY; } +/* +#define COM_TRY_END } \ + catch(const CNewException &) { return E_OUTOFMEMORY; } \ + catch(...) { return HRESULT_FROM_WIN32(ERROR_NOACCESS); } \ +*/ // catch(const CSystemException &e) { return e.ErrorCode; } // catch(...) { return E_FAIL; } diff --git a/3rdparty/lzma/CPP/Common/CommandLineParser.cpp b/3rdparty/lzma/CPP/Common/CommandLineParser.cpp index 1c7f26547c8..465e0fde853 100644 --- a/3rdparty/lzma/CPP/Common/CommandLineParser.cpp +++ b/3rdparty/lzma/CPP/Common/CommandLineParser.cpp @@ -4,20 +4,6 @@ #include "CommandLineParser.h" -static bool IsString1PrefixedByString2_NoCase(const wchar_t *u, const char *a) -{ - for (;;) - { - char c = *a; - if (c == 0) - return true; - if ((unsigned char)MyCharLower_Ascii(c) != MyCharLower_Ascii(*u)) - return false; - a++; - u++; - } -} - namespace NCommandLineParser { bool SplitCommandLine(const UString &src, UString &dest1, UString &dest2) @@ -44,7 +30,7 @@ bool SplitCommandLine(const UString &src, UString &dest1, UString &dest2) void SplitCommandLine(const UString &s, UStringVector &parts) { - UString sTemp = s; + UString sTemp (s); sTemp.Trim(); parts.Clear(); for (;;) @@ -59,18 +45,17 @@ void SplitCommandLine(const UString &s, UStringVector &parts) } -static const char *kStopSwitchParsing = "--"; +static const char * const kStopSwitchParsing = "--"; static bool inline IsItSwitchChar(wchar_t c) { return (c == '-'); } -CParser::CParser(unsigned numSwitches): - _numSwitches(numSwitches), - _switches(0) +CParser::CParser(): + _switches(NULL), + StopSwitchIndex(-1) { - _switches = new CSwitchResult[numSwitches]; } CParser::~CParser() @@ -81,7 +66,7 @@ CParser::~CParser() // if (s) contains switch then function updates switch structures // out: true, if (s) is a switch -bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) +bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsigned numSwitches) { if (s.IsEmpty() || !IsItSwitchChar(s[0])) return false; @@ -90,16 +75,16 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) unsigned switchIndex = 0; int maxLen = -1; - for (unsigned i = 0; i < _numSwitches; i++) + for (unsigned i = 0; i < numSwitches; i++) { - const char *key = switchForms[i].Key; + const char * const key = switchForms[i].Key; unsigned switchLen = MyStringLen(key); if ((int)switchLen <= maxLen || pos + switchLen > s.Len()) continue; - if (IsString1PrefixedByString2_NoCase((const wchar_t *)s + pos, key)) + if (IsString1PrefixedByString2_NoCase_Ascii((const wchar_t *)s + pos, key)) { switchIndex = i; - maxLen = switchLen; + maxLen = (int)switchLen; } } @@ -109,7 +94,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) return false; } - pos += maxLen; + pos += (unsigned)maxLen; CSwitchResult &sw = _switches[switchIndex]; const CSwitchForm &form = switchForms[switchIndex]; @@ -122,7 +107,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) sw.ThereIs = true; - int rem = s.Len() - pos; + const unsigned rem = s.Len() - pos; if (rem < form.MinLen) { ErrorMessage = "Too short switch:"; @@ -161,8 +146,10 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) break; case NSwitchType::kString: - sw.PostStrings.Add((const wchar_t *)s + pos); + { + sw.PostStrings.Add(s.Ptr(pos)); return true; + } } if (pos != s.Len()) @@ -173,23 +160,30 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms) return true; } -bool CParser::ParseStrings(const CSwitchForm *switchForms, const UStringVector &commandStrings) + +bool CParser::ParseStrings(const CSwitchForm *switchForms, unsigned numSwitches, const UStringVector &commandStrings) { + StopSwitchIndex = -1; + ErrorMessage.Empty(); ErrorLine.Empty(); - bool stopSwitch = false; + NonSwitchStrings.Clear(); + delete []_switches; + _switches = NULL; + _switches = new CSwitchResult[numSwitches]; + FOR_VECTOR (i, commandStrings) { const UString &s = commandStrings[i]; - if (!stopSwitch) + if (StopSwitchIndex < 0) { if (s.IsEqualTo(kStopSwitchParsing)) { - stopSwitch = true; + StopSwitchIndex = (int)NonSwitchStrings.Size(); continue; } if (!s.IsEmpty() && IsItSwitchChar(s[0])) { - if (ParseString(s, switchForms)) + if (ParseString(s, switchForms, numSwitches)) continue; ErrorLine = s; return false; diff --git a/3rdparty/lzma/CPP/Common/CommandLineParser.h b/3rdparty/lzma/CPP/Common/CommandLineParser.h index c9fd2956186..fbd4fa58bc7 100644 --- a/3rdparty/lzma/CPP/Common/CommandLineParser.h +++ b/3rdparty/lzma/CPP/Common/CommandLineParser.h @@ -38,24 +38,24 @@ struct CSwitchResult int PostCharIndex; UStringVector PostStrings; - CSwitchResult(): ThereIs(false) {}; + CSwitchResult(): ThereIs(false) {} }; class CParser { - unsigned _numSwitches; CSwitchResult *_switches; - bool ParseString(const UString &s, const CSwitchForm *switchForms); + bool ParseString(const UString &s, const CSwitchForm *switchForms, unsigned numSwitches); public: UStringVector NonSwitchStrings; + int StopSwitchIndex; // NonSwitchStrings[StopSwitchIndex+] are after "--" AString ErrorMessage; UString ErrorLine; - CParser(unsigned numSwitches); + CParser(); ~CParser(); - bool ParseStrings(const CSwitchForm *switchForms, const UStringVector &commandStrings); - const CSwitchResult& operator[](size_t index) const { return _switches[index]; } + bool ParseStrings(const CSwitchForm *switchForms, unsigned numSwitches, const UStringVector &commandStrings); + const CSwitchResult& operator[](unsigned index) const { return _switches[index]; } }; } diff --git a/3rdparty/lzma/CPP/Common/Common.h b/3rdparty/lzma/CPP/Common/Common.h index 9dd30f4bef2..72db7a8bf44 100644 --- a/3rdparty/lzma/CPP/Common/Common.h +++ b/3rdparty/lzma/CPP/Common/Common.h @@ -3,11 +3,55 @@ #ifndef __COMMON_COMMON_H #define __COMMON_COMMON_H +/* +This file is included to all cpp files in 7-Zip. +Each folder contains StdAfx.h file that includes "Common.h". +So 7-Zip includes "Common.h" in both modes: + with precompiled StdAfx.h +and + without precompiled StdAfx.h + +If you use 7-Zip code, you must include "Common.h" before other h files of 7-zip. +If you don't need some things that are used in 7-Zip, +you can change this h file or h files included in this file. +*/ + +// compiler pragmas to disable some warnings #include "../../C/Compiler.h" +// it's or code that defines windows things, if it's not _WIN32 #include "MyWindows.h" + +// NewHandler.h and NewHandler.cpp redefine operator new() to throw exceptions, if compiled with old MSVC compilers #include "NewHandler.h" -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[1])) + + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + + +/* There is BUG in MSVC 6.0 compiler for operator new[]: + It doesn't check overflow, when it calculates size in bytes for allocated array. + So we can use MY_ARRAY_NEW macro instead of new[] operator. */ + +#if defined(_MSC_VER) && (_MSC_VER == 1200) && !defined(_WIN64) + #define MY_ARRAY_NEW(p, T, size) p = new T[((size) > (unsigned)0xFFFFFFFF / sizeof(T)) ? (unsigned)0xFFFFFFFF / sizeof(T) : (size)]; +#else + #define MY_ARRAY_NEW(p, T, size) p = new T[size]; +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 8)) + #define MY_ATTR_NORETURN __attribute__((noreturn)) +#elif (defined(__clang__) && (__clang_major__ >= 3)) + #if __has_feature(cxx_attributes) + #define MY_ATTR_NORETURN [[noreturn]] + #else + #define MY_ATTR_NORETURN __attribute__ ((noreturn)) + #endif +#elif (defined(_MSC_VER) && (_MSC_VER >= 1900)) + #define MY_ATTR_NORETURN [[noreturn]] +#else + #define MY_ATTR_NORETURN +#endif #endif diff --git a/3rdparty/lzma/CPP/Common/CrcReg.cpp b/3rdparty/lzma/CPP/Common/CrcReg.cpp index 4b662f52251..fdbba77bcab 100644 --- a/3rdparty/lzma/CPP/Common/CrcReg.cpp +++ b/3rdparty/lzma/CPP/Common/CrcReg.cpp @@ -16,8 +16,10 @@ typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); extern CRC_FUNC g_CrcUpdate; -extern CRC_FUNC g_CrcUpdateT8; extern CRC_FUNC g_CrcUpdateT4; +extern CRC_FUNC g_CrcUpdateT8; +extern CRC_FUNC g_CrcUpdateT0_32; +extern CRC_FUNC g_CrcUpdateT0_64; EXTERN_C_END @@ -41,25 +43,20 @@ public: bool CCrcHasher::SetFunctions(UInt32 tSize) { - _updateFunc = g_CrcUpdate; + CRC_FUNC f = NULL; + if (tSize == 0) f = g_CrcUpdate; + else if (tSize == 1) f = CrcUpdateT1; + else if (tSize == 4) f = g_CrcUpdateT4; + else if (tSize == 8) f = g_CrcUpdateT8; + else if (tSize == 32) f = g_CrcUpdateT0_32; + else if (tSize == 64) f = g_CrcUpdateT0_64; - if (tSize == 1) - _updateFunc = CrcUpdateT1; - else if (tSize == 4) + if (!f) { - if (g_CrcUpdateT4) - _updateFunc = g_CrcUpdateT4; - else - return false; + _updateFunc = g_CrcUpdate; + return false; } - else if (tSize == 8) - { - if (g_CrcUpdateT8) - _updateFunc = g_CrcUpdateT8; - else - return false; - } - + _updateFunc = f; return true; } diff --git a/3rdparty/lzma/CPP/Common/Defs.h b/3rdparty/lzma/CPP/Common/Defs.h index 1fbd78bda7f..9adb88cfcc6 100644 --- a/3rdparty/lzma/CPP/Common/Defs.h +++ b/3rdparty/lzma/CPP/Common/Defs.h @@ -10,6 +10,7 @@ template inline int MyCompare(T a, T b) { return a == b ? 0 : (a < b ? -1 : 1); } inline int BoolToInt(bool v) { return (v ? 1 : 0); } +inline unsigned BoolToUInt(bool v) { return (v ? (unsigned)1 : (unsigned)0); } inline bool IntToBool(int v) { return (v != 0); } #endif diff --git a/3rdparty/lzma/CPP/Common/DynLimBuf.cpp b/3rdparty/lzma/CPP/Common/DynLimBuf.cpp new file mode 100644 index 00000000000..79141040990 --- /dev/null +++ b/3rdparty/lzma/CPP/Common/DynLimBuf.cpp @@ -0,0 +1,93 @@ +// Common/DynLimBuf.cpp + +#include "StdAfx.h" + +#include "DynLimBuf.h" +#include "MyString.h" + +CDynLimBuf::CDynLimBuf(size_t limit) throw() +{ + _chars = 0; + _pos = 0; + _size = 0; + _sizeLimit = limit; + _error = true; + unsigned size = 1 << 4; + if (size > limit) + size = (unsigned)limit; + _chars = (Byte *)MyAlloc(size); + if (_chars) + { + _size = size; + _error = false; + } +} + +CDynLimBuf & CDynLimBuf::operator+=(char c) throw() +{ + if (_error) + return *this; + if (_size == _pos) + { + size_t n = _sizeLimit - _size; + if (n == 0) + { + _error = true; + return *this; + } + if (n > _size) + n = _size; + + n += _pos; + + Byte *newBuf = (Byte *)MyAlloc(n); + if (!newBuf) + { + _error = true; + return *this; + } + memcpy(newBuf, _chars, _pos); + MyFree(_chars); + _chars = newBuf; + _size = n; + } + _chars[_pos++] = (Byte)c; + return *this; +} + +CDynLimBuf &CDynLimBuf::operator+=(const char *s) throw() +{ + if (_error) + return *this; + unsigned len = MyStringLen(s); + size_t rem = _sizeLimit - _pos; + if (rem < len) + { + len = (unsigned)rem; + _error = true; + } + if (_size - _pos < len) + { + size_t n = _pos + len; + if (n - _size < _size) + { + n = _sizeLimit; + if (n - _size > _size) + n = _size * 2; + } + + Byte *newBuf = (Byte *)MyAlloc(n); + if (!newBuf) + { + _error = true; + return *this; + } + memcpy(newBuf, _chars, _pos); + MyFree(_chars); + _chars = newBuf; + _size = n; + } + memcpy(_chars + _pos, s, len); + _pos += len; + return *this; +} diff --git a/3rdparty/lzma/CPP/Common/DynLimBuf.h b/3rdparty/lzma/CPP/Common/DynLimBuf.h new file mode 100644 index 00000000000..e80a7e7c5e9 --- /dev/null +++ b/3rdparty/lzma/CPP/Common/DynLimBuf.h @@ -0,0 +1,41 @@ +// Common/DynLimBuf.h + +#ifndef __COMMON_DYN_LIM_BUF_H +#define __COMMON_DYN_LIM_BUF_H + +#include + +#include "../../C/Alloc.h" + +#include "MyString.h" + +class CDynLimBuf +{ + Byte *_chars; + size_t _pos; + size_t _size; + size_t _sizeLimit; + bool _error; + + CDynLimBuf(const CDynLimBuf &s); + + // ---------- forbidden functions ---------- + CDynLimBuf &operator+=(wchar_t c); + +public: + CDynLimBuf(size_t limit) throw(); + ~CDynLimBuf() { MyFree(_chars); } + + size_t Len() const { return _pos; } + bool IsError() const { return _error; } + void Empty() { _pos = 0; _error = false; } + + operator const Byte *() const { return _chars; } + // const char *Ptr() const { return _chars; } + + CDynLimBuf &operator+=(char c) throw(); + CDynLimBuf &operator+=(const char *s) throw(); +}; + + +#endif diff --git a/3rdparty/lzma/CPP/Common/DynamicBuffer.h b/3rdparty/lzma/CPP/Common/DynamicBuffer.h index 44e3df7fd4a..f6f6b1572ef 100644 --- a/3rdparty/lzma/CPP/Common/DynamicBuffer.h +++ b/3rdparty/lzma/CPP/Common/DynamicBuffer.h @@ -54,7 +54,7 @@ public: memcpy(GetCurPtrAndGrow(size), data, size * sizeof(T)); } - const size_t GetPos() const { return _pos; } + size_t GetPos() const { return _pos; } // void Empty() { _pos = 0; } }; diff --git a/3rdparty/lzma/CPP/Common/IntToString.cpp b/3rdparty/lzma/CPP/Common/IntToString.cpp index ed217c72c0a..c87643c8760 100644 --- a/3rdparty/lzma/CPP/Common/IntToString.cpp +++ b/3rdparty/lzma/CPP/Common/IntToString.cpp @@ -2,26 +2,28 @@ #include "StdAfx.h" +#include "../../C/CpuArch.h" + #include "IntToString.h" #define CONVERT_INT_TO_STR(charType, tempSize) \ unsigned char temp[tempSize]; unsigned i = 0; \ while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ *s++ = (charType)('0' + (unsigned)val); \ - while (i != 0) { i--; *s++ = temp[i]; } \ - *s = 0; + while (i != 0) { i--; *s++ = (charType)temp[i]; } \ + *s = 0; \ + return s; -void ConvertUInt32ToString(UInt32 val, char *s) throw() +char * ConvertUInt32ToString(UInt32 val, char *s) throw() { CONVERT_INT_TO_STR(char, 16); } -void ConvertUInt64ToString(UInt64 val, char *s) throw() +char * ConvertUInt64ToString(UInt64 val, char *s) throw() { if (val <= (UInt32)0xFFFFFFFF) { - ConvertUInt32ToString((UInt32)val, s); - return; + return ConvertUInt32ToString((UInt32)val, s); } CONVERT_INT_TO_STR(char, 24); } @@ -46,6 +48,12 @@ void ConvertUInt64ToOct(UInt64 val, char *s) throw() while (i); } + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static inline char GetHexChar(unsigned t) { return GET_HEX_CHAR(t); } + + void ConvertUInt32ToHex(UInt32 val, char *s) throw() { UInt32 v = val; @@ -59,13 +67,14 @@ void ConvertUInt32ToHex(UInt32 val, char *s) throw() s[i] = 0; do { - unsigned t = (unsigned)((val & 0xF)); + unsigned t = (unsigned)(val & 0xF); val >>= 4; - s[--i] = (char)((t < 10) ? ('0' + t) : ('A' + (t - 10))); + s[--i] = GET_HEX_CHAR(t); } while (i); } + void ConvertUInt64ToHex(UInt64 val, char *s) throw() { UInt64 v = val; @@ -79,9 +88,9 @@ void ConvertUInt64ToHex(UInt64 val, char *s) throw() s[i] = 0; do { - unsigned t = (unsigned)((val & 0xF)); + unsigned t = (unsigned)(val & 0xF); val >>= 4; - s[--i] = (char)((t < 10) ? ('0' + t) : ('A' + (t - 10))); + s[--i] = GET_HEX_CHAR(t); } while (i); } @@ -93,7 +102,7 @@ void ConvertUInt32ToHex8Digits(UInt32 val, char *s) throw() { unsigned t = val & 0xF; val >>= 4; - s[i] = (char)(((t < 10) ? ('0' + t) : ('A' + (t - 10)))); + s[i] = GET_HEX_CHAR(t);; } } @@ -110,17 +119,16 @@ void ConvertUInt32ToHex8Digits(UInt32 val, wchar_t *s) } */ -void ConvertUInt32ToString(UInt32 val, wchar_t *s) throw() +wchar_t * ConvertUInt32ToString(UInt32 val, wchar_t *s) throw() { CONVERT_INT_TO_STR(wchar_t, 16); } -void ConvertUInt64ToString(UInt64 val, wchar_t *s) throw() +wchar_t * ConvertUInt64ToString(UInt64 val, wchar_t *s) throw() { if (val <= (UInt32)0xFFFFFFFF) { - ConvertUInt32ToString((UInt32)val, s); - return; + return ConvertUInt32ToString((UInt32)val, s); } CONVERT_INT_TO_STR(wchar_t, 24); } @@ -132,7 +140,7 @@ void ConvertInt64ToString(Int64 val, char *s) throw() *s++ = '-'; val = -val; } - ConvertUInt64ToString(val, s); + ConvertUInt64ToString((UInt64)val, s); } void ConvertInt64ToString(Int64 val, wchar_t *s) throw() @@ -142,5 +150,43 @@ void ConvertInt64ToString(Int64 val, wchar_t *s) throw() *s++ = L'-'; val = -val; } - ConvertUInt64ToString(val, s); + ConvertUInt64ToString((UInt64)val, s); +} + + +static void ConvertByteToHex2Digits(unsigned v, char *s) throw() +{ + s[0] = GetHexChar(v >> 4); + s[1] = GetHexChar(v & 0xF); +} + +static void ConvertUInt16ToHex4Digits(UInt32 val, char *s) throw() +{ + ConvertByteToHex2Digits(val >> 8, s); + ConvertByteToHex2Digits(val & 0xFF, s + 2); +} + +char *RawLeGuidToString(const Byte *g, char *s) throw() +{ + ConvertUInt32ToHex8Digits(GetUi32(g ), s); s += 8; *s++ = '-'; + ConvertUInt16ToHex4Digits(GetUi16(g + 4), s); s += 4; *s++ = '-'; + ConvertUInt16ToHex4Digits(GetUi16(g + 6), s); s += 4; *s++ = '-'; + for (unsigned i = 0; i < 8; i++) + { + if (i == 2) + *s++ = '-'; + ConvertByteToHex2Digits(g[8 + i], s); + s += 2; + } + *s = 0; + return s; +} + +char *RawLeGuidToString_Braced(const Byte *g, char *s) throw() +{ + *s++ = '{'; + s = RawLeGuidToString(g, s); + *s++ = '}'; + *s = 0; + return s; } diff --git a/3rdparty/lzma/CPP/Common/IntToString.h b/3rdparty/lzma/CPP/Common/IntToString.h index 69605ab76ce..d0a96ef1e32 100644 --- a/3rdparty/lzma/CPP/Common/IntToString.h +++ b/3rdparty/lzma/CPP/Common/IntToString.h @@ -5,11 +5,13 @@ #include "MyTypes.h" -void ConvertUInt32ToString(UInt32 value, char *s) throw(); -void ConvertUInt64ToString(UInt64 value, char *s) throw(); +// return: the pointer to the "terminating" null character after written characters -void ConvertUInt32ToString(UInt32 value, wchar_t *s) throw(); -void ConvertUInt64ToString(UInt64 value, wchar_t *s) throw(); +char * ConvertUInt32ToString(UInt32 value, char *s) throw(); +char * ConvertUInt64ToString(UInt64 value, char *s) throw(); + +wchar_t * ConvertUInt32ToString(UInt32 value, wchar_t *s) throw(); +wchar_t * ConvertUInt64ToString(UInt64 value, wchar_t *s) throw(); void ConvertUInt64ToOct(UInt64 value, char *s) throw(); @@ -21,4 +23,8 @@ void ConvertUInt32ToHex8Digits(UInt32 value, char *s) throw(); void ConvertInt64ToString(Int64 value, char *s) throw(); void ConvertInt64ToString(Int64 value, wchar_t *s) throw(); +// use RawLeGuid only for RAW bytes that contain stored GUID as Little-endian. +char *RawLeGuidToString(const Byte *guid, char *s) throw(); +char *RawLeGuidToString_Braced(const Byte *guid, char *s) throw(); + #endif diff --git a/3rdparty/lzma/CPP/Common/Lang.h b/3rdparty/lzma/CPP/Common/Lang.h index 22e42574eaa..cc66677d443 100644 --- a/3rdparty/lzma/CPP/Common/Lang.h +++ b/3rdparty/lzma/CPP/Common/Lang.h @@ -15,7 +15,7 @@ class CLang public: CLang(): _text(0) {} ~CLang() { Clear(); } - bool Open(CFSTR fileName, const wchar_t *id); + bool Open(CFSTR fileName, const char *id); void Clear() throw(); const wchar_t *Get(UInt32 id) const throw(); }; diff --git a/3rdparty/lzma/CPP/Common/ListFileUtils.cpp b/3rdparty/lzma/CPP/Common/ListFileUtils.cpp index 3bf4ec29d9f..b361b378891 100644 --- a/3rdparty/lzma/CPP/Common/ListFileUtils.cpp +++ b/3rdparty/lzma/CPP/Common/ListFileUtils.cpp @@ -4,14 +4,19 @@ #include "../../C/CpuArch.h" -#include "../Windows/FileIO.h" - #include "ListFileUtils.h" #include "MyBuffer.h" #include "StringConvert.h" #include "UTFConvert.h" -static const char kQuoteChar = '\"'; +#include "../Windows/FileIO.h" + +#define CSysInFile NWindows::NFile::NIO::CInFile +#define MY_GET_LAST_ERROR ::GetLastError() + + +#define kQuoteChar '\"' + static void AddName(UStringVector &strings, UString &s) { @@ -25,14 +30,39 @@ static void AddName(UStringVector &strings, UString &s) strings.Add(s); } -bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage) + +static bool My_File_Read(CSysInFile &file, void *data, size_t size, DWORD &lastError) +{ + size_t processed; + if (!file.ReadFull(data, size, processed)) + { + lastError = MY_GET_LAST_ERROR; + return false; + } + if (processed != size) + { + lastError = 1; // error: size of listfile was changed + return false; + } + return true; +} + + +bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, DWORD &lastError) { - NWindows::NFile::NIO::CInFile file; + lastError = 0; + CSysInFile file; if (!file.Open(fileName)) + { + lastError = MY_GET_LAST_ERROR; return false; + } UInt64 fileSize; if (!file.GetLength(fileSize)) + { + lastError = MY_GET_LAST_ERROR; return false; + } if (fileSize >= ((UInt32)1 << 31) - 32) return false; UString u; @@ -41,18 +71,17 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage if ((fileSize & 1) != 0) return false; CByteArr buf((size_t)fileSize); - UInt32 processed; - if (!file.Read(buf, (UInt32)fileSize, processed)) - return false; - if (processed != fileSize) + + if (!My_File_Read(file, buf, (size_t)fileSize, lastError)) return false; + file.Close(); - unsigned num = (unsigned)fileSize / 2; + const unsigned num = (unsigned)fileSize / 2; wchar_t *p = u.GetBuf(num); if (codePage == MY__CP_UTF16) for (unsigned i = 0; i < num; i++) { - wchar_t c = GetUi16(buf + i * 2); + wchar_t c = GetUi16(buf + (size_t)i * 2); if (c == 0) return false; p[i] = c; @@ -60,7 +89,7 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage else for (unsigned i = 0; i < num; i++) { - wchar_t c = (wchar_t)GetBe16(buf + i * 2); + wchar_t c = (wchar_t)GetBe16(buf + (size_t)i * 2); if (c == 0) return false; p[i] = c; @@ -72,19 +101,21 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage { AString s; char *p = s.GetBuf((unsigned)fileSize); - UInt32 processed; - if (!file.Read(p, (UInt32)fileSize, processed)) - return false; - if (processed != fileSize) + + if (!My_File_Read(file, p, (size_t)fileSize, lastError)) return false; + file.Close(); - s.ReleaseBuf_CalcLen((unsigned)processed); - if (s.Len() != processed) + s.ReleaseBuf_CalcLen((unsigned)fileSize); + if (s.Len() != fileSize) return false; // #ifdef CP_UTF8 if (codePage == CP_UTF8) { + // we must check UTF8 here, if convert function doesn't check + if (!CheckUTF8_AString(s)) + return false; if (!ConvertUTF8ToUnicode(s, u)) return false; } @@ -94,7 +125,7 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage } const wchar_t kGoodBOM = 0xFEFF; - const wchar_t kBadBOM = 0xFFFE; + // const wchar_t kBadBOM = 0xFFFE; UString s; unsigned i = 0; @@ -102,9 +133,11 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage for (; i < u.Len(); i++) { wchar_t c = u[i]; + /* if (c == kGoodBOM || c == kBadBOM) return false; - if (c == L'\n' || c == 0xD) + */ + if (c == '\n' || c == 0xD) { AddName(strings, s); s.Empty(); diff --git a/3rdparty/lzma/CPP/Common/ListFileUtils.h b/3rdparty/lzma/CPP/Common/ListFileUtils.h index e8d833fdb8c..a91e4b112ab 100644 --- a/3rdparty/lzma/CPP/Common/ListFileUtils.h +++ b/3rdparty/lzma/CPP/Common/ListFileUtils.h @@ -9,6 +9,10 @@ #define MY__CP_UTF16 1200 #define MY__CP_UTF16BE 1201 -bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage = CP_OEMCP); +// bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage = CP_OEMCP); + + // = CP_OEMCP +bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, + DWORD &lastError); #endif diff --git a/3rdparty/lzma/CPP/Common/LzFindPrepare.cpp b/3rdparty/lzma/CPP/Common/LzFindPrepare.cpp new file mode 100644 index 00000000000..8845e4a55bc --- /dev/null +++ b/3rdparty/lzma/CPP/Common/LzFindPrepare.cpp @@ -0,0 +1,7 @@ +// Sha256Prepare.cpp + +#include "StdAfx.h" + +#include "../../C/LzFind.h" + +static struct CLzFindPrepare { CLzFindPrepare() { LzFindPrepare(); } } g_CLzFindPrepare; diff --git a/3rdparty/lzma/CPP/Common/MyBuffer.h b/3rdparty/lzma/CPP/Common/MyBuffer.h index d0378057a95..18ab6fa4089 100644 --- a/3rdparty/lzma/CPP/Common/MyBuffer.h +++ b/3rdparty/lzma/CPP/Common/MyBuffer.h @@ -4,6 +4,10 @@ #define __COMMON_MY_BUFFER_H #include "Defs.h" +#include "MyTypes.h" + +/* 7-Zip now uses CBuffer only as CByteBuffer. + So there is no need to use MY_ARRAY_NEW macro in CBuffer code. */ template class CBuffer { @@ -88,6 +92,12 @@ public: _size = newSize; } + void Wipe() + { + if (_size != 0) + memset(_items, 0, _size * sizeof(T)); + } + CBuffer& operator=(const CBuffer &buffer) { if (&buffer != this) @@ -119,17 +129,28 @@ bool operator!=(const CBuffer& b1, const CBuffer& b2) } -typedef CBuffer CCharBuffer; +// typedef CBuffer CCharBuffer; // typedef CBuffer CWCharBuffer; typedef CBuffer CByteBuffer; +class CByteBuffer_Wipe: public CByteBuffer +{ + CLASS_NO_COPY(CByteBuffer_Wipe) +public: + // CByteBuffer_Wipe(): CBuffer() {} + CByteBuffer_Wipe(size_t size): CBuffer(size) {} + ~CByteBuffer_Wipe() { Wipe(); } +}; + + + template class CObjArray { protected: T *_items; private: - // we disable constructors + // we disable copy CObjArray(const CObjArray &buffer); void operator=(const CObjArray &buffer); public: @@ -138,7 +159,14 @@ public: delete []_items; _items = 0; } - CObjArray(size_t size): _items(0) { if (size != 0) _items = new T[size]; } + CObjArray(size_t size): _items(0) + { + if (size != 0) + { + MY_ARRAY_NEW(_items, T, size) + // _items = new T[size]; + } + } CObjArray(): _items(0) {}; ~CObjArray() { delete []_items; } @@ -149,7 +177,8 @@ public: { delete []_items; _items = 0; - _items = new T[newSize]; + MY_ARRAY_NEW(_items, T, newSize) + // _items = new T[newSize]; } }; @@ -164,6 +193,7 @@ template class CObjArray2 T *_items; unsigned _size; + // we disable copy CObjArray2(const CObjArray2 &buffer); void operator=(const CObjArray2 &buffer); public: @@ -216,7 +246,10 @@ public: return; T *newBuffer = NULL; if (size != 0) - newBuffer = new T[size]; + { + MY_ARRAY_NEW(newBuffer, T, size) + // newBuffer = new T[size]; + } delete []_items; _items = newBuffer; _size = size; diff --git a/3rdparty/lzma/CPP/Common/MyBuffer2.h b/3rdparty/lzma/CPP/Common/MyBuffer2.h new file mode 100644 index 00000000000..372d478c436 --- /dev/null +++ b/3rdparty/lzma/CPP/Common/MyBuffer2.h @@ -0,0 +1,139 @@ +// Common/MyBuffer2.h + +#ifndef __COMMON_MY_BUFFER2_H +#define __COMMON_MY_BUFFER2_H + +#include "../../C/Alloc.h" + +#include "MyTypes.h" + +class CMidBuffer +{ + Byte *_data; + size_t _size; + + CLASS_NO_COPY(CMidBuffer) + +public: + CMidBuffer(): _data(NULL), _size(0) {} + ~CMidBuffer() { ::MidFree(_data); } + + void Free() { ::MidFree(_data); _data = NULL; _size = 0; } + + bool IsAllocated() const { return _data != NULL; } + operator Byte *() { return _data; } + operator const Byte *() const { return _data; } + size_t Size() const { return _size; } + + void Alloc(size_t size) + { + if (!_data || size != _size) + { + ::MidFree(_data); + _size = 0; + _data = NULL; + _data = (Byte *)::MidAlloc(size); + if (_data) + _size = size; + } + } + + void AllocAtLeast(size_t size) + { + if (!_data || size > _size) + { + ::MidFree(_data); + const size_t kMinSize = (size_t)1 << 16; + if (size < kMinSize) + size = kMinSize; + _size = 0; + _data = NULL; + _data = (Byte *)::MidAlloc(size); + if (_data) + _size = size; + } + } +}; + + +class CAlignedBuffer +{ + Byte *_data; + size_t _size; + + CLASS_NO_COPY(CAlignedBuffer) + +public: + CAlignedBuffer(): _data(NULL), _size(0) {} + ~CAlignedBuffer() + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + } + + CAlignedBuffer(size_t size): _size(0) + { + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (!_data) + throw 1; + _size = size; + } + + void Free() + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _data = NULL; + _size = 0; + } + + bool IsAllocated() const { return _data != NULL; } + operator Byte *() { return _data; } + operator const Byte *() const { return _data; } + size_t Size() const { return _size; } + + void Alloc(size_t size) + { + if (!_data || size != _size) + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _size = 0; + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (_data) + _size = size; + } + } + + void AllocAtLeast(size_t size) + { + if (!_data || size > _size) + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _size = 0; + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (_data) + _size = size; + } + } +}; + +/* + CMidAlignedBuffer must return aligned pointer. + - in Windows it uses CMidBuffer(): MidAlloc() : VirtualAlloc() + VirtualAlloc(): Memory allocated is automatically initialized to zero. + MidAlloc(0) returns NULL + - in non-Windows systems it uses g_AlignedAlloc. + g_AlignedAlloc::Alloc(size = 0) can return non NULL. +*/ + +typedef +#ifdef _WIN32 + CMidBuffer +#else + CAlignedBuffer +#endif + CMidAlignedBuffer; + + +#endif diff --git a/3rdparty/lzma/CPP/Common/MyCom.h b/3rdparty/lzma/CPP/Common/MyCom.h index 7c725c9136e..ff5427806e7 100644 --- a/3rdparty/lzma/CPP/Common/MyCom.h +++ b/3rdparty/lzma/CPP/Common/MyCom.h @@ -4,7 +4,7 @@ #define __MY_COM_H #include "MyWindows.h" -#include "NewHandler.h" +#include "MyTypes.h" #ifndef RINOK #define RINOK(x) { HRESULT __result_ = (x); if (__result_ != S_OK) return __result_; } @@ -67,6 +67,7 @@ public: template HRESULT QueryInterface(REFGUID iid, Q** pp) const throw() { + // if (*pp) throw 20220216; // for debug return _p->QueryInterface(iid, (void**)pp); } }; @@ -82,7 +83,7 @@ inline HRESULT StringToBstr(LPCOLESTR src, BSTR *bstr) class CMyComBSTR { BSTR m_str; - + CLASS_NO_COPY(CMyComBSTR) public: CMyComBSTR(): m_str(NULL) {} ~CMyComBSTR() { ::SysFreeString(m_str); } @@ -90,13 +91,23 @@ public: operator LPCOLESTR() const { return m_str; } // operator bool() const { return m_str != NULL; } // bool operator!() const { return m_str == NULL; } + + void Wipe_and_Free() + { + if (m_str) + { + memset(m_str, 0, ::SysStringLen(m_str) * sizeof(*m_str)); + Empty(); + } + } + private: // operator BSTR() const { return m_str; } CMyComBSTR(LPCOLESTR src) { m_str = ::SysAllocString(src); } // CMyComBSTR(int nSize) { m_str = ::SysAllocStringLen(NULL, nSize); } // CMyComBSTR(int nSize, LPCOLESTR sz) { m_str = ::SysAllocStringLen(sz, nSize); } - CMyComBSTR(const CMyComBSTR& src) { m_str = src.MyCopy(); } + // CMyComBSTR(const CMyComBSTR& src) { m_str = src.MyCopy(); } /* CMyComBSTR(REFGUID src) @@ -108,6 +119,7 @@ private: } */ + /* CMyComBSTR& operator=(const CMyComBSTR& src) { if (m_str != src.m_str) @@ -118,6 +130,7 @@ private: } return *this; } + */ CMyComBSTR& operator=(LPCOLESTR src) { @@ -158,17 +171,49 @@ private: } }; -////////////////////////////////////////////////////////// + +class CMyComBSTR_Wipe: public CMyComBSTR +{ + CLASS_NO_COPY(CMyComBSTR_Wipe) +public: + CMyComBSTR_Wipe(): CMyComBSTR() {} + ~CMyComBSTR_Wipe() { Wipe_and_Free(); } +}; + + + +/* + If CMyUnknownImp doesn't use virtual destructor, the code size is smaller. + But if some class_1 derived from CMyUnknownImp + uses MY_ADDREF_RELEASE and IUnknown::Release() + and some another class_2 is derived from class_1, + then class_1 must use virtual destructor: + virtual ~class_1(); + In that case, class_1::Release() calls correct destructor of class_2. + + We use virtual ~CMyUnknownImp() to disable warning + "class has virtual functions, but destructor is not virtual". + + also we can use virtual ~IUnknown() {} in MyWindows.h +*/ class CMyUnknownImp { + CLASS_NO_COPY(CMyUnknownImp) public: ULONG __m_RefCount; CMyUnknownImp(): __m_RefCount(0) {} - // virtual ~CMyUnknownImp() {}; + #ifdef _WIN32 + #if defined(__GNUC__) || defined(__clang__) + virtual // to disable GCC/CLANG varnings + #endif + #endif + ~CMyUnknownImp() {} }; + + #define MY_QUERYINTERFACE_BEGIN STDMETHOD(QueryInterface) \ (REFGUID iid, void **outObject) throw() { *outObject = NULL; @@ -186,8 +231,8 @@ public: #define MY_ADDREF_RELEASE \ STDMETHOD_(ULONG, AddRef)() throw() { return ++__m_RefCount; } \ -STDMETHOD_(ULONG, Release)() { if (--__m_RefCount != 0) \ - return __m_RefCount; delete this; return 0; } +STDMETHOD_(ULONG, Release)() { if (--__m_RefCount != 0) return __m_RefCount; \ + delete this; return 0; } #define MY_UNKNOWN_IMP_SPEC(i) \ MY_QUERYINTERFACE_BEGIN \ diff --git a/3rdparty/lzma/CPP/Common/MyGuidDef.h b/3rdparty/lzma/CPP/Common/MyGuidDef.h index 68745870ee3..38aad6e6d37 100644 --- a/3rdparty/lzma/CPP/Common/MyGuidDef.h +++ b/3rdparty/lzma/CPP/Common/MyGuidDef.h @@ -18,6 +18,9 @@ typedef struct { #define REFGUID const GUID * #endif +// typedef GUID IID; +typedef GUID CLSID; + #define REFCLSID REFGUID #define REFIID REFGUID diff --git a/3rdparty/lzma/CPP/Common/MyInitGuid.h b/3rdparty/lzma/CPP/Common/MyInitGuid.h index 279fba5d612..04d77e21c10 100644 --- a/3rdparty/lzma/CPP/Common/MyInitGuid.h +++ b/3rdparty/lzma/CPP/Common/MyInitGuid.h @@ -19,13 +19,17 @@ Also we need IID_IUnknown that is initialized in some file for linking: Other: we define IID_IUnknown in this file */ +#ifdef __clang__ + #pragma clang diagnostic ignored "-Wmissing-variable-declarations" +#endif + #ifdef _WIN32 #ifdef UNDER_CE #include #endif -#include +#include #ifdef UNDER_CE DEFINE_GUID(IID_IUnknown, diff --git a/3rdparty/lzma/CPP/Common/MyLinux.h b/3rdparty/lzma/CPP/Common/MyLinux.h index 1a918993527..98b453c11d3 100644 --- a/3rdparty/lzma/CPP/Common/MyLinux.h +++ b/3rdparty/lzma/CPP/Common/MyLinux.h @@ -3,6 +3,18 @@ #ifndef __MY_LIN_LINUX_H #define __MY_LIN_LINUX_H +// #include "../../C/7zTypes.h" + +#define MY_LIN_DT_UNKNOWN 0 +#define MY_LIN_DT_FIFO 1 +#define MY_LIN_DT_CHR 2 +#define MY_LIN_DT_DIR 4 +#define MY_LIN_DT_BLK 6 +#define MY_LIN_DT_REG 8 +#define MY_LIN_DT_LNK 10 +#define MY_LIN_DT_SOCK 12 +#define MY_LIN_DT_WHT 14 + #define MY_LIN_S_IFMT 00170000 #define MY_LIN_S_IFSOCK 0140000 #define MY_LIN_S_IFLNK 0120000 @@ -39,4 +51,25 @@ #define MY_LIN_S_IWOTH 00002 #define MY_LIN_S_IXOTH 00001 +/* +// major/minor encoding for makedev(): MMMMMmmmmmmMMMmm: + +inline UInt32 MY_dev_major(UInt64 dev) +{ + return ((UInt32)(dev >> 8) & (UInt32)0xfff) | ((UInt32)(dev >> 32) & ~(UInt32)0xfff); +} + +inline UInt32 MY_dev_minor(UInt64 dev) +{ + return ((UInt32)(dev) & 0xff) | ((UInt32)(dev >> 12) & ~0xff); +} + +inline UInt64 MY_dev_makedev(UInt32 __major, UInt32 __minor) +{ + return (__minor & 0xff) | ((__major & 0xfff) << 8) + | ((UInt64) (__minor & ~0xff) << 12) + | ((UInt64) (__major & ~0xfff) << 32); +} +*/ + #endif diff --git a/3rdparty/lzma/CPP/Common/MyString.cpp b/3rdparty/lzma/CPP/Common/MyString.cpp index a2d4cc4e6d0..bf1638ebe10 100644 --- a/3rdparty/lzma/CPP/Common/MyString.cpp +++ b/3rdparty/lzma/CPP/Common/MyString.cpp @@ -8,6 +8,8 @@ #include #endif +#include "IntToString.h" + #if !defined(_UNICODE) || !defined(USE_UNICODE_FSTRING) #include "StringConvert.h" #endif @@ -28,6 +30,10 @@ inline const char* MyStringGetNextCharPointer(const char *p) throw() } */ +#define MY_STRING_NEW_char(_size_) MY_STRING_NEW(char, (_size_)) +#define MY_STRING_NEW_wchar_t(_size_) MY_STRING_NEW(wchar_t, (_size_)) + + int FindCharPosInString(const char *s, char c) throw() { for (const char *p = s;; p++) @@ -52,7 +58,18 @@ int FindCharPosInString(const wchar_t *s, wchar_t c) throw() } /* -void MyStringUpper_Ascii(wchar_t *s) +void MyStringUpper_Ascii(char *s) throw() +{ + for (;;) + { + char c = *s; + if (c == 0) + return; + *s++ = MyCharUpper_Ascii(c); + } +} + +void MyStringUpper_Ascii(wchar_t *s) throw() { for (;;) { @@ -173,8 +190,8 @@ bool IsString1PrefixedByString2(const char *s1, const char *s2) throw() { for (;;) { - unsigned char c2 = (unsigned char)*s2++; if (c2 == 0) return true; - unsigned char c1 = (unsigned char)*s1++; if (c1 != c2) return false; + const unsigned char c2 = (unsigned char)*s2++; if (c2 == 0) return true; + const unsigned char c1 = (unsigned char)*s1++; if (c1 != c2) return false; } } @@ -182,8 +199,8 @@ bool StringsAreEqualNoCase(const wchar_t *s1, const wchar_t *s2) throw() { for (;;) { - wchar_t c1 = *s1++; - wchar_t c2 = *s2++; + const wchar_t c1 = *s1++; + const wchar_t c2 = *s2++; if (c1 != c2 && MyCharUpper(c1) != MyCharUpper(c2)) return false; if (c1 == 0) return true; } @@ -196,10 +213,10 @@ bool AString::IsPrefixedBy_Ascii_NoCase(const char *s) const throw() const char *s1 = _chars; for (;;) { - char c2 = *s++; + const char c2 = *s++; if (c2 == 0) return true; - char c1 = *s1++; + const char c1 = *s1++; if (MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) return false; @@ -211,20 +228,34 @@ bool UString::IsPrefixedBy_Ascii_NoCase(const char *s) const throw() const wchar_t *s1 = _chars; for (;;) { - char c2 = *s++; + const char c2 = *s++; if (c2 == 0) return true; - wchar_t c1 = *s1++; + const wchar_t c1 = *s1++; if (MyCharLower_Ascii(c1) != (unsigned char)MyCharLower_Ascii(c2)) return false; } } +bool StringsAreEqual_Ascii(const char *u, const char *a) throw() +{ + for (;;) + { + const char c = *a; + if (c != *u) + return false; + if (c == 0) + return true; + a++; + u++; + } +} + bool StringsAreEqual_Ascii(const wchar_t *u, const char *a) throw() { for (;;) { - unsigned char c = *a; + const unsigned char c = (unsigned char)*a; if (c != *u) return false; if (c == 0) @@ -238,8 +269,8 @@ bool StringsAreEqualNoCase_Ascii(const char *s1, const char *s2) throw() { for (;;) { - char c1 = *s1++; - char c2 = *s2++; + const char c1 = *s1++; + const char c2 = *s2++; if (c1 != c2 && MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) return false; if (c1 == 0) @@ -251,8 +282,8 @@ bool StringsAreEqualNoCase_Ascii(const wchar_t *s1, const wchar_t *s2) throw() { for (;;) { - wchar_t c1 = *s1++; - wchar_t c2 = *s2++; + const wchar_t c1 = *s1++; + const wchar_t c2 = *s2++; if (c1 != c2 && MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) return false; if (c1 == 0) @@ -264,8 +295,8 @@ bool StringsAreEqualNoCase_Ascii(const wchar_t *s1, const char *s2) throw() { for (;;) { - wchar_t c1 = *s1++; - char c2 = *s2++; + const wchar_t c1 = *s1++; + const char c2 = *s2++; if (c1 != (unsigned char)c2 && (c1 > 0x7F || MyCharLower_Ascii(c1) != (unsigned char)MyCharLower_Ascii(c2))) return false; if (c1 == 0) @@ -277,8 +308,39 @@ bool IsString1PrefixedByString2(const wchar_t *s1, const wchar_t *s2) throw() { for (;;) { - wchar_t c2 = *s2++; if (c2 == 0) return true; - wchar_t c1 = *s1++; if (c1 != c2) return false; + const wchar_t c2 = *s2++; if (c2 == 0) return true; + const wchar_t c1 = *s1++; if (c1 != c2) return false; + } +} + +bool IsString1PrefixedByString2(const wchar_t *s1, const char *s2) throw() +{ + for (;;) + { + const unsigned char c2 = (unsigned char)(*s2++); if (c2 == 0) return true; + const wchar_t c1 = *s1++; if (c1 != c2) return false; + } +} + +bool IsString1PrefixedByString2_NoCase_Ascii(const char *s1, const char *s2) throw() +{ + for (;;) + { + const char c2 = *s2++; if (c2 == 0) return true; + const char c1 = *s1++; + if (c1 != c2 && MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2)) + return false; + } +} + +bool IsString1PrefixedByString2_NoCase_Ascii(const wchar_t *s1, const char *s2) throw() +{ + for (;;) + { + const char c2 = *s2++; if (c2 == 0) return true; + const wchar_t c1 = *s1++; + if (c1 != (unsigned char)c2 && MyCharLower_Ascii(c1) != (unsigned char)MyCharLower_Ascii(c2)) + return false; } } @@ -286,8 +348,8 @@ bool IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) thr { for (;;) { - wchar_t c2 = *s2++; if (c2 == 0) return true; - wchar_t c1 = *s1++; + const wchar_t c2 = *s2++; if (c2 == 0) return true; + const wchar_t c1 = *s1++; if (c1 != c2 && MyCharUpper(c1) != MyCharUpper(c2)) return false; } @@ -298,12 +360,12 @@ int MyStringCompareNoCase(const wchar_t *s1, const wchar_t *s2) throw() { for (;;) { - wchar_t c1 = *s1++; - wchar_t c2 = *s2++; + const wchar_t c1 = *s1++; + const wchar_t c2 = *s2++; if (c1 != c2) { - wchar_t u1 = MyCharUpper(c1); - wchar_t u2 = MyCharUpper(c2); + const wchar_t u1 = MyCharUpper(c1); + const wchar_t u2 = MyCharUpper(c2); if (u1 < u2) return -1; if (u1 > u2) return 1; } @@ -339,14 +401,13 @@ void AString::InsertSpace(unsigned &index, unsigned size) MoveItems(index + size, index); } -#define k_Alloc_Len_Limit 0x40000000 +#define k_Alloc_Len_Limit (0x40000000 - 2) void AString::ReAlloc(unsigned newLimit) { - if (newLimit < _len || newLimit >= k_Alloc_Len_Limit) throw 20130220; - // MY_STRING_REALLOC(_chars, char, newLimit + 1, _len + 1); - char *newBuf = MY_STRING_NEW(char, newLimit + 1); - memcpy(newBuf, _chars, (size_t)(_len + 1)); \ + // MY_STRING_REALLOC(_chars, char, (size_t)newLimit + 1, (size_t)_len + 1); + char *newBuf = MY_STRING_NEW_char((size_t)newLimit + 1); + memcpy(newBuf, _chars, (size_t)_len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = newLimit; @@ -354,9 +415,9 @@ void AString::ReAlloc(unsigned newLimit) void AString::ReAlloc2(unsigned newLimit) { - if (newLimit >= k_Alloc_Len_Limit) throw 20130220; - // MY_STRING_REALLOC(_chars, char, newLimit + 1, 0); - char *newBuf = MY_STRING_NEW(char, newLimit + 1); + if (newLimit > k_Alloc_Len_Limit) throw 20130220; + // MY_STRING_REALLOC(_chars, char, (size_t)newLimit + 1, 0); + char *newBuf = MY_STRING_NEW_char((size_t)newLimit + 1); newBuf[0] = 0; MY_STRING_DELETE(_chars); _chars = newBuf; @@ -365,8 +426,8 @@ void AString::ReAlloc2(unsigned newLimit) void AString::SetStartLen(unsigned len) { - _chars = 0; - _chars = MY_STRING_NEW(char, len + 1); + _chars = NULL; + _chars = MY_STRING_NEW_char((size_t)len + 1); _len = len; _limit = len; } @@ -377,23 +438,32 @@ void AString::Grow_1() next += next / 2; next += 16; next &= ~(unsigned)15; - ReAlloc(next - 1); + next--; + if (next < _len || next > k_Alloc_Len_Limit) + next = k_Alloc_Len_Limit; + if (next <= _len) + throw 20130220; + ReAlloc(next); + // Grow(1); } void AString::Grow(unsigned n) { - unsigned freeSize = _limit - _len; + const unsigned freeSize = _limit - _len; if (n <= freeSize) return; - unsigned next = _len + n; next += next / 2; next += 16; next &= ~(unsigned)15; - ReAlloc(next - 1); + next--; + if (next < _len || next > k_Alloc_Len_Limit) + next = k_Alloc_Len_Limit; + if (next <= _len || next - _len < n) + throw 20130220; + ReAlloc(next); } -/* AString::AString(unsigned num, const char *s) { unsigned len = MyStringLen(s); @@ -403,7 +473,6 @@ AString::AString(unsigned num, const char *s) memcpy(_chars, s, num); _chars[num] = 0; } -*/ AString::AString(unsigned num, const AString &s) { @@ -421,7 +490,7 @@ AString::AString(const AString &s, char c) unsigned len = s.Len(); memcpy(chars, s, len); chars[len] = c; - chars[len + 1] = 0; + chars[(size_t)len + 1] = 0; } AString::AString(const char *s1, unsigned num1, const char *s2, unsigned num2) @@ -436,20 +505,23 @@ AString operator+(const AString &s1, const AString &s2) { return AString(s1, s1. AString operator+(const AString &s1, const char *s2) { return AString(s1, s1.Len(), s2, MyStringLen(s2)); } AString operator+(const char *s1, const AString &s2) { return AString(s1, MyStringLen(s1), s2, s2.Len()); } +static const unsigned kStartStringCapacity = 4; + AString::AString() { - _chars = 0; - _chars = MY_STRING_NEW(char, 4); + _chars = NULL; + _chars = MY_STRING_NEW_char(kStartStringCapacity); _len = 0; - _limit = 4 - 1; + _limit = kStartStringCapacity - 1; _chars[0] = 0; } AString::AString(char c) { SetStartLen(1); - _chars[0] = c; - _chars[1] = 0; + char *chars = _chars; + chars[0] = c; + chars[1] = 0; } AString::AString(const char *s) @@ -468,14 +540,15 @@ AString &AString::operator=(char c) { if (1 > _limit) { - char *newBuf = MY_STRING_NEW(char, 1 + 1); + char *newBuf = MY_STRING_NEW_char(1 + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = 1; } _len = 1; - _chars[0] = c; - _chars[1] = 0; + char *chars = _chars; + chars[0] = c; + chars[1] = 0; return *this; } @@ -484,7 +557,7 @@ AString &AString::operator=(const char *s) unsigned len = MyStringLen(s); if (len > _limit) { - char *newBuf = MY_STRING_NEW(char, len + 1); + char *newBuf = MY_STRING_NEW_char((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -501,7 +574,7 @@ AString &AString::operator=(const AString &s) unsigned len = s._len; if (len > _limit) { - char *newBuf = MY_STRING_NEW(char, len + 1); + char *newBuf = MY_STRING_NEW_char((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -526,7 +599,7 @@ void AString::SetFromWStr_if_Ascii(const wchar_t *s) } if (len > _limit) { - char *newBuf = MY_STRING_NEW(char, len + 1); + char *newBuf = MY_STRING_NEW_char((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -550,7 +623,7 @@ void AString::SetFromBstr_if_Ascii(BSTR s) } if (len > _limit) { - char *newBuf = MY_STRING_NEW(char, len + 1); + char *newBuf = MY_STRING_NEW_char((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -567,6 +640,7 @@ void AString::SetFromBstr_if_Ascii(BSTR s) void AString::Add_Space() { operator+=(' '); } void AString::Add_Space_if_NotEmpty() { if (!IsEmpty()) Add_Space(); } void AString::Add_LF() { operator+=('\n'); } +void AString::Add_Slash() { operator+=('/'); } AString &AString::operator+=(const char *s) { @@ -577,6 +651,12 @@ AString &AString::operator+=(const char *s) return *this; } +void AString::Add_OptSpaced(const char *s) +{ + Add_Space_if_NotEmpty(); + (*this) += s; +} + AString &AString::operator+=(const AString &s) { Grow(s._len); @@ -585,11 +665,35 @@ AString &AString::operator+=(const AString &s) return *this; } +void AString::Add_UInt32(UInt32 v) +{ + Grow(10); + _len = (unsigned)(ConvertUInt32ToString(v, _chars + _len) - _chars); +} + +void UString::Add_UInt64(UInt64 v) +{ + Grow(20); + _len = (unsigned)(ConvertUInt64ToString(v, _chars + _len) - _chars); +} + +void AString::AddFrom(const char *s, unsigned len) // no check +{ + if (len != 0) + { + Grow(len); + memcpy(_chars + _len, s, len); + len += _len; + _chars[len] = 0; + _len = len; + } +} + void AString::SetFrom(const char *s, unsigned len) // no check { if (len > _limit) { - char *newBuf = MY_STRING_NEW(char, len + 1); + char *newBuf = MY_STRING_NEW_char((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -694,7 +798,7 @@ void AString::TrimRight() throw() unsigned i; for (i = _len; i != 0; i--) { - char c = p[i - 1]; + char c = p[(size_t)i - 1]; if (c != ' ' && c != '\n' && c != '\t') break; } @@ -780,12 +884,13 @@ void AString::Replace(char oldChar, char newChar) throw() return; // 0; // unsigned number = 0; int pos = 0; + char *chars = _chars; while ((unsigned)pos < _len) { - pos = Find(oldChar, pos); + pos = Find(oldChar, (unsigned)pos); if (pos < 0) break; - _chars[(unsigned)pos] = newChar; + chars[(unsigned)pos] = newChar; pos++; // number++; } @@ -804,11 +909,11 @@ void AString::Replace(const AString &oldString, const AString &newString) int pos = 0; while ((unsigned)pos < _len) { - pos = Find(oldString, pos); + pos = Find(oldString, (unsigned)pos); if (pos < 0) break; - Delete(pos, oldLen); - Insert(pos, newString); + Delete((unsigned)pos, oldLen); + Insert((unsigned)pos, newString); pos += newLen; // number++; } @@ -893,9 +998,8 @@ void UString::InsertSpace(unsigned index, unsigned size) void UString::ReAlloc(unsigned newLimit) { - if (newLimit < _len || newLimit >= k_Alloc_Len_Limit) throw 20130221; - // MY_STRING_REALLOC(_chars, wchar_t, newLimit + 1, _len + 1); - wchar_t *newBuf = MY_STRING_NEW(wchar_t, newLimit + 1); + // MY_STRING_REALLOC(_chars, wchar_t, (size_t)newLimit + 1, (size_t)_len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)newLimit + 1); wmemcpy(newBuf, _chars, _len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; @@ -904,9 +1008,9 @@ void UString::ReAlloc(unsigned newLimit) void UString::ReAlloc2(unsigned newLimit) { - if (newLimit >= k_Alloc_Len_Limit) throw 20130221; + if (newLimit > k_Alloc_Len_Limit) throw 20130221; // MY_STRING_REALLOC(_chars, wchar_t, newLimit + 1, 0); - wchar_t *newBuf = MY_STRING_NEW(wchar_t, newLimit + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)newLimit + 1); newBuf[0] = 0; MY_STRING_DELETE(_chars); _chars = newBuf; @@ -916,7 +1020,7 @@ void UString::ReAlloc2(unsigned newLimit) void UString::SetStartLen(unsigned len) { _chars = 0; - _chars = MY_STRING_NEW(wchar_t, len + 1); + _chars = MY_STRING_NEW_wchar_t((size_t)len + 1); _len = len; _limit = len; } @@ -927,19 +1031,28 @@ void UString::Grow_1() next += next / 2; next += 16; next &= ~(unsigned)15; - ReAlloc(next - 1); + next--; + if (next < _len || next > k_Alloc_Len_Limit) + next = k_Alloc_Len_Limit; + if (next <= _len) + throw 20130220; + ReAlloc(next); } void UString::Grow(unsigned n) { - unsigned freeSize = _limit - _len; + const unsigned freeSize = _limit - _len; if (n <= freeSize) return; - unsigned next = _len + n; next += next / 2; next += 16; next &= ~(unsigned)15; + next--; + if (next < _len || next > k_Alloc_Len_Limit) + next = k_Alloc_Len_Limit; + if (next <= _len || next - _len < n) + throw 20130220; ReAlloc(next - 1); } @@ -971,7 +1084,7 @@ UString::UString(const UString &s, wchar_t c) unsigned len = s.Len(); wmemcpy(chars, s, len); chars[len] = c; - chars[len + 1] = 0; + chars[(size_t)len + 1] = 0; } UString::UString(const wchar_t *s1, unsigned num1, const wchar_t *s2, unsigned num2) @@ -989,26 +1102,56 @@ UString operator+(const wchar_t *s1, const UString &s2) { return UString(s1, MyS UString::UString() { _chars = 0; - _chars = MY_STRING_NEW(wchar_t, 4); + _chars = MY_STRING_NEW_wchar_t(kStartStringCapacity); _len = 0; - _limit = 4 - 1; + _limit = kStartStringCapacity - 1; _chars[0] = 0; } UString::UString(wchar_t c) { SetStartLen(1); - _chars[0] = c; - _chars[1] = 0; + wchar_t *chars = _chars; + chars[0] = c; + chars[1] = 0; +} + +UString::UString(char c) +{ + SetStartLen(1); + wchar_t *chars = _chars; + chars[0] = (unsigned char)c; + chars[1] = 0; } UString::UString(const wchar_t *s) { - unsigned len = MyStringLen(s); + const unsigned len = MyStringLen(s); SetStartLen(len); wmemcpy(_chars, s, len + 1); } +UString::UString(const char *s) +{ + const unsigned len = MyStringLen(s); + SetStartLen(len); + wchar_t *chars = _chars; + for (unsigned i = 0; i < len; i++) + chars[i] = (unsigned char)s[i]; + chars[len] = 0; +} + +UString::UString(const AString &s) +{ + const unsigned len = s.Len(); + SetStartLen(len); + wchar_t *chars = _chars; + const char *s2 = s.Ptr(); + for (unsigned i = 0; i < len; i++) + chars[i] = (unsigned char)s2[i]; + chars[len] = 0; +} + UString::UString(const UString &s) { SetStartLen(s._len); @@ -1019,14 +1162,15 @@ UString &UString::operator=(wchar_t c) { if (1 > _limit) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, 1 + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t(1 + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = 1; } _len = 1; - _chars[0] = c; - _chars[1] = 0; + wchar_t *chars = _chars; + chars[0] = c; + chars[1] = 0; return *this; } @@ -1035,7 +1179,7 @@ UString &UString::operator=(const wchar_t *s) unsigned len = MyStringLen(s); if (len > _limit) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -1052,7 +1196,7 @@ UString &UString::operator=(const UString &s) unsigned len = s._len; if (len > _limit) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; @@ -1062,19 +1206,99 @@ UString &UString::operator=(const UString &s) return *this; } -void UString::SetFromBstr(BSTR s) +void UString::SetFrom(const wchar_t *s, unsigned len) // no check { - unsigned len = ::SysStringLen(s); if (len > _limit) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); MY_STRING_DELETE(_chars); _chars = newBuf; _limit = len; } + if (len != 0) + wmemcpy(_chars, s, len); + _chars[len] = 0; _len = len; +} + +void UString::SetFromBstr(LPCOLESTR s) +{ + unsigned len = ::SysStringLen((BSTR)(void *)(s)); + + /* + #if WCHAR_MAX > 0xffff + size_t num_wchars = 0; + for (size_t i = 0; i < len;) + { + wchar_t c = s[i++]; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = s[i]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + i++; + } + } + num_wchars++; + } + len = num_wchars; + #endif + */ + + if (len > _limit) + { + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); + MY_STRING_DELETE(_chars); + _chars = newBuf; + _limit = len; + } + _len = len; + + /* + #if WCHAR_MAX > 0xffff + + wchar_t *chars = _chars; + for (size_t i = 0; i <= len; i++) + { + wchar_t c = *s++; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = *s; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + s++; + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + } + } + chars[i] = c; + } + + #else + */ + // if (s) wmemcpy(_chars, s, len + 1); + + // #endif +} + +UString &UString::operator=(const char *s) +{ + unsigned len = MyStringLen(s); + if (len > _limit) + { + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); + MY_STRING_DELETE(_chars); + _chars = newBuf; + _limit = len; + } + wchar_t *chars = _chars; + for (unsigned i = 0; i < len; i++) + chars[i] = (unsigned char)s[i]; + chars[len] = 0; + _len = len; + return *this; } void UString::Add_Space() { operator+=(L' '); } @@ -1108,39 +1332,7 @@ UString &UString::operator+=(const UString &s) return *this; } -void UString::SetFrom(const wchar_t *s, unsigned len) // no check -{ - if (len > _limit) - { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); - MY_STRING_DELETE(_chars); - _chars = newBuf; - _limit = len; - } - if (len != 0) - wmemcpy(_chars, s, len); - _chars[len] = 0; - _len = len; -} - -void UString::SetFromAscii(const char *s) -{ - unsigned len = MyStringLen(s); - if (len > _limit) - { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); - MY_STRING_DELETE(_chars); - _chars = newBuf; - _limit = len; - } - wchar_t *chars = _chars; - for (unsigned i = 0; i < len; i++) - chars[i] = (unsigned char)s[i]; - chars[len] = 0; - _len = len; -} - -void UString::AddAscii(const char *s) +UString &UString::operator+=(const char *s) { unsigned len = MyStringLen(s); Grow(len); @@ -1149,8 +1341,21 @@ void UString::AddAscii(const char *s) chars[i] = (unsigned char)s[i]; chars[len] = 0; _len += len; + return *this; +} + + +void UString::Add_UInt32(UInt32 v) +{ + Grow(10); + _len = (unsigned)(ConvertUInt32ToString(v, _chars + _len) - _chars); } +void AString::Add_UInt64(UInt64 v) +{ + Grow(20); + _len = (unsigned)(ConvertUInt64ToString(v, _chars + _len) - _chars); +} int UString::Find(const wchar_t *s, unsigned startIndex) const throw() @@ -1238,7 +1443,7 @@ void UString::TrimRight() throw() unsigned i; for (i = _len; i != 0; i--) { - wchar_t c = p[i - 1]; + wchar_t c = p[(size_t)i - 1]; if (c != ' ' && c != '\n' && c != '\t') break; } @@ -1259,7 +1464,7 @@ void UString::InsertAtFront(wchar_t c) } /* -void UString::Insert(unsigned index, wchar_t c) +void UString::Insert_wchar_t(unsigned index, wchar_t c) { InsertSpace(index, 1); _chars[index] = c; @@ -1324,12 +1529,13 @@ void UString::Replace(wchar_t oldChar, wchar_t newChar) throw() return; // 0; // unsigned number = 0; int pos = 0; + wchar_t *chars = _chars; while ((unsigned)pos < _len) { - pos = Find(oldChar, pos); + pos = Find(oldChar, (unsigned)pos); if (pos < 0) break; - _chars[(unsigned)pos] = newChar; + chars[(unsigned)pos] = newChar; pos++; // number++; } @@ -1348,11 +1554,11 @@ void UString::Replace(const UString &oldString, const UString &newString) int pos = 0; while ((unsigned)pos < _len) { - pos = Find(oldString, pos); + pos = Find(oldString, (unsigned)pos); if (pos < 0) break; - Delete(pos, oldLen); - Insert(pos, newString); + Delete((unsigned)pos, oldLen); + Insert((unsigned)pos, newString); pos += newLen; // number++; } @@ -1390,15 +1596,24 @@ void UString::DeleteFrontal(unsigned num) throw() void UString2::ReAlloc2(unsigned newLimit) { - if (newLimit >= k_Alloc_Len_Limit) throw 20130221; + // wrong (_len) is allowed after this function + if (newLimit > k_Alloc_Len_Limit) throw 20130221; // MY_STRING_REALLOC(_chars, wchar_t, newLimit + 1, 0); - _chars = MY_STRING_NEW(wchar_t, newLimit + 1); + if (_chars) + { + MY_STRING_DELETE(_chars); + _chars = NULL; + // _len = 0; + } + _chars = MY_STRING_NEW_wchar_t((size_t)newLimit + 1); + _chars[0] = 0; + // _len = newLimit; } void UString2::SetStartLen(unsigned len) { - _chars = 0; - _chars = MY_STRING_NEW(wchar_t, len + 1); + _chars = NULL; + _chars = MY_STRING_NEW_wchar_t((size_t)len + 1); _len = len; } @@ -1407,14 +1622,15 @@ void UString2::SetStartLen(unsigned len) UString2::UString2(wchar_t c) { SetStartLen(1); - _chars[0] = c; - _chars[1] = 0; + wchar_t *chars = _chars; + chars[0] = c; + chars[1] = 0; } */ UString2::UString2(const wchar_t *s) { - unsigned len = MyStringLen(s); + const unsigned len = MyStringLen(s); SetStartLen(len); wmemcpy(_chars, s, len + 1); } @@ -1433,14 +1649,15 @@ UString2 &UString2::operator=(wchar_t c) { if (1 > _len) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, 1 + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t(1 + 1); if (_chars) MY_STRING_DELETE(_chars); _chars = newBuf; } _len = 1; - _chars[0] = c; - _chars[1] = 0; + wchar_t *chars = _chars; + chars[0] = c; + chars[1] = 0; return *this; } */ @@ -1450,7 +1667,7 @@ UString2 &UString2::operator=(const wchar_t *s) unsigned len = MyStringLen(s); if (len > _len) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); if (_chars) MY_STRING_DELETE(_chars); _chars = newBuf; @@ -1465,7 +1682,7 @@ void UString2::SetFromAscii(const char *s) unsigned len = MyStringLen(s); if (len > _len) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); if (_chars) MY_STRING_DELETE(_chars); _chars = newBuf; @@ -1484,7 +1701,7 @@ UString2 &UString2::operator=(const UString2 &s) unsigned len = s._len; if (len > _len) { - wchar_t *newBuf = MY_STRING_NEW(wchar_t, len + 1); + wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); if (_chars) MY_STRING_DELETE(_chars); _chars = newBuf; @@ -1524,6 +1741,8 @@ int MyStringCompareNoCase(const char *s1, const char *s2) } */ +#if !defined(USE_UNICODE_FSTRING) || !defined(_UNICODE) + static inline UINT GetCurrentCodePage() { #if defined(UNDER_CE) || !defined(_WIN32) @@ -1533,6 +1752,8 @@ static inline UINT GetCurrentCodePage() #endif } +#endif + #ifdef USE_UNICODE_FSTRING #ifndef _UNICODE @@ -1542,18 +1763,28 @@ AString fs2fas(CFSTR s) return UnicodeStringToMultiByte(s, GetCurrentCodePage()); } +FString fas2fs(const char *s) +{ + return MultiByteToUnicodeString(s, GetCurrentCodePage()); +} + FString fas2fs(const AString &s) { return MultiByteToUnicodeString(s, GetCurrentCodePage()); } -#endif +#endif // _UNICODE -#else +#else // USE_UNICODE_FSTRING + +UString fs2us(const FChar *s) +{ + return MultiByteToUnicodeString(s, GetCurrentCodePage()); +} UString fs2us(const FString &s) { - return MultiByteToUnicodeString((AString)s, GetCurrentCodePage()); + return MultiByteToUnicodeString(s, GetCurrentCodePage()); } FString us2fs(const wchar_t *s) @@ -1561,4 +1792,4 @@ FString us2fs(const wchar_t *s) return UnicodeStringToMultiByte(s, GetCurrentCodePage()); } -#endif +#endif // USE_UNICODE_FSTRING diff --git a/3rdparty/lzma/CPP/Common/MyString.h b/3rdparty/lzma/CPP/Common/MyString.h index 02308555a38..c777c8c3ebc 100644 --- a/3rdparty/lzma/CPP/Common/MyString.h +++ b/3rdparty/lzma/CPP/Common/MyString.h @@ -1,7 +1,7 @@ -// Common/String.h +// Common/MyString.h -#ifndef __COMMON_STRING_H -#define __COMMON_STRING_H +#ifndef __COMMON_MY_STRING_H +#define __COMMON_MY_STRING_H #include @@ -14,6 +14,37 @@ #include "MyTypes.h" #include "MyVector.h" + +/* if (DEBUG_FSTRING_INHERITS_ASTRING is defined), then + FString inherits from AString, so we can find bugs related to FString at compile time. + DON'T define DEBUG_FSTRING_INHERITS_ASTRING in release code */ + +// #define DEBUG_FSTRING_INHERITS_ASTRING + +#ifdef DEBUG_FSTRING_INHERITS_ASTRING +class FString; +#endif + + +#ifdef _MSC_VER + #ifdef _NATIVE_WCHAR_T_DEFINED + #define MY_NATIVE_WCHAR_T_DEFINED + #endif +#else + #define MY_NATIVE_WCHAR_T_DEFINED +#endif + +/* + native support for wchar_t: + _MSC_VER == 1600 : /Zc:wchar_t is not supported + _MSC_VER == 1310 (VS2003) + ? _MSC_VER == 1400 (VS2005) : wchar_t <- unsigned short + /Zc:wchar_t : wchar_t <- __wchar_t, _WCHAR_T_DEFINED and _NATIVE_WCHAR_T_DEFINED + _MSC_VER > 1400 (VS2008+) + /Zc:wchar_t[-] + /Zc:wchar_t is on by default +*/ + #ifdef _WIN32 #define IS_PATH_SEPAR(c) ((c) == '\\' || (c) == '/') #else @@ -60,6 +91,12 @@ inline void MyStringCopy(wchar_t *dest, const wchar_t *src) while ((*dest++ = *src++) != 0); } +inline void MyStringCat(wchar_t *dest, const wchar_t *src) +{ + MyStringCopy(dest + MyStringLen(dest), src); +} + + /* inline wchar_t *MyWcpCpy(wchar_t *dest, const wchar_t *src) { @@ -88,13 +125,15 @@ int FindCharPosInString(const wchar_t *s, wchar_t c) throw(); #define STRING_UNICODE_THROW throw() #endif -/* + inline char MyCharUpper_Ascii(char c) { if (c >= 'a' && c <= 'z') - return (char)(c - 0x20); + return (char)((unsigned char)c - 0x20); return c; } + +/* inline wchar_t MyCharUpper_Ascii(wchar_t c) { if (c >= 'a' && c <= 'z') @@ -131,7 +170,7 @@ inline wchar_t MyCharUpper(wchar_t c) throw() return (wchar_t)MyCharUpper_WIN(c); #endif #else - return (wchar_t)towupper(c); + return (wchar_t)towupper((wint_t)c); #endif } @@ -158,6 +197,7 @@ inline wchar_t MyCharLower(wchar_t c) throw() // char *MyStringUpper(char *s) throw(); // char *MyStringLower(char *s) throw(); +// void MyStringUpper_Ascii(char *s) throw(); // void MyStringUpper_Ascii(wchar_t *s) throw(); void MyStringLower_Ascii(char *s) throw(); void MyStringLower_Ascii(wchar_t *s) throw(); @@ -168,13 +208,18 @@ bool StringsAreEqualNoCase(const wchar_t *s1, const wchar_t *s2) throw(); bool IsString1PrefixedByString2(const char *s1, const char *s2) throw(); bool IsString1PrefixedByString2(const wchar_t *s1, const wchar_t *s2) throw(); +bool IsString1PrefixedByString2(const wchar_t *s1, const char *s2) throw(); +bool IsString1PrefixedByString2_NoCase_Ascii(const char *s1, const char *s2) throw(); +bool IsString1PrefixedByString2_NoCase_Ascii(const wchar_t *u, const char *a) throw(); bool IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) throw(); +#define MyStringCompare(s1, s2) wcscmp(s1, s2) int MyStringCompareNoCase(const wchar_t *s1, const wchar_t *s2) throw(); // int MyStringCompareNoCase_N(const wchar_t *s1, const wchar_t *s2, unsigned num) throw(); // ---------- ASCII ---------- // char values in ASCII strings must be less then 128 +bool StringsAreEqual_Ascii(const char *u, const char *a) throw(); bool StringsAreEqual_Ascii(const wchar_t *u, const char *a) throw(); bool StringsAreEqualNoCase_Ascii(const char *s1, const char *s2) throw(); bool StringsAreEqualNoCase_Ascii(const wchar_t *s1, const char *s2) throw(); @@ -183,6 +228,33 @@ bool StringsAreEqualNoCase_Ascii(const wchar_t *s1, const wchar_t *s2) throw(); #define MY_STRING_DELETE(_p_) delete []_p_; // #define MY_STRING_DELETE(_p_) my_delete(_p_); + +#define FORBID_STRING_OPS_2(cls, t) \ + void Find(t) const; \ + void Find(t, unsigned startIndex) const; \ + void ReverseFind(t) const; \ + void InsertAtFront(t); \ + void RemoveChar(t); \ + void Replace(t, t); \ + +#define FORBID_STRING_OPS(cls, t) \ + explicit cls(t); \ + explicit cls(const t *); \ + cls &operator=(t); \ + cls &operator=(const t *); \ + cls &operator+=(t); \ + cls &operator+=(const t *); \ + FORBID_STRING_OPS_2(cls, t) \ + +/* + cls &operator+(t); \ + cls &operator+(const t *); \ +*/ + +#define FORBID_STRING_OPS_AString(t) FORBID_STRING_OPS(AString, t) +#define FORBID_STRING_OPS_UString(t) FORBID_STRING_OPS(UString, t) +#define FORBID_STRING_OPS_UString2(t) FORBID_STRING_OPS(UString2, t) + class AString { char *_chars; @@ -202,12 +274,12 @@ class AString void Grow_1(); void Grow(unsigned n); - // AString(unsigned num, const char *s); + AString(unsigned num, const char *s); AString(unsigned num, const AString &s); AString(const AString &s, char c); // it's for String + char AString(const char *s1, unsigned num1, const char *s2, unsigned num2); - friend AString operator+(const AString &s, char c) { return AString(s, c); } ; + friend AString operator+(const AString &s, char c) { return AString(s, c); } // friend AString operator+(char c, const AString &s); // is not supported friend AString operator+(const AString &s1, const AString &s2); @@ -215,20 +287,30 @@ class AString friend AString operator+(const char *s1, const AString &s2); // ---------- forbidden functions ---------- - AString &operator+=(wchar_t c); - AString &operator=(wchar_t c); - AString(wchar_t c); - void Find(wchar_t c) const; - void Find(wchar_t c, unsigned startIndex) const; - void ReverseFind(wchar_t c) const; - void InsertAtFront(wchar_t c); - void RemoveChar(wchar_t ch); - void Replace(wchar_t oldChar, wchar_t newChar); + + #ifdef MY_NATIVE_WCHAR_T_DEFINED + FORBID_STRING_OPS_AString(wchar_t) + #endif + + FORBID_STRING_OPS_AString(signed char) + FORBID_STRING_OPS_AString(unsigned char) + FORBID_STRING_OPS_AString(short) + FORBID_STRING_OPS_AString(unsigned short) + FORBID_STRING_OPS_AString(int) + FORBID_STRING_OPS_AString(unsigned) + FORBID_STRING_OPS_AString(long) + FORBID_STRING_OPS_AString(unsigned long) + + #ifdef DEBUG_FSTRING_INHERITS_ASTRING + AString(const FString &s); + AString &operator=(const FString &s); + AString &operator+=(const FString &s); + #endif public: - AString(); - AString(char c); - AString(const char *s); + explicit AString(); + explicit AString(char c); + explicit AString(const char *s); AString(const AString &s); ~AString() { MY_STRING_DELETE(_chars); } @@ -237,13 +319,15 @@ public: void Empty() { _len = 0; _chars[0] = 0; } operator const char *() const { return _chars; } + char *Ptr_non_const() const { return _chars; } const char *Ptr() const { return _chars; } const char *Ptr(unsigned pos) const { return _chars + pos; } const char *RightPtr(unsigned num) const { return _chars + _len - num; } - char Back() const { return _chars[_len - 1]; } + char Back() const { return _chars[(size_t)_len - 1]; } void ReplaceOneCharAtPos(unsigned pos, char c) { _chars[pos] = c; } + char *GetBuf() { return _chars; } /* GetBuf(minLen): provides the buffer that can store at least (minLen) characters and additional null terminator. 9.35: GetBuf doesn't preserve old characters and terminator */ @@ -292,20 +376,23 @@ public: void Add_Space(); void Add_Space_if_NotEmpty(); + void Add_OptSpaced(const char *s); void Add_LF(); + void Add_Slash(); void Add_PathSepar() { operator+=(CHAR_PATH_SEPARATOR); } AString &operator+=(const char *s); AString &operator+=(const AString &s); - void AddAscii(const char *s) { operator+=(s); } + void Add_UInt32(UInt32 v); + void Add_UInt64(UInt64 v); + + void AddFrom(const char *s, unsigned len); // no check void SetFrom(const char *s, unsigned len); // no check void SetFrom_CalcLen(const char *s, unsigned len); - // void SetFromAscii(const char *s) { operator+=(s); } AString Mid(unsigned startIndex, unsigned count) const { return AString(count, _chars + startIndex); } AString Left(unsigned count) const { return AString(count, *this); } - // void MakeUpper() { MyStringUpper(_chars); } // void MakeLower() { MyStringLower(_chars); } void MakeLower_Ascii() { MyStringLower_Ascii(_chars); } @@ -373,8 +460,30 @@ public: _chars[index] = 0; } } + + void Wipe_and_Empty() + { + if (_chars) + { + memset(_chars, 0, (_limit + 1) * sizeof(*_chars)); + _len = 0; + } + } +}; + + +class AString_Wipe: public AString +{ + CLASS_NO_COPY(AString_Wipe) +public: + AString_Wipe(): AString() {} + // AString_Wipe(const AString &s): AString(s) {} + // AString_Wipe &operator=(const AString &s) { AString::operator=(s); return *this; } + // AString_Wipe &operator=(const char *s) { AString::operator=(s); return *this; } + ~AString_Wipe() { Wipe_and_Empty(); } }; + bool operator<(const AString &s1, const AString &s2); bool operator>(const AString &s1, const AString &s2); @@ -435,7 +544,7 @@ class UString UString(const UString &s, wchar_t c); // it's for String + char UString(const wchar_t *s1, unsigned num1, const wchar_t *s2, unsigned num2); - friend UString operator+(const UString &s, wchar_t c) { return UString(s, c); } ; + friend UString operator+(const UString &s, wchar_t c) { return UString(s, c); } // friend UString operator+(wchar_t c, const UString &s); // is not supported friend UString operator+(const UString &s1, const UString &s2); @@ -444,28 +553,33 @@ class UString // ---------- forbidden functions ---------- - UString &operator+=(char c); - UString &operator+=(unsigned char c); - UString &operator=(char c); - UString &operator=(unsigned char c); - UString(char c); - UString(unsigned char c); - void Find(char c) const; - void Find(unsigned char c) const; - void Find(char c, unsigned startIndex) const; - void Find(unsigned char c, unsigned startIndex) const; - void ReverseFind(char c) const; - void ReverseFind(unsigned char c) const; - void InsertAtFront(char c); - void InsertAtFront(unsigned char c); - void RemoveChar(char ch); - void RemoveChar(unsigned char ch); - void Replace(char oldChar, char newChar); - void Replace(unsigned char oldChar, unsigned char newChar); + FORBID_STRING_OPS_UString(signed char) + FORBID_STRING_OPS_UString(unsigned char) + FORBID_STRING_OPS_UString(short) + + #ifdef MY_NATIVE_WCHAR_T_DEFINED + FORBID_STRING_OPS_UString(unsigned short) + #endif + + FORBID_STRING_OPS_UString(int) + FORBID_STRING_OPS_UString(unsigned) + FORBID_STRING_OPS_UString(long) + FORBID_STRING_OPS_UString(unsigned long) + + FORBID_STRING_OPS_2(UString, char) + + #ifdef DEBUG_FSTRING_INHERITS_ASTRING + UString(const FString &s); + UString &operator=(const FString &s); + UString &operator+=(const FString &s); + #endif public: UString(); - UString(wchar_t c); + explicit UString(wchar_t c); + explicit UString(char c); + explicit UString(const char *s); + explicit UString(const AString &s); UString(const wchar_t *s); UString(const UString &s); ~UString() { MY_STRING_DELETE(_chars); } @@ -475,13 +589,16 @@ public: void Empty() { _len = 0; _chars[0] = 0; } operator const wchar_t *() const { return _chars; } + wchar_t *Ptr_non_const() const { return _chars; } const wchar_t *Ptr() const { return _chars; } const wchar_t *Ptr(unsigned pos) const { return _chars + pos; } const wchar_t *RightPtr(unsigned num) const { return _chars + _len - num; } - wchar_t Back() const { return _chars[_len - 1]; } + wchar_t Back() const { return _chars[(size_t)_len - 1]; } void ReplaceOneCharAtPos(unsigned pos, wchar_t c) { _chars[pos] = c; } + wchar_t *GetBuf() { return _chars; } + wchar_t *GetBuf(unsigned minLen) { if (minLen > _limit) @@ -508,9 +625,13 @@ public: } UString &operator=(wchar_t c); + UString &operator=(char c) { return (*this)=((wchar_t)(unsigned char)c); } UString &operator=(const wchar_t *s); UString &operator=(const UString &s); - void SetFromBstr(BSTR s); + void SetFrom(const wchar_t *s, unsigned len); // no check + void SetFromBstr(LPCOLESTR s); + UString &operator=(const char *s); + UString &operator=(const AString &s) { return operator=(s.Ptr()); } UString &operator+=(wchar_t c) { @@ -524,6 +645,8 @@ public: return *this; } + UString &operator+=(char c) { return (*this)+=((wchar_t)(unsigned char)c); } + void Add_Space(); void Add_Space_if_NotEmpty(); void Add_LF(); @@ -531,11 +654,11 @@ public: UString &operator+=(const wchar_t *s); UString &operator+=(const UString &s); + UString &operator+=(const char *s); + UString &operator+=(const AString &s) { return operator+=(s.Ptr()); } - void SetFrom(const wchar_t *s, unsigned len); // no check - - void SetFromAscii(const char *s); - void AddAscii(const char *s); + void Add_UInt32(UInt32 v); + void Add_UInt64(UInt64 v); UString Mid(unsigned startIndex, unsigned count) const { return UString(count, _chars + startIndex); } UString Left(unsigned count) const { return UString(count, *this); } @@ -588,7 +711,7 @@ public: } void InsertAtFront(wchar_t c); - // void Insert(unsigned index, wchar_t c); + // void Insert_wchar_t(unsigned index, wchar_t c); void Insert(unsigned index, const wchar_t *s); void Insert(unsigned index, const UString &s); @@ -609,8 +732,30 @@ public: _chars[index] = 0; } } + + void Wipe_and_Empty() + { + if (_chars) + { + memset(_chars, 0, (_limit + 1) * sizeof(*_chars)); + _len = 0; + } + } +}; + + +class UString_Wipe: public UString +{ + CLASS_NO_COPY(UString_Wipe) +public: + UString_Wipe(): UString() {} + // UString_Wipe(const UString &s): UString(s) {} + // UString_Wipe &operator=(const UString &s) { UString::operator=(s); return *this; } + // UString_Wipe &operator=(const wchar_t *s) { UString::operator=(s); return *this; } + ~UString_Wipe() { Wipe_and_Empty(); } }; + bool operator<(const UString &s1, const UString &s2); bool operator>(const UString &s1, const UString &s2); @@ -630,6 +775,12 @@ void operator==(const UString &s1, wchar_t c2); void operator+(wchar_t c, const UString &s); // this function can be OK, but we don't use it +void operator+(const AString &s1, const UString &s2); +void operator+(const UString &s1, const AString &s2); + +void operator+(const UString &s1, const char *s2); +void operator+(const char *s1, const UString &s2); + void operator+(const UString &s, char c); void operator+(const UString &s, unsigned char c); void operator+(char c, const UString &s); @@ -662,15 +813,25 @@ class UString2 // ---------- forbidden functions ---------- - UString2 &operator=(char c); - UString2 &operator=(unsigned char c); + FORBID_STRING_OPS_UString2(char) + FORBID_STRING_OPS_UString2(signed char) + FORBID_STRING_OPS_UString2(unsigned char) + FORBID_STRING_OPS_UString2(short) + UString2 &operator=(wchar_t c); - UString2(char c); - UString2(unsigned char c); + + UString2(const AString &s); + UString2 &operator=(const AString &s); + UString2 &operator+=(const AString &s); + + #ifdef DEBUG_FSTRING_INHERITS_ASTRING + UString2(const FString &s); + UString2 &operator=(const FString &s); + UString2 &operator+=(const FString &s); + #endif public: UString2(): _chars(NULL), _len(0) {} - // UString2(wchar_t c); UString2(const wchar_t *s); UString2(const UString2 &s); ~UString2() { if (_chars) MY_STRING_DELETE(_chars); } @@ -682,6 +843,8 @@ public: // operator const wchar_t *() const { return _chars; } const wchar_t *GetRawPtr() const { return _chars; } + int Compare(const wchar_t *s) const { return wcscmp(_chars, s); } + wchar_t *GetBuf(unsigned minLen) { if (!_chars || minLen > _len) @@ -741,9 +904,11 @@ typedef CObjectVector CSysStringVector; // ---------- FString ---------- +#ifndef DEBUG_FSTRING_INHERITS_ASTRING #ifdef _WIN32 #define USE_UNICODE_FSTRING #endif +#endif #ifdef USE_UNICODE_FSTRING @@ -754,6 +919,7 @@ typedef CObjectVector CSysStringVector; #define fs2us(_x_) (_x_) #define us2fs(_x_) (_x_) + FString fas2fs(const char *s); FString fas2fs(const AString &s); AString fs2fas(const FChar *s); @@ -762,11 +928,55 @@ typedef CObjectVector CSysStringVector; #define __FTEXT(quote) quote typedef char FChar; + + #ifdef DEBUG_FSTRING_INHERITS_ASTRING + + class FString: public AString + { + // FString &operator=(const char *s); + FString &operator=(const AString &s); + // FString &operator+=(const AString &s); + public: + FString(const AString &s): AString(s.Ptr()) {} + FString(const FString &s): AString(s.Ptr()) {} + FString(const char *s): AString(s) {} + FString() {} + FString &operator=(const FString &s) { AString::operator=((const AString &)s); return *this; } + FString &operator=(char c) { AString::operator=(c); return *this; } + FString &operator+=(char c) { AString::operator+=(c); return *this; } + FString &operator+=(const FString &s) { AString::operator+=((const AString &)s); return *this; } + FString Left(unsigned count) const { return FString(AString::Left(count)); } + }; + void operator+(const AString &s1, const FString &s2); + void operator+(const FString &s1, const AString &s2); + + inline FString operator+(const FString &s1, const FString &s2) + { + AString s =(const AString &)s1 + (const AString &)s2; + return FString(s.Ptr()); + // return FString((const AString &)s1 + (const AString &)s2); + } + inline FString operator+(const FString &s1, const FChar *s2) + { + return s1 + (FString)s2; + } + /* + inline FString operator+(const FChar *s1, const FString &s2) + { + return (FString)s1 + s2; + } + */ + + inline FString fas2fs(const char *s) { return FString(s); } + + #else // DEBUG_FSTRING_INHERITS_ASTRING typedef AString FString; + #define fas2fs(_x_) (_x_) + #endif // DEBUG_FSTRING_INHERITS_ASTRING + UString fs2us(const FChar *s); UString fs2us(const FString &s); FString us2fs(const wchar_t *s); - #define fas2fs(_x_) (_x_) #define fs2fas(_x_) (_x_) #endif @@ -775,10 +985,29 @@ typedef CObjectVector CSysStringVector; #define FCHAR_PATH_SEPARATOR FTEXT(CHAR_PATH_SEPARATOR) #define FSTRING_PATH_SEPARATOR FTEXT(STRING_PATH_SEPARATOR) -#define FCHAR_ANY_MASK FTEXT('*') -#define FSTRING_ANY_MASK FTEXT("*") + +// #define FCHAR_ANY_MASK FTEXT('*') +// #define FSTRING_ANY_MASK FTEXT("*") + typedef const FChar *CFSTR; typedef CObjectVector FStringVector; #endif + + + +#if defined(_WIN32) + // #include + // WCHAR_MAX is defined as ((wchar_t)-1) + #define _WCHART_IS_16BIT 1 +#elif (defined(WCHAR_MAX) && (WCHAR_MAX <= 0xffff)) \ + || (defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ == 2)) + #define _WCHART_IS_16BIT 1 +#endif + +#if WCHAR_PATH_SEPARATOR == L'\\' +// WSL scheme +#define WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT ((wchar_t)((unsigned)(0xF000) + (unsigned)'\\')) +// #define WCHAR_IN_FILE_NAME_BACKSLASH_REPLACEMENT '_' +#endif diff --git a/3rdparty/lzma/CPP/Common/MyTypes.h b/3rdparty/lzma/CPP/Common/MyTypes.h index 75806f37dcf..71b8e7fa199 100644 --- a/3rdparty/lzma/CPP/Common/MyTypes.h +++ b/3rdparty/lzma/CPP/Common/MyTypes.h @@ -25,6 +25,12 @@ struct CBoolPair Val = true; Def = true; } + + void SetVal_as_Defined(bool val) + { + Val = val; + Def = true; + } }; #define CLASS_NO_COPY(cls) \ @@ -32,4 +38,15 @@ struct CBoolPair cls(const cls &); \ cls &operator=(const cls &); +class CUncopyable +{ +protected: + CUncopyable() {} // allow constructor + // ~CUncopyable() {} +CLASS_NO_COPY(CUncopyable) +}; + +#define MY_UNCOPYABLE :private CUncopyable +// #define MY_UNCOPYABLE + #endif diff --git a/3rdparty/lzma/CPP/Common/MyVector.h b/3rdparty/lzma/CPP/Common/MyVector.h index 14f7b11a1a7..3417a176bec 100644 --- a/3rdparty/lzma/CPP/Common/MyVector.h +++ b/3rdparty/lzma/CPP/Common/MyVector.h @@ -5,6 +5,8 @@ #include +const unsigned k_VectorSizeMax = ((unsigned)1 << 31) - 1; + template class CRecordVector { @@ -17,29 +19,41 @@ class CRecordVector memmove(_items + destIndex, _items + srcIndex, (size_t)(_size - srcIndex) * sizeof(T)); } - void ReserveOnePosition() + void ReAllocForNewCapacity(const unsigned newCapacity) { - if (_size == _capacity) - { - unsigned newCapacity = _capacity + (_capacity >> 2) + 1; - T *p = new T[newCapacity]; - if (_size != 0) - memcpy(p, _items, (size_t)_size * sizeof(T)); - delete []_items; - _items = p; - _capacity = newCapacity; - } + T *p; + MY_ARRAY_NEW(p, T, newCapacity); + // p = new T[newCapacity]; + if (_size != 0) + memcpy(p, _items, (size_t)_size * sizeof(T)); + delete []_items; + _items = p; + _capacity = newCapacity; } public: - CRecordVector(): _items(0), _size(0), _capacity(0) {} + void ReserveOnePosition() + { + if (_size != _capacity) + return; + if (_capacity >= k_VectorSizeMax) + throw 2021; + const unsigned rem = k_VectorSizeMax - _capacity; + unsigned add = (_capacity >> 2) + 1; + if (add > rem) + add = rem; + ReAllocForNewCapacity(_capacity + add); + } + + CRecordVector(): _items(NULL), _size(0), _capacity(0) {} - CRecordVector(const CRecordVector &v): _items(0), _size(0), _capacity(0) + CRecordVector(const CRecordVector &v): _items(NULL), _size(0), _capacity(0) { - unsigned size = v.Size(); + const unsigned size = v.Size(); if (size != 0) { + // MY_ARRAY_NEW(_items, T, size) _items = new T[size]; _size = size; _capacity = size; @@ -54,7 +68,8 @@ public: { if (size != 0) { - _items = new T[size]; + MY_ARRAY_NEW(_items, T, size) + // _items = new T[size]; _capacity = size; } } @@ -63,24 +78,30 @@ public: { if (newCapacity > _capacity) { - T *p = new T[newCapacity]; - if (_size != 0) - memcpy(p, _items, (size_t)_size * sizeof(T)); - delete []_items; - _items = p; - _capacity = newCapacity; + if (newCapacity > k_VectorSizeMax) + throw 2021; + ReAllocForNewCapacity(newCapacity); } } + void ChangeSize_KeepData(unsigned newSize) + { + Reserve(newSize); + _size = newSize; + } + void ClearAndReserve(unsigned newCapacity) { Clear(); if (newCapacity > _capacity) { + if (newCapacity > k_VectorSizeMax) + throw 2021; delete []_items; _items = NULL; _capacity = 0; - _items = new T[newCapacity]; + MY_ARRAY_NEW(_items, T, newCapacity) + // _items = new T[newCapacity]; _capacity = newCapacity; } } @@ -91,20 +112,6 @@ public: _size = newSize; } - void ChangeSize_KeepData(unsigned newSize) - { - if (newSize > _capacity) - { - T *p = new T[newSize]; - if (_size != 0) - memcpy(p, _items, (size_t)_size * sizeof(T)); - delete []_items; - _items = p; - _capacity = newSize; - } - _size = newSize; - } - void ReserveDown() { if (_size == _capacity) @@ -112,6 +119,7 @@ public: T *p = NULL; if (_size != 0) { + // MY_ARRAY_NEW(p, T, _size) p = new T[_size]; memcpy(p, _items, (size_t)_size * sizeof(T)); } @@ -170,7 +178,7 @@ public: { if (&v == this) return *this; - unsigned size = v.Size(); + const unsigned size = v.Size(); if (size > _capacity) { delete []_items; @@ -188,24 +196,45 @@ public: CRecordVector& operator+=(const CRecordVector &v) { - unsigned size = v.Size(); - Reserve(_size + size); + const unsigned size = v.Size(); if (size != 0) + { + if (_size >= k_VectorSizeMax || size > k_VectorSizeMax - _size) + throw 2021; + const unsigned newSize = _size + size; + Reserve(newSize); memcpy(_items + _size, v._items, (size_t)size * sizeof(T)); - _size += size; + _size = newSize; + } return *this; } unsigned Add(const T item) { ReserveOnePosition(); - _items[_size] = item; - return _size++; + const unsigned size = _size; + _size = size + 1; + _items[size] = item; + return size; } - void AddInReserved(const T item) + /* + unsigned Add2(const T &item) { - _items[_size++] = item; + ReserveOnePosition(); + const unsigned size = _size; + _size = size + 1; + _items[size] = item; + return size; + } + */ + + unsigned AddInReserved(const T item) + { + const unsigned size = _size; + _size = size + 1; + _items[size] = item; + return size; } void Insert(unsigned index, const T item) @@ -216,6 +245,13 @@ public: _size++; } + void InsertInReserved(unsigned index, const T item) + { + MoveItems(index + 1, index); + _items[index] = item; + _size++; + } + void MoveToFront(unsigned index) { if (index != 0) @@ -230,8 +266,8 @@ public: T& operator[](unsigned index) { return _items[index]; } const T& Front() const { return _items[0]; } T& Front() { return _items[0]; } - const T& Back() const { return _items[_size - 1]; } - T& Back() { return _items[_size - 1]; } + const T& Back() const { return _items[(size_t)_size - 1]; } + T& Back() { return _items[(size_t)_size - 1]; } /* void Swap(unsigned i, unsigned j) @@ -246,10 +282,11 @@ public: { while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T midVal = (*this)[mid]; if (item == midVal) - return mid; + return (int)mid; if (item < midVal) right = mid; else @@ -262,11 +299,12 @@ public: { while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T& midVal = (*this)[mid]; - int comp = item.Compare(midVal); + const int comp = item.Compare(midVal); if (comp == 0) - return mid; + return (int)mid; if (comp < 0) right = mid; else @@ -290,7 +328,8 @@ public: unsigned left = 0, right = _size; while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T midVal = (*this)[mid]; if (item == midVal) return mid; @@ -308,9 +347,10 @@ public: unsigned left = 0, right = _size; while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T& midVal = (*this)[mid]; - int comp = item.Compare(midVal); + const int comp = item.Compare(midVal); if (comp == 0) return mid; if (comp < 0) @@ -370,7 +410,7 @@ public: unsigned s = (k << 1); if (s > size) break; - if (s < size && p[s + 1].Compare(p[s]) > 0) + if (s < size && p[(size_t)s + 1].Compare(p[s]) > 0) s++; if (temp.Compare(p[s]) >= 0) break; @@ -420,32 +460,38 @@ public: // void Reserve(unsigned newCapacity) { _v.Reserve(newCapacity); } void ClearAndReserve(unsigned newCapacity) { Clear(); _v.ClearAndReserve(newCapacity); } - CObjectVector() {}; + CObjectVector() {} CObjectVector(const CObjectVector &v) { - unsigned size = v.Size(); + const unsigned size = v.Size(); _v.ConstructReserve(size); for (unsigned i = 0; i < size; i++) - _v.AddInReserved(new T(v[i])); + AddInReserved(v[i]); } CObjectVector& operator=(const CObjectVector &v) { if (&v == this) return *this; Clear(); - unsigned size = v.Size(); + const unsigned size = v.Size(); _v.Reserve(size); for (unsigned i = 0; i < size; i++) - _v.AddInReserved(new T(v[i])); + AddInReserved(v[i]); return *this; } CObjectVector& operator+=(const CObjectVector &v) { - unsigned size = v.Size(); - _v.Reserve(Size() + size); - for (unsigned i = 0; i < size; i++) - _v.AddInReserved(new T(v[i])); + const unsigned addSize = v.Size(); + if (addSize != 0) + { + const unsigned size = Size(); + if (size >= k_VectorSizeMax || addSize > k_VectorSizeMax - size) + throw 2021; + _v.Reserve(size + addSize); + for (unsigned i = 0; i < addSize; i++) + AddInReserved(v[i]); + } return *this; } @@ -453,19 +499,42 @@ public: T& operator[](unsigned index) { return *((T *)_v[index]); } const T& Front() const { return operator[](0); } T& Front() { return operator[](0); } - const T& Back() const { return operator[](_v.Size() - 1); } - T& Back() { return operator[](_v.Size() - 1); } + const T& Back() const { return *(T *)_v.Back(); } + T& Back() { return *(T *)_v.Back(); } void MoveToFront(unsigned index) { _v.MoveToFront(index); } - unsigned Add(const T& item) { return _v.Add(new T(item)); } + unsigned Add(const T& item) + { + _v.ReserveOnePosition(); + return AddInReserved(item); + } + + unsigned AddInReserved(const T& item) + { + return _v.AddInReserved(new T(item)); + } + + void ReserveOnePosition() + { + _v.ReserveOnePosition(); + } + + unsigned AddInReserved_Ptr_of_new(T *ptr) + { + return _v.AddInReserved(ptr); + } + + #define VECTOR_ADD_NEW_OBJECT(v, a) \ + (v).ReserveOnePosition(); \ + (v).AddInReserved_Ptr_of_new(new a); - void AddInReserved(const T& item) { _v.AddInReserved(new T(item)); } T& AddNew() { + _v.ReserveOnePosition(); T *p = new T; - _v.Add(p); + _v.AddInReserved(p); return *p; } @@ -476,12 +545,17 @@ public: return *p; } - void Insert(unsigned index, const T& item) { _v.Insert(index, new T(item)); } + void Insert(unsigned index, const T& item) + { + _v.ReserveOnePosition(); + _v.InsertInReserved(index, new T(item)); + } T& InsertNew(unsigned index) { + _v.ReserveOnePosition(); T *p = new T; - _v.Insert(index, p); + _v.InsertInReserved(index, p); return *p; } @@ -506,7 +580,7 @@ public: void DeleteFrom(unsigned index) { - unsigned size = _v.Size(); + const unsigned size = _v.Size(); for (unsigned i = index; i < size; i++) delete (T *)_v[i]; _v.DeleteFrom(index); @@ -521,7 +595,7 @@ public: void DeleteBack() { - delete (T *)_v[_v.Size() - 1]; + delete (T *)_v.Back(); _v.DeleteBack(); } @@ -556,11 +630,12 @@ public: unsigned left = 0, right = Size(); while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T& midVal = (*this)[mid]; - int comp = item.Compare(midVal); + const int comp = item.Compare(midVal); if (comp == 0) - return mid; + return (int)mid; if (comp < 0) right = mid; else @@ -574,9 +649,10 @@ public: unsigned left = 0, right = Size(); while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T& midVal = (*this)[mid]; - int comp = item.Compare(midVal); + const int comp = item.Compare(midVal); if (comp == 0) return mid; if (comp < 0) @@ -594,9 +670,10 @@ public: unsigned left = 0, right = Size(); while (left != right) { - unsigned mid = (left + right) / 2; + // const unsigned mid = (unsigned)(((size_t)left + (size_t)right) / 2); + const unsigned mid = (left + right) / 2; const T& midVal = (*this)[mid]; - int comp = item.Compare(midVal); + const int comp = item.Compare(midVal); if (comp == 0) { right = mid + 1; @@ -616,9 +693,9 @@ public: { _v.Sort(compare, param); } static int CompareObjectItems(void *const *a1, void *const *a2, void * /* param */) - { return (*(*((const T **)a1))).Compare(*(*((const T **)a2))); } + { return (*(*((const T *const *)a1))).Compare(*(*((const T *const *)a2))); } - void Sort() { _v.Sort(CompareObjectItems, 0); } + void Sort() { _v.Sort(CompareObjectItems, NULL); } }; #define FOR_VECTOR(_i_, _v_) for (unsigned _i_ = 0; _i_ < (_v_).Size(); _i_++) diff --git a/3rdparty/lzma/CPP/Common/MyWindows.cpp b/3rdparty/lzma/CPP/Common/MyWindows.cpp index 463c77c4a37..88f312fb4ce 100644 --- a/3rdparty/lzma/CPP/Common/MyWindows.cpp +++ b/3rdparty/lzma/CPP/Common/MyWindows.cpp @@ -5,6 +5,10 @@ #ifndef _WIN32 #include +#include +#ifdef __GNUC__ +#include +#endif #include "MyWindows.h" @@ -38,11 +42,11 @@ BSTR SysAllocStringByteLen(LPCSTR s, UINT len) /* Original SysAllocStringByteLen in Win32 maybe fills only unaligned null OLECHAR at the end. We provide also aligned null OLECHAR at the end. */ - if (len >= (k_BstrSize_Max - sizeof(OLECHAR) - sizeof(OLECHAR) - sizeof(CBstrSizeType))) + if (len >= (k_BstrSize_Max - (UINT)sizeof(OLECHAR) - (UINT)sizeof(OLECHAR) - (UINT)sizeof(CBstrSizeType))) return NULL; - UINT size = (len + sizeof(OLECHAR) + sizeof(OLECHAR) - 1) & ~(sizeof(OLECHAR) - 1); - void *p = AllocateForBSTR(size + sizeof(CBstrSizeType)); + UINT size = (len + (UINT)sizeof(OLECHAR) + (UINT)sizeof(OLECHAR) - 1) & ~((UINT)sizeof(OLECHAR) - 1); + void *p = AllocateForBSTR(size + (UINT)sizeof(CBstrSizeType)); if (!p) return NULL; *(CBstrSizeType *)p = (CBstrSizeType)len; @@ -56,11 +60,11 @@ BSTR SysAllocStringByteLen(LPCSTR s, UINT len) BSTR SysAllocStringLen(const OLECHAR *s, UINT len) { - if (len >= (k_BstrSize_Max - sizeof(OLECHAR) - sizeof(CBstrSizeType)) / sizeof(OLECHAR)) + if (len >= (k_BstrSize_Max - (UINT)sizeof(OLECHAR) - (UINT)sizeof(CBstrSizeType)) / (UINT)sizeof(OLECHAR)) return NULL; - UINT size = len * sizeof(OLECHAR); - void *p = AllocateForBSTR(size + sizeof(CBstrSizeType) + sizeof(OLECHAR)); + UINT size = len * (UINT)sizeof(OLECHAR); + void *p = AllocateForBSTR(size + (UINT)sizeof(CBstrSizeType) + (UINT)sizeof(OLECHAR)); if (!p) return NULL; *(CBstrSizeType *)p = (CBstrSizeType)size; @@ -98,7 +102,7 @@ UINT SysStringLen(BSTR bstr) { if (!bstr) return 0; - return *((CBstrSizeType *)bstr - 1) / sizeof(OLECHAR); + return *((CBstrSizeType *)bstr - 1) / (UINT)sizeof(OLECHAR); } @@ -139,7 +143,150 @@ LONG CompareFileTime(const FILETIME* ft1, const FILETIME* ft2) DWORD GetLastError() { - return 0; + return (DWORD)errno; +} + +void SetLastError(DWORD dw) +{ + errno = (int)dw; +} + + +static LONG TIME_GetBias() +{ + time_t utc = time(NULL); + struct tm *ptm = localtime(&utc); + int localdaylight = ptm->tm_isdst; /* daylight for local timezone */ + ptm = gmtime(&utc); + ptm->tm_isdst = localdaylight; /* use local daylight, not that of Greenwich */ + LONG bias = (int)(mktime(ptm)-utc); + return bias; +} + +#define TICKS_PER_SEC 10000000 +/* +#define SECS_PER_DAY (24 * 60 * 60) +#define SECS_1601_TO_1970 ((369 * 365 + 89) * (UInt64)SECS_PER_DAY) +#define TICKS_1601_TO_1970 (SECS_1601_TO_1970 * TICKS_PER_SEC) +*/ + +#define GET_TIME_64(pft) ((pft)->dwLowDateTime | ((UInt64)(pft)->dwHighDateTime << 32)) + +#define SET_FILETIME(ft, v64) \ + (ft)->dwLowDateTime = (DWORD)v64; \ + (ft)->dwHighDateTime = (DWORD)(v64 >> 32); + + +BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *localFileTime) +{ + UInt64 v = GET_TIME_64(fileTime); + v = (UInt64)((Int64)v - (Int64)TIME_GetBias() * TICKS_PER_SEC); + SET_FILETIME(localFileTime, v); + return TRUE; +} + +BOOL WINAPI LocalFileTimeToFileTime(const FILETIME *localFileTime, FILETIME *fileTime) +{ + UInt64 v = GET_TIME_64(localFileTime); + v = (UInt64)((Int64)v + (Int64)TIME_GetBias() * TICKS_PER_SEC); + SET_FILETIME(fileTime, v); + return TRUE; +} + +/* +VOID WINAPI GetSystemTimeAsFileTime(FILETIME *ft) +{ + UInt64 t = 0; + timeval tv; + if (gettimeofday(&tv, NULL) == 0) + { + t = tv.tv_sec * (UInt64)TICKS_PER_SEC + TICKS_1601_TO_1970; + t += tv.tv_usec * 10; + } + SET_FILETIME(ft, t); +} +*/ + +DWORD WINAPI GetTickCount(VOID) +{ + #ifndef _WIN32 + // gettimeofday() doesn't work in some MINGWs by unknown reason + timeval tv; + if (gettimeofday(&tv, NULL) == 0) + { + // tv_sec and tv_usec are (long) + return (DWORD)((UInt64)(Int64)tv.tv_sec * (UInt64)1000 + (UInt64)(Int64)tv.tv_usec / 1000); + } + #endif + return (DWORD)time(NULL) * 1000; +} + + +#define PERIOD_4 (4 * 365 + 1) +#define PERIOD_100 (PERIOD_4 * 25 - 1) +#define PERIOD_400 (PERIOD_100 * 4 + 1) + +BOOL WINAPI FileTimeToSystemTime(const FILETIME *ft, SYSTEMTIME *st) +{ + UInt32 v; + UInt64 v64 = GET_TIME_64(ft); + v64 /= 10000; + st->wMilliseconds = (WORD)(v64 % 1000); v64 /= 1000; + st->wSecond = (WORD)(v64 % 60); v64 /= 60; + st->wMinute = (WORD)(v64 % 60); v64 /= 60; + v = (UInt32)v64; + st->wHour = (WORD)(v % 24); v /= 24; + + // 1601-01-01 was Monday + st->wDayOfWeek = (WORD)((v + 1) % 7); + + UInt32 leaps, year, day, mon; + leaps = (3 * ((4 * v + (365 - 31 - 28) * 4 + 3) / PERIOD_400) + 3) / 4; + v += 28188 + leaps; + // leaps - the number of exceptions from PERIOD_4 rules starting from 1600-03-01 + // (1959 / 64) - converts day from 03-01 to month + year = (20 * v - 2442) / (5 * PERIOD_4); + day = v - (year * PERIOD_4) / 4; + mon = (64 * day) / 1959; + st->wDay = (WORD)(day - (1959 * mon) / 64); + mon -= 1; + year += 1524; + if (mon > 12) + { + mon -= 12; + year++; + } + st->wMonth = (WORD)mon; + st->wYear = (WORD)year; + + /* + unsigned year, mon; + unsigned char ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + unsigned t; + + year = (WORD)(1601 + v / PERIOD_400 * 400); + v %= PERIOD_400; + + t = v / PERIOD_100; if (t == 4) t = 3; year += t * 100; v -= t * PERIOD_100; + t = v / PERIOD_4; if (t == 25) t = 24; year += t * 4; v -= t * PERIOD_4; + t = v / 365; if (t == 4) t = 3; year += t; v -= t * 365; + + st->wYear = (WORD)year; + + if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) + ms[1] = 29; + for (mon = 0;; mon++) + { + unsigned d = ms[mon]; + if (v < d) + break; + v -= d; + } + st->wDay = (WORD)(v + 1); + st->wMonth = (WORD)(mon + 1); + */ + + return TRUE; } #endif diff --git a/3rdparty/lzma/CPP/Common/MyWindows.h b/3rdparty/lzma/CPP/Common/MyWindows.h index 139a4e8b976..15db52434f7 100644 --- a/3rdparty/lzma/CPP/Common/MyWindows.h +++ b/3rdparty/lzma/CPP/Common/MyWindows.h @@ -5,21 +5,24 @@ #ifdef _WIN32 -#include +#include #ifdef UNDER_CE #undef VARIANT_TRUE #define VARIANT_TRUE ((VARIANT_BOOL)-1) #endif -#else +#else // _WIN32 #include // for wchar_t #include +// #include // for uintptr_t #include "MyGuidDef.h" +// WINAPI is __stdcall in Windows-MSVC in windef.h #define WINAPI +#define EXTERN_C MY_EXTERN_C typedef char CHAR; typedef unsigned char UCHAR; @@ -34,15 +37,22 @@ typedef unsigned short USHORT; typedef unsigned short WORD; typedef short VARIANT_BOOL; -typedef int INT; -typedef Int32 INT32; -typedef unsigned int UINT; -typedef UInt32 UINT32; -typedef INT32 LONG; // LONG, ULONG and DWORD must be 32-bit -typedef UINT32 ULONG; +#define LOWORD(l) ((WORD)((DWORD_PTR)(l) & 0xffff)) +#define HIWORD(l) ((WORD)((DWORD_PTR)(l) >> 16)) + +// MS uses long for BOOL, but long is 32-bit in MS. So we use int. +// typedef long BOOL; +typedef int BOOL; + +#ifndef FALSE + #define FALSE 0 + #define TRUE 1 +#endif -#undef DWORD -typedef UINT32 DWORD; +// typedef size_t ULONG_PTR; +// typedef size_t DWORD_PTR; +// typedef uintptr_t UINT_PTR; +// typedef ptrdiff_t UINT_PTR; typedef Int64 LONGLONG; typedef UInt64 ULONGLONG; @@ -66,27 +76,33 @@ typedef struct _FILETIME DWORD dwHighDateTime; } FILETIME; -#define HRESULT LONG -#define FAILED(Status) ((HRESULT)(Status)<0) +#define SUCCEEDED(hr) ((HRESULT)(hr) >= 0) +#define FAILED(hr) ((HRESULT)(hr) < 0) typedef ULONG PROPID; typedef LONG SCODE; + #define S_OK ((HRESULT)0x00000000L) #define S_FALSE ((HRESULT)0x00000001L) -#define E_NOTIMPL ((HRESULT)0x80004001L) +#define E_NOTIMPL ((HRESULT)0x80004001L) #define E_NOINTERFACE ((HRESULT)0x80004002L) -#define E_ABORT ((HRESULT)0x80004004L) -#define E_FAIL ((HRESULT)0x80004005L) -#define STG_E_INVALIDFUNCTION ((HRESULT)0x80030001L) -#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) -#define E_INVALIDARG ((HRESULT)0x80070057L) +#define E_ABORT ((HRESULT)0x80004004L) +#define E_FAIL ((HRESULT)0x80004005L) +#define STG_E_INVALIDFUNCTION ((HRESULT)0x80030001L) +#define CLASS_E_CLASSNOTAVAILABLE ((HRESULT)0x80040111L) + #ifdef _MSC_VER #define STDMETHODCALLTYPE __stdcall +#define STDAPICALLTYPE __stdcall #else +// do we need __export here? #define STDMETHODCALLTYPE +#define STDAPICALLTYPE #endif +#define STDAPI EXTERN_C HRESULT STDAPICALLTYPE + #define STDMETHOD_(t, f) virtual t STDMETHODCALLTYPE f #define STDMETHOD(f) STDMETHOD_(HRESULT, f) #define STDMETHODIMP_(type) type STDMETHODCALLTYPE @@ -105,9 +121,8 @@ struct IUnknown STDMETHOD(QueryInterface) (REFIID iid, void **outObject) PURE; STDMETHOD_(ULONG, AddRef)() PURE; STDMETHOD_(ULONG, Release)() PURE; - #ifndef _WIN32 virtual ~IUnknown() {} - #endif + // We use virtual ~IUnknown() here for binary compatibility with 7z.so from p7zip }; typedef IUnknown *LPUNKNOWN; @@ -134,6 +149,7 @@ enum VARENUM VT_VARIANT = 12, VT_UNKNOWN = 13, VT_DECIMAL = 14, + VT_I1 = 16, VT_UI1 = 17, VT_UI2 = 18, @@ -199,8 +215,14 @@ MY_EXTERN_C UINT SysStringByteLen(BSTR bstr); MY_EXTERN_C UINT SysStringLen(BSTR bstr); MY_EXTERN_C DWORD GetLastError(); +MY_EXTERN_C void SetLastError(DWORD dwCode); MY_EXTERN_C LONG CompareFileTime(const FILETIME* ft1, const FILETIME* ft2); +MY_EXTERN_C DWORD GetCurrentThreadId(); +MY_EXTERN_C DWORD GetCurrentProcessId(); + +#define MAX_PATH 1024 + #define CP_ACP 0 #define CP_OEMCP 1 #define CP_UTF8 65001 @@ -212,5 +234,35 @@ typedef enum tagSTREAM_SEEK STREAM_SEEK_END = 2 } STREAM_SEEK; -#endif + + +typedef struct _SYSTEMTIME +{ + WORD wYear; + WORD wMonth; + WORD wDayOfWeek; + WORD wDay; + WORD wHour; + WORD wMinute; + WORD wSecond; + WORD wMilliseconds; +} SYSTEMTIME; + +BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *localFileTime); +BOOL WINAPI LocalFileTimeToFileTime(const FILETIME *localFileTime, FILETIME *fileTime); +BOOL WINAPI FileTimeToSystemTime(const FILETIME *fileTime, SYSTEMTIME *systemTime); +// VOID WINAPI GetSystemTimeAsFileTime(FILETIME *systemTimeAsFileTime); + +DWORD GetTickCount(); + + +#define CREATE_NEW 1 +#define CREATE_ALWAYS 2 +#define OPEN_EXISTING 3 +#define OPEN_ALWAYS 4 +#define TRUNCATE_EXISTING 5 + + +#endif // _WIN32 + #endif diff --git a/3rdparty/lzma/CPP/Common/NewHandler.cpp b/3rdparty/lzma/CPP/Common/NewHandler.cpp index 522c2a75390..65ef41d4f5c 100644 --- a/3rdparty/lzma/CPP/Common/NewHandler.cpp +++ b/3rdparty/lzma/CPP/Common/NewHandler.cpp @@ -10,7 +10,7 @@ #ifndef DEBUG_MEMORY_LEAK -#ifdef _WIN32 +#ifdef _7ZIP_REDEFINE_OPERATOR_NEW /* void * my_new(size_t size) @@ -97,19 +97,33 @@ const int kDebugSize = 1000000; static void *a[kDebugSize]; static int index = 0; +static bool wasInit = false; +static CRITICAL_SECTION cs; + static int numAllocs = 0; void * __cdecl operator new(size_t size) { + if (!wasInit) + { + InitializeCriticalSection(&cs); + wasInit = true; + } + EnterCriticalSection(&cs); + numAllocs++; + int loc = numAllocs; void *p = HeapAlloc(GetProcessHeap(), 0, size); + /* if (index < kDebugSize) { a[index] = p; index++; } + */ + printf("Alloc %6d, size = %8u\n", loc, (unsigned)size); + LeaveCriticalSection(&cs); if (p == 0) throw CNewException(); - printf("Alloc %6d, size = %8u\n", numAllocs, (unsigned)size); return p; } @@ -123,6 +137,7 @@ public: } ~CC() { + printf("\nDestructor: %d\n", numAllocs); for (int i = 0; i < kDebugSize; i++) if (a[i] != 0) return; @@ -134,6 +149,7 @@ void __cdecl operator delete(void *p) { if (p == 0) return; + EnterCriticalSection(&cs); /* for (int i = 0; i < index; i++) if (a[i] == p) @@ -142,6 +158,7 @@ void __cdecl operator delete(void *p) HeapFree(GetProcessHeap(), 0, p); numAllocs--; printf("Free %d\n", numAllocs); + LeaveCriticalSection(&cs); } #endif diff --git a/3rdparty/lzma/CPP/Common/NewHandler.h b/3rdparty/lzma/CPP/Common/NewHandler.h index dcad56886db..aedeca64e30 100644 --- a/3rdparty/lzma/CPP/Common/NewHandler.h +++ b/3rdparty/lzma/CPP/Common/NewHandler.h @@ -4,28 +4,29 @@ #define __COMMON_NEW_HANDLER_H /* -This file must be included before any code that uses operators "delete" or "new". -Also you must compile and link "NewHandler.cpp", if you use MSVC 6.0. -The operator "new" in MSVC 6.0 doesn't throw exception "bad_alloc". -So we define another version of operator "new" that throws "CNewException" on failure. +NewHandler.h and NewHandler.cpp allows to solve problem with compilers that +don't throw exception in operator new(). -If you use compiler that throws exception in "new" operator (GCC or new version of MSVC), -you can compile without "NewHandler.cpp". So standard exception "bad_alloc" will be used. +This file must be included before any code that uses operators new() or delete() +and you must compile and link "NewHandler.cpp", if you use some old MSVC compiler. -It's still allowed to use redefined version of operator "new" from "NewHandler.cpp" -with any compiler. 7-Zip's code can work with "bad_alloc" and "CNewException" exceptions. -But if you use some additional code (outside of 7-Zip's code), you must check -that redefined version of operator "new" (that throws CNewException) is not -problem for your code. +The operator new() in some MSVC versions doesn't throw exception std::bad_alloc. +MSVC 6.0 (_MSC_VER == 1200) doesn't throw exception. +The code produced by some another MSVC compilers also can be linked +to library that doesn't throw exception. +We suppose that code compiled with VS2015+ (_MSC_VER >= 1900) throws exception std::bad_alloc. +For older _MSC_VER versions we redefine operator new() and operator delete(). +Our version of operator new() throws CNewException() exception on failure. -Also we declare delete(void *p) throw() that creates smaller code. +It's still allowed to use redefined version of operator new() from "NewHandler.cpp" +with any compiler. 7-Zip's code can work with std::bad_alloc and CNewException() exceptions. +But if you use some additional code (outside of 7-Zip's code), you must check +that redefined version of operator new() is not problem for your code. */ #include -class CNewException {}; - -#ifdef WIN32 +#ifdef _WIN32 // We can compile my_new and my_delete with _fastcall /* void * my_new(size_t size); @@ -34,7 +35,19 @@ void my_delete(void *p) throw(); */ #endif -#ifdef _WIN32 + +#if defined(_MSC_VER) && (_MSC_VER < 1900) + // If you want to use default operator new(), you can disable the following line + #define _7ZIP_REDEFINE_OPERATOR_NEW +#endif + + +#ifdef _7ZIP_REDEFINE_OPERATOR_NEW + +// std::bad_alloc can require additional DLL dependency. +// So we don't define CNewException as std::bad_alloc here. + +class CNewException {}; void * #ifdef _MSC_VER @@ -48,6 +61,12 @@ __cdecl #endif operator delete(void *p) throw(); +#else + +#include + +#define CNewException std::bad_alloc + #endif /* diff --git a/3rdparty/lzma/CPP/Common/Sha256Prepare.cpp b/3rdparty/lzma/CPP/Common/Sha256Prepare.cpp new file mode 100644 index 00000000000..1ec242b5772 --- /dev/null +++ b/3rdparty/lzma/CPP/Common/Sha256Prepare.cpp @@ -0,0 +1,7 @@ +// Sha256Prepare.cpp + +#include "StdAfx.h" + +#include "../../C/Sha256.h" + +static struct CSha256Prepare { CSha256Prepare() { Sha256Prepare(); } } g_Sha256Prepare; diff --git a/3rdparty/lzma/CPP/Common/Sha256Reg.cpp b/3rdparty/lzma/CPP/Common/Sha256Reg.cpp index 66941699e61..5f3a35b081c 100644 --- a/3rdparty/lzma/CPP/Common/Sha256Reg.cpp +++ b/3rdparty/lzma/CPP/Common/Sha256Reg.cpp @@ -4,37 +4,67 @@ #include "../../C/Sha256.h" +#include "../Common/MyBuffer2.h" #include "../Common/MyCom.h" #include "../7zip/Common/RegisterCodec.h" class CSha256Hasher: public IHasher, + public ICompressSetCoderProperties, public CMyUnknownImp { - CSha256 _sha; + CAlignedBuffer _buf; Byte mtDummy[1 << 7]; + CSha256 *Sha() { return (CSha256 *)(void *)(Byte *)_buf; } public: - CSha256Hasher() { Sha256_Init(&_sha); } + CSha256Hasher(): + _buf(sizeof(CSha256)) + { + Sha256_SetFunction(Sha(), 0); + Sha256_InitState(Sha()); + } - MY_UNKNOWN_IMP1(IHasher) + MY_UNKNOWN_IMP2(IHasher, ICompressSetCoderProperties) INTERFACE_IHasher(;) + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); }; STDMETHODIMP_(void) CSha256Hasher::Init() throw() { - Sha256_Init(&_sha); + Sha256_InitState(Sha()); } STDMETHODIMP_(void) CSha256Hasher::Update(const void *data, UInt32 size) throw() { - Sha256_Update(&_sha, (const Byte *)data, size); + Sha256_Update(Sha(), (const Byte *)data, size); } STDMETHODIMP_(void) CSha256Hasher::Final(Byte *digest) throw() { - Sha256_Final(&_sha, digest); + Sha256_Final(Sha(), digest); +} + + +STDMETHODIMP CSha256Hasher::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) +{ + unsigned algo = 0; + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT &prop = coderProps[i]; + if (propIDs[i] == NCoderPropID::kDefaultProp) + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + if (prop.ulVal > 2) + return E_NOTIMPL; + algo = (unsigned)prop.ulVal; + } + } + if (!Sha256_SetFunction(Sha(), algo)) + return E_NOTIMPL; + return S_OK; } REGISTER_HASHER(CSha256Hasher, 0xA, "SHA256", SHA256_DIGEST_SIZE) diff --git a/3rdparty/lzma/CPP/Common/StdInStream.cpp b/3rdparty/lzma/CPP/Common/StdInStream.cpp index c35747bbe69..abad34b683e 100644 --- a/3rdparty/lzma/CPP/Common/StdInStream.cpp +++ b/3rdparty/lzma/CPP/Common/StdInStream.cpp @@ -2,28 +2,32 @@ #include "StdAfx.h" +#ifdef _WIN32 #include +#endif #include "StdInStream.h" #include "StringConvert.h" #include "UTFConvert.h" -static const char kNewLineChar = '\n'; +// #define kEOFMessage "Unexpected end of input stream" +// #define kReadErrorMessage "Error reading input stream" +// #define kIllegalCharMessage "Illegal zero character in input stream" -static const char *kEOFMessage = "Unexpected end of input stream"; -static const char *kReadErrorMessage ="Error reading input stream"; -static const char *kIllegalCharMessage = "Illegal character in input stream"; - -static LPCTSTR kFileOpenMode = TEXT("r"); - -extern int g_CodePage; +#define kFileOpenMode TEXT("r") CStdInStream g_StdIn(stdin); bool CStdInStream::Open(LPCTSTR fileName) throw() { Close(); - _stream = _tfopen(fileName, kFileOpenMode); + _stream = + #ifdef _WIN32 + _tfopen + #else + fopen + #endif + (fileName, kFileOpenMode); _streamIsOpen = (_stream != 0); return _streamIsOpen; } @@ -36,59 +40,56 @@ bool CStdInStream::Close() throw() return !_streamIsOpen; } -AString CStdInStream::ScanStringUntilNewLine(bool allowEOF) +bool CStdInStream::ScanAStringUntilNewLine(AString &s) { - AString s; + s.Empty(); for (;;) { int intChar = GetChar(); if (intChar == EOF) - { - if (allowEOF) - break; - throw kEOFMessage; - } + return true; char c = (char)intChar; if (c == 0) - throw kIllegalCharMessage; - if (c == kNewLineChar) - break; + return false; + if (c == '\n') + return true; s += c; } - return s; } -UString CStdInStream::ScanUStringUntilNewLine() +bool CStdInStream::ScanUStringUntilNewLine(UString &dest) { - AString s = ScanStringUntilNewLine(true); - int codePage = g_CodePage; + dest.Empty(); + AString s; + bool res = ScanAStringUntilNewLine(s); + int codePage = CodePage; if (codePage == -1) codePage = CP_OEMCP; - UString dest; if (codePage == CP_UTF8) ConvertUTF8ToUnicode(s, dest); else - dest = MultiByteToUnicodeString(s, (UINT)codePage); - return dest; + MultiByteToUnicodeString2(dest, s, (UINT)codePage); + return res; } -void CStdInStream::ReadToString(AString &resultString) +/* +bool CStdInStream::ReadToString(AString &resultString) { resultString.Empty(); - int c; - while ((c = GetChar()) != EOF) - resultString += (char)c; -} - -bool CStdInStream::Eof() throw() -{ - return (feof(_stream) != 0); + for (;;) + { + int intChar = GetChar(); + if (intChar == EOF) + return !Error(); + char c = (char)intChar; + if (c == 0) + return false; + resultString += c; + } } +*/ int CStdInStream::GetChar() { - int c = fgetc(_stream); // getc() doesn't work in BeOS? - if (c == EOF && !Eof()) - throw kReadErrorMessage; - return c; + return fgetc(_stream); // getc() doesn't work in BeOS? } diff --git a/3rdparty/lzma/CPP/Common/StdInStream.h b/3rdparty/lzma/CPP/Common/StdInStream.h index 6d9ed67040d..71578eb4d76 100644 --- a/3rdparty/lzma/CPP/Common/StdInStream.h +++ b/3rdparty/lzma/CPP/Common/StdInStream.h @@ -13,18 +13,29 @@ class CStdInStream FILE *_stream; bool _streamIsOpen; public: - CStdInStream(): _stream(0), _streamIsOpen(false) {}; - CStdInStream(FILE *stream): _stream(stream), _streamIsOpen(false) {}; + int CodePage; + + CStdInStream(FILE *stream = NULL): + _stream(stream), + _streamIsOpen(false), + CodePage(-1) + {}; + ~CStdInStream() { Close(); } bool Open(LPCTSTR fileName) throw(); bool Close() throw(); - AString ScanStringUntilNewLine(bool allowEOF = false); - void ReadToString(AString &resultString); - UString ScanUStringUntilNewLine(); + // returns: + // false, if ZERO character in stream + // true, if EOF or '\n' + bool ScanAStringUntilNewLine(AString &s); + bool ScanUStringUntilNewLine(UString &s); + // bool ReadToString(AString &resultString); + + bool Eof() const throw() { return (feof(_stream) != 0); } + bool Error() const throw() { return (ferror(_stream) != 0); } - bool Eof() throw(); int GetChar(); }; diff --git a/3rdparty/lzma/CPP/Common/StdOutStream.cpp b/3rdparty/lzma/CPP/Common/StdOutStream.cpp index 6aed31a3143..40799e22642 100644 --- a/3rdparty/lzma/CPP/Common/StdOutStream.cpp +++ b/3rdparty/lzma/CPP/Common/StdOutStream.cpp @@ -2,18 +2,16 @@ #include "StdAfx.h" +#ifdef _WIN32 #include +#endif #include "IntToString.h" #include "StdOutStream.h" #include "StringConvert.h" #include "UTFConvert.h" -static const char kNewLineChar = '\n'; - -static const char *kFileOpenMode = "wt"; - -extern int g_CodePage; +#define kFileOpenMode "wt" CStdOutStream g_StdOut(stdout); CStdOutStream g_StdErr(stderr); @@ -44,39 +42,93 @@ bool CStdOutStream::Flush() throw() CStdOutStream & endl(CStdOutStream & outStream) throw() { - return outStream << kNewLineChar; + return outStream << '\n'; } CStdOutStream & CStdOutStream::operator<<(const wchar_t *s) { - int codePage = g_CodePage; + AString temp; + UString s2(s); + PrintUString(s2, temp); + return *this; +} + +void CStdOutStream::PrintUString(const UString &s, AString &temp) +{ + Convert_UString_to_AString(s, temp); + *this << (const char *)temp; +} + +void CStdOutStream::Convert_UString_to_AString(const UString &src, AString &dest) +{ + int codePage = CodePage; if (codePage == -1) codePage = CP_OEMCP; - AString dest; if (codePage == CP_UTF8) - ConvertUnicodeToUTF8(s, dest); + ConvertUnicodeToUTF8(src, dest); else - UnicodeStringToMultiByte2(dest, s, (UINT)codePage); - return operator<<((const char *)dest); + UnicodeStringToMultiByte2(dest, src, (UINT)codePage); } -void StdOut_Convert_UString_to_AString(const UString &s, AString &temp) + +static const wchar_t kReplaceChar = '_'; + +void CStdOutStream::Normalize_UString__LF_Allowed(UString &s) { - int codePage = g_CodePage; - if (codePage == -1) - codePage = CP_OEMCP; - if (codePage == CP_UTF8) - ConvertUnicodeToUTF8(s, temp); + unsigned len = s.Len(); + wchar_t *d = s.GetBuf(); + + if (IsTerminalMode) + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c <= 13 && c >= 7 && c != '\n') + d[i] = kReplaceChar; + } +} + +void CStdOutStream::Normalize_UString(UString &s) +{ + unsigned len = s.Len(); + wchar_t *d = s.GetBuf(); + + if (IsTerminalMode) + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c <= 13 && c >= 7) + d[i] = kReplaceChar; + } else - UnicodeStringToMultiByte2(temp, s, (UINT)codePage); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c == '\n') + d[i] = kReplaceChar; + } } -void CStdOutStream::PrintUString(const UString &s, AString &temp) +void CStdOutStream::NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA) { - StdOut_Convert_UString_to_AString(s, temp); - *this << (const char *)temp; + tempU = s; + Normalize_UString(tempU); + PrintUString(tempU, tempA); } +void CStdOutStream::NormalizePrint_UString(const UString &s) +{ + NormalizePrint_wstr(s); +} + +void CStdOutStream::NormalizePrint_wstr(const wchar_t *s) +{ + UString tempU = s; + Normalize_UString(tempU); + AString tempA; + PrintUString(tempU, tempA); +} + + CStdOutStream & CStdOutStream::operator<<(Int32 number) throw() { char s[32]; diff --git a/3rdparty/lzma/CPP/Common/StdOutStream.h b/3rdparty/lzma/CPP/Common/StdOutStream.h index 1837bfb0adc..93f1dfa4e55 100644 --- a/3rdparty/lzma/CPP/Common/StdOutStream.h +++ b/3rdparty/lzma/CPP/Common/StdOutStream.h @@ -13,8 +13,16 @@ class CStdOutStream FILE *_stream; bool _streamIsOpen; public: - CStdOutStream(): _stream(0), _streamIsOpen(false) {}; - CStdOutStream(FILE *stream): _stream(stream), _streamIsOpen(false) {}; + bool IsTerminalMode; + int CodePage; + + CStdOutStream(FILE *stream = 0): + _stream(stream), + _streamIsOpen(false), + IsTerminalMode(false), + CodePage(-1) + {}; + ~CStdOutStream() { Close(); } // void AttachStdStream(FILE *stream) { _stream = stream; _streamIsOpen = false; } @@ -50,6 +58,14 @@ public: CStdOutStream & operator<<(const wchar_t *s); void PrintUString(const UString &s, AString &temp); + void Convert_UString_to_AString(const UString &src, AString &dest); + + void Normalize_UString__LF_Allowed(UString &s); + void Normalize_UString(UString &s); + + void NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA); + void NormalizePrint_UString(const UString &s); + void NormalizePrint_wstr(const wchar_t *s); }; CStdOutStream & endl(CStdOutStream & outStream) throw(); @@ -57,6 +73,4 @@ CStdOutStream & endl(CStdOutStream & outStream) throw(); extern CStdOutStream g_StdOut; extern CStdOutStream g_StdErr; -void StdOut_Convert_UString_to_AString(const UString &s, AString &temp); - #endif diff --git a/3rdparty/lzma/CPP/Common/StringConvert.cpp b/3rdparty/lzma/CPP/Common/StringConvert.cpp index 30d95aa7b56..c0bde0fa121 100644 --- a/3rdparty/lzma/CPP/Common/StringConvert.cpp +++ b/3rdparty/lzma/CPP/Common/StringConvert.cpp @@ -5,9 +5,18 @@ #include "StringConvert.h" #ifndef _WIN32 +// #include #include #endif +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) +#include "UTFConvert.h" +#endif + +#ifdef ENV_HAVE_LOCALE +#include +#endif + static const char k_DefultChar = '_'; #ifdef _WIN32 @@ -71,7 +80,7 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) d[i] = 0; dest.ReleaseBuf_SetLen(i); */ - unsigned len = MultiByteToWideChar(codePage, 0, src, src.Len(), NULL, 0); + unsigned len = (unsigned)MultiByteToWideChar(codePage, 0, src, (int)src.Len(), NULL, 0); if (len == 0) { if (GetLastError() != 0) @@ -79,7 +88,7 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) } else { - len = MultiByteToWideChar(codePage, 0, src, src.Len(), dest.GetBuf(len), len); + len = (unsigned)MultiByteToWideChar(codePage, 0, src, (int)src.Len(), dest.GetBuf(len), (int)len); if (len == 0) throw 282228; dest.ReleaseBuf_SetEnd(len); @@ -175,7 +184,7 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT co } */ - unsigned len = WideCharToMultiByte(codePage, 0, src, src.Len(), NULL, 0, NULL, NULL); + unsigned len = (unsigned)WideCharToMultiByte(codePage, 0, src, (int)src.Len(), NULL, 0, NULL, NULL); if (len == 0) { if (GetLastError() != 0) @@ -186,8 +195,8 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT co BOOL defUsed = FALSE; bool isUtf = (codePage == CP_UTF8 || codePage == CP_UTF7); // defaultChar = defaultChar; - len = WideCharToMultiByte(codePage, 0, src, src.Len(), - dest.GetBuf(len), len, + len = (unsigned)WideCharToMultiByte(codePage, 0, src, (int)src.Len(), + dest.GetBuf(len), (int)len, (isUtf ? NULL : &defaultChar), (isUtf ? NULL : &defUsed) ); @@ -213,23 +222,137 @@ AString SystemStringToOemString(const CSysString &src) #endif */ -#else +#else // _WIN32 + +// #include +/* + if (wchar_t is 32-bit (#if WCHAR_MAX > 0xffff), + and utf-8 string contains big unicode character > 0xffff), + then we still use 16-bit surrogate pair in UString. + It simplifies another code where utf-16 encoding is used. + So we use surrogate-conversion code only in is file. +*/ + +/* + mbstowcs() returns error if there is error in utf-8 stream, + mbstowcs() returns error if there is single surrogates point (d800-dfff) in utf-8 stream +*/ + +/* +static void MultiByteToUnicodeString2_Native(UString &dest, const AString &src) +{ + dest.Empty(); + if (src.IsEmpty()) + return; + + const size_t limit = ((size_t)src.Len() + 1) * 2; + wchar_t *d = dest.GetBuf((unsigned)limit); + const size_t len = mbstowcs(d, src, limit); + if (len != (size_t)-1) + { + dest.ReleaseBuf_SetEnd((unsigned)len); + return; + } + dest.ReleaseBuf_SetEnd(0); +} +*/ + +bool g_ForceToUTF8 = true; // false; -void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePage */) +void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) { dest.Empty(); if (src.IsEmpty()) return; - size_t limit = ((size_t)src.Len() + 1) * 2; + if (codePage == CP_UTF8 || g_ForceToUTF8) + { + ConvertUTF8ToUnicode(src, dest); + return; + } + + const size_t limit = ((size_t)src.Len() + 1) * 2; wchar_t *d = dest.GetBuf((unsigned)limit); - size_t len = mbstowcs(d, src, limit); + const size_t len = mbstowcs(d, src, limit); if (len != (size_t)-1) { dest.ReleaseBuf_SetEnd((unsigned)len); + + #if WCHAR_MAX > 0xffff + d = dest.GetBuf(); + for (size_t i = 0;; i++) + { + // wchar_t c = dest[i]; + wchar_t c = d[i]; + if (c == 0) + break; + if (c >= 0x10000 && c < 0x110000) + { + /* + c -= 0x10000; + unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); + dest.ReplaceOneCharAtPos(i, c0); + i++; + c = 0xdc00 + (c & 0x3FF); + dest.Insert_wchar_t(i, c); + */ + UString temp = d + i; + + for (size_t t = 0;; t++) + { + wchar_t w = temp[t]; + if (w == 0) + break; + if (i == limit) + break; // unexpected error + if (w >= 0x10000 && w < 0x110000) + { + if (i + 1 == limit) + break; // unexpected error + w -= 0x10000; + d[i++] = (unsigned)0xd800 + (((unsigned)w >> 10) & 0x3FF); + w = 0xdc00 + (w & 0x3FF); + } + d[i++] = w; + } + dest.ReleaseBuf_SetEnd((unsigned)i); + } + } + + #endif + + /* + printf("\nMultiByteToUnicodeString2 (%d) %s\n", (int)src.Len(), src.Ptr()); + printf("char: "); + for (unsigned i = 0; i < src.Len(); i++) + printf (" %02x", (int)(Byte)src[i]); + printf("\n"); + printf("\n-> (%d) %ls\n", (int)dest.Len(), dest.Ptr()); + printf("wchar_t: "); + for (unsigned i = 0; i < dest.Len(); i++) + { + printf (" %02x", (int)dest[i]); + } + printf("\n"); + */ + return; } + + /* if there is mbstowcs() error, we have two ways: + + 1) change 0x80+ characters to some character: '_' + in that case we lose data, but we have correct UString() + and that scheme can show errors to user in early stages, + when file converted back to mbs() cannot be found + + 2) transfer bad characters in some UTF-16 range. + it can be non-original Unicode character. + but later we still can restore original character. + */ + + // printf("\nmbstowcs ERROR !!!!!! s=%s\n", src.Ptr()); { unsigned i; const char *s = (const char *)src; @@ -238,6 +361,8 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePa Byte c = (Byte)s[i]; if (c == 0) break; + // we can use ascii compatibilty character '_' + // if (c > 0x7F) c = '_'; // we replace "bad: character d[i++] = (wchar_t)c; } d[i] = 0; @@ -245,43 +370,131 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT /* codePa } } -static void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT /* codePage */, char defaultChar, bool &defaultCharWasUsed) +static void UnicodeStringToMultiByte2_Native(AString &dest, const UString &src) { dest.Empty(); - defaultCharWasUsed = false; if (src.IsEmpty()) return; - size_t limit = ((size_t)src.Len() + 1) * 6; + const size_t limit = ((size_t)src.Len() + 1) * 6; char *d = dest.GetBuf((unsigned)limit); - size_t len = wcstombs(d, src, limit); + + const size_t len = wcstombs(d, src, limit); + if (len != (size_t)-1) { dest.ReleaseBuf_SetEnd((unsigned)len); return; } + dest.ReleaseBuf_SetEnd(0); +} + + +static void UnicodeStringToMultiByte2(AString &dest, const UString &src2, UINT codePage, char defaultChar, bool &defaultCharWasUsed) +{ + // if (codePage == 1234567) // for debug purposes + if (codePage == CP_UTF8 || g_ForceToUTF8) + { + defaultCharWasUsed = false; + ConvertUnicodeToUTF8(src2, dest); + return; + } + + UString src = src2; + #if WCHAR_MAX > 0xffff + { + src.Empty(); + for (unsigned i = 0; i < src2.Len();) + { + wchar_t c = src2[i]; + if (c >= 0xd800 && c < 0xdc00 && i + 1 != src2.Len()) + { + const wchar_t c2 = src2[i + 1]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + // printf("%4x\n", (int)c); + i++; + } + } + src += c; + i++; + } + } + #endif + + dest.Empty(); + defaultCharWasUsed = false; + if (src.IsEmpty()) + return; + + const size_t len = wcstombs(NULL, src, 0); + + if (len != (size_t)-1) + { + const unsigned limit = ((unsigned)len); + if (limit == len) + { + char *d = dest.GetBuf(limit); + + /* + { + printf("\nwcstombs; len = %d %ls \n", (int)src.Len(), src.Ptr()); + for (unsigned i = 0; i < src.Len(); i++) + printf (" %02x", (int)src[i]); + printf("\n"); + printf("\ndest Limit = %d \n", limit); + } + */ + + const size_t len2 = wcstombs(d, src, len + 1); + + if (len2 != (size_t)-1 && len2 <= limit) + { + /* + printf("\nOK : destLen = %d : %s\n", (int)len, dest.Ptr()); + for (unsigned i = 0; i < len2; i++) + printf(" %02x", (int)(Byte)dest[i]); + printf("\n"); + */ + dest.ReleaseBuf_SetEnd((unsigned)len2); + return; + } + } + } { const wchar_t *s = (const wchar_t *)src; + char *d = dest.GetBuf(src.Len()); + unsigned i; for (i = 0;;) { wchar_t c = s[i]; if (c == 0) break; - if (c >= 0x100) + if (c >= + 0x100 + // 0x80 + ) { c = defaultChar; defaultCharWasUsed = true; } + d[i++] = (char)c; } d[i] = 0; dest.ReleaseBuf_SetLen(i); + /* + printf("\nUnicodeStringToMultiByte2; len = %d \n", (int)src.Len()); + printf("ERROR: %s\n", dest.Ptr()); + */ } } -#endif +#endif // _WIN32 UString MultiByteToUnicodeString(const AString &src, UINT codePage) @@ -291,6 +504,12 @@ UString MultiByteToUnicodeString(const AString &src, UINT codePage) return dest; } +UString MultiByteToUnicodeString(const char *src, UINT codePage) +{ + return MultiByteToUnicodeString(AString(src), codePage); +} + + void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT codePage) { bool defaultCharWasUsed; @@ -311,3 +530,228 @@ AString UnicodeStringToMultiByte(const UString &src, UINT codePage) UnicodeStringToMultiByte2(dest, src, codePage, k_DefultChar, defaultCharWasUsed); return dest; } + + + + + +#ifdef _WIN32 +#define U_to_A(a, b, c) UnicodeStringToMultiByte2 +// #define A_to_U(a, b, c) MultiByteToUnicodeString2 +#else +// void MultiByteToUnicodeString2_Native(UString &dest, const AString &src); +#define U_to_A(a, b, c) UnicodeStringToMultiByte2_Native(a, b) +// #define A_to_U(a, b, c) MultiByteToUnicodeString2_Native(a, b) +#endif + +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) + +bool IsNativeUTF8() +{ + UString u; + AString a, a2; + // for (unsigned c = 0x80; c < (UInt32)0x10000; c += (c >> 9) + 1) + for (unsigned c = 0x80; c < (UInt32)0xD000; c += (c >> 2) + 1) + { + u.Empty(); + u += (wchar_t)c; + /* + if (Unicode_Is_There_Utf16SurrogateError(u)) + continue; + #ifndef _WIN32 + if (Unicode_Is_There_BmpEscape(u)) + continue; + #endif + */ + ConvertUnicodeToUTF8(u, a); + U_to_A(a2, u, CP_OEMCP); + if (a != a2) + return false; + } + return true; +} + +#endif + + +#ifdef ENV_HAVE_LOCALE + +const char *GetLocale(void) +{ + #ifdef ENV_HAVE_LOCALE + // printf("\n\nsetlocale(LC_CTYPE, NULL) : return : "); + const char *s = setlocale(LC_CTYPE, NULL); + if (!s) + { + // printf("[NULL]\n"); + s = "C"; + } + else + { + // ubuntu returns "C" after program start + // printf("\"%s\"\n", s); + } + return s; + #elif defined(LOCALE_IS_UTF8) + return "utf8"; + #else + return "C"; + #endif +} + +#ifdef _WIN32 + static void Set_ForceToUTF8(bool) {} +#else + static void Set_ForceToUTF8(bool val) { g_ForceToUTF8 = val; } +#endif + +static bool Is_Default_Basic_Locale(const char *locale) +{ + const AString a (locale); + if (a.IsEqualTo_Ascii_NoCase("") + || a.IsEqualTo_Ascii_NoCase("C") + || a.IsEqualTo_Ascii_NoCase("POSIX")) + return true; + return false; +} + +static bool Is_Default_Basic_Locale() +{ + return Is_Default_Basic_Locale(GetLocale()); +} + + +void MY_SetLocale() +{ + #ifdef ENV_HAVE_LOCALE + /* + { + const char *s = GetLocale(); + printf("\nGetLocale() : returned : \"%s\"\n", s); + } + */ + + unsigned start = 0; + // unsigned lim = 0; + unsigned lim = 3; + + /* + #define MY_SET_LOCALE_FLAGS__FROM_ENV 1 + #define MY_SET_LOCALE_FLAGS__TRY_UTF8 2 + + unsigned flags = + MY_SET_LOCALE_FLAGS__FROM_ENV | + MY_SET_LOCALE_FLAGS__TRY_UTF8 + + if (flags != 0) + { + if (flags & MY_SET_LOCALE_FLAGS__FROM_ENV) + lim = (flags & MY_SET_LOCALE_FLAGS__TRY_UTF8) ? 3 : 1; + else + { + start = 1; + lim = 2; + } + } + */ + + for (unsigned i = start; i < lim; i++) + { + /* + man7: "If locale is an empty string, "", each part of the locale that + should be modified is set according to the environment variables. + for glibc: glibc, first from the user's environment variables: + 1) the environment variable LC_ALL, + 2) environment variable with the same name as the category (see the + 3) the environment variable LANG + The locale "C" or "POSIX" is a portable locale; it exists on all conforming systems. + + for WIN32 : MSDN : + Sets the locale to the default, which is the user-default + ANSI code page obtained from the operating system. + The locale name is set to the value returned by GetUserDefaultLocaleName. + The code page is set to the value returned by GetACP + */ + const char *newLocale = ""; + + #ifdef __APPLE__ + + /* look also CFLocale + there is no C.UTF-8 in macos + macos has UTF-8 locale only with some language like en_US.UTF-8 + what is best way to set UTF-8 locale in macos? */ + if (i == 1) + newLocale = "en_US.UTF-8"; + + /* file open with non-utf8 sequencies return + #define EILSEQ 92 // "Illegal byte sequence" + */ +#else + // newLocale = "C"; + if (i == 1) + { + newLocale = "C.UTF-8"; // main UTF-8 locale in ubuntu + // newLocale = ".utf8"; // supported in new Windows 10 build 17134 (April 2018 Update), the Universal C Runtime + // newLocale = "en_US.utf8"; // supported by ubuntu ? + // newLocale = "en_US.UTF-8"; + /* setlocale() in ubuntu allows locales with minor chracter changes in strings + "en_US.UTF-8" / "en_US.utf8" */ + } + +#endif + + // printf("\nsetlocale(LC_ALL, \"%s\") : returned: ", newLocale); + + // const char *s = + setlocale(LC_ALL, newLocale); + + /* + if (!s) + printf("NULL: can't set locale"); + else + printf("\"%s\"\n", s); + */ + + // request curent locale of program + const char *locale = GetLocale(); + if (locale) + { + AString a (locale); + a.MakeLower_Ascii(); + // if (a.Find("utf") >= 0) + { + if (IsNativeUTF8()) + { + Set_ForceToUTF8(true); + return; + } + } + if (!Is_Default_Basic_Locale(locale)) + { + // if there is some non-default and non-utf locale, we want to use it + break; // comment it for debug + } + } + } + + if (IsNativeUTF8()) + { + Set_ForceToUTF8(true); + return; + } + + if (Is_Default_Basic_Locale()) + { + Set_ForceToUTF8(true); + return; + } + + Set_ForceToUTF8(false); + + #elif defined(LOCALE_IS_UTF8) + // assume LC_CTYPE="utf8" + #else + // assume LC_CTYPE="C" + #endif +} +#endif diff --git a/3rdparty/lzma/CPP/Common/StringConvert.h b/3rdparty/lzma/CPP/Common/StringConvert.h index 924012450c6..865c02543c4 100644 --- a/3rdparty/lzma/CPP/Common/StringConvert.h +++ b/3rdparty/lzma/CPP/Common/StringConvert.h @@ -6,72 +6,105 @@ #include "MyString.h" #include "MyWindows.h" -UString MultiByteToUnicodeString(const AString &srcString, UINT codePage = CP_ACP); +UString MultiByteToUnicodeString(const AString &src, UINT codePage = CP_ACP); +UString MultiByteToUnicodeString(const char *src, UINT codePage = CP_ACP); // optimized versions that work faster for ASCII strings -void MultiByteToUnicodeString2(UString &dest, const AString &srcString, UINT codePage = CP_ACP); +void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage = CP_ACP); // void UnicodeStringToMultiByte2(AString &dest, const UString &s, UINT codePage, char defaultChar, bool &defaultCharWasUsed); -void UnicodeStringToMultiByte2(AString &dest, const UString &srcString, UINT codePage); - -AString UnicodeStringToMultiByte(const UString &srcString, UINT codePage, char defaultChar, bool &defaultCharWasUsed); -AString UnicodeStringToMultiByte(const UString &srcString, UINT codePage = CP_ACP); - -inline const wchar_t* GetUnicodeString(const wchar_t* unicodeString) - { return unicodeString; } -inline const UString& GetUnicodeString(const UString &unicodeString) - { return unicodeString; } -inline UString GetUnicodeString(const AString &ansiString) - { return MultiByteToUnicodeString(ansiString); } -inline UString GetUnicodeString(const AString &multiByteString, UINT codePage) - { return MultiByteToUnicodeString(multiByteString, codePage); } -inline const wchar_t* GetUnicodeString(const wchar_t* unicodeString, UINT) - { return unicodeString; } -inline const UString& GetUnicodeString(const UString &unicodeString, UINT) - { return unicodeString; } - -inline const char* GetAnsiString(const char* ansiString) - { return ansiString; } -inline const AString& GetAnsiString(const AString &ansiString) - { return ansiString; } -inline AString GetAnsiString(const UString &unicodeString) - { return UnicodeStringToMultiByte(unicodeString); } - -inline const char* GetOemString(const char* oemString) - { return oemString; } -inline const AString& GetOemString(const AString &oemString) - { return oemString; } -inline AString GetOemString(const UString &unicodeString) - { return UnicodeStringToMultiByte(unicodeString, CP_OEMCP); } +void UnicodeStringToMultiByte2(AString &dest, const UString &src, UINT codePage); +AString UnicodeStringToMultiByte(const UString &src, UINT codePage, char defaultChar, bool &defaultCharWasUsed); +AString UnicodeStringToMultiByte(const UString &src, UINT codePage = CP_ACP); + +inline const wchar_t* GetUnicodeString(const wchar_t *u) { return u; } +inline const UString& GetUnicodeString(const UString &u) { return u; } + +inline UString GetUnicodeString(const AString &a) { return MultiByteToUnicodeString(a); } +inline UString GetUnicodeString(const char *a) { return MultiByteToUnicodeString(a); } + +inline UString GetUnicodeString(const AString &a, UINT codePage) + { return MultiByteToUnicodeString(a, codePage); } +inline UString GetUnicodeString(const char *a, UINT codePage) + { return MultiByteToUnicodeString(a, codePage); } + +inline const wchar_t* GetUnicodeString(const wchar_t *u, UINT) { return u; } +inline const UString& GetUnicodeString(const UString &u, UINT) { return u; } + +inline const char* GetAnsiString(const char *a) { return a; } +inline const AString& GetAnsiString(const AString &a) { return a; } + +inline AString GetAnsiString(const wchar_t *u) { return UnicodeStringToMultiByte(UString(u)); } +inline AString GetAnsiString(const UString &u) { return UnicodeStringToMultiByte(u); } + +/* +inline const char* GetOemString(const char* oem) + { return oem; } +inline const AString& GetOemString(const AString &oem) + { return oem; } +*/ +const char* GetOemString(const char* oem); +const AString& GetOemString(const AString &oem); +inline AString GetOemString(const UString &u) + { return UnicodeStringToMultiByte(u, CP_OEMCP); } #ifdef _UNICODE - inline const wchar_t* GetSystemString(const wchar_t* unicodeString) - { return unicodeString;} - inline const UString& GetSystemString(const UString &unicodeString) - { return unicodeString;} - inline const wchar_t* GetSystemString(const wchar_t* unicodeString, UINT /* codePage */) - { return unicodeString;} - inline const UString& GetSystemString(const UString &unicodeString, UINT /* codePage */) - { return unicodeString;} - inline UString GetSystemString(const AString &multiByteString, UINT codePage) - { return MultiByteToUnicodeString(multiByteString, codePage);} - inline UString GetSystemString(const AString &multiByteString) - { return MultiByteToUnicodeString(multiByteString);} + inline const wchar_t* GetSystemString(const wchar_t *u) { return u;} + inline const UString& GetSystemString(const UString &u) { return u;} + inline const wchar_t* GetSystemString(const wchar_t *u, UINT /* codePage */) { return u;} + inline const UString& GetSystemString(const UString &u, UINT /* codePage */) { return u;} + + inline UString GetSystemString(const AString &a, UINT codePage) { return MultiByteToUnicodeString(a, codePage); } + inline UString GetSystemString(const char *a, UINT codePage) { return MultiByteToUnicodeString(a, codePage); } + inline UString GetSystemString(const AString &a) { return MultiByteToUnicodeString(a); } + inline UString GetSystemString(const char *a) { return MultiByteToUnicodeString(a); } #else - inline const char* GetSystemString(const char *ansiString) - { return ansiString; } - inline const AString& GetSystemString(const AString &multiByteString, UINT) - { return multiByteString; } - inline const char * GetSystemString(const char *multiByteString, UINT) - { return multiByteString; } - inline AString GetSystemString(const UString &unicodeString) - { return UnicodeStringToMultiByte(unicodeString); } - inline AString GetSystemString(const UString &unicodeString, UINT codePage) - { return UnicodeStringToMultiByte(unicodeString, codePage); } + inline const char* GetSystemString(const char *a) { return a; } + inline const AString& GetSystemString(const AString &a) { return a; } + inline const char* GetSystemString(const char *a, UINT) { return a; } + inline const AString& GetSystemString(const AString &a, UINT) { return a; } + + inline AString GetSystemString(const wchar_t *u) { return UnicodeStringToMultiByte(UString(u)); } + inline AString GetSystemString(const UString &u) { return UnicodeStringToMultiByte(u); } + inline AString GetSystemString(const UString &u, UINT codePage) { return UnicodeStringToMultiByte(u, codePage); } + + + + /* + inline AString GetSystemString(const wchar_t *u) + { + UString s; + s = u; + return UnicodeStringToMultiByte(s); + } + */ + #endif #ifndef UNDER_CE -AString SystemStringToOemString(const CSysString &srcString); +AString SystemStringToOemString(const CSysString &src); +#endif + + +#ifdef _WIN32 +/* we don't need locale functions in Windows + but we can define ENV_HAVE_LOCALE here for debug purposes */ +// #define ENV_HAVE_LOCALE +#else +#define ENV_HAVE_LOCALE +#endif + +#ifdef ENV_HAVE_LOCALE +void MY_SetLocale(); +const char *GetLocale(void); +#endif + +#if !defined(_WIN32) || defined(ENV_HAVE_LOCALE) +bool IsNativeUTF8(); +#endif + +#ifndef _WIN32 +extern bool g_ForceToUTF8; #endif #endif diff --git a/3rdparty/lzma/CPP/Common/StringToInt.cpp b/3rdparty/lzma/CPP/Common/StringToInt.cpp index dfa5cc3bff4..bc4926e5d1b 100644 --- a/3rdparty/lzma/CPP/Common/StringToInt.cpp +++ b/3rdparty/lzma/CPP/Common/StringToInt.cpp @@ -17,7 +17,7 @@ static const UInt64 k_UInt64_max = UINT64_CONST(0xFFFFFFFFFFFFFFFF); if (c < '0' || c > '9') { if (end) *end = s; return res; } \ if (res > (k_ ## uintType ## _max) / 10) return 0; \ res *= 10; \ - unsigned v = (c - '0'); \ + unsigned v = (unsigned)(c - '0'); \ if (res > (k_ ## uintType ## _max) - v) return 0; \ res += v; }} @@ -26,6 +26,33 @@ CONVERT_STRING_TO_UINT_FUNC(UInt32, wchar_t, wchar_t) CONVERT_STRING_TO_UINT_FUNC(UInt64, char, Byte) CONVERT_STRING_TO_UINT_FUNC(UInt64, wchar_t, wchar_t) +/* +Int32 ConvertStringToInt32(const char *s, const char **end) throw() +{ + if (end) + *end = s; + const char *s2 = s; + if (*s == '-') + s2++; + if (*s2 == 0) + return 0; + const char *end2; + UInt32 res = ConvertStringToUInt32(s2, &end2); + if (*s == '-') + { + if (res > ((UInt32)1 << (32 - 1))) + return 0; + } + else if ((res & ((UInt32)1 << (32 - 1))) != 0) + return 0; + if (end) + *end = end2; + if (*s == '-') + return -(Int32)res; + return (Int32)res; +} +*/ + Int32 ConvertStringToInt32(const wchar_t *s, const wchar_t **end) throw() { if (end) diff --git a/3rdparty/lzma/CPP/Common/StringToInt.h b/3rdparty/lzma/CPP/Common/StringToInt.h index 5c5d7d7fe75..4057e494e98 100644 --- a/3rdparty/lzma/CPP/Common/StringToInt.h +++ b/3rdparty/lzma/CPP/Common/StringToInt.h @@ -10,6 +10,7 @@ UInt64 ConvertStringToUInt64(const char *s, const char **end) throw(); UInt32 ConvertStringToUInt32(const wchar_t *s, const wchar_t **end) throw(); UInt64 ConvertStringToUInt64(const wchar_t *s, const wchar_t **end) throw(); +// Int32 ConvertStringToInt32(const char *s, const char **end) throw(); Int32 ConvertStringToInt32(const wchar_t *s, const wchar_t **end) throw(); UInt32 ConvertOctStringToUInt32(const char *s, const char **end) throw(); diff --git a/3rdparty/lzma/CPP/Common/TextConfig.cpp b/3rdparty/lzma/CPP/Common/TextConfig.cpp index 9ae4074b690..1428aab5c85 100644 --- a/3rdparty/lzma/CPP/Common/TextConfig.cpp +++ b/3rdparty/lzma/CPP/Common/TextConfig.cpp @@ -61,7 +61,7 @@ bool GetTextConfig(const AString &s, CObjectVector &pairs) break; CTextConfigPair pair; unsigned finishPos; - AString temp = GetIDString(((const char *)s) + pos, finishPos); + const AString temp (GetIDString(((const char *)s) + pos, finishPos)); if (!ConvertUTF8ToUnicode(temp, pair.ID)) return false; if (finishPos == 0) @@ -107,15 +107,15 @@ bool GetTextConfig(const AString &s, CObjectVector &pairs) return true; } -int FindTextConfigItem(const CObjectVector &pairs, const UString &id) throw() +int FindTextConfigItem(const CObjectVector &pairs, const char *id) throw() { FOR_VECTOR (i, pairs) - if (pairs[i].ID == id) + if (pairs[i].ID.IsEqualTo(id)) return i; return -1; } -UString GetTextConfigValue(const CObjectVector &pairs, const UString &id) +UString GetTextConfigValue(const CObjectVector &pairs, const char *id) { int index = FindTextConfigItem(pairs, id); if (index < 0) diff --git a/3rdparty/lzma/CPP/Common/TextConfig.h b/3rdparty/lzma/CPP/Common/TextConfig.h index 0129ad963a1..cc7ce412776 100644 --- a/3rdparty/lzma/CPP/Common/TextConfig.h +++ b/3rdparty/lzma/CPP/Common/TextConfig.h @@ -13,7 +13,7 @@ struct CTextConfigPair bool GetTextConfig(const AString &text, CObjectVector &pairs); -int FindTextConfigItem(const CObjectVector &pairs, const UString &id) throw(); -UString GetTextConfigValue(const CObjectVector &pairs, const UString &id); +int FindTextConfigItem(const CObjectVector &pairs, const char *id) throw(); +UString GetTextConfigValue(const CObjectVector &pairs, const char *id); #endif diff --git a/3rdparty/lzma/CPP/Common/UTFConvert.cpp b/3rdparty/lzma/CPP/Common/UTFConvert.cpp index b772164aab4..ac069dbadb9 100644 --- a/3rdparty/lzma/CPP/Common/UTFConvert.cpp +++ b/3rdparty/lzma/CPP/Common/UTFConvert.cpp @@ -2,11 +2,17 @@ #include "StdAfx.h" +// #include + #include "MyTypes.h" #include "UTFConvert.h" -#ifdef _WIN32 -#define _WCHART_IS_16BIT 1 + +#ifndef _WCHART_IS_16BIT +#ifndef __APPLE__ + // we define it if the system supports files with non-utf8 symbols: + #define _UTF8_RAW_NON_UTF8_SUPPORTED +#endif #endif /* @@ -18,78 +24,332 @@ 1 : 0xC0 : 11 : 2 : 0xE0 : 16 : Basic Multilingual Plane 3 : 0xF0 : 21 : Unicode space - 3 : 0xF8 : 26 : - 5 : 0xFC : 31 : UCS-4 + 4 : 0xF8 : 26 : + 5 : 0xFC : 31 : UCS-4 : wcstombs() in ubuntu is limited to that value 6 : 0xFE : 36 : We can use it, if we want to encode any 32-bit value 7 : 0xFF : */ #define _UTF8_START(n) (0x100 - (1 << (7 - (n)))) -#define _UTF8_HEAD_PARSE2(n) if (c < _UTF8_START((n) + 1)) { numBytes = (n); c -= _UTF8_START(n); } +#define _UTF8_HEAD_PARSE2(n) \ + if (c < _UTF8_START((n) + 1)) \ + { numBytes = (n); val -= _UTF8_START(n); } + +#ifndef _WCHART_IS_16BIT + +/* + if (wchar_t is 32-bit), we can support large points in long UTF-8 sequence, + when we convert wchar_t strings to UTF-8: + (_UTF8_NUM_TAIL_BYTES_MAX == 3) : (21-bits points) - Unicode + (_UTF8_NUM_TAIL_BYTES_MAX == 5) : (31-bits points) - UCS-4 + (_UTF8_NUM_TAIL_BYTES_MAX == 6) : (36-bit hack) +*/ + +#define _UTF8_NUM_TAIL_BYTES_MAX 5 +#endif +/* #define _UTF8_HEAD_PARSE \ + UInt32 val = c; \ _UTF8_HEAD_PARSE2(1) \ else _UTF8_HEAD_PARSE2(2) \ else _UTF8_HEAD_PARSE2(3) \ else _UTF8_HEAD_PARSE2(4) \ else _UTF8_HEAD_PARSE2(5) \ + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + else _UTF8_HEAD_PARSE2(6) + #endif +*/ + +#define _UTF8_HEAD_PARSE_MAX_3_BYTES \ + UInt32 val = c; \ + _UTF8_HEAD_PARSE2(1) \ + else _UTF8_HEAD_PARSE2(2) \ + else { numBytes = 3; val -= _UTF8_START(3); } + + +#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) + + +#define START_POINT_FOR_SURROGATE 0x10000 + + +/* we use 128 bytes block in 16-bit BMP-PLANE to encode non-UTF-8 Escapes + Also we can use additional HIGH-PLANE (we use 21-bit points above 0x1f0000) + to simplify internal intermediate conversion in Linux: + RAW-UTF-8 <-> internal wchar_t utf-16 strings <-> RAW-UTF-UTF-8 +*/ + + +#if defined(_WCHART_IS_16BIT) + +#define UTF_ESCAPE_PLANE 0 + +#else + +/* +we can place 128 ESCAPE chars to + ef 80 - ee be 80 (3-bytes utf-8) : similar to WSL + ef ff - ee bf bf + +1f ef 80 - f7 be be 80 (4-bytes utf-8) : last 4-bytes utf-8 plane (out of Unicode) +1f ef ff - f7 be bf bf (4-bytes utf-8) : last 4-bytes utf-8 plane (out of Unicode) +*/ + +// #define UTF_ESCAPE_PLANE_HIGH (0x1f << 16) +// #define UTF_ESCAPE_PLANE UTF_ESCAPE_PLANE_HIGH +#define UTF_ESCAPE_PLANE 0 + +/* + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is set) + { + if (UTF_ESCAPE_PLANE is UTF_ESCAPE_PLANE_HIGH) + { + we can restore any 8-bit Escape from ESCAPE-PLANE-21 plane. + But ESCAPE-PLANE-21 point cannot be stored to utf-16 (7z archive) + So we still need a way to extract 8-bit Escapes and BMP-Escapes-8 + from same BMP-Escapes-16 stored in 7z. + And if we want to restore any 8-bit from 7z archive, + we still must use UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT for (utf-8 -> utf-16) + Also we need additional Conversions to tranform from utf-16 to utf-16-With-Escapes-21 + } + else (UTF_ESCAPE_PLANE == 0) + { + we must convert original 3-bytes utf-8 BMP-Escape point to sequence + of 3 BMP-Escape-16 points with UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + so we can extract original RAW-UTF-8 from UTFD-16 later. + } + } +*/ + +#endif + + + +#define UTF_ESCAPE_BASE 0xef00 + + +#ifdef UTF_ESCAPE_BASE +#define IS_ESCAPE_POINT(v, plane) (((v) & (UInt32)0xffffff80) == (plane) + UTF_ESCAPE_BASE + 0x80) +#endif + +#define IS_SURROGATE_POINT(v) (((v) & (UInt32)0xfffff800) == 0xd800) +#define IS_LOW_SURROGATE_POINT(v) (((v) & (UInt32)0xfffffC00) == 0xdc00) + + +#define _ERROR_UTF8_CHECK \ + { NonUtf = true; continue; } + +void CUtf8Check::Check_Buf(const char *src, size_t size) throw() +{ + Clear(); + // Byte maxByte = 0; + + for (;;) + { + if (size == 0) + break; + + const Byte c = (Byte)(*src++); + size--; + + if (c == 0) + { + ZeroChar = true; + continue; + } + + /* + if (c > maxByte) + maxByte = c; + */ + + if (c < 0x80) + continue; + + if (c < 0xc0 + 2)// it's limit for 0x140000 unicode codes : win32 compatibility + _ERROR_UTF8_CHECK + + unsigned numBytes; - // else _UTF8_HEAD_PARSE2(6) + UInt32 val = c; + _UTF8_HEAD_PARSE2(1) + else _UTF8_HEAD_PARSE2(2) + else _UTF8_HEAD_PARSE2(4) + else _UTF8_HEAD_PARSE2(5) + else + { + _ERROR_UTF8_CHECK + } + unsigned pos = 0; + do + { + if (pos == size) + break; + unsigned c2 = (Byte)src[pos]; + c2 -= 0x80; + if (c2 >= 0x40) + break; + val <<= 6; + val |= c2; + if (pos == 0) + if (val < (((unsigned)1 << 7) >> numBytes)) + break; + pos++; + } + while (--numBytes); + + if (numBytes != 0) + { + if (pos == size) + Truncated = true; + else + _ERROR_UTF8_CHECK + } + + #ifdef UTF_ESCAPE_BASE + if (IS_ESCAPE_POINT(val, 0)) + Escape = true; + #endif + + if (MaxHighPoint < val) + MaxHighPoint = val; + + if (IS_SURROGATE_POINT(val)) + SingleSurrogate = true; + + src += pos; + size -= pos; + } + + // MaxByte = maxByte; +} + +bool Check_UTF8_Buf(const char *src, size_t size, bool allowReduced) throw() +{ + CUtf8Check check; + check.Check_Buf(src, size); + return check.IsOK(allowReduced); +} + +/* +bool CheckUTF8_chars(const char *src, bool allowReduced) throw() +{ + CUtf8Check check; + check.CheckBuf(src, strlen(src)); + return check.IsOK(allowReduced); +} +*/ + +bool CheckUTF8_AString(const AString &s) throw() +{ + CUtf8Check check; + check.Check_AString(s); + return check.IsOK(); +} + + +/* bool CheckUTF8(const char *src, bool allowReduced) throw() { + // return Check_UTF8_Buf(src, strlen(src), allowReduced); + for (;;) { - Byte c = *src++; + const Byte c = (Byte)(*src++); if (c == 0) return true; if (c < 0x80) continue; - if (c < 0xC0) // (c < 0xC0 + 2) // if we support only optimal encoding chars + if (c < 0xC0 + 2 || c >= 0xf5) return false; unsigned numBytes; _UTF8_HEAD_PARSE else return false; - - UInt32 val = c; + unsigned pos = 0; + do { - Byte c2 = *src++; + Byte c2 = (Byte)(*src++); if (c2 < 0x80 || c2 >= 0xC0) return allowReduced && c2 == 0; val <<= 6; val |= (c2 - 0x80); + pos++; } while (--numBytes); - + + if (val < _UTF8_RANGE(pos - 1)) + return false; + if (val >= 0x110000) return false; } } +*/ + +// in case of UTF-8 error we have two ways: +// 21.01- : old : 0xfffd: REPLACEMENT CHARACTER : old version +// 21.02+ : new : 0xef00 + (c) : similar to WSL scheme for low symbols + +#define UTF_REPLACEMENT_CHAR 0xfffd + +#define UTF_ESCAPE(c) \ + ((flags & UTF_FLAG__FROM_UTF8__USE_ESCAPE) ? \ + UTF_ESCAPE_PLANE + UTF_ESCAPE_BASE + (c) : UTF_REPLACEMENT_CHAR) + +/* +#define _HARD_ERROR_UTF8 + { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \ + destPos++; ok = false; continue; } +*/ + +// we ignore utf errors, and don't change (ok) variable! + #define _ERROR_UTF8 \ - { if (dest) dest[destPos] = (wchar_t)0xFFFD; destPos++; ok = false; continue; } + { if (dest) dest[destPos] = (wchar_t)UTF_ESCAPE(c); \ + destPos++; continue; } -static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim) throw() +// we store UTF-16 in wchar_t strings. So we use surrogates for big unicode points: + +// for debug puposes only we can store UTF-32 in wchar_t: +// #define START_POINT_FOR_SURROGATE ((UInt32)0 - 1) + + +/* + WIN32 MultiByteToWideChar(CP_UTF8) emits 0xfffd point, if utf-8 error was found. + Ant it can emit single 0xfffd from 2 src bytes. + It doesn't emit single 0xfffd from 3-4 src bytes. + We can + 1) emit Escape point for each incorrect byte. So we can data recover later + 2) emit 0xfffd for each incorrect byte. + That scheme is similar to Escape scheme, but we emit 0xfffd + instead of each Escape point. + 3) emit single 0xfffd from 1-2 incorrect bytes, as WIN32 MultiByteToWideChar scheme +*/ + +static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const char *srcLim, unsigned flags) throw() { size_t destPos = 0; bool ok = true; for (;;) { - Byte c; if (src == srcLim) { *destLen = destPos; return ok; } - c = *src++; + + const Byte c = (Byte)(*src++); if (c < 0x80) { @@ -98,68 +358,127 @@ static bool Utf8_To_Utf16(wchar_t *dest, size_t *destLen, const char *src, const destPos++; continue; } - if (c < 0xC0) + + if (c < 0xc0 + 2 + || c >= 0xf5) // it's limit for 0x140000 unicode codes : win32 compatibility + { _ERROR_UTF8 + } unsigned numBytes; - _UTF8_HEAD_PARSE - else - _ERROR_UTF8 - - UInt32 val = c; + _UTF8_HEAD_PARSE_MAX_3_BYTES + + unsigned pos = 0; do { - Byte c2; - if (src == srcLim) + if (src + pos == srcLim) break; - c2 = *src; - if (c2 < 0x80 || c2 >= 0xC0) + unsigned c2 = (Byte)src[pos]; + c2 -= 0x80; + if (c2 >= 0x40) break; - src++; val <<= 6; - val |= (c2 - 0x80); + val |= c2; + pos++; + if (pos == 1) + { + if (val < (((unsigned)1 << 7) >> numBytes)) + break; + if (numBytes == 2) + { + if (flags & UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) + if ((val & (0xF800 >> 6)) == (0xd800 >> 6)) + break; + } + else if (numBytes == 3 && val >= (0x110000 >> 12)) + break; + } } while (--numBytes); if (numBytes != 0) + { + if ((flags & UTF_FLAG__FROM_UTF8__USE_ESCAPE) == 0) + { + // the following code to emit the 0xfffd chars as win32 Utf8 function. + // disable the folling line, if you need 0xfffd for each incorrect byte as in Escape mode + src += pos; + } + _ERROR_UTF8 + } + + /* + if (val < _UTF8_RANGE(pos - 1)) _ERROR_UTF8 + */ + + #ifdef UTF_ESCAPE_BASE + + if ((flags & UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT) + && IS_ESCAPE_POINT(val, 0)) + { + // We will emit 3 utf16-Escape-16-21 points from one Escape-16 point (3 bytes) + _ERROR_UTF8 + } + + #endif - if (val < 0x10000) + /* + We don't expect virtual Escape-21 points in UTF-8 stream. + And we don't check for Escape-21. + So utf8-Escape-21 will be converted to another 3 utf16-Escape-21 points. + Maybe we could convert virtual utf8-Escape-21 to one utf16-Escape-21 point in some cases? + */ + + if (val < START_POINT_FOR_SURROGATE) { + /* + if ((flags & UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) + && IS_SURROGATE_POINT(val)) + { + // We will emit 3 utf16-Escape-16-21 points from one Surrogate-16 point (3 bytes) + _ERROR_UTF8 + } + */ if (dest) dest[destPos] = (wchar_t)val; destPos++; } else { - val -= 0x10000; - if (val >= 0x100000) + /* + if (val >= 0x110000) + { + // We will emit utf16-Escape-16-21 point from each source byte _ERROR_UTF8 + } + */ if (dest) { - dest[destPos + 0] = (wchar_t)(0xD800 + (val >> 10)); - dest[destPos + 1] = (wchar_t)(0xDC00 + (val & 0x3FF)); + dest[destPos + 0] = (wchar_t)(0xd800 - (0x10000 >> 10) + (val >> 10)); + dest[destPos + 1] = (wchar_t)(0xdc00 + (val & 0x3ff)); } destPos += 2; } + src += pos; } } -#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) + #define _UTF8_HEAD(n, val) ((char)(_UTF8_START(n) + (val >> (6 * (n))))) #define _UTF8_CHAR(n, val) ((char)(0x80 + (((val) >> (6 * (n))) & 0x3F))) -static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) +static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim, unsigned flags) { - size_t size = srcLim - src; + size_t size = (size_t)(srcLim - src); for (;;) { if (src == srcLim) return size; - UInt32 val = *src++; + UInt32 val = (UInt32)(*src++); if (val < 0x80) continue; @@ -170,15 +489,32 @@ static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) continue; } - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) + #ifdef UTF_ESCAPE_BASE + + #if UTF_ESCAPE_PLANE != 0 + if (flags & UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE) + if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE)) + continue; + #endif + + if (flags & UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE) + if (IS_ESCAPE_POINT(val, 0)) + continue; + + #endif + + if (IS_SURROGATE_POINT(val)) { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) + // it's hack to UTF-8 encoding + + if (val < 0xdc00 && src != srcLim) { - src++; - size += 2; - continue; + const UInt32 c2 = (UInt32)*src; + if (c2 >= 0xdc00 && c2 < 0xe000) + src++; } + size += 2; + continue; } #ifdef _WCHART_IS_16BIT @@ -191,20 +527,26 @@ static size_t Utf16_To_Utf8_Calc(const wchar_t *src, const wchar_t *srcLim) else if (val < _UTF8_RANGE(3)) size += 3; else if (val < _UTF8_RANGE(4)) size += 4; else if (val < _UTF8_RANGE(5)) size += 5; - else size += 6; + else + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + size += 6; + #else + size += 3; + #endif #endif } } -static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim) + +static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim, unsigned flags) { for (;;) { if (src == srcLim) return dest; - UInt32 val = *src++; + UInt32 val = (UInt32)*src++; if (val < 0x80) { @@ -220,22 +562,57 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim continue; } - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) + #ifdef UTF_ESCAPE_BASE + + #if UTF_ESCAPE_PLANE != 0 + /* + if (wchar_t is 32-bit) + && (UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE is set) + && (point is virtual escape plane) + we extract 8-bit byte from virtual HIGH-ESCAPE PLANE. + */ + if (flags & UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE) + if (IS_ESCAPE_POINT(val, UTF_ESCAPE_PLANE)) { - src++; - val = (((val - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000; - dest[0] = _UTF8_HEAD(3, val); - dest[1] = _UTF8_CHAR(2, val); - dest[2] = _UTF8_CHAR(1, val); - dest[3] = _UTF8_CHAR(0, val); - dest += 4; + *dest++ = (char)(val); + continue; + } + #endif // UTF_ESCAPE_PLANE != 0 + + /* if (UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE is defined) + we extract 8-bit byte from BMP-ESCAPE PLANE. */ + + if (flags & UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE) + if (IS_ESCAPE_POINT(val, 0)) + { + *dest++ = (char)(val); continue; } - } + #endif // UTF_ESCAPE_BASE + + if (IS_SURROGATE_POINT(val)) + { + // it's hack to UTF-8 encoding + if (val < 0xdc00 && src != srcLim) + { + const UInt32 c2 = (UInt32)*src; + if (IS_LOW_SURROGATE_POINT(c2)) + { + src++; + val = (((val - 0xd800) << 10) | (c2 - 0xdc00)) + 0x10000; + dest[0] = _UTF8_HEAD(3, val); + dest[1] = _UTF8_CHAR(2, val); + dest[2] = _UTF8_CHAR(1, val); + dest[3] = _UTF8_CHAR(0, val); + dest += 4; + continue; + } + } + if (flags & UTF_FLAG__TO_UTF8__SURROGATE_ERROR) + val = UTF_REPLACEMENT_CHAR; // WIN32 function does it + } + #ifndef _WCHART_IS_16BIT if (val < _UTF8_RANGE(2)) #endif @@ -249,14 +626,25 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim #ifndef _WCHART_IS_16BIT - UInt32 b; + // we don't expect this case. so we can throw exception + // throw 20210407; + + char b; unsigned numBits; if (val < _UTF8_RANGE(3)) { numBits = 6 * 3; b = _UTF8_HEAD(3, val); } else if (val < _UTF8_RANGE(4)) { numBits = 6 * 4; b = _UTF8_HEAD(4, val); } else if (val < _UTF8_RANGE(5)) { numBits = 6 * 5; b = _UTF8_HEAD(5, val); } - else { numBits = 6 * 6; b = _UTF8_START(6); } - - *dest++ = (Byte)b; + #if _UTF8_NUM_TAIL_BYTES_MAX >= 6 + else { numBits = 6 * 6; b = (char)_UTF8_START(6); } + #else + else + { + val = UTF_REPLACEMENT_CHAR; + { numBits = 6 * 3; b = _UTF8_HEAD(3, val); } + } + #endif + + *dest++ = b; do { @@ -269,20 +657,207 @@ static char *Utf16_To_Utf8(char *dest, const wchar_t *src, const wchar_t *srcLim } } -bool ConvertUTF8ToUnicode(const AString &src, UString &dest) +bool Convert_UTF8_Buf_To_Unicode(const char *src, size_t srcSize, UString &dest, unsigned flags) { dest.Empty(); size_t destLen = 0; - Utf8_To_Utf16(NULL, &destLen, src, src.Ptr(src.Len())); - bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src.Ptr(src.Len())); + Utf8_To_Utf16(NULL, &destLen, src, src + srcSize, flags); + bool res = Utf8_To_Utf16(dest.GetBuf((unsigned)destLen), &destLen, src, src + srcSize, flags); dest.ReleaseBuf_SetEnd((unsigned)destLen); return res; } -void ConvertUnicodeToUTF8(const UString &src, AString &dest) +bool ConvertUTF8ToUnicode_Flags(const AString &src, UString &dest, unsigned flags) +{ + return Convert_UTF8_Buf_To_Unicode(src, src.Len(), dest, flags); +} + + +static +unsigned g_UTF8_To_Unicode_Flags = + UTF_FLAG__FROM_UTF8__USE_ESCAPE + #ifndef _WCHART_IS_16BIT + | UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + #ifdef _UTF8_RAW_NON_UTF8_SUPPORTED + | UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + #endif + #endif + ; + + +/* +bool ConvertUTF8ToUnicode_boolRes(const AString &src, UString &dest) +{ + return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags); +} +*/ + +bool ConvertUTF8ToUnicode(const AString &src, UString &dest) { + return ConvertUTF8ToUnicode_Flags(src, dest, g_UTF8_To_Unicode_Flags); +} + +void Print_UString(const UString &a); + +void ConvertUnicodeToUTF8_Flags(const UString &src, AString &dest, unsigned flags) +{ + /* + if (src.Len()== 24) + throw "202104"; + */ dest.Empty(); - size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len())); - Utf16_To_Utf8(dest.GetBuf((unsigned)destLen), src, src.Ptr(src.Len())); + const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags); + char *destStart = dest.GetBuf((unsigned)destLen); + const char *destEnd = Utf16_To_Utf8(destStart, src, src.Ptr(src.Len()), flags); dest.ReleaseBuf_SetEnd((unsigned)destLen); + // printf("\nlen = %d\n", src.Len()); + if (destLen != (size_t)(destEnd - destStart)) + { + /* + // dest.ReleaseBuf_SetEnd((unsigned)(destEnd - destStart)); + printf("\nlen = %d\n", (unsigned)destLen); + printf("\n(destEnd - destStart) = %d\n", (unsigned)(destEnd - destStart)); + printf("\n"); + // Print_UString(src); + printf("\n"); + // printf("\nlen = %d\n", destLen); + */ + throw 20210406; + } +} + + + +unsigned g_Unicode_To_UTF8_Flags = + // UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE + 0 + #ifndef _WIN32 + #ifdef _UTF8_RAW_NON_UTF8_SUPPORTED + | UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE + #else + | UTF_FLAG__TO_UTF8__SURROGATE_ERROR; + #endif + #endif + ; + +void ConvertUnicodeToUTF8(const UString &src, AString &dest) +{ + ConvertUnicodeToUTF8_Flags(src, dest, g_Unicode_To_UTF8_Flags); +} + +void Convert_Unicode_To_UTF8_Buf(const UString &src, CByteBuffer &dest) +{ + const unsigned flags = g_Unicode_To_UTF8_Flags; + dest.Free(); + const size_t destLen = Utf16_To_Utf8_Calc(src, src.Ptr(src.Len()), flags); + dest.Alloc(destLen); + const char *destEnd = Utf16_To_Utf8((char *)(void *)(Byte *)dest, src, src.Ptr(src.Len()), flags); + if (destLen != (size_t)(destEnd - (char *)(void *)(Byte *)dest)) + throw 202104; } + +/* + +#ifndef _WIN32 +void Convert_UTF16_To_UTF32(const UString &src, UString &dest) +{ + dest.Empty(); + for (size_t i = 0; i < src.Len();) + { + wchar_t c = src[i++]; + if (c >= 0xd800 && c < 0xdc00 && i < src.Len()) + { + const wchar_t c2 = src[i]; + if (c2 >= 0xdc00 && c2 < 0x10000) + { + // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + // printf("%4x\n", (int)c); + i++; + } + } + dest += c; + } +} + +void Convert_UTF32_To_UTF16(const UString &src, UString &dest) +{ + dest.Empty(); + for (size_t i = 0; i < src.Len();) + { + wchar_t w = src[i++]; + if (w >= 0x10000 && w < 0x110000) + { + w -= 0x10000; + dest += (wchar_t)((unsigned)0xd800 + (((unsigned)w >> 10) & 0x3ff)); + w = 0xdc00 + (w & 0x3ff); + } + dest += w; + } +} + +bool UTF32_IsThere_BigPoint(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 c = (UInt32)src[i++]; + if (c >= 0x110000) + return true; + } + return false; +} + +bool Unicode_IsThere_BmpEscape(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 c = (UInt32)src[i++]; + if (IS_ESCAPE_POINT(c, 0)) + return true; + } + return false; +} + + +#endif + +bool Unicode_IsThere_Utf16SurrogateError(const UString &src) +{ + for (size_t i = 0; i < src.Len();) + { + const UInt32 val = (UInt32)src[i++]; + if (IS_SURROGATE_POINT(val)) + { + // it's hack to UTF-8 encoding + if (val >= 0xdc00 || i == src.Len()) + return true; + const UInt32 c2 = (UInt32)*src; + if (!IS_LOW_SURROGATE_POINT(c2)) + return true; + } + } + return false; +} +*/ + +#ifndef _WCHART_IS_16BIT + +void Convert_UnicodeEsc16_To_UnicodeEscHigh +#if UTF_ESCAPE_PLANE == 0 + (UString &) {} +#else + (UString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + { + wchar_t c = s[i]; + if (IS_ESCAPE_POINT(c, 0)) + { + c += UTF_ESCAPE_PLANE; + s.ReplaceOneCharAtPos(i, c); + } + } +} +#endif +#endif diff --git a/3rdparty/lzma/CPP/Common/UTFConvert.h b/3rdparty/lzma/CPP/Common/UTFConvert.h index 827f3dcfe19..37c4975a9dd 100644 --- a/3rdparty/lzma/CPP/Common/UTFConvert.h +++ b/3rdparty/lzma/CPP/Common/UTFConvert.h @@ -3,10 +3,382 @@ #ifndef __COMMON_UTF_CONVERT_H #define __COMMON_UTF_CONVERT_H +#include "MyBuffer.h" #include "MyString.h" -bool CheckUTF8(const char *src, bool allowReduced = false) throw(); -bool ConvertUTF8ToUnicode(const AString &utfString, UString &resultString); -void ConvertUnicodeToUTF8(const UString &unicodeString, AString &resultString); +struct CUtf8Check +{ + // Byte MaxByte; // in original src stream + bool NonUtf; + bool ZeroChar; + bool SingleSurrogate; + bool Escape; + bool Truncated; + UInt32 MaxHighPoint; // only for points >= 0x80 + + CUtf8Check() { Clear(); } + + void Clear() + { + // MaxByte = 0; + NonUtf = false; + ZeroChar = false; + SingleSurrogate = false; + Escape = false; + Truncated = false; + MaxHighPoint = 0; + } + + void Update(const CUtf8Check &c) + { + if (c.NonUtf) NonUtf = true; + if (c.ZeroChar) ZeroChar = true; + if (c.SingleSurrogate) SingleSurrogate = true; + if (c.Escape) Escape = true; + if (c.Truncated) Truncated = true; + if (MaxHighPoint < c.MaxHighPoint) MaxHighPoint = c.MaxHighPoint; + } + + void PrintStatus(AString &s) const + { + s.Empty(); + + // s.Add_OptSpaced("MaxByte="); + // s.Add_UInt32(MaxByte); + + if (NonUtf) s.Add_OptSpaced("non-UTF8"); + if (ZeroChar) s.Add_OptSpaced("ZeroChar"); + if (SingleSurrogate) s.Add_OptSpaced("SingleSurrogate"); + if (Escape) s.Add_OptSpaced("Escape"); + if (Truncated) s.Add_OptSpaced("Truncated"); + + if (MaxHighPoint != 0) + { + s.Add_OptSpaced("MaxUnicode="); + s.Add_UInt32(MaxHighPoint); + } + } + + + bool IsOK(bool allowReduced = false) const + { + if (NonUtf || SingleSurrogate || ZeroChar) + return false; + if (MaxHighPoint >= 0x110000) + return false; + if (Truncated && !allowReduced) + return false; + return true; + } + + // it checks full buffer as specified in (size) and it doesn't stop on zero char + void Check_Buf(const char *src, size_t size) throw(); + + void Check_AString(const AString &s) throw() + { + Check_Buf(s.Ptr(), s.Len()); + } +}; + +/* +if (allowReduced == false) - all UTF-8 character sequences must be finished. +if (allowReduced == true) - it allows truncated last character-Utf8-sequence +*/ + +bool Check_UTF8_Buf(const char *src, size_t size, bool allowReduced) throw(); +bool CheckUTF8_AString(const AString &s) throw(); + +#define UTF_FLAG__FROM_UTF8__SURROGATE_ERROR (1 << 0) +#define UTF_FLAG__FROM_UTF8__USE_ESCAPE (1 << 1) +#define UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT (1 << 2) + +/* +UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + + if (flag is NOT set) + { + it processes SINGLE-SURROGATE-8 as valid Unicode point. + it converts SINGLE-SURROGATE-8 to SINGLE-SURROGATE-16 + Note: some sequencies of two SINGLE-SURROGATE-8 points + will generate correct SURROGATE-16-PAIR, and + that SURROGATE-16-PAIR later will be converted to correct + UTF8-SURROGATE-21 point. So we don't restore original + STR-8 sequence in that case. + } + + if (flag is set) + { + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is defined) + it generates ESCAPE for SINGLE-SURROGATE-8, + if (UTF_FLAG__FROM_UTF8__USE_ESCAPE is not defined) + it generates U+fffd for SINGLE-SURROGATE-8, + } + + +UTF_FLAG__FROM_UTF8__USE_ESCAPE + + if (flag is NOT set) + it generates (U+fffd) code for non-UTF-8 (invalid) characters + + if (flag is set) + { + It generates (ESCAPE) codes for NON-UTF-8 (invalid) characters. + And later we can restore original UTF-8-RAW characters from (ESCAPE-16-21) codes. + } + +UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + + if (flag is NOT set) + { + it process ESCAPE-8 points as another Unicode points. + In Linux: ESCAPE-16 will mean two different ESCAPE-8 seqences, + so we need HIGH-ESCAPE-PLANE-21 to restore UTF-8-RAW -> UTF-16 -> UTF-8-RAW + } + + if (flag is set) + { + it generates ESCAPE-16-21 for ESCAPE-8 points + so we can restore UTF-8-RAW -> UTF-16 -> UTF-8-RAW without HIGH-ESCAPE-PLANE-21. + } + + +Main USE CASES with UTF-8 <-> UTF-16 conversions: + + WIN32: UTF-16-RAW -> UTF-8 (Archive) -> UTF-16-RAW + { + set UTF_FLAG__FROM_UTF8__USE_ESCAPE + Do NOT set UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + Do NOT set UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + + So we restore original SINGLE-SURROGATE-16 from single SINGLE-SURROGATE-8. + } + + Linux: UTF-8-RAW -> UTF-16 (Intermediate / Archive) -> UTF-8-RAW + { + we want restore original UTF-8-RAW sequence later from that ESCAPE-16. + Set the flags: + UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + UTF_FLAG__FROM_UTF8__USE_ESCAPE + UTF_FLAG__FROM_UTF8__BMP_ESCAPE_CONVERT + } + + MacOS: UTF-8-RAW -> UTF-16 (Intermediate / Archive) -> UTF-8-RAW + { + we want to restore correct UTF-8 without any BMP processing: + Set the flags: + UTF_FLAG__FROM_UTF8__SURROGATE_ERROR + UTF_FLAG__FROM_UTF8__USE_ESCAPE + } + +*/ + +// zero char is not allowed in (src) buf +bool Convert_UTF8_Buf_To_Unicode(const char *src, size_t srcSize, UString &dest, unsigned flags = 0); + +bool ConvertUTF8ToUnicode_Flags(const AString &src, UString &dest, unsigned flags = 0); +bool ConvertUTF8ToUnicode(const AString &src, UString &dest); + +#define UTF_FLAG__TO_UTF8__SURROGATE_ERROR (1 << 8) +#define UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE (1 << 9) +// #define UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE (1 << 10) + +/* +UTF_FLAG__TO_UTF8__SURROGATE_ERROR + + if (flag is NOT set) + { + we extract SINGLE-SURROGATE as normal UTF-8 + + In Windows : for UTF-16-RAW <-> UTF-8 (archive) <-> UTF-16-RAW in . + + In Linux : + use-case-1: UTF-8 -> UTF-16 -> UTF-8 doesn't generate UTF-16 SINGLE-SURROGATE, + if (UTF_FLAG__FROM_UTF8__SURROGATE_ERROR) is used. + use-case 2: UTF-16-7z (with SINGLE-SURROGATE from Windows) -> UTF-8 (Linux) + will generate SINGLE-SURROGATE-UTF-8 here. + } + + if (flag is set) + { + we generate UTF_REPLACEMENT_CHAR (0xfffd) for SINGLE_SURROGATE + it can be used for compatibility mode with WIN32 UTF function + or if we want UTF-8 stream without any errors + } + + +UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE + + if (flag is NOT set) it doesn't extract raw 8-bit symbol from Escape-Plane-16 + if (flag is set) it extracts raw 8-bit symbol from Escape-Plane-16 + + in Linux we need some way to extract NON-UTF8 RAW 8-bits from BMP (UTF-16 7z archive): + if (we use High-Escape-Plane), we can transfer BMP escapes to High-Escape-Plane. + if (we don't use High-Escape-Plane), we must use UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE. + + +UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE + // that flag affects the code only if (wchar_t is 32-bit) + // that mode with high-escape can be disabled now in UTFConvert.cpp + if (flag is NOT set) + it doesn't extract raw 8-bit symbol from High-Escape-Plane + if (flag is set) + it extracts raw 8-bit symbol from High-Escape-Plane + +Main use cases: + +WIN32 : UTF-16-RAW -> UTF-8 (archive) -> UTF-16-RAW + { + Do NOT set UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE. + Do NOT set UTF_FLAG__TO_UTF8__SURROGATE_ERROR. + So we restore original UTF-16-RAW. + } + +Linix : UTF-8 with Escapes -> UTF-16 (7z archive) -> UTF-8 with Escapes + set UTF_FLAG__TO_UTF8__EXTRACT_BMP_ESCAPE to extract non-UTF from 7z archive + set UTF_FLAG__TO_UTF8__PARSE_HIGH_ESCAPE for intermediate UTF-16. + Note: high esacape mode can be ignored now in UTFConvert.cpp + +macOS: + the system doesn't support incorrect UTF-8 in file names. + set UTF_FLAG__TO_UTF8__SURROGATE_ERROR +*/ + +extern unsigned g_Unicode_To_UTF8_Flags; + +void ConvertUnicodeToUTF8_Flags(const UString &src, AString &dest, unsigned flags = 0); +void ConvertUnicodeToUTF8(const UString &src, AString &dest); + +void Convert_Unicode_To_UTF8_Buf(const UString &src, CByteBuffer &dest); + +/* +#ifndef _WIN32 +void Convert_UTF16_To_UTF32(const UString &src, UString &dest); +void Convert_UTF32_To_UTF16(const UString &src, UString &dest); +bool UTF32_IsThere_BigPoint(const UString &src); +bool Unicode_IsThere_BmpEscape(const UString &src); +#endif + +bool Unicode_IsThere_Utf16SurrogateError(const UString &src); +*/ + +#ifdef _WCHART_IS_16BIT +#define Convert_UnicodeEsc16_To_UnicodeEscHigh(s) +#else +void Convert_UnicodeEsc16_To_UnicodeEscHigh(UString &s); +#endif + +/* +// #include "../../C/CpuArch.h" + +// ---------- Utf16 Little endian functions ---------- + +// We store 16-bit surrogates even in 32-bit WCHARs in Linux. +// So now we don't use the following code: + +#if WCHAR_MAX > 0xffff + +// void *p : pointer to src bytes stream +// size_t len : num Utf16 characters : it can include or not include NULL character + +inline size_t Utf16LE__Get_Num_WCHARs(const void *p, size_t len) +{ + #if WCHAR_MAX > 0xffff + size_t num_wchars = 0; + for (size_t i = 0; i < len; i++) + { + wchar_t c = GetUi16(p); + p = (const void *)((const Byte *)p + 2); + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = GetUi16(p); + if (c2 >= 0xdc00 && c2 < 0xe000) + { + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + p = (const void *)((const Byte *)p + 2); + i++; + } + } + num_wchars++; + } + return num_wchars; + #else + UNUSED_VAR(p) + return len; + #endif +} + +// #include + +inline wchar_t *Utf16LE__To_WCHARs_Sep(const void *p, size_t len, wchar_t *dest) +{ + for (size_t i = 0; i < len; i++) + { + wchar_t c = GetUi16(p); + p = (const void *)((const Byte *)p + 2); + + #if WCHAR_PATH_SEPARATOR != L'/' + if (c == L'/') + c = WCHAR_PATH_SEPARATOR; + #endif + + #if WCHAR_MAX > 0xffff + + if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) + { + wchar_t c2 = GetUi16(p); + if (c2 >= 0xdc00 && c2 < 0xe000) + { + // printf("\nSurragate : %4x %4x -> ", (int)c, (int)c2); + c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); + p = (const void *)((const Byte *)p + 2); + i++; + // printf("%4x\n", (int)c); + } + } + + #endif + + *dest++ = c; + } + return dest; +} + + +inline size_t Get_Num_Utf16_chars_from_wchar_string(const wchar_t *p) +{ + size_t num = 0; + for (;;) + { + wchar_t c = *p++; + if (c == 0) + return num; + num += ((c >= 0x10000 && c < 0x110000) ? 2 : 1); + } + return num; +} + +inline Byte *wchars_to_Utf16LE(const wchar_t *p, Byte *dest) +{ + for (;;) + { + wchar_t c = *p++; + if (c == 0) + return dest; + if (c >= 0x10000 && c < 0x110000) + { + SetUi16(dest , (UInt16)(0xd800 + ((c >> 10) & 0x3FF))); + SetUi16(dest + 2, (UInt16)(0xdc00 + ( c & 0x3FF))); + dest += 4; + } + else + { + SetUi16(dest, c); + dest += 2; + } + } +} + +#endif +*/ #endif diff --git a/3rdparty/lzma/CPP/Common/Wildcard.cpp b/3rdparty/lzma/CPP/Common/Wildcard.cpp index 5e6ddfc6e3f..861f3f71cd3 100644 --- a/3rdparty/lzma/CPP/Common/Wildcard.cpp +++ b/3rdparty/lzma/CPP/Common/Wildcard.cpp @@ -4,9 +4,17 @@ #include "Wildcard.h" +extern +bool g_CaseSensitive; bool g_CaseSensitive = #ifdef _WIN32 false; + #elif defined (__APPLE__) + #ifdef TARGET_OS_IPHONE + true; + #else + false; + #endif #else true; #endif @@ -19,19 +27,92 @@ bool IsPath1PrefixedByPath2(const wchar_t *s1, const wchar_t *s2) return IsString1PrefixedByString2_NoCase(s1, s2); } +// #include + +/* +static int MyStringCompare_PathLinux(const wchar_t *s1, const wchar_t *s2) throw() +{ + for (;;) + { + wchar_t c1 = *s1++; + wchar_t c2 = *s2++; + if (c1 != c2) + { + if (c1 == 0) return -1; + if (c2 == 0) return 1; + if (c1 == '/') c1 = 0; + if (c2 == '/') c2 = 0; + if (c1 < c2) return -1; + if (c1 > c2) return 1; + continue; + } + if (c1 == 0) return 0; + } +} +*/ + +static int MyStringCompare_Path(const wchar_t *s1, const wchar_t *s2) throw() +{ + for (;;) + { + wchar_t c1 = *s1++; + wchar_t c2 = *s2++; + if (c1 != c2) + { + if (c1 == 0) return -1; + if (c2 == 0) return 1; + if (IS_PATH_SEPAR(c1)) c1 = 0; + if (IS_PATH_SEPAR(c2)) c2 = 0; + if (c1 < c2) return -1; + if (c1 > c2) return 1; + continue; + } + if (c1 == 0) return 0; + } +} + +static int MyStringCompareNoCase_Path(const wchar_t *s1, const wchar_t *s2) throw() +{ + for (;;) + { + wchar_t c1 = *s1++; + wchar_t c2 = *s2++; + if (c1 != c2) + { + if (c1 == 0) return -1; + if (c2 == 0) return 1; + if (IS_PATH_SEPAR(c1)) c1 = 0; + if (IS_PATH_SEPAR(c2)) c2 = 0; + c1 = MyCharUpper(c1); + c2 = MyCharUpper(c2); + if (c1 < c2) return -1; + if (c1 > c2) return 1; + continue; + } + if (c1 == 0) return 0; + } +} + int CompareFileNames(const wchar_t *s1, const wchar_t *s2) STRING_UNICODE_THROW { + /* + printf("\nCompareFileNames"); + printf("\n S1: %ls", s1); + printf("\n S2: %ls", s2); + printf("\n"); + */ + // 21.07 : we parse PATH_SEPARATOR so: 0 < PATH_SEPARATOR < 1 if (g_CaseSensitive) - return wcscmp(s1, s2); - return MyStringCompareNoCase(s1, s2); + return MyStringCompare_Path(s1, s2); + return MyStringCompareNoCase_Path(s1, s2); } #ifndef USE_UNICODE_FSTRING int CompareFileNames(const char *s1, const char *s2) { - if (g_CaseSensitive) - return wcscmp(fs2us(s1), fs2us(s2)); - return MyStringCompareNoCase(fs2us(s1), fs2us(s2)); + const UString u1 = fs2us(s1); + const UString u2 = fs2us(s2); + return CompareFileNames(u1, u2); } #endif @@ -120,24 +201,16 @@ void SplitPathToParts_Smart(const UString &path, UString &dirPrefix, UString &na name = p; } +/* UString ExtractDirPrefixFromPath(const UString &path) { - const wchar_t *start = path; - const wchar_t *p = start + path.Len(); - for (; p != start; p--) - if (IsPathSepar(*(p - 1))) - break; - return path.Left((unsigned)(p - start)); + return path.Left(path.ReverseFind_PathSepar() + 1)); } +*/ UString ExtractFileNameFromPath(const UString &path) { - const wchar_t *start = path; - const wchar_t *p = start + path.Len(); - for (; p != start; p--) - if (IsPathSepar(*(p - 1))) - break; - return p; + return UString(path.Ptr((unsigned)(path.ReverseFind_PathSepar() + 1))); } @@ -235,12 +308,12 @@ bool CItem::CheckPath(const UStringVector &pathParts, bool isFile) const { if (WildcardMatching) { - if (!DoesWildcardMatchName(PathParts[i], pathParts[i + d])) + if (!DoesWildcardMatchName(PathParts[i], pathParts[i + (unsigned)d])) break; } else { - if (CompareFileNames(PathParts[i], pathParts[i + d]) != 0) + if (CompareFileNames(PathParts[i], pathParts[i + (unsigned)d]) != 0) break; } } @@ -264,16 +337,14 @@ int CCensorNode::FindSubNode(const UString &name) const { FOR_VECTOR (i, SubNodes) if (CompareFileNames(SubNodes[i].Name, name) == 0) - return i; + return (int)i; return -1; } void CCensorNode::AddItemSimple(bool include, CItem &item) { - if (include) - IncludeItems.Add(item); - else - ExcludeItems.Add(item); + CObjectVector &items = include ? IncludeItems : ExcludeItems; + items.Add(item); } void CCensorNode::AddItem(bool include, CItem &item, int ignoreWildcardIndex) @@ -288,6 +359,7 @@ void CCensorNode::AddItem(bool include, CItem &item, int ignoreWildcardIndex) AddItemSimple(include, item); return; } + const UString &front = item.PathParts.Front(); // WIN32 doesn't support wildcards in file names @@ -298,23 +370,23 @@ void CCensorNode::AddItem(bool include, CItem &item, int ignoreWildcardIndex) AddItemSimple(include, item); return; } - int index = FindSubNode(front); - if (index < 0) - index = SubNodes.Add(CCensorNode(front, this)); + CCensorNode &subNode = Find_SubNode_Or_Add_New(front); item.PathParts.Delete(0); - SubNodes[index].AddItem(include, item, ignoreWildcardIndex - 1); + subNode.AddItem(include, item, ignoreWildcardIndex - 1); } -void CCensorNode::AddItem(bool include, const UString &path, bool recursive, bool forFile, bool forDir, bool wildcardMatching) +/* +void CCensorNode::AddItem(bool include, const UString &path, const CCensorPathProps &props) { CItem item; SplitPathToParts(path, item.PathParts); - item.Recursive = recursive; - item.ForFile = forFile; - item.ForDir = forDir; - item.WildcardMatching = wildcardMatching; + item.Recursive = props.Recursive; + item.ForFile = props.ForFile; + item.ForDir = props.ForDir; + item.WildcardMatching = props.WildcardMatching; AddItem(include, item); } +*/ bool CCensorNode::NeedCheckSubDirs() const { @@ -353,18 +425,19 @@ bool CCensorNode::CheckPathVect(const UStringVector &pathParts, bool isFile, boo include = false; return true; } - include = true; - bool finded = CheckPathCurrent(true, pathParts, isFile); - if (pathParts.Size() <= 1) - return finded; - int index = FindSubNode(pathParts.Front()); - if (index >= 0) + if (pathParts.Size() > 1) { - UStringVector pathParts2 = pathParts; - pathParts2.Delete(0); - if (SubNodes[index].CheckPathVect(pathParts2, isFile, include)) - return true; + int index = FindSubNode(pathParts.Front()); + if (index >= 0) + { + UStringVector pathParts2 = pathParts; + pathParts2.Delete(0); + if (SubNodes[(unsigned)index].CheckPathVect(pathParts2, isFile, include)) + return true; + } } + bool finded = CheckPathCurrent(true, pathParts, isFile); + include = finded; // if (!finded), then (true) is allowed also return finded; } @@ -400,14 +473,26 @@ bool CCensorNode::CheckPath(bool isAltStream, const UString &path, bool isFile) } */ -bool CCensorNode::CheckPathToRoot(bool include, UStringVector &pathParts, bool isFile) const +bool CCensorNode::CheckPathToRoot_Change(bool include, UStringVector &pathParts, bool isFile) const { if (CheckPathCurrent(include, pathParts, isFile)) return true; - if (Parent == 0) + if (!Parent) return false; pathParts.Insert(0, Name); - return Parent->CheckPathToRoot(include, pathParts, isFile); + return Parent->CheckPathToRoot_Change(include, pathParts, isFile); +} + +bool CCensorNode::CheckPathToRoot(bool include, const UStringVector &pathParts, bool isFile) const +{ + if (CheckPathCurrent(include, pathParts, isFile)) + return true; + if (!Parent) + return false; + UStringVector pathParts2; + pathParts2.Add(Name); + pathParts2 += pathParts; + return Parent->CheckPathToRoot_Change(include, pathParts2, isFile); } /* @@ -419,39 +504,21 @@ bool CCensorNode::CheckPathToRoot(bool include, const UString &path, bool isFile } */ -void CCensorNode::AddItem2(bool include, const UString &path, bool recursive, bool wildcardMatching) -{ - if (path.IsEmpty()) - return; - bool forFile = true; - bool forFolder = true; - UString path2 = path; - if (IsPathSepar(path.Back())) - { - path2.DeleteBack(); - forFile = false; - } - AddItem(include, path2, recursive, forFile, forFolder, wildcardMatching); -} - void CCensorNode::ExtendExclude(const CCensorNode &fromNodes) { ExcludeItems += fromNodes.ExcludeItems; FOR_VECTOR (i, fromNodes.SubNodes) { const CCensorNode &node = fromNodes.SubNodes[i]; - int subNodeIndex = FindSubNode(node.Name); - if (subNodeIndex < 0) - subNodeIndex = SubNodes.Add(CCensorNode(node.Name, this)); - SubNodes[subNodeIndex].ExtendExclude(node); + Find_SubNode_Or_Add_New(node.Name).ExtendExclude(node); } } -int CCensor::FindPrefix(const UString &prefix) const +int CCensor::FindPairForPrefix(const UString &prefix) const { FOR_VECTOR (i, Pairs) if (CompareFileNames(Pairs[i].Prefix, prefix) == 0) - return i; + return (int)i; return -1; } @@ -460,7 +527,11 @@ int CCensor::FindPrefix(const UString &prefix) const bool IsDriveColonName(const wchar_t *s) { wchar_t c = s[0]; - return c != 0 && s[1] == ':' && s[2] == 0 && (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'); + return c != 0 + && s[1] == ':' + && s[2] == 0 + && ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z')); } unsigned GetNumPrefixParts_if_DrivePath(UStringVector &pathParts) @@ -488,9 +559,12 @@ static unsigned GetNumPrefixParts(const UStringVector &pathParts) { if (pathParts.IsEmpty()) return 0; + + /* empty last part could be removed already from (pathParts), + if there was tail path separator (slash) in original full path string. */ #ifdef _WIN32 - + if (IsDriveColonName(pathParts[0])) return 1; if (!pathParts[0].IsEmpty()) @@ -531,7 +605,8 @@ static unsigned GetNumPrefixParts(const UStringVector &pathParts) #endif } -void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &path, bool recursive, bool wildcardMatching) +void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &path, + const CCensorPathProps &props) { if (path.IsEmpty()) throw "Empty file path"; @@ -539,12 +614,26 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat UStringVector pathParts; SplitPathToParts(path, pathParts); + CCensorPathProps props2 = props; + bool forFile = true; - if (pathParts.Back().IsEmpty()) + bool forDir = true; + const UString &back = pathParts.Back(); + if (back.IsEmpty()) { + // we have tail path separator. So it's directory. + // we delete tail path separator here even for "\" and "c:\" forFile = false; pathParts.DeleteBack(); } + else + { + if (props.MarkMode == kMark_StrictFile + || (props.MarkMode == kMark_StrictFile_IfWildcard + && DoesNameContainWildcard(back))) + forDir = false; + } + UString prefix; @@ -561,6 +650,7 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat if (pathMode != k_AbsPath) { + // detection of the number of Skip Parts for prefix ignoreWildcardIndex = -1; const unsigned numPrefixParts = GetNumPrefixParts(pathParts); @@ -568,6 +658,7 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat if (pathMode != k_FullPath) { + // if absolute path, then all parts before last part will be in prefix if (numPrefixParts != 0 && pathParts.Size() > numPrefixParts) numSkipParts = pathParts.Size() - 1; } @@ -577,22 +668,25 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat { const UString &part = pathParts[i]; if (part == L".." || part == L".") - dotsIndex = i; + dotsIndex = (int)i; } if (dotsIndex >= 0) + { if (dotsIndex == (int)pathParts.Size() - 1) numSkipParts = pathParts.Size(); else numSkipParts = pathParts.Size() - 1; + } } + // we split (pathParts) to (prefix) and (pathParts). for (unsigned i = 0; i < numSkipParts; i++) { { const UString &front = pathParts.Front(); // WIN32 doesn't support wildcards in file names - if (wildcardMatching) + if (props.WildcardMatching) if (i >= numPrefixParts && DoesNameContainWildcard(front)) break; prefix += front; @@ -602,30 +696,45 @@ void CCensor::AddItem(ECensorPathMode pathMode, bool include, const UString &pat } } - int index = FindPrefix(prefix); + int index = FindPairForPrefix(prefix); if (index < 0) - index = Pairs.Add(CPair(prefix)); + { + index = (int)Pairs.Size(); + Pairs.AddNew().Prefix = prefix; + } if (pathMode != k_AbsPath) { - if (pathParts.IsEmpty() || pathParts.Size() == 1 && pathParts[0].IsEmpty()) + if (pathParts.IsEmpty() || (pathParts.Size() == 1 && pathParts[0].IsEmpty())) { // we create universal item, if we skip all parts as prefix (like \ or L:\ ) pathParts.Clear(); - pathParts.Add(L"*"); + pathParts.Add(UString("*")); forFile = true; - wildcardMatching = true; - recursive = false; + forDir = true; + props2.WildcardMatching = true; + props2.Recursive = false; } } + /* + // not possible now + if (!forDir && !forFile) + { + UString s ("file path was blocked for files and directories: "); + s += path; + throw s; + // return; // for debug : ignore item (don't create Item) + } + */ + CItem item; item.PathParts = pathParts; - item.ForDir = true; + item.ForDir = forDir; item.ForFile = forFile; - item.Recursive = recursive; - item.WildcardMatching = wildcardMatching; - Pairs[index].Head.AddItem(include, item, ignoreWildcardIndex); + item.Recursive = props2.Recursive; + item.WildcardMatching = props2.WildcardMatching; + Pairs[(unsigned)index].Head.AddItem(include, item, ignoreWildcardIndex); } /* @@ -665,18 +774,17 @@ void CCensor::AddPathsToCensor(ECensorPathMode censorPathMode) FOR_VECTOR(i, CensorPaths) { const CCensorPath &cp = CensorPaths[i]; - AddItem(censorPathMode, cp.Include, cp.Path, cp.Recursive, cp.WildcardMatching); + AddItem(censorPathMode, cp.Include, cp.Path, cp.Props); } CensorPaths.Clear(); } -void CCensor::AddPreItem(bool include, const UString &path, bool recursive, bool wildcardMatching) +void CCensor::AddPreItem(bool include, const UString &path, const CCensorPathProps &props) { CCensorPath &cp = CensorPaths.AddNew(); cp.Path = path; cp.Include = include; - cp.Recursive = recursive; - cp.WildcardMatching = wildcardMatching; + cp.Props = props; } } diff --git a/3rdparty/lzma/CPP/Common/Wildcard.h b/3rdparty/lzma/CPP/Common/Wildcard.h index 512687d1e67..0fa48873d14 100644 --- a/3rdparty/lzma/CPP/Common/Wildcard.h +++ b/3rdparty/lzma/CPP/Common/Wildcard.h @@ -51,50 +51,117 @@ struct CItem bool CheckPath(const UStringVector &pathParts, bool isFile) const; }; -class CCensorNode + + +const Byte kMark_FileOrDir = 0; +const Byte kMark_StrictFile = 1; +const Byte kMark_StrictFile_IfWildcard = 2; + +struct CCensorPathProps +{ + bool Recursive; + bool WildcardMatching; + Byte MarkMode; + + CCensorPathProps(): + Recursive(false), + WildcardMatching(true), + MarkMode(kMark_FileOrDir) + {} +}; + + +class CCensorNode MY_UNCOPYABLE { CCensorNode *Parent; bool CheckPathCurrent(bool include, const UStringVector &pathParts, bool isFile) const; void AddItemSimple(bool include, CItem &item); public: - bool CheckPathVect(const UStringVector &pathParts, bool isFile, bool &include) const; + // bool ExcludeDirItems; + + CCensorNode(): + Parent(NULL) + // , ExcludeDirItems(false) + {}; - CCensorNode(): Parent(0) { }; - CCensorNode(const UString &name, CCensorNode *parent): Name(name), Parent(parent) { }; + CCensorNode(const UString &name, CCensorNode *parent): + Parent(parent) + // , ExcludeDirItems(false) + , Name(name) + {} UString Name; // WIN32 doesn't support wildcards in file names CObjectVector SubNodes; CObjectVector IncludeItems; CObjectVector ExcludeItems; + CCensorNode &Find_SubNode_Or_Add_New(const UString &name) + { + int i = FindSubNode(name); + if (i >= 0) + return SubNodes[(unsigned)i]; + // return SubNodes.Add(CCensorNode(name, this)); + CCensorNode &node = SubNodes.AddNew(); + node.Parent = this; + node.Name = name; + return node; + } + bool AreAllAllowed() const; int FindSubNode(const UString &path) const; void AddItem(bool include, CItem &item, int ignoreWildcardIndex = -1); - void AddItem(bool include, const UString &path, bool recursive, bool forFile, bool forDir, bool wildcardMatching); - void AddItem2(bool include, const UString &path, bool recursive, bool wildcardMatching); + // void AddItem(bool include, const UString &path, const CCensorPathProps &props); + void Add_Wildcard() + { + CItem item; + item.PathParts.Add(L"*"); + item.Recursive = false; + item.ForFile = true; + item.ForDir = true; + item.WildcardMatching = true; + AddItem( + true // include + , item); + } + // NeedCheckSubDirs() returns true, if there are IncludeItems rules that affect items in subdirs bool NeedCheckSubDirs() const; bool AreThereIncludeItems() const; + /* + CheckPathVect() doesn't check path in Parent CCensorNode + so use CheckPathVect() for root CCensorNode + OUT: + returns (true) && (include = false) - file in exlude list + returns (true) && (include = true) - file in include list and is not in exlude list + returns (false) - file is not in (include/exlude) list + */ + bool CheckPathVect(const UStringVector &pathParts, bool isFile, bool &include) const; + // bool CheckPath2(bool isAltStream, const UString &path, bool isFile, bool &include) const; // bool CheckPath(bool isAltStream, const UString &path, bool isFile) const; - bool CheckPathToRoot(bool include, UStringVector &pathParts, bool isFile) const; + // CheckPathToRoot_Change() changes pathParts !!! + bool CheckPathToRoot_Change(bool include, UStringVector &pathParts, bool isFile) const; + bool CheckPathToRoot(bool include, const UStringVector &pathParts, bool isFile) const; + // bool CheckPathToRoot(const UString &path, bool isFile, bool include) const; void ExtendExclude(const CCensorNode &fromNodes); }; -struct CPair + +struct CPair MY_UNCOPYABLE { UString Prefix; CCensorNode Head; - CPair(const UString &prefix): Prefix(prefix) { }; + // CPair(const UString &prefix): Prefix(prefix) { }; }; + enum ECensorPathMode { k_RelatPath, // absolute prefix as Prefix, remain path in Tree @@ -102,48 +169,63 @@ enum ECensorPathMode k_AbsPath // full path in Tree }; + struct CCensorPath { UString Path; bool Include; - bool Recursive; - bool WildcardMatching; + CCensorPathProps Props; CCensorPath(): - Include(true), - Recursive(false), - WildcardMatching(true) - {} + Include(true) + {} }; -class CCensor + +class CCensor MY_UNCOPYABLE { - int FindPrefix(const UString &prefix) const; + int FindPairForPrefix(const UString &prefix) const; public: CObjectVector Pairs; + bool ExcludeDirItems; + bool ExcludeFileItems; + + CCensor(): + ExcludeDirItems(false), + ExcludeFileItems(false) + {} + CObjectVector CensorPaths; bool AllAreRelative() const { return (Pairs.Size() == 1 && Pairs.Front().Prefix.IsEmpty()); } - void AddItem(ECensorPathMode pathMode, bool include, const UString &path, bool recursive, bool wildcardMatching); + void AddItem(ECensorPathMode pathMode, bool include, const UString &path, const CCensorPathProps &props); // bool CheckPath(bool isAltStream, const UString &path, bool isFile) const; void ExtendExclude(); void AddPathsToCensor(NWildcard::ECensorPathMode censorPathMode); - void AddPreItem(bool include, const UString &path, bool recursive, bool wildcardMatching); - void AddPreItem(const UString &path) + void AddPreItem(bool include, const UString &path, const CCensorPathProps &props); + + void AddPreItem_NoWildcard(const UString &path) { - AddPreItem(true, path, false, false); + CCensorPathProps props; + props.WildcardMatching = false; + AddPreItem( + true, // include + path, props); } void AddPreItem_Wildcard() { - AddPreItem(true, L"*", false, true); + CCensorPathProps props; + // props.WildcardMatching = true; + AddPreItem( + true, // include + UString("*"), props); } }; - } #endif diff --git a/3rdparty/lzma/CPP/Common/XzCrc64Init.cpp b/3rdparty/lzma/CPP/Common/XzCrc64Init.cpp new file mode 100644 index 00000000000..5cb8e674489 --- /dev/null +++ b/3rdparty/lzma/CPP/Common/XzCrc64Init.cpp @@ -0,0 +1,7 @@ +// XzCrc64Init.cpp + +#include "StdAfx.h" + +#include "../../C/XzCrc64.h" + +static struct CCrc64Gen { CCrc64Gen() { Crc64GenerateTable(); } } g_Crc64TableInit; diff --git a/3rdparty/lzma/CPP/Windows/Clipboard.cpp b/3rdparty/lzma/CPP/Windows/Clipboard.cpp new file mode 100644 index 00000000000..bc7e201dd8d --- /dev/null +++ b/3rdparty/lzma/CPP/Windows/Clipboard.cpp @@ -0,0 +1,130 @@ +// Windows/Clipboard.cpp + +#include "StdAfx.h" + +#ifdef UNDER_CE +#include +#endif + +#include "../Common/StringConvert.h" + +#include "Clipboard.h" +#include "Defs.h" +#include "MemoryGlobal.h" +#include "Shell.h" + +namespace NWindows { + +bool CClipboard::Open(HWND wndNewOwner) throw() +{ + m_Open = BOOLToBool(::OpenClipboard(wndNewOwner)); + return m_Open; +} + +bool CClipboard::Close() throw() +{ + if (!m_Open) + return true; + m_Open = !BOOLToBool(CloseClipboard()); + return !m_Open; +} + +bool ClipboardIsFormatAvailableHDROP() +{ + return BOOLToBool(IsClipboardFormatAvailable(CF_HDROP)); +} + +/* +bool ClipboardGetTextString(AString &s) +{ + s.Empty(); + if (!IsClipboardFormatAvailable(CF_TEXT)) + return false; + CClipboard clipboard; + + if (!clipboard.Open(NULL)) + return false; + + HGLOBAL h = ::GetClipboardData(CF_TEXT); + if (h != NULL) + { + NMemory::CGlobalLock globalLock(h); + const char *p = (const char *)globalLock.GetPointer(); + if (p != NULL) + { + s = p; + return true; + } + } + return false; +} +*/ + +/* +bool ClipboardGetFileNames(UStringVector &names) +{ + names.Clear(); + if (!IsClipboardFormatAvailable(CF_HDROP)) + return false; + CClipboard clipboard; + + if (!clipboard.Open(NULL)) + return false; + + HGLOBAL h = ::GetClipboardData(CF_HDROP); + if (h != NULL) + { + NMemory::CGlobalLock globalLock(h); + void *p = (void *)globalLock.GetPointer(); + if (p != NULL) + { + NShell::CDrop drop(false); + drop.Attach((HDROP)p); + drop.QueryFileNames(names); + return true; + } + } + return false; +} +*/ + +static bool ClipboardSetData(UINT uFormat, const void *data, size_t size) throw() +{ + NMemory::CGlobal global; + if (!global.Alloc(GMEM_DDESHARE | GMEM_MOVEABLE, size)) + return false; + { + NMemory::CGlobalLock globalLock(global); + LPVOID p = globalLock.GetPointer(); + if (!p) + return false; + memcpy(p, data, size); + } + if (::SetClipboardData(uFormat, global) == NULL) + return false; + global.Detach(); + return true; +} + +bool ClipboardSetText(HWND owner, const UString &s) +{ + CClipboard clipboard; + if (!clipboard.Open(owner)) + return false; + if (!::EmptyClipboard()) + return false; + + bool res; + res = ClipboardSetData(CF_UNICODETEXT, (const wchar_t *)s, (s.Len() + 1) * sizeof(wchar_t)); + #ifndef _UNICODE + AString a (UnicodeStringToMultiByte(s, CP_ACP)); + if (ClipboardSetData(CF_TEXT, (const char *)a, (a.Len() + 1) * sizeof(char))) + res = true; + a = UnicodeStringToMultiByte(s, CP_OEMCP); + if (ClipboardSetData(CF_OEMTEXT, (const char *)a, (a.Len() + 1) * sizeof(char))) + res = true; + #endif + return res; +} + +} diff --git a/3rdparty/lzma/CPP/Windows/Clipboard.h b/3rdparty/lzma/CPP/Windows/Clipboard.h new file mode 100644 index 00000000000..60fedc13b14 --- /dev/null +++ b/3rdparty/lzma/CPP/Windows/Clipboard.h @@ -0,0 +1,28 @@ +// Windows/Clipboard.h + +#ifndef __CLIPBOARD_H +#define __CLIPBOARD_H + +#include "../Common/MyString.h" + +namespace NWindows { + +class CClipboard +{ + bool m_Open; +public: + CClipboard(): m_Open(false) {}; + ~CClipboard() { Close(); } + bool Open(HWND wndNewOwner) throw(); + bool Close() throw(); +}; + +bool ClipboardIsFormatAvailableHDROP(); + +// bool ClipboardGetFileNames(UStringVector &names); +// bool ClipboardGetTextString(AString &s); +bool ClipboardSetText(HWND owner, const UString &s); + +} + +#endif diff --git a/3rdparty/lzma/CPP/Windows/CommonDialog.cpp b/3rdparty/lzma/CPP/Windows/CommonDialog.cpp index 8f4f56d39a4..eaaecada727 100644 --- a/3rdparty/lzma/CPP/Windows/CommonDialog.cpp +++ b/3rdparty/lzma/CPP/Windows/CommonDialog.cpp @@ -67,8 +67,11 @@ bool CDoubleZeroStringListW::Add(LPCWSTR s) throw() return true; } + +#ifdef UNDER_CE #define MY__OFN_PROJECT 0x00400000 #define MY__OFN_SHOW_ALL 0x01000000 +#endif /* if (lpstrFilter == NULL && nFilterIndex == 0) MSDN : "the system doesn't show any files", @@ -91,14 +94,34 @@ So we use size of old version of structure. */ #if defined(UNDER_CE) || defined(_WIN64) || (_WIN32_WINNT < 0x0500) // || !defined(WINVER) + #ifndef _UNICODE #define my_compatib_OPENFILENAMEA_size sizeof(OPENFILENAMEA) + #endif #define my_compatib_OPENFILENAMEW_size sizeof(OPENFILENAMEW) #else + + // MinGW doesn't support some required macros. So we define them here: + #ifndef CDSIZEOF_STRUCT + #define CDSIZEOF_STRUCT(structname, member) (((int)((LPBYTE)(&((structname*)0)->member) - ((LPBYTE)((structname*)0)))) + sizeof(((structname*)0)->member)) + #endif + #ifndef _UNICODE + #ifndef OPENFILENAME_SIZE_VERSION_400A + #define OPENFILENAME_SIZE_VERSION_400A CDSIZEOF_STRUCT(OPENFILENAMEA,lpTemplateName) + #endif + #endif + #ifndef OPENFILENAME_SIZE_VERSION_400W + #define OPENFILENAME_SIZE_VERSION_400W CDSIZEOF_STRUCT(OPENFILENAMEW,lpTemplateName) + #endif + + #ifndef _UNICODE #define my_compatib_OPENFILENAMEA_size OPENFILENAME_SIZE_VERSION_400A + #endif #define my_compatib_OPENFILENAMEW_size OPENFILENAME_SIZE_VERSION_400W #endif +#ifndef _UNICODE #define CONV_U_To_A(dest, src, temp) AString temp; if (src) { temp = GetSystemString(src); dest = temp; } +#endif bool MyGetOpenFileName(HWND hwnd, LPCWSTR title, LPCWSTR initialDir, diff --git a/3rdparty/lzma/CPP/Windows/Control/ComboBox.cpp b/3rdparty/lzma/CPP/Windows/Control/ComboBox.cpp index febc61ef992..f6ed8d346e4 100644 --- a/3rdparty/lzma/CPP/Windows/Control/ComboBox.cpp +++ b/3rdparty/lzma/CPP/Windows/Control/ComboBox.cpp @@ -43,10 +43,10 @@ LRESULT CComboBox::GetLBText(int index, UString &s) s.Empty(); if (g_IsNT) { - LRESULT len = SendMsgW(CB_GETLBTEXTLEN, index, 0); + LRESULT len = SendMsgW(CB_GETLBTEXTLEN, MY__int_TO_WPARAM(index), 0); if (len == CB_ERR) return len; - LRESULT len2 = SendMsgW(CB_GETLBTEXT, index, (LPARAM)s.GetBuf((unsigned)len)); + LRESULT len2 = SendMsgW(CB_GETLBTEXT, MY__int_TO_WPARAM(index), (LPARAM)s.GetBuf((unsigned)len)); if (len2 == CB_ERR) return len; if (len > len2) diff --git a/3rdparty/lzma/CPP/Windows/Control/ComboBox.h b/3rdparty/lzma/CPP/Windows/Control/ComboBox.h index 1d5a482134c..f08b1f7ce28 100644 --- a/3rdparty/lzma/CPP/Windows/Control/ComboBox.h +++ b/3rdparty/lzma/CPP/Windows/Control/ComboBox.h @@ -5,13 +5,15 @@ #include "../../Common/MyWindows.h" -#include +#include #include "../Window.h" namespace NWindows { namespace NControl { +#define MY__int_TO_WPARAM(i) ((WPARAM)(INT_PTR)(i)) + class CComboBox: public CWindow { public: @@ -20,19 +22,25 @@ public: #ifndef _UNICODE LRESULT AddString(LPCWSTR s); #endif - LRESULT SetCurSel(int index) { return SendMsg(CB_SETCURSEL, index, 0); } + + /* If this parameter is -1, any current selection in the list is removed and the edit control is cleared.*/ + LRESULT SetCurSel(int index) { return SendMsg(CB_SETCURSEL, MY__int_TO_WPARAM(index), 0); } + + /* If no item is selected, it returns CB_ERR (-1) */ int GetCurSel() { return (int)SendMsg(CB_GETCURSEL, 0, 0); } + + /* If an error occurs, it is CB_ERR (-1) */ int GetCount() { return (int)SendMsg(CB_GETCOUNT, 0, 0); } - LRESULT GetLBTextLen(int index) { return SendMsg(CB_GETLBTEXTLEN, index, 0); } - LRESULT GetLBText(int index, LPTSTR s) { return SendMsg(CB_GETLBTEXT, index, (LPARAM)s); } + LRESULT GetLBTextLen(int index) { return SendMsg(CB_GETLBTEXTLEN, MY__int_TO_WPARAM(index), 0); } + LRESULT GetLBText(int index, LPTSTR s) { return SendMsg(CB_GETLBTEXT, MY__int_TO_WPARAM(index), (LPARAM)s); } LRESULT GetLBText(int index, CSysString &s); #ifndef _UNICODE LRESULT GetLBText(int index, UString &s); #endif - LRESULT SetItemData(int index, LPARAM lParam) { return SendMsg(CB_SETITEMDATA, index, lParam); } - LRESULT GetItemData(int index) { return SendMsg(CB_GETITEMDATA, index, 0); } + LRESULT SetItemData(int index, LPARAM lParam) { return SendMsg(CB_SETITEMDATA, MY__int_TO_WPARAM(index), lParam); } + LRESULT GetItemData(int index) { return SendMsg(CB_GETITEMDATA, MY__int_TO_WPARAM(index), 0); } LRESULT GetItemData_of_CurSel() { return GetItemData(GetCurSel()); } @@ -46,7 +54,11 @@ class CComboBoxEx: public CComboBox public: bool SetUnicodeFormat(bool fUnicode) { return LRESULTToBool(SendMsg(CBEM_SETUNICODEFORMAT, BOOLToBool(fUnicode), 0)); } - LRESULT DeleteItem(int index) { return SendMsg(CBEM_DELETEITEM, index, 0); } + /* Returns: + an INT value that represents the number of items remaining in the control. + If (index) is invalid, the message returns CB_ERR. */ + LRESULT DeleteItem(int index) { return SendMsg(CBEM_DELETEITEM, MY__int_TO_WPARAM(index), 0); } + LRESULT InsertItem(COMBOBOXEXITEM *item) { return SendMsg(CBEM_INSERTITEM, 0, (LPARAM)item); } #ifndef _UNICODE LRESULT InsertItem(COMBOBOXEXITEMW *item) { return SendMsg(CBEM_INSERTITEMW, 0, (LPARAM)item); } diff --git a/3rdparty/lzma/CPP/Windows/Control/Dialog.cpp b/3rdparty/lzma/CPP/Windows/Control/Dialog.cpp index 9df3ef5ef24..9ddd23427d8 100644 --- a/3rdparty/lzma/CPP/Windows/Control/Dialog.cpp +++ b/3rdparty/lzma/CPP/Windows/Control/Dialog.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +// #include "../../Windows/DLL.h" + #ifndef _UNICODE #include "../../Common/StringConvert.h" #endif @@ -26,6 +28,14 @@ static INT_PTR APIENTRY DialogProcedure(HWND dialogHWND, UINT message, WPARAM wP return FALSE; if (message == WM_INITDIALOG) dialog->Attach(dialogHWND); + + /* MSDN: The dialog box procedure should return + TRUE - if it processed the message + FALSE - if it did not process the message + If the dialog box procedure returns FALSE, + the dialog manager performs the default dialog operation in response to the message. + */ + try { return BoolToBOOL(dialog->OnMessage(message, wParam, lParam)); } catch(...) { return TRUE; } } @@ -39,6 +49,7 @@ bool CDialog::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) case WM_NOTIFY: return OnNotify((UINT)wParam, (LPNMHDR) lParam); case WM_TIMER: return OnTimer(wParam, lParam); case WM_SIZE: return OnSize(wParam, LOWORD(lParam), HIWORD(lParam)); + case WM_DESTROY: return OnDestroy(); case WM_HELP: OnHelp(); return true; /* OnHelp( @@ -72,29 +83,62 @@ bool CDialog::OnButtonClicked(int buttonID, HWND /* buttonHWND */) { case IDOK: OnOK(); break; case IDCANCEL: OnCancel(); break; + case IDCLOSE: OnClose(); break; case IDHELP: OnHelp(); break; default: return false; } return true; } -static bool GetWorkAreaRect(RECT *rect) + +static bool GetWorkAreaRect(RECT *rect, HWND hwnd) { - // use another function for multi-monitor. + if (hwnd) + { + #ifndef UNDER_CE + /* MonitorFromWindow() is supported in Win2000+ + MonitorFromWindow() : retrieves a handle to the display monitor that has the + largest area of intersection with the bounding rectangle of a specified window. + dwFlags: Determines the function's return value if the window does not intersect any display monitor. + MONITOR_DEFAULTTONEAREST : Returns display that is nearest to the window. + MONITOR_DEFAULTTONULL : Returns NULL. + MONITOR_DEFAULTTOPRIMARY : Returns the primary display monitor. + */ + const HMONITOR hmon = MonitorFromWindow(hwnd, MONITOR_DEFAULTTOPRIMARY); + if (hmon) + { + MONITORINFO mi; + memset(&mi, 0, sizeof(mi)); + mi.cbSize = sizeof(mi); + if (GetMonitorInfoA(hmon, &mi)) + { + *rect = mi.rcWork; + return true; + } + } + #endif + } + + /* Retrieves the size of the work area on the primary display monitor. + The work area is the portion of the screen not obscured + by the system taskbar or by application desktop toolbars. + Any DPI virtualization mode of the caller has no effect on this output. */ + return BOOLToBool(::SystemParametersInfo(SPI_GETWORKAREA, 0, rect, 0)); } -bool IsDialogSizeOK(int xSize, int ySize) + +bool IsDialogSizeOK(int xSize, int ySize, HWND hwnd) { // it returns for system font. Real font uses another values - LONG v = GetDialogBaseUnits(); - int x = LOWORD(v); - int y = HIWORD(v); + const LONG v = GetDialogBaseUnits(); + const int x = LOWORD(v); + const int y = HIWORD(v); RECT rect; - GetWorkAreaRect(&rect); - int wx = RECT_SIZE_X(rect); - int wy = RECT_SIZE_Y(rect); + GetWorkAreaRect(&rect, hwnd); + const int wx = RECT_SIZE_X(rect); + const int wy = RECT_SIZE_Y(rect); return xSize / 4 * x <= wx && ySize / 8 * y <= wy; @@ -149,47 +193,167 @@ bool CDialog::MoveItem(int id, int x, int y, int width, int height, bool repaint return BOOLToBool(::MoveWindow(GetItem(id), x, y, width, height, BoolToBOOL(repaint))); } + +/* +typedef BOOL (WINAPI * Func_DwmGetWindowAttribute)( + HWND hwnd, DWORD dwAttribute, PVOID pvAttribute, DWORD cbAttribute); + +static bool GetWindowsRect_DWM(HWND hwnd, RECT *rect) +{ + // dll load and free is too slow : 300 calls in second. + NDLL::CLibrary dll; + if (!dll.Load(FTEXT("dwmapi.dll"))) + return false; + Func_DwmGetWindowAttribute f = (Func_DwmGetWindowAttribute)dll.GetProc("DwmGetWindowAttribute" ); + if (f) + { + #define MY__DWMWA_EXTENDED_FRAME_BOUNDS 9 + // 30000 per second + RECT r; + if (f(hwnd, MY__DWMWA_EXTENDED_FRAME_BOUNDS, &r, sizeof(RECT)) == S_OK) + { + *rect = r; + return true; + } + } + return false; +} +*/ + + +static bool IsRect_Small_Inside_Big(const RECT &sm, const RECT &big) +{ + return sm.left >= big.left + && sm.right <= big.right + && sm.top >= big.top + && sm.bottom <= big.bottom; +} + + +static bool AreRectsOverlapped(const RECT &r1, const RECT &r2) +{ + return r1.left < r2.right + && r1.right > r2.left + && r1.top < r2.bottom + && r1.bottom > r2.top; +} + + +static bool AreRectsEqual(const RECT &r1, const RECT &r2) +{ + return r1.left == r2.left + && r1.right == r2.right + && r1.top == r2.top + && r1.bottom == r2.bottom; +} + + void CDialog::NormalizeSize(bool fullNormalize) { RECT workRect; - GetWorkAreaRect(&workRect); - int xSize = RECT_SIZE_X(workRect); - int ySize = RECT_SIZE_Y(workRect); + if (!GetWorkAreaRect(&workRect, *this)) + return; RECT rect; - GetWindowRect(&rect); - int xSize2 = RECT_SIZE_X(rect); - int ySize2 = RECT_SIZE_Y(rect); - bool needMove = (xSize2 > xSize || ySize2 > ySize); - if (xSize2 > xSize || (needMove && fullNormalize)) + if (!GetWindowRect(&rect)) + return; + int xs = RECT_SIZE_X(rect); + int ys = RECT_SIZE_Y(rect); + + // we don't want to change size using workRect, if window is outside of WorkArea + if (!AreRectsOverlapped(rect, workRect)) + return; + + /* here rect and workRect are overlapped, but it can be false + overlapping of small shadow when window in another display. */ + + const int xsW = RECT_SIZE_X(workRect); + const int ysW = RECT_SIZE_Y(workRect); + if (xs <= xsW && ys <= ysW) + return; // size of window is OK + if (fullNormalize) { - rect.left = workRect.left; - rect.right = workRect.right; - xSize2 = xSize; + Show(SW_SHOWMAXIMIZED); + return; } - if (ySize2 > ySize || (needMove && fullNormalize)) + int x = workRect.left; + int y = workRect.top; + if (xs < xsW) x += (xsW - xs) / 2; else xs = xsW; + if (ys < ysW) y += (ysW - ys) / 2; else ys = ysW; + Move(x, y, xs, ys, true); +} + + +void CDialog::NormalizePosition() +{ + RECT workRect; + if (!GetWorkAreaRect(&workRect, *this)) + return; + + RECT rect2 = workRect; + bool useWorkArea = true; + const HWND parentHWND = GetParent(); + + if (parentHWND) { - rect.top = workRect.top; - rect.bottom = workRect.bottom; - ySize2 = ySize; + RECT workRectParent; + if (!GetWorkAreaRect(&workRectParent, parentHWND)) + return; + + // if windows are in different monitors, we use only workArea of current window + + if (AreRectsEqual(workRectParent, workRect)) + { + // RECT rect3; if (GetWindowsRect_DWM(parentHWND, &rect3)) {} + CWindow wnd(parentHWND); + if (wnd.GetWindowRect(&rect2)) + { + // it's same monitor. So we try to use parentHWND rect. + /* we don't want to change position, if parent window is not inside work area. + In Win10 : parent window rect is 8 pixels larger for each corner than window size for shadow. + In maximize mode : window is outside of workRect. + if parent window is inside workRect, we will use parent window instead of workRect */ + if (IsRect_Small_Inside_Big(rect2, workRect)) + useWorkArea = false; + } + } } - if (needMove) + + RECT rect; + if (!GetWindowRect(&rect)) + return; + + if (useWorkArea) { - if (fullNormalize) - Show(SW_SHOWMAXIMIZED); - else - Move(rect.left, rect.top, xSize2, ySize2, true); + // we don't want to move window, if it's already inside. + if (IsRect_Small_Inside_Big(rect, workRect)) + return; + // we don't want to move window, if it's outside of workArea + if (!AreRectsOverlapped(rect, workRect)) + return; + rect2 = workRect; } -} -void CDialog::NormalizePosition() -{ - RECT workRect, rect; - GetWorkAreaRect(&workRect); - GetWindowRect(&rect); - if (rect.bottom > workRect.bottom && rect.top > workRect.top) - Move(rect.left, workRect.top, RECT_SIZE_X(rect), RECT_SIZE_Y(rect), true); + { + const int xs = RECT_SIZE_X(rect); + const int ys = RECT_SIZE_Y(rect); + const int xs2 = RECT_SIZE_X(rect2); + const int ys2 = RECT_SIZE_Y(rect2); + // we don't want to change position if parent is smaller. + if (xs <= xs2 && ys <= ys2) + { + const int x = rect2.left + (xs2 - xs) / 2; + const int y = rect2.top + (ys2 - ys) / 2; + + if (x != rect.left || y != rect.top) + Move(x, y, xs, ys, true); + // SetWindowPos(*this, HWND_TOP, x, y, 0, 0, SWP_NOSIZE); + return; + } + } } + + bool CModelessDialog::Create(LPCTSTR templateName, HWND parentWindow) { HWND aHWND = CreateDialogParam(g_hInstance, templateName, parentWindow, DialogProcedure, (LPARAM)this); diff --git a/3rdparty/lzma/CPP/Windows/Control/Dialog.h b/3rdparty/lzma/CPP/Windows/Control/Dialog.h index 59b9f41935c..8a39e996fa0 100644 --- a/3rdparty/lzma/CPP/Windows/Control/Dialog.h +++ b/3rdparty/lzma/CPP/Windows/Control/Dialog.h @@ -31,6 +31,12 @@ public: bool SetItemText(int itemID, LPCTSTR s) { return BOOLToBool(SetDlgItemText(_window, itemID, s)); } + bool SetItemTextA(int itemID, LPCSTR s) + { return BOOLToBool(SetDlgItemTextA(_window, itemID, s)); } + + bool SetItemText_Empty(int itemID) + { return SetItemText(itemID, TEXT("")); } + #ifndef _UNICODE bool SetItemText(int itemID, LPCWSTR s) { @@ -51,6 +57,12 @@ public: */ #endif + bool GetItemText(int itemID, UString &s) + { + CWindow window(GetItem(itemID)); + return window.GetText(s); + } + bool SetItemInt(int itemID, UINT value, bool isSigned) { return BOOLToBool(SetDlgItemInt(_window, itemID, value, BoolToBOOL(isSigned))); } bool GetItemInt(int itemID, bool isSigned, UINT &value) @@ -65,6 +77,13 @@ public: HWND GetNextTabItem(HWND control, bool previous) { return GetNextDlgTabItem(_window, control, BoolToBOOL(previous)); } + LRESULT SendMsg_NextDlgCtl(WPARAM wParam, LPARAM lParam) + { return SendMsg(WM_NEXTDLGCTL, wParam, lParam); } + LRESULT SendMsg_NextDlgCtl_HWND(HWND hwnd) { return SendMsg_NextDlgCtl((WPARAM)hwnd, TRUE); } + LRESULT SendMsg_NextDlgCtl_CtlId(int id) { return SendMsg_NextDlgCtl_HWND(GetItem(id)); } + LRESULT SendMsg_NextDlgCtl_Next() { return SendMsg_NextDlgCtl(0, FALSE); } + LRESULT SendMsg_NextDlgCtl_Prev() { return SendMsg_NextDlgCtl(1, FALSE); } + bool MapRect(LPRECT rect) { return BOOLToBool(MapDialogRect(_window, rect)); } @@ -92,6 +111,7 @@ public: virtual bool OnCommand(WPARAM wParam, LPARAM lParam); virtual bool OnCommand(int code, int itemID, LPARAM lParam); virtual bool OnSize(WPARAM /* wParam */, int /* xSize */, int /* ySize */) { return false; } + virtual bool OnDestroy() { return false; } /* #ifdef UNDER_CE @@ -105,6 +125,7 @@ public: virtual bool OnButtonClicked(int buttonID, HWND buttonHWND); virtual void OnOK() {}; virtual void OnCancel() {}; + virtual void OnClose() {} virtual bool OnNotify(UINT /* controlID */, LPNMHDR /* lParam */) { return false; } virtual bool OnTimer(WPARAM /* timerID */, LPARAM /* callback */) { return false; } @@ -133,6 +154,7 @@ public: #endif virtual void OnOK() { Destroy(); } virtual void OnCancel() { Destroy(); } + virtual void OnClose() { Destroy(); } }; class CModalDialog: public CDialog @@ -147,6 +169,7 @@ public: bool End(INT_PTR result) { return BOOLToBool(::EndDialog(_window, result)); } virtual void OnOK() { End(IDOK); } virtual void OnCancel() { End(IDCANCEL); } + virtual void OnClose() { End(IDCLOSE); } }; class CDialogChildControl: public NWindows::CWindow @@ -160,7 +183,7 @@ public: } }; -bool IsDialogSizeOK(int xSize, int ySize); +bool IsDialogSizeOK(int xSize, int ySize, HWND hwnd = NULL); }} diff --git a/3rdparty/lzma/CPP/Windows/Control/ImageList.h b/3rdparty/lzma/CPP/Windows/Control/ImageList.h index 0d9c9313103..19feb117d66 100644 --- a/3rdparty/lzma/CPP/Windows/Control/ImageList.h +++ b/3rdparty/lzma/CPP/Windows/Control/ImageList.h @@ -3,7 +3,7 @@ #ifndef __WINDOWS_CONTROL_IMAGE_LIST_H #define __WINDOWS_CONTROL_IMAGE_LIST_H -#include +#include #include "../Defs.h" diff --git a/3rdparty/lzma/CPP/Windows/Control/ListView.cpp b/3rdparty/lzma/CPP/Windows/Control/ListView.cpp index 6d916591a51..16cfd396ede 100644 --- a/3rdparty/lzma/CPP/Windows/Control/ListView.cpp +++ b/3rdparty/lzma/CPP/Windows/Control/ListView.cpp @@ -35,7 +35,7 @@ int CListView::InsertColumn(int columnIndex, LPCTSTR text, int width) { LVCOLUMN ci; ci.mask = LVCF_TEXT | LVCF_WIDTH | LVCF_SUBITEM; - ci.pszText = (LPTSTR)text; + ci.pszText = (LPTSTR)(void *)text; ci.iSubItem = columnIndex; ci.cx = width; return InsertColumn(columnIndex, &ci); @@ -47,7 +47,7 @@ int CListView::InsertItem(int index, LPCTSTR text) item.mask = LVIF_TEXT | LVIF_PARAM; item.iItem = index; item.lParam = index; - item.pszText = (LPTSTR)text; + item.pszText = (LPTSTR)(void *)text; item.iSubItem = 0; return InsertItem(&item); } @@ -57,7 +57,7 @@ int CListView::SetSubItem(int index, int subIndex, LPCTSTR text) LVITEM item; item.mask = LVIF_TEXT; item.iItem = index; - item.pszText = (LPTSTR)text; + item.pszText = (LPTSTR)(void *)text; item.iSubItem = subIndex; return SetItem(&item); } @@ -68,7 +68,7 @@ int CListView::InsertColumn(int columnIndex, LPCWSTR text, int width) { LVCOLUMNW ci; ci.mask = LVCF_TEXT | LVCF_WIDTH | LVCF_SUBITEM; - ci.pszText = (LPWSTR)text; + ci.pszText = (LPWSTR)(void *)text; ci.iSubItem = columnIndex; ci.cx = width; return InsertColumn(columnIndex, &ci); @@ -80,7 +80,7 @@ int CListView::InsertItem(int index, LPCWSTR text) item.mask = LVIF_TEXT | LVIF_PARAM; item.iItem = index; item.lParam = index; - item.pszText = (LPWSTR)text; + item.pszText = (LPWSTR)(void *)text; item.iSubItem = 0; return InsertItem(&item); } @@ -90,7 +90,7 @@ int CListView::SetSubItem(int index, int subIndex, LPCWSTR text) LVITEMW item; item.mask = LVIF_TEXT; item.iItem = index; - item.pszText = (LPWSTR)text; + item.pszText = (LPWSTR)(void *)text; item.iSubItem = subIndex; return SetItem(&item); } diff --git a/3rdparty/lzma/CPP/Windows/Control/ListView.h b/3rdparty/lzma/CPP/Windows/Control/ListView.h index 9a3abe70fa1..a13b1041fd9 100644 --- a/3rdparty/lzma/CPP/Windows/Control/ListView.h +++ b/3rdparty/lzma/CPP/Windows/Control/ListView.h @@ -5,7 +5,7 @@ #include "../../Common/MyWindows.h" -#include +#include #include "../Window.h" @@ -32,7 +32,8 @@ public: int InsertColumn(int columnIndex, const LVCOLUMN *columnInfo) { return ListView_InsertColumn(_window, columnIndex, columnInfo); } int InsertColumn(int columnIndex, LPCTSTR text, int width); - bool SetColumnOrderArray(int count, const int *columns) { return BOOLToBool(ListView_SetColumnOrderArray(_window, count, columns)); } + bool SetColumnOrderArray(int count, const int *columns) + { return BOOLToBool(ListView_SetColumnOrderArray(_window, count, (int *)(void *)columns)); } /* int GetNumColumns() diff --git a/3rdparty/lzma/CPP/Windows/Control/ProgressBar.h b/3rdparty/lzma/CPP/Windows/Control/ProgressBar.h index 38ebcb6131b..03743067448 100644 --- a/3rdparty/lzma/CPP/Windows/Control/ProgressBar.h +++ b/3rdparty/lzma/CPP/Windows/Control/ProgressBar.h @@ -5,7 +5,7 @@ #include "../../Common/MyWindows.h" -#include +#include #include "../Window.h" diff --git a/3rdparty/lzma/CPP/Windows/Control/PropertyPage.cpp b/3rdparty/lzma/CPP/Windows/Control/PropertyPage.cpp index bbac74ad732..ce8696d4018 100644 --- a/3rdparty/lzma/CPP/Windows/Control/PropertyPage.cpp +++ b/3rdparty/lzma/CPP/Windows/Control/PropertyPage.cpp @@ -115,7 +115,7 @@ INT_PTR MyPropertySheet(const CObjectVector &pagesInfo, HWND hwndPare sheet.dwFlags = PSH_PROPSHEETPAGE; sheet.hwndParent = hwndParent; sheet.hInstance = g_hInstance; - AString titleA = GetSystemString(title); + AString titleA (GetSystemString(title)); sheet.pszCaption = titleA; sheet.nPages = pagesInfo.Size(); sheet.nStartPage = 0; diff --git a/3rdparty/lzma/CPP/Windows/Control/PropertyPage.h b/3rdparty/lzma/CPP/Windows/Control/PropertyPage.h index 4c4ddad9113..b68fd8fee24 100644 --- a/3rdparty/lzma/CPP/Windows/Control/PropertyPage.h +++ b/3rdparty/lzma/CPP/Windows/Control/PropertyPage.h @@ -5,7 +5,7 @@ #include "../../Common/MyWindows.h" -#include +#include #include "Dialog.h" diff --git a/3rdparty/lzma/CPP/Windows/DLL.cpp b/3rdparty/lzma/CPP/Windows/DLL.cpp index a71a8017b87..cf5d01a302c 100644 --- a/3rdparty/lzma/CPP/Windows/DLL.cpp +++ b/3rdparty/lzma/CPP/Windows/DLL.cpp @@ -4,6 +4,8 @@ #include "DLL.h" +#ifdef _WIN32 + #ifndef _UNICODE extern bool g_IsNT; #endif @@ -97,14 +99,93 @@ FString GetModuleDirPrefix() { int pos = s.ReverseFind_PathSepar(); if (pos >= 0) - { - s.DeleteFrom(pos + 1); - return s; - } + s.DeleteFrom((unsigned)(pos + 1)); } - return FTEXT(".") FSTRING_PATH_SEPARATOR; + if (s.IsEmpty()) + s = "." STRING_PATH_SEPARATOR; + return s; } #endif }} + +#else + +#include +#include + +namespace NWindows { +namespace NDLL { + +bool CLibrary::Free() throw() +{ + if (_module == NULL) + return true; + int ret = dlclose(_module); + if (ret != 0) + return false; + _module = NULL; + return true; +} + +static +// FARPROC +void * +local_GetProcAddress(HMODULE module, LPCSTR procName) +{ + void *ptr = NULL; + if (module) + { + ptr = dlsym(module, procName); + } + return ptr; +} + +bool CLibrary::Load(CFSTR path) throw() +{ + if (!Free()) + return false; + + int options = 0; + + #ifdef RTLD_LOCAL + options |= RTLD_LOCAL; + #endif + + #ifdef RTLD_NOW + options |= RTLD_NOW; + #endif + + #ifdef RTLD_GROUP + #if ! (defined(hpux) || defined(__hpux)) + options |= RTLD_GROUP; // mainly for solaris but not for HPUX + #endif + #endif + + void *handler = dlopen(path, options); + + if (handler) + { + // here we can transfer some settings to DLL + } + else + { + } + + _module = handler; + + return (_module != NULL); +} + +// FARPROC +void * CLibrary::GetProc(LPCSTR procName) const +{ + // return My_GetProcAddress(_module, procName); + return local_GetProcAddress(_module, procName); + // return NULL; +} + +}} + +#endif diff --git a/3rdparty/lzma/CPP/Windows/DLL.h b/3rdparty/lzma/CPP/Windows/DLL.h index 984a1d33b58..0c093eeddb6 100644 --- a/3rdparty/lzma/CPP/Windows/DLL.h +++ b/3rdparty/lzma/CPP/Windows/DLL.h @@ -8,10 +8,12 @@ namespace NWindows { namespace NDLL { +#ifdef _WIN32 + #ifdef UNDER_CE -#define My_GetProcAddress(module, procName) ::GetProcAddressA(module, procName) +#define My_GetProcAddress(module, procName) (void *)::GetProcAddressA(module, procName) #else -#define My_GetProcAddress(module, procName) ::GetProcAddress(module, procName) +#define My_GetProcAddress(module, procName) (void *)::GetProcAddress(module, procName) #endif /* Win32: Don't call CLibrary::Free() and FreeLibrary() from another @@ -46,9 +48,33 @@ public: bool Free() throw(); bool LoadEx(CFSTR path, DWORD flags = LOAD_LIBRARY_AS_DATAFILE) throw(); bool Load(CFSTR path) throw(); - FARPROC GetProc(LPCSTR procName) const { return My_GetProcAddress(_module, procName); } + // FARPROC + void *GetProc(LPCSTR procName) const { return My_GetProcAddress(_module, procName); } }; +#else + +typedef void * HMODULE; +// typedef int (*FARPROC)(); +// typedef void *FARPROC; + +class CLibrary +{ + HMODULE _module; + + // CLASS_NO_COPY(CLibrary); +public: + CLibrary(): _module(NULL) {}; + ~CLibrary() { Free(); } + + bool Free() throw(); + bool Load(CFSTR path) throw(); + // FARPROC + void *GetProc(LPCSTR procName) const; // { return My_GetProcAddress(_module, procName); } +}; + +#endif + bool MyGetModuleFileName(FString &path); FString GetModuleDirPrefix(); diff --git a/3rdparty/lzma/CPP/Windows/Defs.h b/3rdparty/lzma/CPP/Windows/Defs.h index 281c40c33eb..1d96078dd60 100644 --- a/3rdparty/lzma/CPP/Windows/Defs.h +++ b/3rdparty/lzma/CPP/Windows/Defs.h @@ -7,10 +7,11 @@ #ifdef _WIN32 inline bool LRESULTToBool(LRESULT v) { return (v != FALSE); } -inline bool BOOLToBool(BOOL v) { return (v != FALSE); } inline BOOL BoolToBOOL(bool v) { return (v ? TRUE: FALSE); } #endif +inline bool BOOLToBool(BOOL v) { return (v != FALSE); } + inline VARIANT_BOOL BoolToVARIANT_BOOL(bool v) { return (v ? VARIANT_TRUE: VARIANT_FALSE); } inline bool VARIANT_BOOLToBool(VARIANT_BOOL v) { return (v != VARIANT_FALSE); } diff --git a/3rdparty/lzma/CPP/Windows/ErrorMsg.cpp b/3rdparty/lzma/CPP/Windows/ErrorMsg.cpp index c9de4a6c0a7..f6343a5cf3b 100644 --- a/3rdparty/lzma/CPP/Windows/ErrorMsg.cpp +++ b/3rdparty/lzma/CPP/Windows/ErrorMsg.cpp @@ -2,21 +2,33 @@ #include "StdAfx.h" -#ifndef _UNICODE +#if !defined(_UNICODE) || !defined(_WIN32) #include "../Common/StringConvert.h" #endif #include "ErrorMsg.h" -#ifndef _UNICODE +#ifdef _WIN32 +#if !defined(_UNICODE) extern bool g_IsNT; #endif +#endif namespace NWindows { namespace NError { static bool MyFormatMessage(DWORD errorCode, UString &message) { + #ifndef _SFX + if ((HRESULT)errorCode == MY_HRES_ERROR__INTERNAL_ERROR) + { + message = "Internal Error: The failure in hardware (RAM or CPU), OS or program"; + return true; + } + #endif + + #ifdef _WIN32 + LPVOID msgBuf; #ifndef _UNICODE if (!g_IsNT) @@ -38,8 +50,63 @@ static bool MyFormatMessage(DWORD errorCode, UString &message) } ::LocalFree(msgBuf); return true; + + #else // _WIN32 + + AString m; + + const char *s = NULL; + + switch ((Int32)errorCode) + { + // case ERROR_NO_MORE_FILES : s = "No more files"; break; + // case ERROR_DIRECTORY : s = "Error Directory"; break; + case E_NOTIMPL : s = "E_NOTIMPL : Not implemented"; break; + case E_NOINTERFACE : s = "E_NOINTERFACE : No such interface supported"; break; + case E_ABORT : s = "E_ABORT : Operation aborted"; break; + case E_FAIL : s = "E_FAIL : Unspecified error"; break; + + case STG_E_INVALIDFUNCTION : s = "STG_E_INVALIDFUNCTION"; break; + case CLASS_E_CLASSNOTAVAILABLE : s = "CLASS_E_CLASSNOTAVAILABLE"; break; + + case E_OUTOFMEMORY : s = "E_OUTOFMEMORY : Can't allocate required memory"; break; + case E_INVALIDARG : s = "E_INVALIDARG : One or more arguments are invalid"; break; + + // case MY__E_ERROR_NEGATIVE_SEEK : s = "MY__E_ERROR_NEGATIVE_SEEK"; break; + default: + break; + } + + /* strerror() for unknown errors still shows message "Unknown error -12345678") + So we must transfer error codes before strerror() */ + if (!s) + { + if ((errorCode & 0xFFFF0000) == (UInt32)((MY__FACILITY__WRes << 16) | 0x80000000)) + errorCode &= 0xFFFF; + else if ((errorCode & ((UInt32)1 << 31))) + return false; // we will show hex error later for that case + + s = strerror((int)errorCode); + + // if (!s) + { + m += "errno="; + m.Add_UInt32(errorCode); + if (s) + m += " : "; + } + } + + if (s) + m += s; + + MultiByteToUnicodeString2(message, m); + return true; + + #endif } + UString MyFormatMessage(DWORD errorCode) { UString m; @@ -53,8 +120,8 @@ UString MyFormatMessage(DWORD errorCode) s[7 - i] = (char)((t < 10) ? ('0' + t) : ('A' + (t - 10))); } s[8] = 0; - m.AddAscii("Error #"); - m.AddAscii(s); + m += "Error #"; + m += s; } else if (m.Len() >= 2 && m[m.Len() - 1] == 0x0A diff --git a/3rdparty/lzma/CPP/Windows/ErrorMsg.h b/3rdparty/lzma/CPP/Windows/ErrorMsg.h index 0957c696ad1..01204eb9567 100644 --- a/3rdparty/lzma/CPP/Windows/ErrorMsg.h +++ b/3rdparty/lzma/CPP/Windows/ErrorMsg.h @@ -9,6 +9,7 @@ namespace NWindows { namespace NError { UString MyFormatMessage(DWORD errorCode); +inline UString MyFormatMessage(HRESULT errorCode) { return MyFormatMessage((DWORD)errorCode); } }} diff --git a/3rdparty/lzma/CPP/Windows/FileDir.cpp b/3rdparty/lzma/CPP/Windows/FileDir.cpp index da71b710bdc..cce2638587e 100644 --- a/3rdparty/lzma/CPP/Windows/FileDir.cpp +++ b/3rdparty/lzma/CPP/Windows/FileDir.cpp @@ -2,8 +2,21 @@ #include "StdAfx.h" -#ifndef _UNICODE + +#ifndef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "../Common/StringConvert.h" +#include "../Common/C_FileIO.h" #endif #include "FileDir.h" @@ -18,10 +31,36 @@ using namespace NWindows; using namespace NFile; using namespace NName; +#ifndef _WIN32 + +static bool FiTime_To_timespec(const CFiTime *ft, timespec &ts) +{ + if (ft) + { + ts = *ft; + return true; + } + // else + { + ts.tv_sec = 0; + ts.tv_nsec = + #ifdef UTIME_OMIT + UTIME_OMIT; // -2 keep old timesptamp + #else + // UTIME_NOW; -1 // set to the current time + 0; + #endif + return false; + } +} +#endif + namespace NWindows { namespace NFile { namespace NDir { +#ifdef _WIN32 + #ifndef UNDER_CE bool GetWindowsDir(FString &path) @@ -67,9 +106,10 @@ bool GetSystemDir(FString &path) } return (needLength > 0 && needLength <= MAX_PATH); } -#endif +#endif // UNDER_CE -bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) + +bool SetDirTime(CFSTR path, const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) { #ifndef _UNICODE if (!g_IsNT) @@ -102,6 +142,8 @@ bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const return res; } + + bool SetFileAttrib(CFSTR path, DWORD attrib) { #ifndef _UNICODE @@ -128,6 +170,17 @@ bool SetFileAttrib(CFSTR path, DWORD attrib) return false; } + +bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) +{ + #ifdef _WIN32 + if ((attrib & 0xF0000000) != 0) + attrib &= 0x3FFF; + #endif + return SetFileAttrib(path, attrib); +} + + bool RemoveDir(CFSTR path) { #ifndef _UNICODE @@ -154,6 +207,7 @@ bool RemoveDir(CFSTR path) return false; } + bool MyMoveFile(CFSTR oldFile, CFSTR newFile) { #ifndef _UNICODE @@ -166,8 +220,10 @@ bool MyMoveFile(CFSTR oldFile, CFSTR newFile) #endif { IF_USE_MAIN_PATH_2(oldFile, newFile) + { if (::MoveFileW(fs2us(oldFile), fs2us(newFile))) return true; + } #ifdef WIN_LONG_PATH if (USE_SUPER_PATH_2) { @@ -181,7 +237,6 @@ bool MyMoveFile(CFSTR oldFile, CFSTR newFile) } #ifndef UNDER_CE - EXTERN_C_BEGIN typedef BOOL (WINAPI *Func_CreateHardLinkW)( LPCWSTR lpFileName, @@ -189,6 +244,7 @@ typedef BOOL (WINAPI *Func_CreateHardLinkW)( LPSECURITY_ATTRIBUTES lpSecurityAttributes ); EXTERN_C_END +#endif // UNDER_CE bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) { @@ -206,12 +262,14 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) #endif { Func_CreateHardLinkW my_CreateHardLinkW = (Func_CreateHardLinkW) - ::GetProcAddress(::GetModuleHandleW(L"kernel32.dll"), "CreateHardLinkW"); + (void *)::GetProcAddress(::GetModuleHandleW(L"kernel32.dll"), "CreateHardLinkW"); if (!my_CreateHardLinkW) return false; IF_USE_MAIN_PATH_2(newFileName, existFileName) + { if (my_CreateHardLinkW(fs2us(newFileName), fs2us(existFileName), NULL)) return true; + } #ifdef WIN_LONG_PATH if (USE_SUPER_PATH_2) { @@ -224,7 +282,6 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) return false; } -#endif /* WinXP-64 CreateDir(): @@ -324,6 +381,10 @@ static bool CreateDir2(CFSTR path) return fi.IsDir(); } +#endif // _WIN32 + +static bool CreateDir2(CFSTR path); + bool CreateComplexDir(CFSTR _path) { #ifdef _WIN32 @@ -339,13 +400,23 @@ bool CreateComplexDir(CFSTR _path) if (IsDriveRootPath_SuperAllowed(_path)) return false; - unsigned prefixSize = GetRootPrefixSize(_path); + const unsigned prefixSize = GetRootPrefixSize(_path); - #endif + #endif // UNDER_CE + + #else // _WIN32 + + // Posix + NFind::CFileInfo fi; + if (fi.Find(_path)) + { + if (fi.IsDir()) + return true; + } - #endif + #endif // _WIN32 - FString path = _path; + FString path (_path); int pos = path.ReverseFind_PathSepar(); if (pos >= 0 && (unsigned)pos == path.Len() - 1) @@ -355,8 +426,8 @@ bool CreateComplexDir(CFSTR _path) path.DeleteBack(); } - const FString path2 = path; - pos = path.Len(); + const FString path2 (path); + pos = (int)path.Len(); for (;;) { @@ -375,17 +446,17 @@ bool CreateComplexDir(CFSTR _path) return false; #endif - path.DeleteFrom(pos); + path.DeleteFrom((unsigned)pos); } while (pos < (int)path2.Len()) { - int pos2 = NName::FindSepar(path2.Ptr(pos + 1)); + int pos2 = NName::FindSepar(path2.Ptr((unsigned)pos + 1)); if (pos2 < 0) - pos = path2.Len(); + pos = (int)path2.Len(); else pos += 1 + pos2; - path.SetFrom(path2, pos); + path.SetFrom(path2, (unsigned)pos); if (!CreateDir(path)) return false; } @@ -393,6 +464,9 @@ bool CreateComplexDir(CFSTR _path) return true; } + +#ifdef _WIN32 + bool DeleteFileAlways(CFSTR path) { /* If alt stream, we also need to clear READ-ONLY attribute of main file before delete. @@ -403,7 +477,7 @@ bool DeleteFileAlways(CFSTR path) && (attrib & FILE_ATTRIBUTE_DIRECTORY) == 0 && (attrib & FILE_ATTRIBUTE_READONLY) != 0) { - if (!SetFileAttrib(path, attrib & ~FILE_ATTRIBUTE_READONLY)) + if (!SetFileAttrib(path, attrib & ~(DWORD)FILE_ATTRIBUTE_READONLY)) return false; } } @@ -434,6 +508,8 @@ bool DeleteFileAlways(CFSTR path) return false; } + + bool RemoveDirWithSubItems(const FString &path) { bool needRemoveSubItems = true; @@ -452,12 +528,14 @@ bool RemoveDirWithSubItems(const FString &path) if (needRemoveSubItems) { - FString s = path; + FString s (path); s.Add_PathSepar(); - unsigned prefixSize = s.Len(); - s += FCHAR_ANY_MASK; - NFind::CEnumerator enumerator(s); - NFind::CFileInfo fi; + const unsigned prefixSize = s.Len(); + NFind::CEnumerator enumerator; + enumerator.SetDirPrefix(s); + NFind::CDirEntry fi; + bool isError = false; + DWORD lastError = 0; while (enumerator.Next(fi)) { s.DeleteFrom(prefixSize); @@ -465,18 +543,32 @@ bool RemoveDirWithSubItems(const FString &path) if (fi.IsDir()) { if (!RemoveDirWithSubItems(s)) - return false; + { + lastError = GetLastError(); + isError = true; + } } else if (!DeleteFileAlways(s)) - return false; + { + lastError = GetLastError(); + isError = false; + } + } + if (isError) + { + SetLastError(lastError); + return false; } } + // we clear read-only attrib to remove read-only dir if (!SetFileAttrib(path, 0)) return false; return RemoveDir(path); } +#endif // _WIN32 + #ifdef UNDER_CE bool MyGetFullPathName(CFSTR path, FString &resFullPath) @@ -492,6 +584,8 @@ bool MyGetFullPathName(CFSTR path, FString &resFullPath) return GetFullPath(path, resFullPath); } +#ifdef _WIN32 + bool SetCurrentDir(CFSTR path) { // SetCurrentDirectory doesn't support \\?\ prefix @@ -507,9 +601,11 @@ bool SetCurrentDir(CFSTR path) } } + bool GetCurrentDir(FString &path) { path.Empty(); + DWORD needLength; #ifndef _UNICODE if (!g_IsNT) @@ -530,7 +626,9 @@ bool GetCurrentDir(FString &path) return (needLength > 0 && needLength <= MAX_PATH); } -#endif +#endif // _WIN32 +#endif // UNDER_CE + bool GetFullPathAndSplit(CFSTR path, FString &resDirPrefix, FString &resFileName) { @@ -538,8 +636,9 @@ bool GetFullPathAndSplit(CFSTR path, FString &resDirPrefix, FString &resFileName if (!res) resDirPrefix = path; int pos = resDirPrefix.ReverseFind_PathSepar(); - resFileName = resDirPrefix.Ptr(pos + 1); - resDirPrefix.DeleteFrom(pos + 1); + pos++; + resFileName = resDirPrefix.Ptr((unsigned)pos); + resDirPrefix.DeleteFrom((unsigned)pos); return res; } @@ -551,6 +650,7 @@ bool GetOnlyDirPrefix(CFSTR path, FString &resDirPrefix) bool MyGetTempPath(FString &path) { + #ifdef _WIN32 path.Empty(); DWORD needLength; #ifndef _UNICODE @@ -570,28 +670,44 @@ bool MyGetTempPath(FString &path) path = us2fs(s); } return (needLength > 0 && needLength <= MAX_PATH); + + #else + + // FIXME: improve that code + path = "/tmp/"; + if (!NFind::DoesDirExist_FollowLink(path)) + path = "./"; + return true; + #endif } + static bool CreateTempFile(CFSTR prefix, bool addRandom, FString &path, NIO::COutFile *outFile) { - UInt32 d = (GetTickCount() << 12) ^ (GetCurrentThreadId() << 14) ^ GetCurrentProcessId(); + UInt32 d = + #ifdef _WIN32 + (GetTickCount() << 12) ^ (GetCurrentThreadId() << 14) ^ GetCurrentProcessId(); + #else + (UInt32)(time(NULL) << 12) ^ ((UInt32)getppid() << 14) ^ (UInt32)(getpid()); + #endif + for (unsigned i = 0; i < 100; i++) { path = prefix; if (addRandom) { - FChar s[16]; - UInt32 value = d; + char s[16]; + UInt32 val = d; unsigned k; for (k = 0; k < 8; k++) { - unsigned t = value & 0xF; - value >>= 4; - s[k] = (FChar)((t < 10) ? ('0' + t) : ('A' + (t - 10))); + unsigned t = val & 0xF; + val >>= 4; + s[k] = (char)((t < 10) ? ('0' + t) : ('A' + (t - 10))); } s[k] = '\0'; if (outFile) - path += FChar('.'); + path += '.'; path += s; UInt32 step = GetTickCount() + 2; if (step == 0) @@ -600,7 +716,7 @@ static bool CreateTempFile(CFSTR prefix, bool addRandom, FString &path, NIO::COu } addRandom = true; if (outFile) - path += FTEXT(".tmp"); + path += ".tmp"; if (NFind::DoesFileOrDirExist(path)) { SetLastError(ERROR_ALREADY_EXISTS); @@ -658,14 +774,30 @@ bool CTempFile::Remove() bool CTempFile::MoveTo(CFSTR name, bool deleteDestBefore) { + // DWORD attrib = 0; if (deleteDestBefore) - if (NFind::DoesFileExist(name)) + { + if (NFind::DoesFileExist_Raw(name)) + { + // attrib = NFind::GetFileAttrib(name); if (!DeleteFileAlways(name)) return false; + } + } DisableDeleting(); return MyMoveFile(_path, name); + + /* + if (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_READONLY)) + { + DWORD attrib2 = NFind::GetFileAttrib(name); + if (attrib2 != INVALID_FILE_ATTRIBUTES) + SetFileAttrib(name, attrib2 | FILE_ATTRIBUTE_READONLY); + } + */ } +#ifdef _WIN32 bool CTempDir::Create(CFSTR prefix) { if (!Remove()) @@ -686,5 +818,317 @@ bool CTempDir::Remove() _mustBeDeleted = !RemoveDirWithSubItems(_path); return !_mustBeDeleted; } +#endif + + + +#ifndef _WIN32 + +bool RemoveDir(CFSTR path) +{ + return (rmdir(path) == 0); +} + + +static BOOL My__CopyFile(CFSTR oldFile, CFSTR newFile) +{ + NWindows::NFile::NIO::COutFile outFile; + if (!outFile.Create(newFile, false)) + return FALSE; + + NWindows::NFile::NIO::CInFile inFile; + if (!inFile.Open(oldFile)) + return FALSE; + + char buf[1 << 14]; + + for (;;) + { + const ssize_t num = inFile.read_part(buf, sizeof(buf)); + if (num == 0) + return TRUE; + if (num < 0) + return FALSE; + size_t processed; + const ssize_t num2 = outFile.write_full(buf, (size_t)num, processed); + if (num2 != num || processed != (size_t)num) + return FALSE; + } +} + + +bool MyMoveFile(CFSTR oldFile, CFSTR newFile) +{ + int res = rename(oldFile, newFile); + if (res == 0) + return true; + if (errno != EXDEV) // (oldFile and newFile are not on the same mounted filesystem) + return false; + + if (My__CopyFile(oldFile, newFile) == FALSE) + return false; + + struct stat info_file; + res = stat(oldFile, &info_file); + if (res != 0) + return false; + + /* + ret = chmod(dst,info_file.st_mode & g_umask.mask); + */ + return (unlink(oldFile) == 0); +} + + +bool CreateDir(CFSTR path) +{ + return (mkdir(path, 0777) == 0); // change it +} + +static bool CreateDir2(CFSTR path) +{ + return (mkdir(path, 0777) == 0); // change it +} + + +bool DeleteFileAlways(CFSTR path) +{ + return (remove(path) == 0); +} + +bool SetCurrentDir(CFSTR path) +{ + return (chdir(path) == 0); +} + + +bool GetCurrentDir(FString &path) +{ + path.Empty(); + + #define MY__PATH_MAX PATH_MAX + // #define MY__PATH_MAX 1024 + + char s[MY__PATH_MAX + 1]; + char *res = getcwd(s, MY__PATH_MAX); + if (res) + { + path = fas2fs(s); + return true; + } + { + // if (errno != ERANGE) return false; + #if defined(__GLIBC__) || defined(__APPLE__) + /* As an extension to the POSIX.1-2001 standard, glibc's getcwd() + allocates the buffer dynamically using malloc(3) if buf is NULL. */ + res = getcwd(NULL, 0); + if (res) + { + path = fas2fs(res); + ::free(res); + return true; + } + #endif + return false; + } +} + + + +// #undef UTIME_OMIT // to debug + +#ifndef UTIME_OMIT + /* we can define UTIME_OMIT for debian and another systems. + Is it OK to define UTIME_OMIT to -2 here, if UTIME_OMIT is not defined? */ + // #define UTIME_OMIT -2 +#endif + + + + + +bool SetDirTime(CFSTR path, const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) +{ + // need testing + /* + struct utimbuf buf; + struct stat st; + UNUSED_VAR(cTime) + + printf("\nstat = %s\n", path); + int ret = stat(path, &st); + + if (ret == 0) + { + buf.actime = st.st_atime; + buf.modtime = st.st_mtime; + } + else + { + time_t cur_time = time(0); + buf.actime = cur_time; + buf.modtime = cur_time; + } + + if (aTime) + { + UInt32 ut; + if (NTime::FileTimeToUnixTime(*aTime, ut)) + buf.actime = ut; + } + + if (mTime) + { + UInt32 ut; + if (NTime::FileTimeToUnixTime(*mTime, ut)) + buf.modtime = ut; + } + + return utime(path, &buf) == 0; + */ + + // if (!aTime && !mTime) return true; + + struct timespec times[2]; + UNUSED_VAR(cTime) + + bool needChange; + needChange = FiTime_To_timespec(aTime, times[0]); + needChange |= FiTime_To_timespec(mTime, times[1]); + + /* + if (mTime) + { + printf("\n time = %ld.%9ld\n", mTime->tv_sec, mTime->tv_nsec); + } + */ + + if (!needChange) + return true; + const int flags = 0; // follow link + // = AT_SYMLINK_NOFOLLOW; // don't follow link + return utimensat(AT_FDCWD, path, times, flags) == 0; +} + + + +struct C_umask +{ + mode_t mask; + + C_umask() + { + /* by security reasons we restrict attributes according + with process's file mode creation mask (umask) */ + const mode_t um = umask(0); // octal :0022 is expected + mask = 0777 & (~um); // octal: 0755 is expected + umask(um); // restore the umask + // printf("\n umask = 0%03o mask = 0%03o\n", um, mask); + + // mask = 0777; // debug we can disable the restriction: + } +}; + +static C_umask g_umask; + +// #define PRF(x) x; +#define PRF(x) + +#define TRACE_SetFileAttrib(msg) \ + PRF(printf("\nSetFileAttrib(%s, %x) : %s\n", (const char *)path, attrib, msg)); + +#define TRACE_chmod(s, mode) \ + PRF(printf("\n chmod(%s, %o)\n", (const char *)path, (unsigned)(mode))); + +int my_chown(CFSTR path, uid_t owner, gid_t group) +{ + return chown(path, owner, group); +} + +bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib) +{ + TRACE_SetFileAttrib(""); + + struct stat st; + + bool use_lstat = true; + if (use_lstat) + { + if (lstat(path, &st) != 0) + { + TRACE_SetFileAttrib("bad lstat()"); + return false; + } + // TRACE_chmod("lstat", st.st_mode); + } + else + { + if (stat(path, &st) != 0) + { + TRACE_SetFileAttrib("bad stat()"); + return false; + } + } + + if (attrib & FILE_ATTRIBUTE_UNIX_EXTENSION) + { + TRACE_SetFileAttrib("attrib & FILE_ATTRIBUTE_UNIX_EXTENSION"); + st.st_mode = attrib >> 16; + if (S_ISDIR(st.st_mode)) + { + // user/7z must be able to create files in this directory + st.st_mode |= (S_IRUSR | S_IWUSR | S_IXUSR); + } + else if (!S_ISREG(st.st_mode)) + return true; + } + else if (S_ISLNK(st.st_mode)) + { + /* for most systems: permissions for symlinks are fixed to rwxrwxrwx. + so we don't need chmod() for symlinks. */ + return true; + // SetLastError(ENOSYS); + // return false; + } + else + { + TRACE_SetFileAttrib("Only Windows Attributes"); + // Only Windows Attributes + if (S_ISDIR(st.st_mode) + || (attrib & FILE_ATTRIBUTE_READONLY) == 0) + return true; + st.st_mode &= ~(mode_t)(S_IWUSR | S_IWGRP | S_IWOTH); // octal: ~0222; // disable write permissions + } + + int res; + /* + if (S_ISLNK(st.st_mode)) + { + printf("\nfchmodat()\n"); + TRACE_chmod(path, (st.st_mode) & g_umask.mask); + // AT_SYMLINK_NOFOLLOW is not implemted still in Linux. + res = fchmodat(AT_FDCWD, path, (st.st_mode) & g_umask.mask, + S_ISLNK(st.st_mode) ? AT_SYMLINK_NOFOLLOW : 0); + } + else + */ + { + TRACE_chmod(path, (st.st_mode) & g_umask.mask); + res = chmod(path, (st.st_mode) & g_umask.mask); + } + // TRACE_SetFileAttrib("End") + return (res == 0); +} + + +bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) +{ + PRF(printf("\nhard link() %s -> %s\n", newFileName, existFileName)); + return (link(existFileName, newFileName) == 0); +} + +#endif // !_WIN32 + +// #endif }}} diff --git a/3rdparty/lzma/CPP/Windows/FileDir.h b/3rdparty/lzma/CPP/Windows/FileDir.h index b13d1ccada2..08281aaa986 100644 --- a/3rdparty/lzma/CPP/Windows/FileDir.h +++ b/3rdparty/lzma/CPP/Windows/FileDir.h @@ -14,8 +14,33 @@ namespace NDir { bool GetWindowsDir(FString &path); bool GetSystemDir(FString &path); -bool SetDirTime(CFSTR path, const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime); +/* +WIN32 API : SetFileTime() doesn't allow to set zero timestamps in file +but linux : allows unix time = 0 in filesystem +*/ + +bool SetDirTime(CFSTR path, const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime); + + +#ifdef _WIN32 + bool SetFileAttrib(CFSTR path, DWORD attrib); + +/* + Some programs store posix attributes in high 16 bits of windows attributes field. + Also some programs use additional flag markers: 0x8000 or 0x4000. + SetFileAttrib_PosixHighDetect() tries to detect posix field, and it extracts only attribute + bits that are related to current system only. +*/ +#else + +int my_chown(CFSTR path, uid_t owner, gid_t group); + +#endif + +bool SetFileAttrib_PosixHighDetect(CFSTR path, DWORD attrib); + + bool MyMoveFile(CFSTR existFileName, CFSTR newFileName); #ifndef UNDER_CE @@ -48,7 +73,7 @@ bool GetCurrentDir(FString &resultPath); bool MyGetTempPath(FString &resultPath); -class CTempFile +class CTempFile MY_UNCOPYABLE { bool _mustBeDeleted; FString _path; @@ -63,7 +88,9 @@ public: bool MoveTo(CFSTR name, bool deleteDestBefore); }; -class CTempDir + +#ifdef _WIN32 +class CTempDir MY_UNCOPYABLE { bool _mustBeDeleted; FString _path; @@ -75,9 +102,11 @@ public: bool Create(CFSTR namePrefix) ; bool Remove(); }; +#endif + #if !defined(UNDER_CE) -class CCurrentDirRestorer +class CCurrentDirRestorer MY_UNCOPYABLE { FString _path; public: diff --git a/3rdparty/lzma/CPP/Windows/FileFind.cpp b/3rdparty/lzma/CPP/Windows/FileFind.cpp index 05196f927bd..c6557599a2e 100644 --- a/3rdparty/lzma/CPP/Windows/FileFind.cpp +++ b/3rdparty/lzma/CPP/Windows/FileFind.cpp @@ -2,8 +2,13 @@ #include "StdAfx.h" -#ifndef _UNICODE -#include "../Common/StringConvert.h" +// #include + +#ifndef _WIN32 +#include /* Definition of AT_* constants */ +#include "TimeUtils.h" +// for major +// #include #endif #include "FileFind.h" @@ -34,27 +39,62 @@ typedef struct WCHAR cStreamName[MAX_PATH + 36]; } MY_WIN32_FIND_STREAM_DATA, *MY_PWIN32_FIND_STREAM_DATA; -typedef WINBASEAPI HANDLE (WINAPI *FindFirstStreamW_Ptr)(LPCWSTR fileName, MY_STREAM_INFO_LEVELS infoLevel, +typedef HANDLE (WINAPI *FindFirstStreamW_Ptr)(LPCWSTR fileName, MY_STREAM_INFO_LEVELS infoLevel, LPVOID findStreamData, DWORD flags); -typedef WINBASEAPI BOOL (APIENTRY *FindNextStreamW_Ptr)(HANDLE findStream, LPVOID findStreamData); +typedef BOOL (APIENTRY *FindNextStreamW_Ptr)(HANDLE findStream, LPVOID findStreamData); EXTERN_C_END -#endif +#endif // defined(_WIN32) && !defined(UNDER_CE) + namespace NWindows { namespace NFile { + +#ifdef _WIN32 #ifdef SUPPORT_DEVICE_FILE namespace NSystem { bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); } #endif +#endif namespace NFind { +/* +#ifdef _WIN32 +#define MY_CLEAR_FILETIME(ft) ft.dwLowDateTime = ft.dwHighDateTime = 0; +#else +#define MY_CLEAR_FILETIME(ft) ft.tv_sec = 0; ft.tv_nsec = 0; +#endif +*/ + +void CFileInfoBase::ClearBase() throw() +{ + Size = 0; + FiTime_Clear(CTime); + FiTime_Clear(ATime); + FiTime_Clear(MTime); + + #ifdef _WIN32 + Attrib = 0; + // ReparseTag = 0; + IsAltStream = false; + IsDevice = false; + #else + dev = 0; + ino = 0; + mode = 0; + nlink = 0; + uid = 0; + gid = 0; + rdev = 0; + #endif +} + bool CFileInfo::IsDots() const throw() { if (!IsDir() || Name.IsEmpty()) @@ -64,12 +104,17 @@ bool CFileInfo::IsDots() const throw() return Name.Len() == 1 || (Name.Len() == 2 && Name[1] == '.'); } + +#ifdef _WIN32 + + #define WIN_FD_TO_MY_FI(fi, fd) \ fi.Attrib = fd.dwFileAttributes; \ fi.CTime = fd.ftCreationTime; \ fi.ATime = fd.ftLastAccessTime; \ fi.MTime = fd.ftLastWriteTime; \ fi.Size = (((UInt64)fd.nFileSizeHigh) << 32) + fd.nFileSizeLow; \ + /* fi.ReparseTag = fd.dwReserved0; */ \ fi.IsAltStream = false; \ fi.IsDevice = false; @@ -91,7 +136,6 @@ static void Convert_WIN32_FIND_DATA_to_FileInfo(const WIN32_FIND_DATAW &fd, CFil } #ifndef _UNICODE - static void Convert_WIN32_FIND_DATA_to_FileInfo(const WIN32_FIND_DATA &fd, CFileInfo &fi) { WIN_FD_TO_MY_FI(fi, fd); @@ -143,7 +187,8 @@ WinXP-64 FindFirstFile(): \\Server\Share_RootDrive - ERROR_INVALID_NAME \\Server\Share_RootDrive\ - ERROR_INVALID_NAME - c:\* - ERROR_FILE_NOT_FOUND, if thare are no item in that folder + e:\* - ERROR_FILE_NOT_FOUND, if there are no items in that root folder + w:\* - ERROR_PATH_NOT_FOUND, if there is no such drive w: */ bool CFindFile::FindFirst(CFSTR path, CFileInfo &fi) @@ -210,12 +255,13 @@ bool CFindFile::FindNext(CFileInfo &fi) static FindFirstStreamW_Ptr g_FindFirstStreamW; static FindNextStreamW_Ptr g_FindNextStreamW; -struct CFindStreamLoader +static struct CFindStreamLoader { CFindStreamLoader() { - g_FindFirstStreamW = (FindFirstStreamW_Ptr)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "FindFirstStreamW"); - g_FindNextStreamW = (FindNextStreamW_Ptr)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "FindNextStreamW"); + HMODULE hm = ::GetModuleHandleA("kernel32.dll"); + g_FindFirstStreamW = (FindFirstStreamW_Ptr)(void *)::GetProcAddress(hm, "FindFirstStreamW"); + g_FindNextStreamW = (FindNextStreamW_Ptr)(void *)::GetProcAddress(hm, "FindNextStreamW"); } } g_FindStreamLoader; @@ -227,7 +273,7 @@ bool CStreamInfo::IsMainStream() const throw() UString CStreamInfo::GetReducedName() const { // remove ":$DATA" postfix, but keep postfix, if Name is "::$DATA" - UString s = Name; + UString s (Name); if (s.Len() > 6 + 1 && StringsAreEqualNoCase_Ascii(s.RightPtr(6), ":$DATA")) s.DeleteFrom(s.Len() - 6); return s; @@ -245,7 +291,7 @@ UString CStreamInfo::GetReducedName2() const static void Convert_WIN32_FIND_STREAM_DATA_to_StreamInfo(const MY_WIN32_FIND_STREAM_DATA &sd, CStreamInfo &si) { - si.Size = sd.StreamSize.QuadPart; + si.Size = (UInt64)sd.StreamSize.QuadPart; si.Name = sd.cStreamName; } @@ -336,19 +382,6 @@ bool CStreamEnumerator::Next(CStreamInfo &si, bool &found) #endif -#define MY_CLEAR_FILETIME(ft) ft.dwLowDateTime = ft.dwHighDateTime = 0; - -void CFileInfoBase::ClearBase() throw() -{ - Size = 0; - MY_CLEAR_FILETIME(CTime); - MY_CLEAR_FILETIME(ATime); - MY_CLEAR_FILETIME(MTime); - Attrib = 0; - IsAltStream = false; - IsDevice = false; -} - /* WinXP-64 GetFileAttributes(): If the function fails, it returns INVALID_FILE_ATTRIBUTES and use GetLastError() to get error code @@ -416,9 +449,23 @@ also we support paths that are not supported by FindFirstFile: c::stream - Name = c::stream */ -bool CFileInfo::Find(CFSTR path) +bool CFileInfo::Find(CFSTR path, bool followLink) { #ifdef SUPPORT_DEVICE_FILE + + if (IS_PATH_SEPAR(path[0]) && + IS_PATH_SEPAR(path[1]) && + path[2] == '.' && + path[3] == 0) + { + // 22.00 : it's virtual directory for devices + // IsDevice = true; + ClearBase(); + Name = path + 2; + Attrib = FILE_ATTRIBUTE_DIRECTORY; + return true; + } + if (IsDevicePath(path)) { ClearBase(); @@ -449,12 +496,12 @@ bool CFileInfo::Find(CFSTR path) #if defined(_WIN32) && !defined(UNDER_CE) - int colonPos = FindAltStreamColon(path); + const int colonPos = FindAltStreamColon(path); if (colonPos >= 0 && path[(unsigned)colonPos + 1] != 0) { UString streamName = fs2us(path + (unsigned)colonPos); - FString filePath = path; - filePath.DeleteFrom(colonPos); + FString filePath (path); + filePath.DeleteFrom((unsigned)colonPos); /* we allow both cases: name:stream name:stream:$DATA @@ -462,12 +509,12 @@ bool CFileInfo::Find(CFSTR path) const unsigned kPostfixSize = 6; if (streamName.Len() <= kPostfixSize || !StringsAreEqualNoCase_Ascii(streamName.RightPtr(kPostfixSize), ":$DATA")) - streamName += L":$DATA"; + streamName += ":$DATA"; bool isOk = true; if (IsDrivePath2(filePath) && - (colonPos == 2 || colonPos == 3 && filePath[2] == '\\')) + (colonPos == 2 || (colonPos == 3 && filePath[2] == '\\'))) { // FindFirstFile doesn't work for "c:\" and for "c:" (if current dir is ROOT) ClearBase(); @@ -476,11 +523,11 @@ bool CFileInfo::Find(CFSTR path) Name = filePath; } else - isOk = Find(filePath); + isOk = Find(filePath, followLink); // check it (followLink) if (isOk) { - Attrib &= ~(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); + Attrib &= ~(DWORD)(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); Size = 0; CStreamEnumerator enumerator(filePath); for (;;) @@ -536,11 +583,15 @@ bool CFileInfo::Find(CFSTR path) ClearBase(); Attrib = attrib; Name = path + rootSize; - Name.DeleteFrom(2); // we don't need backslash (C:) + Name.DeleteFrom(2); + if (!Fill_From_ByHandleFileInfo(path)) + { + } return true; } } else if (IS_PATH_SEPAR(path[0])) + { if (path[1] == 0) { DWORD attrib = GetFileAttrib(path); @@ -559,10 +610,15 @@ bool CFileInfo::Find(CFSTR path) { if (NName::FindSepar(path + prefixSize) < 0) { - FString s = path; + if (Fill_From_ByHandleFileInfo(path)) + { + Name = path + prefixSize; + return true; + } + + FString s (path); s.Add_PathSepar(); - s += FCHAR_ANY_MASK; - + s += '*'; // CHAR_ANY_MASK bool isOK = false; if (finder.FindFirst(s, *this)) { @@ -577,7 +633,7 @@ bool CFileInfo::Find(CFSTR path) } { DWORD attrib = GetFileAttrib(path); - if (isOK || attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY) != 0) + if (isOK || (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY) != 0)) { ClearBase(); if (attrib != INVALID_FILE_ATTRIBUTES) @@ -592,23 +648,105 @@ bool CFileInfo::Find(CFSTR path) } } } + } } #endif - return finder.FindFirst(path, *this); + bool res = finder.FindFirst(path, *this); + if (!followLink + || !res + || !HasReparsePoint()) + return res; + + // return FollowReparse(path, IsDir()); + return Fill_From_ByHandleFileInfo(path); } +bool CFileInfoBase::Fill_From_ByHandleFileInfo(CFSTR path) +{ + BY_HANDLE_FILE_INFORMATION info; + if (!NIO::CFileBase::GetFileInformation(path, &info)) + return false; + { + Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + CTime = info.ftCreationTime; + ATime = info.ftLastAccessTime; + MTime = info.ftLastWriteTime; + Attrib = info.dwFileAttributes; + return true; + } +} -bool DoesFileExist(CFSTR name) +/* +bool CFileInfo::FollowReparse(CFSTR path, bool isDir) +{ + if (isDir) + { + FString prefix = path; + prefix.Add_PathSepar(); + + // "folder/." refers to folder itself. So we can't use that path + // we must use enumerator and search "." item + CEnumerator enumerator; + enumerator.SetDirPrefix(prefix); + for (;;) + { + CFileInfo fi; + if (!enumerator.NextAny(fi)) + break; + if (fi.Name.IsEqualTo_Ascii_NoCase(".")) + { + // we can copy preperies; + CTime = fi.CTime; + ATime = fi.ATime; + MTime = fi.MTime; + Attrib = fi.Attrib; + Size = fi.Size; + return true; + } + break; + } + // LastError(lastError); + return false; + } + + { + NIO::CInFile inFile; + if (inFile.Open(path)) + { + BY_HANDLE_FILE_INFORMATION info; + if (inFile.GetFileInformation(&info)) + { + ClearBase(); + Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + CTime = info.ftCreationTime; + ATime = info.ftLastAccessTime; + MTime = info.ftLastWriteTime; + Attrib = info.dwFileAttributes; + return true; + } + } + return false; + } +} +*/ + +bool DoesFileExist_Raw(CFSTR name) { CFileInfo fi; return fi.Find(name) && !fi.IsDir(); } -bool DoesDirExist(CFSTR name) +bool DoesFileExist_FollowLink(CFSTR name) +{ + CFileInfo fi; + return fi.Find_FollowLink(name) && !fi.IsDir(); +} + +bool DoesDirExist(CFSTR name, bool followLink) { CFileInfo fi; - return fi.Find(name) && fi.IsDir(); + return fi.Find(name, followLink) && fi.IsDir(); } bool DoesFileOrDirExist(CFSTR name) @@ -618,6 +756,12 @@ bool DoesFileOrDirExist(CFSTR name) } +void CEnumerator::SetDirPrefix(const FString &dirPrefix) +{ + _wildcard = dirPrefix; + _wildcard += '*'; +} + bool CEnumerator::NextAny(CFileInfo &fi) { if (_findFile.IsHandleAllocated()) @@ -639,15 +783,46 @@ bool CEnumerator::Next(CFileInfo &fi) bool CEnumerator::Next(CFileInfo &fi, bool &found) { + /* + for (;;) + { + if (!NextAny(fi)) + break; + if (!fi.IsDots()) + { + found = true; + return true; + } + } + */ + if (Next(fi)) { found = true; return true; } + found = false; - return (::GetLastError() == ERROR_NO_MORE_FILES); + DWORD lastError = ::GetLastError(); + if (_findFile.IsHandleAllocated()) + return (lastError == ERROR_NO_MORE_FILES); + // we support the case for empty root folder: FindFirstFile("c:\\*") returns ERROR_FILE_NOT_FOUND + if (lastError == ERROR_FILE_NOT_FOUND) + return true; + if (lastError == ERROR_ACCESS_DENIED) + { + // here we show inaccessible root system folder as empty folder to eliminate redundant user warnings + const char *s = "System Volume Information" STRING_PATH_SEPARATOR "*"; + const int len = (int)strlen(s); + const int delta = (int)_wildcard.Len() - len; + if (delta == 0 || (delta > 0 && IS_PATH_SEPAR(_wildcard[(unsigned)delta - 1]))) + if (StringsAreEqual_Ascii(_wildcard.Ptr((unsigned)delta), s)) + return true; + } + return false; } + //////////////////////////////// // CFindChangeNotification // FindFirstChangeNotification can return 0. MSDN doesn't tell about it. @@ -738,6 +913,413 @@ bool MyGetLogicalDriveStrings(CObjectVector &driveStrings) } } -#endif +#endif // UNDER_CE + + + +#else // _WIN32 + +// ---------- POSIX ---------- + +static int MY__lstat(CFSTR path, struct stat *st, bool followLink) +{ + memset(st, 0, sizeof(*st)); + int res; + // #ifdef ENV_HAVE_LSTAT + if (/* global_use_lstat && */ !followLink) + { + // printf("\nlstat\n"); + res = lstat(path, st); + } + else + // #endif + { + // printf("\nstat\n"); + res = stat(path, st); + } + /* + printf("\nres = %d\n", res); + printf("\n st_dev = %lld \n", (long long)(st->st_dev)); + printf("\n st_ino = %lld \n", (long long)(st->st_ino)); + printf("\n st_mode = %lld \n", (long long)(st->st_mode)); + printf("\n st_nlink = %lld \n", (long long)(st->st_nlink)); + printf("\n st_uid = %lld \n", (long long)(st->st_uid)); + printf("\n st_gid = %lld \n", (long long)(st->st_gid)); + printf("\n st_size = %lld \n", (long long)(st->st_size)); + printf("\n st_blksize = %lld \n", (long long)(st->st_blksize)); + printf("\n st_blocks = %lld \n", (long long)(st->st_blocks)); + */ + + return res; +} + + +static const char *Get_Name_from_Path(CFSTR path) throw() +{ + size_t len = strlen(path); + if (len == 0) + return path; + const char *p = path + len - 1; + { + if (p == path) + return path; + p--; + } + for (;;) + { + char c = *p; + if (IS_PATH_SEPAR(c)) + return p + 1; + if (p == path) + return path; + p--; + } +} + + +UInt32 Get_WinAttribPosix_From_PosixMode(UInt32 mode) +{ + UInt32 attrib = S_ISDIR(mode) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + if ((mode & 0222) == 0) // S_IWUSR in p7zip + attrib |= FILE_ATTRIBUTE_READONLY; + return attrib | FILE_ATTRIBUTE_UNIX_EXTENSION | ((mode & 0xFFFF) << 16); +} + +/* +UInt32 Get_WinAttrib_From_stat(const struct stat &st) +{ + UInt32 attrib = S_ISDIR(st.st_mode) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + + if ((st.st_mode & 0222) == 0) // check it !!! + attrib |= FILE_ATTRIBUTE_READONLY; + + attrib |= FILE_ATTRIBUTE_UNIX_EXTENSION + ((st.st_mode & 0xFFFF) << 16); + return attrib; +} +*/ + +void CFileInfo::SetFrom_stat(const struct stat &st) +{ + // IsDevice = false; + + if (S_ISDIR(st.st_mode)) + { + Size = 0; + } + else + { + Size = (UInt64)st.st_size; // for a symbolic link, size = size of filename + } + + // Attrib = Get_WinAttribPosix_From_PosixMode(st.st_mode); + + // NTime::UnixTimeToFileTime(st.st_ctime, CTime); + // NTime::UnixTimeToFileTime(st.st_mtime, MTime); + // NTime::UnixTimeToFileTime(st.st_atime, ATime); + #ifdef __APPLE__ + // #ifdef _DARWIN_FEATURE_64_BIT_INODE + /* + here we can use birthtime instead of st_ctimespec. + but we use st_ctimespec for compatibility with previous versions and p7zip. + st_birthtimespec in OSX + st_birthtim : at FreeBSD, NetBSD + */ + // timespec_To_FILETIME(st.st_birthtimespec, CTime); + // #else + // timespec_To_FILETIME(st.st_ctimespec, CTime); + // #endif + // timespec_To_FILETIME(st.st_mtimespec, MTime); + // timespec_To_FILETIME(st.st_atimespec, ATime); + CTime = st.st_ctimespec; + MTime = st.st_mtimespec; + ATime = st.st_atimespec; + + #else + // timespec_To_FILETIME(st.st_ctim, CTime, &CTime_ns100); + // timespec_To_FILETIME(st.st_mtim, MTime, &MTime_ns100); + // timespec_To_FILETIME(st.st_atim, ATime, &ATime_ns100); + CTime = st.st_ctim; + MTime = st.st_mtim; + ATime = st.st_atim; + + #endif + + dev = st.st_dev; + ino = st.st_ino; + mode = st.st_mode; + nlink = st.st_nlink; + uid = st.st_uid; + gid = st.st_gid; + rdev = st.st_rdev; + + /* + printf("\n sizeof timespec = %d", (int)sizeof(timespec)); + printf("\n sizeof st_rdev = %d", (int)sizeof(rdev)); + printf("\n sizeof st_ino = %d", (int)sizeof(ino)); + printf("\n sizeof mode_t = %d", (int)sizeof(mode_t)); + printf("\n sizeof nlink_t = %d", (int)sizeof(nlink_t)); + printf("\n sizeof uid_t = %d", (int)sizeof(uid_t)); + printf("\n"); + */ + /* + printf("\n st_rdev = %llx", (long long)rdev); + printf("\n st_dev = %llx", (long long)dev); + printf("\n dev : major = %5x minor = %5x", (unsigned)major(dev), (unsigned)minor(dev)); + printf("\n st_ino = %lld", (long long)(ino)); + printf("\n rdev : major = %5x minor = %5x", (unsigned)major(rdev), (unsigned)minor(rdev)); + printf("\n size = %lld \n", (long long)(Size)); + printf("\n"); + */ +} + +/* +int Uid_To_Uname(uid_t uid, AString &name) +{ + name.Empty(); + struct passwd *passwd; + + if (uid != 0 && uid == cached_no_such_uid) + { + *uname = xstrdup (""); + return; + } + + if (!cached_uname || uid != cached_uid) + { + passwd = getpwuid (uid); + if (passwd) + { + cached_uid = uid; + assign_string (&cached_uname, passwd->pw_name); + } + else + { + cached_no_such_uid = uid; + *uname = xstrdup (""); + return; + } + } + *uname = xstrdup (cached_uname); +} +*/ + +bool CFileInfo::Find_DontFill_Name(CFSTR path, bool followLink) +{ + struct stat st; + if (MY__lstat(path, &st, followLink) != 0) + return false; + // printf("\nFind_DontFill_Name : name=%s\n", path); + SetFrom_stat(st); + return true; +} + + +bool CFileInfo::Find(CFSTR path, bool followLink) +{ + // printf("\nCEnumerator::Find() name = %s\n", path); + if (!Find_DontFill_Name(path, followLink)) + return false; + + // printf("\nOK\n"); + + Name = Get_Name_from_Path(path); + if (!Name.IsEmpty()) + { + char c = Name.Back(); + if (IS_PATH_SEPAR(c)) + Name.DeleteBack(); + } + return true; +} + + +bool DoesFileExist_Raw(CFSTR name) +{ + // FIXME for symbolic links. + struct stat st; + if (MY__lstat(name, &st, false) != 0) + return false; + return !S_ISDIR(st.st_mode); +} + +bool DoesFileExist_FollowLink(CFSTR name) +{ + // FIXME for symbolic links. + struct stat st; + if (MY__lstat(name, &st, true) != 0) + return false; + return !S_ISDIR(st.st_mode); +} + +bool DoesDirExist(CFSTR name, bool followLink) +{ + struct stat st; + if (MY__lstat(name, &st, followLink) != 0) + return false; + return S_ISDIR(st.st_mode); +} + +bool DoesFileOrDirExist(CFSTR name) +{ + struct stat st; + if (MY__lstat(name, &st, false) != 0) + return false; + return true; +} + + +CEnumerator::~CEnumerator() +{ + if (_dir) + closedir(_dir); +} + +void CEnumerator::SetDirPrefix(const FString &dirPrefix) +{ + _wildcard = dirPrefix; +} + +bool CDirEntry::IsDots() const throw() +{ + /* some systems (like CentOS 7.x on XFS) have (Type == DT_UNKNOWN) + we can call fstatat() for that case, but we use only (Name) check here */ + + #if !defined(_AIX) + if (Type != DT_DIR && Type != DT_UNKNOWN) + return false; + #endif + + return Name.Len() != 0 + && Name.Len() <= 2 + && Name[0] == '.' + && (Name.Len() == 1 || Name[1] == '.'); +} + + +bool CEnumerator::NextAny(CDirEntry &fi, bool &found) +{ + found = false; + + if (!_dir) + { + const char *w = "./"; + if (!_wildcard.IsEmpty()) + w = _wildcard.Ptr(); + _dir = ::opendir((const char *)w); + if (_dir == NULL) + return false; + } + + // To distinguish end of stream from an error, we must set errno to zero before readdir() + errno = 0; + + struct dirent *de = readdir(_dir); + if (!de) + { + if (errno == 0) + return true; // it's end of stream, and we report it with (found = false) + // it's real error + return false; + } + + fi.iNode = de->d_ino; + + #if !defined(_AIX) + fi.Type = de->d_type; + /* some systems (like CentOS 7.x on XFS) have (Type == DT_UNKNOWN) + we can set (Type) from fstatat() in that case. + But (Type) is not too important. So we don't set it here with slow fstatat() */ + /* + // fi.Type = DT_UNKNOWN; // for debug + if (fi.Type == DT_UNKNOWN) + { + struct stat st; + if (fstatat(dirfd(_dir), de->d_name, &st, AT_SYMLINK_NOFOLLOW) == 0) + if (S_ISDIR(st.st_mode)) + fi.Type = DT_DIR; + } + */ + #endif + + /* + if (de->d_type == DT_DIR) + fi.Attrib = FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_UNIX_EXTENSION | ((UInt32)(S_IFDIR) << 16); + else if (de->d_type < 16) + fi.Attrib = FILE_ATTRIBUTE_UNIX_EXTENSION | ((UInt32)(de->d_type) << (16 + 12)); + */ + fi.Name = de->d_name; + + /* + printf("\nCEnumerator::NextAny; len = %d %s \n", (int)fi.Name.Len(), fi.Name.Ptr()); + for (unsigned i = 0; i < fi.Name.Len(); i++) + printf (" %02x", (unsigned)(Byte)de->d_name[i]); + printf("\n"); + */ + + found = true; + return true; +} + + +bool CEnumerator::Next(CDirEntry &fi, bool &found) +{ + // printf("\nCEnumerator::Next()\n"); + // PrintName("Next", ""); + for (;;) + { + if (!NextAny(fi, found)) + return false; + if (!found) + return true; + if (!fi.IsDots()) + { + /* + if (!NeedFullStat) + return true; + // we silently skip error file here - it can be wrong link item + if (fi.Find_DontFill_Name(path)) + return true; + */ + return true; + } + } +} + +/* +bool CEnumerator::Next(CDirEntry &fileInfo, bool &found) +{ + bool found; + if (!Next(fi, found)) + return false; + return found; +} +*/ + +bool CEnumerator::Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) const +{ + // printf("\nCEnumerator::Fill_FileInfo()\n"); + struct stat st; + // probably it's OK to use fstatat() even if it changes file position dirfd(_dir) + int res = fstatat(dirfd(_dir), de.Name, &st, followLink ? 0 : AT_SYMLINK_NOFOLLOW); + // if fstatat() is not supported, we can use stat() / lstat() + + /* + const FString path = _wildcard + s; + int res = MY__lstat(path, &st, followLink); + */ + + if (res != 0) + return false; + // printf("\nname=%s\n", de.Name.Ptr()); + fileInfo.SetFrom_stat(st); + fileInfo.Name = de.Name; + return true; +} + +#endif // _WIN32 }}} diff --git a/3rdparty/lzma/CPP/Windows/FileFind.h b/3rdparty/lzma/CPP/Windows/FileFind.h index 615c2dfe24c..fcfe02c7e64 100644 --- a/3rdparty/lzma/CPP/Windows/FileFind.h +++ b/3rdparty/lzma/CPP/Windows/FileFind.h @@ -3,13 +3,41 @@ #ifndef __WINDOWS_FILE_FIND_H #define __WINDOWS_FILE_FIND_H +#ifndef _WIN32 +#include +#include +#include +#endif + +#include "../Common/MyLinux.h" #include "../Common/MyString.h" +#include "../Common/MyWindows.h" + #include "Defs.h" +#include "FileIO.h" + namespace NWindows { namespace NFile { namespace NFind { +// bool DoesFileExist(CFSTR name, bool followLink); +bool DoesFileExist_Raw(CFSTR name); +bool DoesFileExist_FollowLink(CFSTR name); +bool DoesDirExist(CFSTR name, bool followLink); + +inline bool DoesDirExist(CFSTR name) + { return DoesDirExist(name, false); } +inline bool DoesDirExist_FollowLink(CFSTR name) + { return DoesDirExist(name, true); } + +// it's always _Raw +bool DoesFileOrDirExist(CFSTR name); + +DWORD GetFileAttrib(CFSTR path); + +#ifdef _WIN32 + namespace NAttributes { inline bool IsReadOnly(DWORD attrib) { return (attrib & FILE_ATTRIBUTE_READONLY) != 0; } @@ -19,16 +47,34 @@ namespace NAttributes inline bool IsArchived(DWORD attrib) { return (attrib & FILE_ATTRIBUTE_ARCHIVE) != 0; } inline bool IsCompressed(DWORD attrib) { return (attrib & FILE_ATTRIBUTE_COMPRESSED) != 0; } inline bool IsEncrypted(DWORD attrib) { return (attrib & FILE_ATTRIBUTE_ENCRYPTED) != 0; } + + inline UInt32 Get_PosixMode_From_WinAttrib(DWORD attrib) + { + UInt32 v = IsDir(attrib) ? MY_LIN_S_IFDIR : MY_LIN_S_IFREG; + /* 21.06: as WSL we allow write permissions (0222) for directories even for (FILE_ATTRIBUTE_READONLY). + So extracting at Linux will be allowed to write files inside (0777) directories. */ + v |= ((IsReadOnly(attrib) && !IsDir(attrib)) ? 0555 : 0777); + return v; + } } +#else + +UInt32 Get_WinAttribPosix_From_PosixMode(UInt32 mode); + +#endif + class CFileInfoBase { + #ifdef _WIN32 bool MatchesMask(UINT32 mask) const { return ((Attrib & mask) != 0); } + #endif public: UInt64 Size; - FILETIME CTime; - FILETIME ATime; - FILETIME MTime; + CFiTime CTime; + CFiTime ATime; + CFiTime MTime; + #ifdef _WIN32 DWORD Attrib; bool IsAltStream; bool IsDevice; @@ -40,11 +86,25 @@ public: UINT32 ReparseTag; #endif */ + #else + dev_t dev; /* ID of device containing file */ + ino_t ino; + mode_t mode; + nlink_t nlink; + uid_t uid; /* user ID of owner */ + gid_t gid; /* group ID of owner */ + dev_t rdev; /* device ID (defined, if S_ISCHR(mode) || S_ISBLK(mode)) */ + // bool Use_lstat; + #endif CFileInfoBase() { ClearBase(); } void ClearBase() throw(); - void SetAsDir() { Attrib = FILE_ATTRIBUTE_DIRECTORY; } + #ifdef _WIN32 + + bool Fill_From_ByHandleFileInfo(CFSTR path); + void SetAsDir() { Attrib = FILE_ATTRIBUTE_DIRECTORY; } // |= (FILE_ATTRIBUTE_UNIX_EXTENSION + (S_IFDIR << 16)); + void SetAsFile() { Attrib = 0; } bool IsArchived() const { return MatchesMask(FILE_ATTRIBUTE_ARCHIVE); } bool IsCompressed() const { return MatchesMask(FILE_ATTRIBUTE_COMPRESSED); } @@ -58,6 +118,43 @@ public: bool IsSparse() const { return MatchesMask(FILE_ATTRIBUTE_SPARSE_FILE); } bool IsSystem() const { return MatchesMask(FILE_ATTRIBUTE_SYSTEM); } bool IsTemporary() const { return MatchesMask(FILE_ATTRIBUTE_TEMPORARY); } + + UInt32 GetWinAttrib() const { return Attrib; } + UInt32 GetPosixAttrib() const + { + return NAttributes::Get_PosixMode_From_WinAttrib(Attrib); + } + bool Has_Attrib_ReparsePoint() const { return (Attrib & FILE_ATTRIBUTE_REPARSE_POINT) != 0; } + + #else + + UInt32 GetPosixAttrib() const { return mode; } + UInt32 GetWinAttrib() const { return Get_WinAttribPosix_From_PosixMode(mode); } + + bool IsDir() const { return S_ISDIR(mode); } + void SetAsDir() { mode = S_IFDIR; } + void SetAsFile() { mode = S_IFREG; } + + bool IsReadOnly() const + { + // does linux support writing to ReadOnly files? + if ((mode & 0222) == 0) // S_IWUSR in p7zip + return true; + return false; + } + + bool IsPosixLink() const { return S_ISLNK(mode); } + + #endif + + bool IsOsSymLink() const + { + #ifdef _WIN32 + return HasReparsePoint(); + #else + return IsPosixLink(); + #endif + } }; struct CFileInfo: public CFileInfoBase @@ -68,10 +165,22 @@ struct CFileInfo: public CFileInfoBase #endif bool IsDots() const throw(); - bool Find(CFSTR path); + bool Find(CFSTR path, bool followLink = false); + bool Find_FollowLink(CFSTR path) { return Find(path, true); } + + #ifdef _WIN32 + // bool Fill_From_ByHandleFileInfo(CFSTR path); + // bool FollowReparse(CFSTR path, bool isDir); + #else + bool Find_DontFill_Name(CFSTR path, bool followLink = false); + void SetFrom_stat(const struct stat &st); + #endif }; -class CFindFileBase + +#ifdef _WIN32 + +class CFindFileBase MY_UNCOPYABLE { protected: HANDLE _handle; @@ -108,38 +217,34 @@ public: bool FindNext(CStreamInfo &streamInfo); }; -class CStreamEnumerator +class CStreamEnumerator MY_UNCOPYABLE { CFindStream _find; FString _filePath; - bool NextAny(CFileInfo &fileInfo); + bool NextAny(CFileInfo &fileInfo, bool &found); public: CStreamEnumerator(const FString &filePath): _filePath(filePath) {} bool Next(CStreamInfo &streamInfo, bool &found); }; -#endif - -bool DoesFileExist(CFSTR name); -bool DoesDirExist(CFSTR name); -bool DoesFileOrDirExist(CFSTR name); +#endif // defined(_WIN32) && !defined(UNDER_CE) -DWORD GetFileAttrib(CFSTR path); -class CEnumerator +class CEnumerator MY_UNCOPYABLE { CFindFile _findFile; FString _wildcard; bool NextAny(CFileInfo &fileInfo); public: - CEnumerator(const FString &wildcard): _wildcard(wildcard) {} + void SetDirPrefix(const FString &dirPrefix); bool Next(CFileInfo &fileInfo); bool Next(CFileInfo &fileInfo, bool &found); }; -class CFindChangeNotification + +class CFindChangeNotification MY_UNCOPYABLE { HANDLE _handle; public: @@ -156,6 +261,80 @@ public: bool MyGetLogicalDriveStrings(CObjectVector &driveStrings); #endif +typedef CFileInfo CDirEntry; + + +#else // WIN32 + + +struct CDirEntry +{ + ino_t iNode; + #if !defined(_AIX) + Byte Type; + #endif + FString Name; + + /* + #if !defined(_AIX) + bool IsDir() const + { + // (Type == DT_UNKNOWN) on some systems + return Type == DT_DIR; + } + #endif + */ + + bool IsDots() const throw(); +}; + +class CEnumerator MY_UNCOPYABLE +{ + DIR *_dir; + FString _wildcard; + + bool NextAny(CDirEntry &fileInfo, bool &found); +public: + CEnumerator(): _dir(NULL) {} + ~CEnumerator(); + void SetDirPrefix(const FString &dirPrefix); + + bool Next(CDirEntry &fileInfo, bool &found); + bool Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) const; + bool DirEntry_IsDir(const CDirEntry &de, bool followLink) const + { + #if !defined(_AIX) + if (de.Type == DT_DIR) + return true; + if (de.Type != DT_UNKNOWN) + return false; + #endif + CFileInfo fileInfo; + if (Fill_FileInfo(de, fileInfo, followLink)) + { + return fileInfo.IsDir(); + } + return false; // change it + } +}; + +/* +inline UInt32 Get_WinAttrib_From_PosixMode(UInt32 mode) +{ + UInt32 attrib = S_ISDIR(mode) ? + FILE_ATTRIBUTE_DIRECTORY : + FILE_ATTRIBUTE_ARCHIVE; + if ((st.st_mode & 0222) == 0) // check it !!! + attrib |= FILE_ATTRIBUTE_READONLY; + return attrib; +} +*/ + +// UInt32 Get_WinAttrib_From_stat(const struct stat &st); + + +#endif // WIN32 + }}} #endif diff --git a/3rdparty/lzma/CPP/Windows/FileIO.cpp b/3rdparty/lzma/CPP/Windows/FileIO.cpp index 56e6ca45796..e51b0eb31b8 100644 --- a/3rdparty/lzma/CPP/Windows/FileIO.cpp +++ b/3rdparty/lzma/CPP/Windows/FileIO.cpp @@ -6,9 +6,29 @@ #include "../../C/Alloc.h" #endif +// #include + +/* +#ifndef _WIN32 +// for ioctl BLKGETSIZE64 +#include +#include +#endif +*/ + #include "FileIO.h" #include "FileName.h" +HRESULT GetLastError_noZero_HRESULT() +{ + DWORD res = ::GetLastError(); + if (res == 0) + return E_FAIL; + return HRESULT_FROM_WIN32(res); +} + +#ifdef _WIN32 + #ifndef _UNICODE extern bool g_IsNT; #endif @@ -78,6 +98,42 @@ bool CFileBase::Create(CFSTR path, DWORD desiredAccess, } #endif } + + /* + #ifndef UNDER_CE + #ifndef _SFX + if (_handle == INVALID_HANDLE_VALUE) + { + // it's debug hack to open symbolic links in Windows XP and WSL links in Windows 10 + DWORD lastError = GetLastError(); + if (lastError == ERROR_CANT_ACCESS_FILE) + { + CByteBuffer buf; + if (NIO::GetReparseData(path, buf, NULL)) + { + CReparseAttr attr; + if (attr.Parse(buf, buf.Size())) + { + FString dirPrefix, fileName; + if (NDir::GetFullPathAndSplit(path, dirPrefix, fileName)) + { + FString fullPath; + if (GetFullPath(dirPrefix, us2fs(attr.GetPath()), fullPath)) + { + // FIX IT: recursion levels must be restricted + return Create(fullPath, desiredAccess, + shareMode, creationDisposition, flagsAndAttributes); + } + } + } + } + SetLastError(lastError); + } + } + #endif + #endif + */ + return (_handle != INVALID_HANDLE_VALUE); } @@ -91,11 +147,6 @@ bool CFileBase::Close() throw() return true; } -bool CFileBase::GetPosition(UInt64 &position) const throw() -{ - return Seek(0, FILE_CURRENT, position); -} - bool CFileBase::GetLength(UInt64 &length) const throw() { #ifdef SUPPORT_DEVICE_FILE @@ -106,13 +157,64 @@ bool CFileBase::GetLength(UInt64 &length) const throw() } #endif - DWORD sizeHigh; - DWORD sizeLow = ::GetFileSize(_handle, &sizeHigh); - if (sizeLow == 0xFFFFFFFF) + DWORD high = 0; + const DWORD low = ::GetFileSize(_handle, &high); + if (low == INVALID_FILE_SIZE) if (::GetLastError() != NO_ERROR) return false; - length = (((UInt64)sizeHigh) << 32) + sizeLow; + length = (((UInt64)high) << 32) + low; + return true; + + /* + LARGE_INTEGER fileSize; + // GetFileSizeEx() is unsupported in 98/ME/NT, and supported in Win2000+ + if (!GetFileSizeEx(_handle, &fileSize)) + return false; + length = (UInt64)fileSize.QuadPart; return true; + */ +} + + +/* Specification for SetFilePointer(): + + If a new file pointer is a negative value, + { + the function fails, + the file pointer is not moved, + the code returned by GetLastError() is ERROR_NEGATIVE_SEEK. + } + + If the hFile handle is opened with the FILE_FLAG_NO_BUFFERING flag set + { + an application can move the file pointer only to sector-aligned positions. + A sector-aligned position is a position that is a whole number multiple of + the volume sector size. + An application can obtain a volume sector size by calling the GetDiskFreeSpace. + } + + It is not an error to set a file pointer to a position beyond the end of the file. + The size of the file does not increase until you call the SetEndOfFile, WriteFile, or WriteFileEx function. + + If the return value is INVALID_SET_FILE_POINTER and if lpDistanceToMoveHigh is non-NULL, + an application must call GetLastError to determine whether or not the function has succeeded or failed. +*/ + +bool CFileBase::GetPosition(UInt64 &position) const throw() +{ + LONG high = 0; + const DWORD low = ::SetFilePointer(_handle, 0, &high, FILE_CURRENT); + if (low == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) + { + // for error case we can set (position) to (-1) or (0) or leave (position) unchanged. + // position = (UInt64)(Int64)-1; // for debug + position = 0; + return false; + } + position = (((UInt64)(UInt32)high) << 32) + low; + return true; + // we don't want recursed GetPosition() + // return Seek(0, FILE_CURRENT, position); } bool CFileBase::Seek(Int64 distanceToMove, DWORD moveMethod, UInt64 &newPosition) const throw() @@ -126,23 +228,31 @@ bool CFileBase::Seek(Int64 distanceToMove, DWORD moveMethod, UInt64 &newPosition #endif LONG high = (LONG)(distanceToMove >> 32); - DWORD low = ::SetFilePointer(_handle, (LONG)(distanceToMove & 0xFFFFFFFF), &high, moveMethod); - if (low == 0xFFFFFFFF) - if (::GetLastError() != NO_ERROR) + const DWORD low = ::SetFilePointer(_handle, (LONG)(distanceToMove & 0xFFFFFFFF), &high, moveMethod); + if (low == INVALID_SET_FILE_POINTER) + { + const DWORD lastError = ::GetLastError(); + if (lastError != NO_ERROR) + { + // 21.07: we set (newPosition) to real position even after error. + GetPosition(newPosition); + SetLastError(lastError); // restore LastError return false; + } + } newPosition = (((UInt64)(UInt32)high) << 32) + low; return true; } bool CFileBase::Seek(UInt64 position, UInt64 &newPosition) const throw() { - return Seek(position, FILE_BEGIN, newPosition); + return Seek((Int64)position, FILE_BEGIN, newPosition); } bool CFileBase::SeekToBegin() const throw() { - UInt64 newPosition; - return Seek(0, newPosition); + UInt64 newPosition = 0; + return Seek(0, newPosition) && (newPosition == 0); } bool CFileBase::SeekToEnd(UInt64 &newPosition) const throw() @@ -258,7 +368,7 @@ void CInFile::CalcDeviceSize(CFSTR s) if (GetPartitionInfo(&partInfo)) { - Size = partInfo.PartitionLength.QuadPart; + Size = (UInt64)partInfo.PartitionLength.QuadPart; SizeDefined = true; needCorrectSize = false; if ((s)[0] == '\\' && (s)[1] == '\\' && (s)[2] == '.' && (s)[3] == '\\' && (s)[5] == ':' && (s)[6] == 0) @@ -277,7 +387,7 @@ void CInFile::CalcDeviceSize(CFSTR s) my_DISK_GEOMETRY_EX geomEx; SizeDefined = GetGeometryEx(&geomEx); if (SizeDefined) - Size = geomEx.DiskSize.QuadPart; + Size = (UInt64)geomEx.DiskSize.QuadPart; else { DISK_GEOMETRY geom; @@ -285,7 +395,7 @@ void CInFile::CalcDeviceSize(CFSTR s) if (!SizeDefined) SizeDefined = GetCdRomGeometry(&geom); if (SizeDefined) - Size = geom.Cylinders.QuadPart * geom.TracksPerCylinder * geom.SectorsPerTrack * geom.BytesPerSector; + Size = (UInt64)geom.Cylinders.QuadPart * geom.TracksPerCylinder * geom.SectorsPerTrack * geom.BytesPerSector; } } @@ -310,7 +420,24 @@ void CInFile::CalcDeviceSize(CFSTR s) bool CInFile::Open(CFSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) { - bool res = Create(fileName, GENERIC_READ, shareMode, creationDisposition, flagsAndAttributes); + DWORD desiredAccess = GENERIC_READ; + + #ifdef _WIN32 + if (PreserveATime) + desiredAccess |= FILE_WRITE_ATTRIBUTES; + #endif + + bool res = Create(fileName, desiredAccess, shareMode, creationDisposition, flagsAndAttributes); + + #ifdef _WIN32 + if (res && PreserveATime) + { + FILETIME ft; + ft.dwHighDateTime = ft.dwLowDateTime = 0xFFFFFFFF; + ::SetFileTime(_handle, NULL, &ft, NULL); + } + #endif + MY_DEVICE_EXTRA_CODE return res; } @@ -330,7 +457,7 @@ bool CInFile::Open(CFSTR fileName) // for 32 MB (maybe also for 16 MB). // And message can be "Network connection was lost" -static UInt32 kChunkSizeMax = (1 << 22); +static const UInt32 kChunkSizeMax = (1 << 22); bool CInFile::Read1(void *data, UInt32 size, UInt32 &processedSize) throw() { @@ -366,6 +493,26 @@ bool CInFile::Read(void *data, UInt32 size, UInt32 &processedSize) throw() return true; } +bool CInFile::ReadFull(void *data, size_t size, size_t &processedSize) throw() +{ + processedSize = 0; + do + { + UInt32 processedLoc = 0; + const UInt32 sizeLoc = (size > kChunkSizeMax ? (UInt32)kChunkSizeMax : (UInt32)size); + const bool res = Read1(data, sizeLoc, processedLoc); + processedSize += processedLoc; + if (!res) + return false; + if (processedLoc == 0) + return true; + data = (void *)((unsigned char *)data + processedLoc); + size -= processedLoc; + } + while (size > 0); + return true; +} + // ---------- COutFile --------- static inline DWORD GetCreationDisposition(bool createAlways) @@ -413,7 +560,24 @@ bool COutFile::Write(const void *data, UInt32 size, UInt32 &processedSize) throw data = (const void *)((const unsigned char *)data + processedLoc); size -= processedLoc; } - while (size > 0); + while (size != 0); + return true; +} + +bool COutFile::WriteFull(const void *data, size_t size) throw() +{ + do + { + UInt32 processedLoc = 0; + const UInt32 sizeCur = (size > kChunkSizeMax ? kChunkSizeMax : (UInt32)size); + if (!WritePart(data, sizeCur, processedLoc)) + return false; + if (processedLoc == 0) + return (size == 0); + data = (const void *)((const unsigned char *)data + processedLoc); + size -= processedLoc; + } + while (size != 0); return true; } @@ -429,4 +593,313 @@ bool COutFile::SetLength(UInt64 length) throw() return SetEndOfFile(); } +bool COutFile::SetLength_KeepPosition(UInt64 length) throw() +{ + UInt64 currentPos = 0; + if (!GetPosition(currentPos)) + return false; + DWORD lastError = 0; + const bool result = SetLength(length); + if (!result) + lastError = GetLastError(); + UInt64 currentPos2; + const bool result2 = Seek(currentPos, currentPos2); + if (lastError != 0) + SetLastError(lastError); + return (result && result2); +} + }}} + +#else // _WIN32 + + +// POSIX + +#include +#include + +namespace NWindows { +namespace NFile { + +namespace NDir { +bool SetDirTime(CFSTR path, const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime); +} + +namespace NIO { + +bool CFileBase::OpenBinary(const char *name, int flags) +{ + #ifdef O_BINARY + flags |= O_BINARY; + #endif + + Close(); + _handle = ::open(name, flags, 0666); + return _handle != -1; + + /* + if (_handle == -1) + return false; + if (IsString1PrefixedByString2(name, "/dev/")) + { + // /dev/sda + // IsDeviceFile = true; // for debug + // SizeDefined = false; + // SizeDefined = (GetDeviceSize_InBytes(Size) == 0); + } + return true; + */ +} + +bool CFileBase::Close() +{ + if (_handle == -1) + return true; + if (close(_handle) != 0) + return false; + _handle = -1; + /* + IsDeviceFile = false; + SizeDefined = false; + */ + return true; +} + +bool CFileBase::GetLength(UInt64 &length) const +{ + length = 0; + // length = (UInt64)(Int64)-1; // for debug + const off_t curPos = seekToCur(); + if (curPos == -1) + return false; + const off_t lengthTemp = seek(0, SEEK_END); + seek(curPos, SEEK_SET); + length = (UInt64)lengthTemp; + + /* + // 22.00: + if (lengthTemp == 1) + if (IsDeviceFile && SizeDefined) + { + length = Size; + return true; + } + */ + + return (lengthTemp != -1); +} + +off_t CFileBase::seek(off_t distanceToMove, int moveMethod) const +{ + /* + if (IsDeviceFile && SizeDefined && moveMethod == SEEK_END) + { + printf("\n seek : IsDeviceFile moveMethod = %d distanceToMove = %ld\n", moveMethod, distanceToMove); + distanceToMove += Size; + moveMethod = SEEK_SET; + } + */ + + // printf("\nCFileBase::seek() moveMethod = %d, distanceToMove = %lld", moveMethod, (long long)distanceToMove); + // off_t res = ::lseek(_handle, distanceToMove, moveMethod); + // printf("\n lseek : moveMethod = %d distanceToMove = %ld\n", moveMethod, distanceToMove); + return ::lseek(_handle, distanceToMove, moveMethod); + // return res; +} + +off_t CFileBase::seekToBegin() const throw() +{ + return seek(0, SEEK_SET); +} + +off_t CFileBase::seekToCur() const throw() +{ + return seek(0, SEEK_CUR); +} + +/* +bool CFileBase::SeekToBegin() const throw() +{ + return (::seek(0, SEEK_SET) != -1); +} +*/ + + +///////////////////////// +// CInFile + +bool CInFile::Open(const char *name) +{ + return CFileBase::OpenBinary(name, O_RDONLY); +} + +bool CInFile::OpenShared(const char *name, bool) +{ + return Open(name); +} + + +/* +int CFileBase::my_ioctl_BLKGETSIZE64(unsigned long long *numBlocks) +{ + // we can read "/sys/block/sda/size" "/sys/block/sda/sda1/size" - partition + // #include + return ioctl(_handle, BLKGETSIZE64, numBlocks); + // in block size +} + +int CFileBase::GetDeviceSize_InBytes(UInt64 &size) +{ + size = 0; + unsigned long long numBlocks; + int res = my_ioctl_BLKGETSIZE64(&numBlocks); + if (res == 0) + size = numBlocks; // another blockSize s possible? + printf("\nGetDeviceSize_InBytes res = %d, size = %lld\n", res, (long long)size); + return res; +} +*/ + +/* +On Linux (32-bit and 64-bit): +read(), write() (and similar system calls) will transfer at most +0x7ffff000 = (2GiB - 4 KiB) bytes, returning the number of bytes actually transferred. +*/ + +static const size_t kChunkSizeMax = ((size_t)1 << 22); + +ssize_t CInFile::read_part(void *data, size_t size) throw() +{ + if (size > kChunkSizeMax) + size = kChunkSizeMax; + return ::read(_handle, data, size); +} + +bool CInFile::ReadFull(void *data, size_t size, size_t &processed) throw() +{ + processed = 0; + do + { + const ssize_t res = read_part(data, size); + if (res < 0) + return false; + if (res == 0) + break; + data = (void *)((unsigned char *)data + (size_t)res); + size -= (size_t)res; + processed += (size_t)res; + } + while (size > 0); + return true; +} + + +///////////////////////// +// COutFile + +bool COutFile::Create(const char *name, bool createAlways) +{ + Path = name; // change it : set it only if open is success. + if (createAlways) + { + Close(); + _handle = ::creat(name, 0666); + return _handle != -1; + } + return OpenBinary(name, O_CREAT | O_EXCL | O_WRONLY); +} + +bool COutFile::Open(const char *name, DWORD creationDisposition) +{ + UNUSED_VAR(creationDisposition) // FIXME + return Create(name, false); +} + +ssize_t COutFile::write_part(const void *data, size_t size) throw() +{ + if (size > kChunkSizeMax) + size = kChunkSizeMax; + return ::write(_handle, data, size); +} + +ssize_t COutFile::write_full(const void *data, size_t size, size_t &processed) throw() +{ + processed = 0; + do + { + const ssize_t res = write_part(data, size); + if (res < 0) + return res; + if (res == 0) + break; + data = (const void *)((const unsigned char *)data + (size_t)res); + size -= (size_t)res; + processed += (size_t)res; + } + while (size > 0); + return (ssize_t)processed; +} + +bool COutFile::SetLength(UInt64 length) throw() +{ + const off_t len2 = (off_t)length; + if ((Int64)length != len2) + { + SetLastError(EFBIG); + return false; + } + // The value of the seek pointer shall not be modified by a call to ftruncate(). + int iret = ftruncate(_handle, len2); + return (iret == 0); +} + +bool COutFile::Close() +{ + bool res = CFileBase::Close(); + if (!res) + return res; + if (CTime_defined || ATime_defined || MTime_defined) + { + /* bool res2 = */ NWindows::NFile::NDir::SetDirTime(Path, + CTime_defined ? &CTime : NULL, + ATime_defined ? &ATime : NULL, + MTime_defined ? &MTime : NULL); + } + return res; +} + +bool COutFile::SetTime(const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) throw() +{ + // On some OS (cygwin, MacOSX ...), you must close the file before updating times + // return true; + + if (cTime) { CTime = *cTime; CTime_defined = true; } else CTime_defined = false; + if (aTime) { ATime = *aTime; ATime_defined = true; } else ATime_defined = false; + if (mTime) { MTime = *mTime; MTime_defined = true; } else MTime_defined = false; + return true; + + /* + struct timespec times[2]; + UNUSED_VAR(cTime) + if (!aTime && !mTime) + return true; + bool needChange; + needChange = FiTime_To_timespec(aTime, times[0]); + needChange |= FiTime_To_timespec(mTime, times[1]); + if (!needChange) + return true; + return futimens(_handle, times) == 0; + */ +} + +bool COutFile::SetMTime(const CFiTime *mTime) throw() +{ + if (mTime) { MTime = *mTime; MTime_defined = true; } else MTime_defined = false; + return true; +} + +}}} + + +#endif diff --git a/3rdparty/lzma/CPP/Windows/FileIO.h b/3rdparty/lzma/CPP/Windows/FileIO.h index 32c285fb9cc..80509653e99 100644 --- a/3rdparty/lzma/CPP/Windows/FileIO.h +++ b/3rdparty/lzma/CPP/Windows/FileIO.h @@ -5,28 +5,46 @@ #include "../Common/MyWindows.h" +#define _my_IO_REPARSE_TAG_MOUNT_POINT (0xA0000003L) +#define _my_IO_REPARSE_TAG_SYMLINK (0xA000000CL) +#define _my_IO_REPARSE_TAG_LX_SYMLINK (0xA000001DL) + +#define _my_SYMLINK_FLAG_RELATIVE 1 + +// what the meaning of that FLAG or field (2)? +#define _my_LX_SYMLINK_FLAG 2 + +#ifdef _WIN32 + #if defined(_WIN32) && !defined(UNDER_CE) -#include +#include +#endif + +#else + +#include +#include + #endif #include "../Common/MyString.h" #include "../Common/MyBuffer.h" +#include "../Windows/TimeUtils.h" + #include "Defs.h" -#define _my_IO_REPARSE_TAG_MOUNT_POINT (0xA0000003L) -#define _my_IO_REPARSE_TAG_SYMLINK (0xA000000CL) +HRESULT GetLastError_noZero_HRESULT(); -#define _my_SYMLINK_FLAG_RELATIVE 1 - -#define my_FSCTL_SET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 41, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) // REPARSE_DATA_BUFFER -#define my_FSCTL_GET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 42, METHOD_BUFFERED, FILE_ANY_ACCESS) // REPARSE_DATA_BUFFER +#define my_FSCTL_SET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 41, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) // REPARSE_DATA_BUFFER +#define my_FSCTL_GET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 42, METHOD_BUFFERED, FILE_ANY_ACCESS) // REPARSE_DATA_BUFFER +#define my_FSCTL_DELETE_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 43, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) // REPARSE_DATA_BUFFER namespace NWindows { namespace NFile { #if defined(_WIN32) && !defined(UNDER_CE) -bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink); +bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink, bool isWSL); #endif struct CReparseShortInfo @@ -44,24 +62,56 @@ struct CReparseAttr UString SubsName; UString PrintName; + AString WslName; + + bool HeaderError; + bool TagIsUnknown; + bool MinorError; + DWORD ErrorCode; + CReparseAttr(): Tag(0), Flags(0) {} + + // Parse() + // returns (true) and (ErrorCode = 0), if (it'a correct known link) + // returns (false) and (ErrorCode = ERROR_REPARSE_TAG_INVALID), if unknown tag bool Parse(const Byte *p, size_t size); - bool IsMountPoint() const { return Tag == _my_IO_REPARSE_TAG_MOUNT_POINT; } // it's Junction - bool IsSymLink() const { return Tag == _my_IO_REPARSE_TAG_SYMLINK; } - bool IsRelative() const { return Flags == _my_SYMLINK_FLAG_RELATIVE; } + bool IsMountPoint() const { return Tag == _my_IO_REPARSE_TAG_MOUNT_POINT; } // it's Junction + bool IsSymLink_Win() const { return Tag == _my_IO_REPARSE_TAG_SYMLINK; } + bool IsSymLink_WSL() const { return Tag == _my_IO_REPARSE_TAG_LX_SYMLINK; } + + bool IsRelative_Win() const { return Flags == _my_SYMLINK_FLAG_RELATIVE; } + + bool IsRelative_WSL() const + { + if (WslName.IsEmpty()) + return true; + char c = WslName[0]; + return !IS_PATH_SEPAR(c); + } + // bool IsVolume() const; bool IsOkNamePair() const; UString GetPath() const; }; +#ifdef _WIN32 +#define CFiInfo BY_HANDLE_FILE_INFORMATION +#define ST_MTIME(st) (st).ftLastWriteTime +#else +#define CFiInfo stat +#endif + +#ifdef _WIN32 + namespace NIO { bool GetReparseData(CFSTR path, CByteBuffer &reparseData, BY_HANDLE_FILE_INFORMATION *fileInfo = NULL); bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size); +bool DeleteReparseData(CFSTR path); -class CFileBase +class CFileBase MY_UNCOPYABLE { protected: HANDLE _handle; @@ -90,15 +140,18 @@ public: } public: + bool PreserveATime; #ifdef SUPPORT_DEVICE_FILE bool IsDeviceFile; bool SizeDefined; UInt64 Size; // it can be larger than real available size #endif - CFileBase(): _handle(INVALID_HANDLE_VALUE) {}; + CFileBase(): _handle(INVALID_HANDLE_VALUE), PreserveATime(false) {}; ~CFileBase() { Close(); } + HANDLE GetHandle() const { return _handle; } + bool Close() throw(); bool GetPosition(UInt64 &position) const throw(); @@ -114,6 +167,7 @@ public: static bool GetFileInformation(CFSTR path, BY_HANDLE_FILE_INFORMATION *info) { + // probably it can work for complex paths: unsupported by another things NIO::CFileBase file; if (!file.Create(path, 0, FILE_SHARE_READ, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS)) return false; @@ -169,9 +223,23 @@ public: #ifndef UNDER_CE + bool Open_for_ReadAttributes(CFSTR fileName) + { + return Create(fileName, FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + // we must use (FILE_FLAG_BACKUP_SEMANTICS) to open handle of directory. + } + bool OpenReparse(CFSTR fileName) { - return Open(fileName, FILE_SHARE_READ, OPEN_EXISTING, + // 17.02 fix: to support Windows XP compatibility junctions: + // we use Create() with (desiredAccess = 0) instead of Open() with GENERIC_READ + return + Create(fileName, 0, + // Open(fileName, + FILE_SHARE_READ, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS); } @@ -180,6 +248,7 @@ public: bool Read1(void *data, UInt32 size, UInt32 &processedSize) throw(); bool ReadPart(void *data, UInt32 size, UInt32 &processedSize) throw(); bool Read(void *data, UInt32 size, UInt32 &processedSize) throw(); + bool ReadFull(void *data, size_t size, size_t &processedSize) throw(); }; class COutFile: public CFileBase @@ -190,14 +259,119 @@ public: bool Create(CFSTR fileName, bool createAlways); bool CreateAlways(CFSTR fileName, DWORD flagsAndAttributes); - bool SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) throw(); - bool SetMTime(const FILETIME *mTime) throw(); + bool SetTime(const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) throw(); + bool SetMTime(const CFiTime *mTime) throw(); bool WritePart(const void *data, UInt32 size, UInt32 &processedSize) throw(); bool Write(const void *data, UInt32 size, UInt32 &processedSize) throw(); + bool WriteFull(const void *data, size_t size) throw(); bool SetEndOfFile() throw(); bool SetLength(UInt64 length) throw(); + bool SetLength_KeepPosition(UInt64 length) throw(); }; -}}} +} + + +#else // _WIN32 + +namespace NIO { + +bool GetReparseData(CFSTR path, CByteBuffer &reparseData); +// bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size); + +// parameters are in reverse order of symlink() function !!! +bool SetSymLink(CFSTR from, CFSTR to); +bool SetSymLink_UString(CFSTR from, const UString &to); + + +class CFileBase +{ +protected: + int _handle; + + /* + bool IsDeviceFile; + bool SizeDefined; + UInt64 Size; // it can be larger than real available size + */ + + bool OpenBinary(const char *name, int flags); +public: + bool PreserveATime; + + CFileBase(): _handle(-1), PreserveATime(false) {}; + ~CFileBase() { Close(); } + bool Close(); + bool GetLength(UInt64 &length) const; + off_t seek(off_t distanceToMove, int moveMethod) const; + off_t seekToBegin() const throw(); + off_t seekToCur() const throw(); + // bool SeekToBegin() throw(); + int my_fstat(struct stat *st) const { return fstat(_handle, st); } + /* + int my_ioctl_BLKGETSIZE64(unsigned long long *val); + int GetDeviceSize_InBytes(UInt64 &size); + void CalcDeviceSize(CFSTR s); + */ +}; + +class CInFile: public CFileBase +{ +public: + bool Open(const char *name); + bool OpenShared(const char *name, bool shareForWrite); + ssize_t read_part(void *data, size_t size) throw(); + // ssize_t read_full(void *data, size_t size, size_t &processed); + bool ReadFull(void *data, size_t size, size_t &processedSize) throw(); +}; + +class COutFile: public CFileBase +{ + bool CTime_defined; + bool ATime_defined; + bool MTime_defined; + + CFiTime CTime; + CFiTime ATime; + CFiTime MTime; + + AString Path; + ssize_t write_part(const void *data, size_t size) throw(); +public: + COutFile(): + CTime_defined(false), + ATime_defined(false), + MTime_defined(false) + {} + + bool Close(); + bool Create(const char *name, bool createAlways); + bool Open(const char *name, DWORD creationDisposition); + ssize_t write_full(const void *data, size_t size, size_t &processed) throw(); + + bool WriteFull(const void *data, size_t size) throw() + { + size_t processed; + ssize_t res = write_full(data, size, processed); + if (res == -1) + return false; + return processed == size; + } + + bool SetLength(UInt64 length) throw(); + bool SetLength_KeepPosition(UInt64 length) throw() + { + return SetLength(length); + } + bool SetTime(const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) throw(); + bool SetMTime(const CFiTime *mTime) throw(); +}; + +} + +#endif // _WIN32 + +}} + #endif diff --git a/3rdparty/lzma/CPP/Windows/FileLink.cpp b/3rdparty/lzma/CPP/Windows/FileLink.cpp index 25d49ac3ed6..8ce634fd01a 100644 --- a/3rdparty/lzma/CPP/Windows/FileLink.cpp +++ b/3rdparty/lzma/CPP/Windows/FileLink.cpp @@ -4,10 +4,17 @@ #include "../../C/CpuArch.h" +#ifndef _WIN32 +#include +#endif + #ifdef SUPPORT_DEVICE_FILE #include "../../C/Alloc.h" #endif +#include "../Common/UTFConvert.h" +#include "../Common/StringConvert.h" + #include "FileDir.h" #include "FileFind.h" #include "FileIO.h" @@ -54,6 +61,12 @@ using namespace NName; Substitute Path */ +/* +Win10 WSL2: +admin rights + sudo: it creates normal windows symbolic link. +in another cases : it creates IO_REPARSE_TAG_LX_SYMLINK repare point. +*/ + /* static const UInt32 kReparseFlags_Alias = (1 << 29); static const UInt32 kReparseFlags_HighLatency = (1 << 30); @@ -71,26 +84,28 @@ static const UInt32 kReparseFlags_Microsoft = ((UInt32)1 << 31); #define Get16(p) GetUi16(p) #define Get32(p) GetUi32(p) -#define Set16(p, v) SetUi16(p, v) -#define Set32(p, v) SetUi32(p, v) - -static const wchar_t *k_LinkPrefix = L"\\??\\"; +static const wchar_t * const k_LinkPrefix = L"\\??\\"; static const unsigned k_LinkPrefix_Size = 4; -static const bool IsLinkPrefix(const wchar_t *s) +static bool IsLinkPrefix(const wchar_t *s) { return IsString1PrefixedByString2(s, k_LinkPrefix); } /* -static const wchar_t *k_VolumePrefix = L"Volume{"; +static const wchar_t * const k_VolumePrefix = L"Volume{"; static const bool IsVolumeName(const wchar_t *s) { return IsString1PrefixedByString2(s, k_VolumePrefix); } */ -void WriteString(Byte *dest, const wchar_t *path) +#if defined(_WIN32) && !defined(UNDER_CE) + +#define Set16(p, v) SetUi16(p, v) +#define Set32(p, v) SetUi32(p, v) + +static void WriteString(Byte *dest, const wchar_t *path) { for (;;) { @@ -102,14 +117,32 @@ void WriteString(Byte *dest, const wchar_t *path) } } -#if defined(_WIN32) && !defined(UNDER_CE) - -bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) +bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink, bool isWSL) { bool isAbs = IsAbsolutePath(path); if (!isAbs && !isSymLink) return false; + if (isWSL) + { + // unsupported characters probably use Replacement Character UTF-16 0xFFFD + AString utf; + ConvertUnicodeToUTF8(path, utf); + const size_t size = 4 + utf.Len(); + if (size != (UInt16)size) + return false; + dest.Alloc(8 + size); + Byte *p = dest; + Set32(p, _my_IO_REPARSE_TAG_LX_SYMLINK); + Set16(p + 4, (UInt16)(size)); + Set16(p + 6, 0); + Set32(p + 8, _my_LX_SYMLINK_FLAG); + memcpy(p + 12, utf.Ptr(), utf.Len()); + return true; + } + + // usual symbolic LINK (NOT WSL) + bool needPrintName = true; if (IsSuperPath(path)) @@ -121,12 +154,12 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) const unsigned add_Prefix_Len = isAbs ? k_LinkPrefix_Size : 0; - unsigned len2 = MyStringLen(path) * 2; - const unsigned len1 = len2 + add_Prefix_Len * 2; + size_t len2 = (size_t)MyStringLen(path) * 2; + const size_t len1 = len2 + add_Prefix_Len * 2; if (!needPrintName) len2 = 0; - unsigned totalNamesSize = (len1 + len2); + size_t totalNamesSize = (len1 + len2); /* some WIM imagex software uses old scheme for symbolic links. so we can old scheme for byte to byte compatibility */ @@ -138,6 +171,8 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) totalNamesSize += 2 * 2; const size_t size = 8 + 8 + (isSymLink ? 4 : 0) + totalNamesSize; + if (size != (UInt16)size) + return false; dest.Alloc(size); memset(dest, 0, size); const UInt32 tag = isSymLink ? @@ -152,9 +187,9 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) unsigned subOffs = 0; unsigned printOffs = 0; if (newOrderScheme) - subOffs = len2; + subOffs = (unsigned)len2; else - printOffs = len1 + 2; + printOffs = (unsigned)len1 + 2; Set16(p + 0, (UInt16)subOffs); Set16(p + 2, (UInt16)len1); @@ -177,7 +212,8 @@ bool FillLinkData(CByteBuffer &dest, const wchar_t *path, bool isSymLink) return true; } -#endif +#endif // defined(_WIN32) && !defined(UNDER_CE) + static void GetString(const Byte *p, unsigned len, UString &res) { @@ -196,30 +232,67 @@ static void GetString(const Byte *p, unsigned len, UString &res) bool CReparseAttr::Parse(const Byte *p, size_t size) { + ErrorCode = (DWORD)ERROR_INVALID_REPARSE_DATA; + HeaderError = true; + TagIsUnknown = true; + MinorError = false; + if (size < 8) return false; Tag = Get32(p); UInt32 len = Get16(p + 4); - if (len + 8 > size) + if (len + 8 != size) + // if (len + 8 > size) return false; /* if ((type & kReparseFlags_Alias) == 0 || (type & kReparseFlags_Microsoft) == 0 || (type & 0xFFFF) != 3) */ - if (Tag != _my_IO_REPARSE_TAG_MOUNT_POINT && - Tag != _my_IO_REPARSE_TAG_SYMLINK) - // return true; - return false; if (Get16(p + 6) != 0) // padding return false; - + + HeaderError = false; + + if ( Tag != _my_IO_REPARSE_TAG_MOUNT_POINT + && Tag != _my_IO_REPARSE_TAG_SYMLINK + && Tag != _my_IO_REPARSE_TAG_LX_SYMLINK) + { + // for unsupported reparse points + ErrorCode = (DWORD)ERROR_REPARSE_TAG_INVALID; // ERROR_REPARSE_TAG_MISMATCH + // errorCode = ERROR_REPARSE_TAG_MISMATCH; // ERROR_REPARSE_TAG_INVALID + return false; + } + + TagIsUnknown = false; + p += 8; size -= 8; - if (len != size) // do we need that check? - return false; + if (Tag == _my_IO_REPARSE_TAG_LX_SYMLINK) + { + if (len < 4) + return false; + Flags = Get32(p); // maybe it's not Flags + if (Flags != _my_LX_SYMLINK_FLAG) + return false; + len -= 4; + p += 4; + char *s = WslName.GetBuf(len); + unsigned i; + for (i = 0; i < len; i++) + { + char c = (char)p[i]; + s[i] = c; + if (c == 0) + break; + } + WslName.ReleaseBuf_SetEnd(i); + MinorError = (i != len); + ErrorCode = 0; + return true; + } if (len < 8) return false; @@ -247,9 +320,11 @@ bool CReparseAttr::Parse(const Byte *p, size_t size) GetString(p + subOffs, subLen >> 1, SubsName); GetString(p + printOffs, printLen >> 1, PrintName); + ErrorCode = 0; return true; } + bool CReparseShortInfo::Parse(const Byte *p, size_t size) { const Byte *start = p; @@ -332,26 +407,34 @@ bool CReparseAttr::IsVolume() const UString CReparseAttr::GetPath() const { - UString s = SubsName; + if (IsSymLink_WSL()) + { + UString u; + // if (CheckUTF8(attr.WslName) + if (!ConvertUTF8ToUnicode(WslName, u)) + MultiByteToUnicodeString2(u, WslName); + return u; + } + + UString s (SubsName); if (IsLinkPrefix(s)) { - s.ReplaceOneCharAtPos(1, '\\'); + s.ReplaceOneCharAtPos(1, '\\'); // we normalize prefix from "\??\" to "\\?\" if (IsDrivePath(s.Ptr(k_LinkPrefix_Size))) s.DeleteFrontal(k_LinkPrefix_Size); } return s; } - #ifdef SUPPORT_DEVICE_FILE namespace NSystem { bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); } -#endif +#endif // SUPPORT_DEVICE_FILE -#ifndef UNDER_CE +#if defined(_WIN32) && !defined(UNDER_CE) namespace NIO { @@ -376,7 +459,7 @@ bool GetReparseData(CFSTR path, CByteBuffer &reparseData, BY_HANDLE_FILE_INFORMA static bool CreatePrefixDirOfFile(CFSTR path) { - FString path2 = path; + FString path2 (path); int pos = path2.ReverseFind_PathSepar(); if (pos < 0) return true; @@ -384,11 +467,26 @@ static bool CreatePrefixDirOfFile(CFSTR path) if (pos == 2 && path2[1] == L':') return true; // we don't create Disk folder; #endif - path2.DeleteFrom(pos); + path2.DeleteFrom((unsigned)pos); return NDir::CreateComplexDir(path2); } -// If there is Reprase data already, it still writes new Reparse data + +static bool OutIoReparseData(DWORD controlCode, CFSTR path, void *data, DWORD size) +{ + COutFile file; + if (!file.Open(path, + FILE_SHARE_WRITE, + OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)) + return false; + + DWORD returnedSize; + return file.DeviceIoControl(controlCode, data, size, NULL, 0, &returnedSize); +} + + +// If there is Reparse data already, it still writes new Reparse data bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) { NFile::NFind::CFileInfo fi; @@ -416,21 +514,100 @@ bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) } } - COutFile file; - if (!file.Open(path, - FILE_SHARE_WRITE, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)) + return OutIoReparseData(my_FSCTL_SET_REPARSE_POINT, path, (void *)(const Byte *)(data), size); +} + + +bool DeleteReparseData(CFSTR path) +{ + CByteBuffer reparseData; + if (!GetReparseData(path, reparseData, NULL)) return false; + /* MSDN: The tag specified in the ReparseTag member of this structure + must match the tag of the reparse point to be deleted, + and the ReparseDataLength member must be zero */ + #define my_REPARSE_DATA_BUFFER_HEADER_SIZE 8 + if (reparseData.Size() < my_REPARSE_DATA_BUFFER_HEADER_SIZE) + { + SetLastError(ERROR_INVALID_REPARSE_DATA); + return false; + } + BYTE buf[my_REPARSE_DATA_BUFFER_HEADER_SIZE]; + memset(buf, 0, sizeof(buf)); + memcpy(buf, reparseData, 4); // tag + return OutIoReparseData(my_FSCTL_DELETE_REPARSE_POINT, path, buf, sizeof(buf)); +} - DWORD returnedSize; - if (!file.DeviceIoControl(my_FSCTL_SET_REPARSE_POINT, (void *)data, size, NULL, 0, &returnedSize)) +} + +#endif // defined(_WIN32) && !defined(UNDER_CE) + + +#ifndef _WIN32 + +namespace NIO { + +bool GetReparseData(CFSTR path, CByteBuffer &reparseData) +{ + reparseData.Free(); + + #define MAX_PATHNAME_LEN 1024 + char buf[MAX_PATHNAME_LEN + 2]; + const size_t request = sizeof(buf) - 1; + + // printf("\nreadlink() path = %s \n", path); + const ssize_t size = readlink(path, buf, request); + // there is no tail zero + + if (size < 0) return false; + if ((size_t)size >= request) + { + SetLastError(EINVAL); // check it: ENAMETOOLONG + return false; + } + + // printf("\nreadlink() res = %s size = %d \n", buf, (int)size); + reparseData.CopyFrom((const Byte *)buf, (size_t)size); return true; } + +/* +// If there is Reparse data already, it still writes new Reparse data +bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) +{ + // AString s; + // s.SetFrom_CalcLen(data, size); + // return (symlink(s, path) == 0); + UNUSED_VAR(path) + UNUSED_VAR(isDir) + UNUSED_VAR(data) + UNUSED_VAR(size) + SetLastError(ENOSYS); + return false; } +*/ -#endif +bool SetSymLink(CFSTR from, CFSTR to) +{ + // printf("\nsymlink() %s -> %s\n", from, to); + int ir; + // ir = unlink(path); + // if (ir == 0) + ir = symlink(to, from); + return (ir == 0); +} + +bool SetSymLink_UString(CFSTR from, const UString &to) +{ + AString utf; + ConvertUnicodeToUTF8(to, utf); + return SetSymLink(from, utf); +} + +} + +#endif // !_WIN32 }} diff --git a/3rdparty/lzma/CPP/Windows/FileName.cpp b/3rdparty/lzma/CPP/Windows/FileName.cpp index 908ed53f4d0..d61ff7e1dd6 100644 --- a/3rdparty/lzma/CPP/Windows/FileName.cpp +++ b/3rdparty/lzma/CPP/Windows/FileName.cpp @@ -2,6 +2,13 @@ #include "StdAfx.h" +#ifndef _WIN32 +#include +#include +#include "../Common/StringConvert.h" +#include "FileDir.h" +#endif + #include "FileName.h" #ifndef _UNICODE @@ -58,7 +65,32 @@ void NormalizeDirPathPrefix(UString &dirPath) dirPath.Add_PathSepar(); } -#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z') +#ifdef _WIN32 + +#ifndef USE_UNICODE_FSTRING +#ifdef WIN_LONG_PATH +static void NormalizeDirSeparators(UString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + if (s[i] == '/') + s.ReplaceOneCharAtPos(i, WCHAR_PATH_SEPARATOR); +} +#endif +#endif + +void NormalizeDirSeparators(FString &s) +{ + const unsigned len = s.Len(); + for (unsigned i = 0; i < len; i++) + if (s[i] == '/') + s.ReplaceOneCharAtPos(i, FCHAR_PATH_SEPARATOR); +} + +#endif + + +#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) bool IsDrivePath(const wchar_t *s) throw() { return IS_LETTER_CHAR(s[0]) && s[1] == ':' && IS_SEPAR(s[2]); } @@ -87,8 +119,10 @@ bool IsAltPathPrefix(CFSTR s) throw() #if defined(_WIN32) && !defined(UNDER_CE) -const wchar_t *kSuperPathPrefix = L"\\\\?\\"; -static const wchar_t *kSuperUncPrefix = L"\\\\?\\UNC\\"; +const char * const kSuperPathPrefix = "\\\\?\\"; +#ifdef WIN_LONG_PATH +static const char * const kSuperUncPrefix = "\\\\?\\UNC\\"; +#endif #define IS_DEVICE_PATH(s) (IS_SEPAR((s)[0]) && IS_SEPAR((s)[1]) && (s)[2] == '.' && IS_SEPAR((s)[3])) #define IS_SUPER_PREFIX(s) (IS_SEPAR((s)[0]) && IS_SEPAR((s)[1]) && (s)[2] == '?' && IS_SEPAR((s)[3])) @@ -109,7 +143,7 @@ bool IsDevicePath(CFSTR s) throw() /* // actually we don't know the way to open device file in WinCE. unsigned len = MyStringLen(s); - if (len < 5 || len > 5 || memcmp(s, FTEXT("DSK"), 3 * sizeof(FChar)) != 0) + if (len < 5 || len > 5 || !IsString1PrefixedByString2(s, "DSK")) return false; if (s[4] != ':') return false; @@ -123,7 +157,7 @@ bool IsDevicePath(CFSTR s) throw() unsigned len = MyStringLen(s); if (len == 6 && s[5] == ':') return true; - if (len < 18 || len > 22 || memcmp(s + kDevicePathPrefixSize, FTEXT("PhysicalDrive"), 13 * sizeof(FChar)) != 0) + if (len < 18 || len > 22 || !IsString1PrefixedByString2(s + kDevicePathPrefixSize, "PhysicalDrive")) return false; for (unsigned i = 17; i < len; i++) if (s[i] < '0' || s[i] > '9') @@ -160,7 +194,7 @@ unsigned GetNetworkServerPrefixSize(CFSTR s) throw() int pos = FindSepar(s + prefixSize); if (pos < 0) return 0; - return prefixSize + pos + 1; + return prefixSize + (unsigned)(pos + 1); } bool IsNetworkShareRootPath(CFSTR s) throw() @@ -191,14 +225,12 @@ bool IsSuperPath(CFSTR s) throw() { return IS_SUPER_PREFIX(s); } bool IsSuperOrDevicePath(CFSTR s) throw() { return IS_SUPER_OR_DEVICE_PATH(s); } #endif // USE_UNICODE_FSTRING -/* -bool IsDrivePath_SuperAllowed(CFSTR s) +bool IsDrivePath_SuperAllowed(CFSTR s) throw() { if (IsSuperPath(s)) s += kSuperPathPrefixSize; return IsDrivePath(s); } -*/ bool IsDriveRootPath_SuperAllowed(CFSTR s) throw() { @@ -212,7 +244,7 @@ bool IsAbsolutePath(const wchar_t *s) throw() return IS_SEPAR(s[0]) || IsDrivePath2(s); } -int FindAltStreamColon(CFSTR path) +int FindAltStreamColon(CFSTR path) throw() { unsigned i = 0; if (IsDrivePath2(path)) @@ -226,7 +258,7 @@ int FindAltStreamColon(CFSTR path) if (c == ':') { if (colonPos < 0) - colonPos = i; + colonPos = (int)i; continue; } if (IS_SEPAR(c)) @@ -274,7 +306,7 @@ static unsigned GetRootPrefixSize_Of_SuperPath(CFSTR s) return kSuperPathPrefixSize + pos + 1; } -unsigned GetRootPrefixSize(CFSTR s) +unsigned GetRootPrefixSize(CFSTR s) throw() { if (IS_DEVICE_PATH(s)) return kDevicePathPrefixSize; @@ -285,7 +317,7 @@ unsigned GetRootPrefixSize(CFSTR s) #endif // USE_UNICODE_FSTRING -static unsigned GetRootPrefixSize_Of_NetworkPath(const wchar_t *s) +static unsigned GetRootPrefixSize_Of_NetworkPath(const wchar_t *s) throw() { // Network path: we look "server\path\" as root prefix int pos = FindSepar(s); @@ -294,10 +326,10 @@ static unsigned GetRootPrefixSize_Of_NetworkPath(const wchar_t *s) int pos2 = FindSepar(s + (unsigned)pos + 1); if (pos2 < 0) return 0; - return pos + pos2 + 2; + return (unsigned)(pos + pos2 + 2); } -static unsigned GetRootPrefixSize_Of_SimplePath(const wchar_t *s) +static unsigned GetRootPrefixSize_Of_SimplePath(const wchar_t *s) throw() { if (IsDrivePath(s)) return kDrivePrefixSize; @@ -309,7 +341,7 @@ static unsigned GetRootPrefixSize_Of_SimplePath(const wchar_t *s) return (size == 0) ? 0 : 2 + size; } -static unsigned GetRootPrefixSize_Of_SuperPath(const wchar_t *s) +static unsigned GetRootPrefixSize_Of_SuperPath(const wchar_t *s) throw() { if (IS_UNC_WITH_SLASH(s + kSuperPathPrefixSize)) { @@ -320,7 +352,7 @@ static unsigned GetRootPrefixSize_Of_SuperPath(const wchar_t *s) int pos = FindSepar(s + kSuperPathPrefixSize); if (pos < 0) return 0; - return kSuperPathPrefixSize + pos + 1; + return kSuperPathPrefixSize + (unsigned)(pos + 1); } unsigned GetRootPrefixSize(const wchar_t *s) throw() @@ -334,12 +366,13 @@ unsigned GetRootPrefixSize(const wchar_t *s) throw() #else // _WIN32 -bool IsAbsolutePath(const wchar_t *s) { return IS_SEPAR(s[0]); } +bool IsAbsolutePath(const wchar_t *s) throw() { return IS_SEPAR(s[0]); } #ifndef USE_UNICODE_FSTRING -unsigned GetRootPrefixSize(CFSTR s) { return IS_SEPAR(s[0]) ? 1 : 0; } +unsigned GetRootPrefixSize(CFSTR s) throw(); +unsigned GetRootPrefixSize(CFSTR s) throw() { return IS_SEPAR(s[0]) ? 1 : 0; } #endif -unsigned GetRootPrefixSize(const wchar_t *s) { return IS_SEPAR(s[0]) ? 1 : 0; } +unsigned GetRootPrefixSize(const wchar_t *s) throw() { return IS_SEPAR(s[0]) ? 1 : 0; } #endif // _WIN32 @@ -349,6 +382,9 @@ unsigned GetRootPrefixSize(const wchar_t *s) { return IS_SEPAR(s[0]) ? 1 : 0; } static bool GetCurDir(UString &path) { path.Empty(); + + #ifdef _WIN32 + DWORD needLength; #ifndef _UNICODE if (!g_IsNT) @@ -367,6 +403,16 @@ static bool GetCurDir(UString &path) path = s; } return (needLength > 0 && needLength <= MAX_PATH); + + #else + + FString s; + if (!NDir::GetCurrentDir(s)) + return false; + path = GetUnicodeString(s); + return true; + + #endif } static bool ResolveDotsFolders(UString &s) @@ -390,7 +436,7 @@ static bool ResolveDotsFolders(UString &s) { if (i == 0) return false; - int k = i - 2; + int k = (int)i - 2; i += 2; for (;; k--) @@ -409,8 +455,8 @@ static bool ResolveDotsFolders(UString &s) if (k >= 0) { - num = i - k; - i = k; + num = i - (unsigned)k; + i = (unsigned)k; } else { @@ -530,6 +576,7 @@ int GetUseSuperPathType(CFSTR s) throw() } + /* returns false in two cases: - if GetCurDir was used, and GetCurDir returned error. @@ -540,7 +587,6 @@ int GetUseSuperPathType(CFSTR s) throw() for absolute paths, returns true, res is Super path. */ - static bool GetSuperPathBase(CFSTR s, UString &res) { res.Empty(); @@ -628,7 +674,7 @@ static bool GetSuperPathBase(CFSTR s, UString &res) unsigned fixedSizeStart = 0; unsigned fixedSize = 0; - const wchar_t *superMarker = NULL; + const char *superMarker = NULL; if (IsSuperPath(curDir)) { fixedSize = GetRootPrefixSize_Of_SuperPath(curDir); @@ -683,7 +729,7 @@ static bool GetSuperPathBase(CFSTR s, UString &res) true false * use Super path true true true don't use any path, we already used mainPath true true false use main path as Super Path, we don't try mainMath - That case is possible now if GetCurDir returns unknow + That case is possible now if GetCurDir returns unknown type of path (not drive and not network) We can change that code if we want to try mainPath, if GetSuperPathBase returns error, @@ -704,6 +750,8 @@ bool GetSuperPath(CFSTR path, UString &superPath, bool onlyIfNew) return false; superPath = fs2us(path); } + + NormalizeDirSeparators(superPath); return true; } return false; @@ -714,6 +762,10 @@ bool GetSuperPaths(CFSTR s1, CFSTR s2, UString &d1, UString &d2, bool onlyIfNew) if (!GetSuperPathBase(s1, d1) || !GetSuperPathBase(s2, d2)) return false; + + NormalizeDirSeparators(d1); + NormalizeDirSeparators(d2); + if (d1.IsEmpty() && d2.IsEmpty() && onlyIfNew) return false; if (d1.IsEmpty()) d1 = fs2us(s1); diff --git a/3rdparty/lzma/CPP/Windows/FileName.h b/3rdparty/lzma/CPP/Windows/FileName.h index 6f9b6e03932..de8bd13488f 100644 --- a/3rdparty/lzma/CPP/Windows/FileName.h +++ b/3rdparty/lzma/CPP/Windows/FileName.h @@ -17,13 +17,17 @@ int FindSepar(const FChar *s) throw(); void NormalizeDirPathPrefix(FString &dirPath); // ensures that it ended with '\\', if dirPath is not epmty void NormalizeDirPathPrefix(UString &dirPath); +#ifdef _WIN32 +void NormalizeDirSeparators(FString &s); +#endif + bool IsDrivePath(const wchar_t *s) throw(); // first 3 chars are drive chars like "a:\\" bool IsAltPathPrefix(CFSTR s) throw(); /* name: */ #if defined(_WIN32) && !defined(UNDER_CE) -extern const wchar_t *kSuperPathPrefix; /* \\?\ */ +extern const char * const kSuperPathPrefix; /* \\?\ */ const unsigned kDevicePathPrefixSize = 4; const unsigned kSuperPathPrefixSize = 4; const unsigned kSuperUncPathPrefixSize = kSuperPathPrefixSize + 4; @@ -42,7 +46,7 @@ unsigned GetNetworkServerPrefixSize(CFSTR s) throw(); bool IsNetworkShareRootPath(CFSTR s) throw(); /* \\?\UNC\SERVER\share or \\SERVER\share or with slash */ -// bool IsDrivePath_SuperAllowed(CFSTR s) throw(); // first chars are drive chars like "a:\" or "\\?\a:\" +bool IsDrivePath_SuperAllowed(CFSTR s) throw(); // first chars are drive chars like "a:\" or "\\?\a:\" bool IsDriveRootPath_SuperAllowed(CFSTR s) throw(); // exact drive root path "a:\" or "\\?\a:\" bool IsDrivePath2(const wchar_t *s) throw(); // first 2 chars are drive chars like "a:" @@ -71,7 +75,7 @@ unsigned GetRootPrefixSize(CFSTR s) throw(); #endif -int FindAltStreamColon(CFSTR path); +int FindAltStreamColon(CFSTR path) throw(); #endif // _WIN32 diff --git a/3rdparty/lzma/CPP/Windows/FileSystem.cpp b/3rdparty/lzma/CPP/Windows/FileSystem.cpp index 6c1f48a206c..6259453225e 100644 --- a/3rdparty/lzma/CPP/Windows/FileSystem.cpp +++ b/3rdparty/lzma/CPP/Windows/FileSystem.cpp @@ -19,6 +19,8 @@ namespace NWindows { namespace NFile { namespace NSystem { +#ifdef _WIN32 + bool MyGetVolumeInformation( CFSTR rootPath, UString &volumeName, @@ -90,7 +92,7 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, #ifndef _UNICODE if (!g_IsNT) { - GetDiskFreeSpaceExA_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExA_Pointer)GetProcAddress( + GetDiskFreeSpaceExA_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExA_Pointer)(void *)GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetDiskFreeSpaceExA"); if (pGetDiskFreeSpaceEx) { @@ -105,7 +107,7 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, else #endif { - GetDiskFreeSpaceExW_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExW_Pointer)GetProcAddress( + GetDiskFreeSpaceExW_Pointer pGetDiskFreeSpaceEx = (GetDiskFreeSpaceExW_Pointer)(void *)GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetDiskFreeSpaceExW"); if (pGetDiskFreeSpaceEx) { @@ -126,6 +128,8 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, return true; } +#endif + }}} #endif diff --git a/3rdparty/lzma/CPP/Windows/FileSystem.h b/3rdparty/lzma/CPP/Windows/FileSystem.h index 9076ea13f4f..9b49a025a3b 100644 --- a/3rdparty/lzma/CPP/Windows/FileSystem.h +++ b/3rdparty/lzma/CPP/Windows/FileSystem.h @@ -10,6 +10,8 @@ namespace NWindows { namespace NFile { namespace NSystem { +#ifdef _WIN32 + bool MyGetVolumeInformation( CFSTR rootPath , UString &volumeName, @@ -22,6 +24,8 @@ UINT MyGetDriveType(CFSTR pathName); bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); +#endif + }}} #endif diff --git a/3rdparty/lzma/CPP/Windows/Handle.h b/3rdparty/lzma/CPP/Windows/Handle.h index bb7cb705da0..5878c830448 100644 --- a/3rdparty/lzma/CPP/Windows/Handle.h +++ b/3rdparty/lzma/CPP/Windows/Handle.h @@ -3,9 +3,11 @@ #ifndef __WINDOWS_HANDLE_H #define __WINDOWS_HANDLE_H +#include "../Common/MyTypes.h" + namespace NWindows { -class CHandle +class CHandle MY_UNCOPYABLE { protected: HANDLE _handle; diff --git a/3rdparty/lzma/CPP/Windows/MemoryGlobal.cpp b/3rdparty/lzma/CPP/Windows/MemoryGlobal.cpp new file mode 100644 index 00000000000..2a22394b39d --- /dev/null +++ b/3rdparty/lzma/CPP/Windows/MemoryGlobal.cpp @@ -0,0 +1,36 @@ +// Windows/MemoryGlobal.cpp + +#include "StdAfx.h" + +#include "MemoryGlobal.h" + +namespace NWindows { +namespace NMemory { + +bool CGlobal::Alloc(UINT flags, SIZE_T size) throw() +{ + HGLOBAL newBlock = ::GlobalAlloc(flags, size); + if (newBlock == NULL) + return false; + _global = newBlock; + return true; +} + +bool CGlobal::Free() throw() +{ + if (_global == NULL) + return true; + _global = ::GlobalFree(_global); + return (_global == NULL); +} + +bool CGlobal::ReAlloc(SIZE_T size) throw() +{ + HGLOBAL newBlock = ::GlobalReAlloc(_global, size, GMEM_MOVEABLE); + if (newBlock == NULL) + return false; + _global = newBlock; + return true; +} + +}} diff --git a/3rdparty/lzma/CPP/Windows/MemoryGlobal.h b/3rdparty/lzma/CPP/Windows/MemoryGlobal.h new file mode 100644 index 00000000000..c217510e584 --- /dev/null +++ b/3rdparty/lzma/CPP/Windows/MemoryGlobal.h @@ -0,0 +1,55 @@ +// Windows/MemoryGlobal.h + +#ifndef __WINDOWS_MEMORY_GLOBAL_H +#define __WINDOWS_MEMORY_GLOBAL_H + +#include "../Common/MyWindows.h" + +namespace NWindows { +namespace NMemory { + +class CGlobal +{ + HGLOBAL _global; +public: + CGlobal(): _global(NULL){}; + ~CGlobal() { Free(); } + operator HGLOBAL() const { return _global; } + void Attach(HGLOBAL hGlobal) + { + Free(); + _global = hGlobal; + } + HGLOBAL Detach() + { + HGLOBAL h = _global; + _global = NULL; + return h; + } + bool Alloc(UINT flags, SIZE_T size) throw(); + bool Free() throw(); + LPVOID Lock() const { return GlobalLock(_global); } + void Unlock() const { GlobalUnlock(_global); } + bool ReAlloc(SIZE_T size) throw(); +}; + +class CGlobalLock +{ + HGLOBAL _global; + LPVOID _ptr; +public: + LPVOID GetPointer() const { return _ptr; } + CGlobalLock(HGLOBAL hGlobal): _global(hGlobal) + { + _ptr = GlobalLock(hGlobal); + }; + ~CGlobalLock() + { + if (_ptr != NULL) + GlobalUnlock(_global); + } +}; + +}} + +#endif diff --git a/3rdparty/lzma/CPP/Windows/MemoryLock.cpp b/3rdparty/lzma/CPP/Windows/MemoryLock.cpp index b628a03e8f9..24866b1aeb3 100644 --- a/3rdparty/lzma/CPP/Windows/MemoryLock.cpp +++ b/3rdparty/lzma/CPP/Windows/MemoryLock.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../C/CpuArch.h" + #include "MemoryLock.h" namespace NWindows { @@ -19,7 +21,7 @@ typedef BOOL (WINAPI * Func_LookupPrivilegeValue)(LPCTSTR lpSystemName, LPCTSTR typedef BOOL (WINAPI * Func_AdjustTokenPrivileges)(HANDLE TokenHandle, BOOL DisableAllPrivileges, PTOKEN_PRIVILEGES NewState, DWORD BufferLength, PTOKEN_PRIVILEGES PreviousState, PDWORD ReturnLength); } -#define GET_PROC_ADDR(fff, name) Func_ ## fff my_ ## fff = (Func_ ## fff)GetProcAddress(hModule, name) +#define GET_PROC_ADDR(fff, name) Func_ ## fff my_ ## fff = (Func_ ## fff) (void(*)()) GetProcAddress(hModule, name) #endif bool EnablePrivilege(LPCTSTR privilegeName, bool enable) @@ -67,6 +69,44 @@ bool EnablePrivilege(LPCTSTR privilegeName, bool enable) return res; } + + +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); + +/* + We suppose that Window 10 works incorrectly with "Large Pages" at: + - Windows 10 1703 (15063) : incorrect allocating after VirtualFree() + - Windows 10 1709 (16299) : incorrect allocating after VirtualFree() + - Windows 10 1809 (17763) : the failures for blocks of 1 GiB and larger, + if CPU doesn't support 1 GB pages. + Windows 10 1903 (18362) probably works correctly. +*/ + +unsigned Get_LargePages_RiskLevel() +{ + OSVERSIONINFOEXW vi; + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return 0; + Func_RtlGetVersion func = (Func_RtlGetVersion)(void *)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return 0; + func(&vi); + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + return 0; + if (vi.dwMajorVersion + vi.dwMinorVersion != 10) + return 0; + if (vi.dwBuildNumber <= 16299) + return 1; + + #ifdef MY_CPU_X86_OR_AMD64 + if (vi.dwBuildNumber < 18362 && !CPU_IsSupported_PageGB()) + return 1; + #endif + + return 0; +} + #endif }} diff --git a/3rdparty/lzma/CPP/Windows/MemoryLock.h b/3rdparty/lzma/CPP/Windows/MemoryLock.h index 4a5dafcf01d..dcaf182e95e 100644 --- a/3rdparty/lzma/CPP/Windows/MemoryLock.h +++ b/3rdparty/lzma/CPP/Windows/MemoryLock.h @@ -31,6 +31,8 @@ inline void EnablePrivilege_SymLink() // Do we need to set SE_BACKUP_NAME ? } +unsigned Get_LargePages_RiskLevel(); + #endif }} diff --git a/3rdparty/lzma/CPP/Windows/NtCheck.h b/3rdparty/lzma/CPP/Windows/NtCheck.h index a1b89ef2d1f..0af32911909 100644 --- a/3rdparty/lzma/CPP/Windows/NtCheck.h +++ b/3rdparty/lzma/CPP/Windows/NtCheck.h @@ -17,6 +17,8 @@ static inline bool IsItWindowsNT() #endif #ifndef _UNICODE + extern + bool g_IsNT; #if defined(_WIN64) || defined(UNDER_CE) bool g_IsNT = true; #define SET_IS_NT diff --git a/3rdparty/lzma/CPP/Windows/PropVariant.cpp b/3rdparty/lzma/CPP/Windows/PropVariant.cpp index e95cbd2b8d5..2b179500479 100644 --- a/3rdparty/lzma/CPP/Windows/PropVariant.cpp +++ b/3rdparty/lzma/CPP/Windows/PropVariant.cpp @@ -91,7 +91,7 @@ CPropVariant& CPropVariant::operator=(BSTR bstrSrc) return *this; } -static const char *kMemException = "out of memory"; +static const char * const kMemException = "out of memory"; CPropVariant& CPropVariant::operator=(LPCOLESTR lpszSrc) { @@ -192,80 +192,108 @@ BSTR CPropVariant::AllocBstr(unsigned numChars) return bstrVal; } +#define SET_PROP_id_dest(id, dest) \ + if (vt != id) { InternalClear(); vt = id; } dest = value; wReserved1 = 0; + +void CPropVariant::Set_Int32(Int32 value) throw() +{ + SET_PROP_id_dest(VT_I4, lVal); +} + +void CPropVariant::Set_Int64(Int64 value) throw() +{ + SET_PROP_id_dest(VT_I8, hVal.QuadPart); +} + #define SET_PROP_FUNC(type, id, dest) \ CPropVariant& CPropVariant::operator=(type value) throw() \ - { if (vt != id) { InternalClear(); vt = id; } \ - dest = value; return *this; } + { SET_PROP_id_dest(id, dest); return *this; } SET_PROP_FUNC(Byte, VT_UI1, bVal) // SET_PROP_FUNC(Int16, VT_I2, iVal) -SET_PROP_FUNC(Int32, VT_I4, lVal) +// SET_PROP_FUNC(Int32, VT_I4, lVal) SET_PROP_FUNC(UInt32, VT_UI4, ulVal) SET_PROP_FUNC(UInt64, VT_UI8, uhVal.QuadPart) -SET_PROP_FUNC(Int64, VT_I8, hVal.QuadPart) +// SET_PROP_FUNC(Int64, VT_I8, hVal.QuadPart) SET_PROP_FUNC(const FILETIME &, VT_FILETIME, filetime) +#define CASE_SIMPLE_VT_VALUES \ + case VT_EMPTY: \ + case VT_BOOL: \ + case VT_FILETIME: \ + case VT_UI8: \ + case VT_UI4: \ + case VT_UI2: \ + case VT_UI1: \ + case VT_I8: \ + case VT_I4: \ + case VT_I2: \ + case VT_I1: \ + case VT_UINT: \ + case VT_INT: \ + case VT_NULL: \ + case VT_ERROR: \ + case VT_R4: \ + case VT_R8: \ + case VT_CY: \ + case VT_DATE: \ + + +/* + ::VariantClear() and ::VariantCopy() don't work, if (vt == VT_FILETIME) + So we handle VT_FILETIME and another simple types directly + we call system functions for VT_BSTR and for unknown typed +*/ + +CPropVariant::~CPropVariant() +{ + switch ((unsigned)vt) + { + CASE_SIMPLE_VT_VALUES + // vt = VT_EMPTY; // it's optional + return; + } + ::VariantClear((tagVARIANT *)this); +} + HRESULT PropVariant_Clear(PROPVARIANT *prop) throw() { - switch (prop->vt) + switch ((unsigned)prop->vt) { - case VT_EMPTY: - case VT_UI1: - case VT_I1: - case VT_I2: - case VT_UI2: - case VT_BOOL: - case VT_I4: - case VT_UI4: - case VT_R4: - case VT_INT: - case VT_UINT: - case VT_ERROR: - case VT_FILETIME: - case VT_UI8: - case VT_R8: - case VT_CY: - case VT_DATE: + CASE_SIMPLE_VT_VALUES prop->vt = VT_EMPTY; - prop->wReserved1 = 0; - prop->wReserved2 = 0; - prop->wReserved3 = 0; - prop->uhVal.QuadPart = 0; - return S_OK; + break; + default: + { + const HRESULT res = ::VariantClear((VARIANTARG *)prop); + if (res != S_OK || prop->vt != VT_EMPTY) + return res; + break; + } } - return ::VariantClear((VARIANTARG *)prop); - // return ::PropVariantClear(prop); - // PropVariantClear can clear VT_BLOB. + prop->wReserved1 = 0; + prop->wReserved2 = 0; + prop->wReserved3 = 0; + prop->uhVal.QuadPart = 0; + return S_OK; } HRESULT CPropVariant::Clear() throw() { if (vt == VT_EMPTY) + { + wReserved1 = 0; return S_OK; + } return PropVariant_Clear(this); } HRESULT CPropVariant::Copy(const PROPVARIANT* pSrc) throw() { - ::VariantClear((tagVARIANT *)this); - switch (pSrc->vt) + Clear(); + switch ((unsigned)pSrc->vt) { - case VT_UI1: - case VT_I1: - case VT_I2: - case VT_UI2: - case VT_BOOL: - case VT_I4: - case VT_UI4: - case VT_R4: - case VT_INT: - case VT_UINT: - case VT_ERROR: - case VT_FILETIME: - case VT_UI8: - case VT_R8: - case VT_CY: - case VT_DATE: + CASE_SIMPLE_VT_VALUES memmove((PROPVARIANT*)this, pSrc, sizeof(PROPVARIANT)); return S_OK; } @@ -275,11 +303,13 @@ HRESULT CPropVariant::Copy(const PROPVARIANT* pSrc) throw() HRESULT CPropVariant::Attach(PROPVARIANT *pSrc) throw() { - HRESULT hr = Clear(); + const HRESULT hr = Clear(); if (FAILED(hr)) return hr; - memcpy(this, pSrc, sizeof(PROPVARIANT)); + // memcpy((PROPVARIANT *)this, pSrc, sizeof(PROPVARIANT)); + *(PROPVARIANT *)this = *pSrc; pSrc->vt = VT_EMPTY; + pSrc->wReserved1 = 0; return S_OK; } @@ -287,20 +317,25 @@ HRESULT CPropVariant::Detach(PROPVARIANT *pDest) throw() { if (pDest->vt != VT_EMPTY) { - HRESULT hr = PropVariant_Clear(pDest); + const HRESULT hr = PropVariant_Clear(pDest); if (FAILED(hr)) return hr; } - memcpy(pDest, this, sizeof(PROPVARIANT)); + // memcpy(pDest, this, sizeof(PROPVARIANT)); + *pDest = *(PROPVARIANT *)this; vt = VT_EMPTY; + wReserved1 = 0; return S_OK; } HRESULT CPropVariant::InternalClear() throw() { if (vt == VT_EMPTY) + { + wReserved1 = 0; return S_OK; - HRESULT hr = Clear(); + } + const HRESULT hr = Clear(); if (FAILED(hr)) { vt = VT_ERROR; @@ -311,7 +346,7 @@ HRESULT CPropVariant::InternalClear() throw() void CPropVariant::InternalCopy(const PROPVARIANT *pSrc) { - HRESULT hr = Copy(pSrc); + const HRESULT hr = Copy(pSrc); if (FAILED(hr)) { if (hr == E_OUTOFMEMORY) @@ -321,11 +356,12 @@ void CPropVariant::InternalCopy(const PROPVARIANT *pSrc) } } + int CPropVariant::Compare(const CPropVariant &a) throw() { if (vt != a.vt) return MyCompare(vt, a.vt); - switch (vt) + switch ((unsigned)vt) { case VT_EMPTY: return 0; // case VT_I1: return MyCompare(cVal, a.cVal); @@ -338,7 +374,15 @@ int CPropVariant::Compare(const CPropVariant &a) throw() case VT_I8: return MyCompare(hVal.QuadPart, a.hVal.QuadPart); case VT_UI8: return MyCompare(uhVal.QuadPart, a.uhVal.QuadPart); case VT_BOOL: return -MyCompare(boolVal, a.boolVal); - case VT_FILETIME: return ::CompareFileTime(&filetime, &a.filetime); + case VT_FILETIME: + { + const int res = CompareFileTime(&filetime, &a.filetime); + if (res != 0) + return res; + const unsigned v1 = Get_Ns100(); + const unsigned v2 = a.Get_Ns100(); + return MyCompare(v1, v2); + } case VT_BSTR: return 0; // Not implemented default: return 0; } diff --git a/3rdparty/lzma/CPP/Windows/PropVariant.h b/3rdparty/lzma/CPP/Windows/PropVariant.h index 3610d6a8085..171402f743a 100644 --- a/3rdparty/lzma/CPP/Windows/PropVariant.h +++ b/3rdparty/lzma/CPP/Windows/PropVariant.h @@ -29,11 +29,14 @@ inline void PropVarEm_Set_UInt64(PROPVARIANT *p, UInt64 v) throw() p->uhVal.QuadPart = v; } -inline void PropVarEm_Set_FileTime64(PROPVARIANT *p, UInt64 v) throw() +inline void PropVarEm_Set_FileTime64_Prec(PROPVARIANT *p, UInt64 v, unsigned prec) throw() { p->vt = VT_FILETIME; p->filetime.dwLowDateTime = (DWORD)v; p->filetime.dwHighDateTime = (DWORD)(v >> 32); + p->wReserved1 = (WORD)prec; + p->wReserved2 = 0; + p->wReserved3 = 0; } inline void PropVarEm_Set_Bool(PROPVARIANT *p, bool b) throw() @@ -45,6 +48,14 @@ inline void PropVarEm_Set_Bool(PROPVARIANT *p, bool b) throw() class CPropVariant : public tagPROPVARIANT { + // ---------- forbidden functions ---------- + CPropVariant(const char *s); + // CPropVariant(const UString &s); + #ifdef DEBUG_FSTRING_INHERITS_ASTRING + CPropVariant(const FString &s); + CPropVariant& operator=(const FString &s); + #endif + public: CPropVariant() { @@ -55,7 +66,51 @@ public: // uhVal.QuadPart = 0; bstrVal = 0; } - ~CPropVariant() throw() { Clear(); } + + + void Set_FtPrec(unsigned prec) + { + wReserved1 = (WORD)prec; + wReserved2 = 0; + wReserved3 = 0; + } + + void SetAsTimeFrom_FT_Prec(const FILETIME &ft, unsigned prec) + { + operator=(ft); + Set_FtPrec(prec); + } + + void SetAsTimeFrom_Ft64_Prec(UInt64 v, unsigned prec) + { + FILETIME ft; + ft.dwLowDateTime = (DWORD)(UInt32)v; + ft.dwHighDateTime = (DWORD)(UInt32)(v >> 32); + operator=(ft); + Set_FtPrec(prec); + } + + void SetAsTimeFrom_FT_Prec_Ns100(const FILETIME &ft, unsigned prec, unsigned ns100) + { + operator=(ft); + wReserved1 = (WORD)prec; + wReserved2 = (WORD)ns100; + wReserved3 = 0; + } + + unsigned Get_Ns100() const + { + const unsigned prec = wReserved1; + const unsigned ns100 = wReserved2; + if (prec == 0 + && prec <= k_PropVar_TimePrec_1ns + && ns100 < 100 + && wReserved3 == 0) + return ns100; + return 0; + } + + ~CPropVariant(); CPropVariant(const PROPVARIANT &varSrc); CPropVariant(const CPropVariant &varSrc); CPropVariant(BSTR bstrSrc); @@ -64,13 +119,14 @@ public: CPropVariant(Byte value) { vt = VT_UI1; wReserved1 = 0; bVal = value; } private: + CPropVariant(UInt16 value); // { vt = VT_UI2; wReserved1 = 0; uiVal = value; } CPropVariant(Int16 value); // { vt = VT_I2; wReserved1 = 0; iVal = value; } CPropVariant(Int32 value); // { vt = VT_I4; wReserved1 = 0; lVal = value; } + CPropVariant(Int64 value); // { vt = VT_I8; wReserved1 = 0; hVal.QuadPart = value; } public: CPropVariant(UInt32 value) { vt = VT_UI4; wReserved1 = 0; ulVal = value; } CPropVariant(UInt64 value) { vt = VT_UI8; wReserved1 = 0; uhVal.QuadPart = value; } - CPropVariant(Int64 value) { vt = VT_I8; wReserved1 = 0; hVal.QuadPart = value; } CPropVariant(const FILETIME &value) { vt = VT_FILETIME; wReserved1 = 0; filetime = value; } CPropVariant& operator=(const CPropVariant &varSrc); @@ -80,20 +136,26 @@ public: CPropVariant& operator=(const UString &s); CPropVariant& operator=(const UString2 &s); CPropVariant& operator=(const char *s); + CPropVariant& operator=(const AString &s) + { return (*this)=(const char *)s; } CPropVariant& operator=(bool bSrc) throw(); CPropVariant& operator=(Byte value) throw(); private: CPropVariant& operator=(Int16 value) throw(); + CPropVariant& operator=(UInt16 value) throw(); + CPropVariant& operator=(Int32 value) throw(); + CPropVariant& operator=(Int64 value) throw(); public: - CPropVariant& operator=(Int32 value) throw(); CPropVariant& operator=(UInt32 value) throw(); CPropVariant& operator=(UInt64 value) throw(); - CPropVariant& operator=(Int64 value) throw(); CPropVariant& operator=(const FILETIME &value) throw(); + void Set_Int32(Int32 value) throw(); + void Set_Int64(Int64 value) throw(); + BSTR AllocBstr(unsigned numChars); HRESULT Clear() throw(); @@ -103,7 +165,6 @@ public: HRESULT InternalClear() throw(); void InternalCopy(const PROPVARIANT *pSrc); - int Compare(const CPropVariant &a) throw(); }; diff --git a/3rdparty/lzma/CPP/Windows/PropVariantConv.cpp b/3rdparty/lzma/CPP/Windows/PropVariantConv.cpp index fe86f9365f0..3c9bbd1702f 100644 --- a/3rdparty/lzma/CPP/Windows/PropVariantConv.cpp +++ b/3rdparty/lzma/CPP/Windows/PropVariantConv.cpp @@ -1,4 +1,4 @@ -// PropVariantConvert.cpp +// PropVariantConv.cpp #include "StdAfx.h" @@ -9,21 +9,27 @@ #define UINT_TO_STR_2(c, val) { s[0] = (c); s[1] = (char)('0' + (val) / 10); s[2] = (char)('0' + (val) % 10); s += 3; } -bool ConvertFileTimeToString(const FILETIME &ft, char *s, bool includeTime, bool includeSeconds) throw() +bool ConvertUtcFileTimeToString2(const FILETIME &utc, unsigned ns100, char *s, int level) throw() { + *s = 0; + FILETIME ft; + if (!FileTimeToLocalFileTime(&utc, &ft)) + return false; + SYSTEMTIME st; if (!BOOLToBool(FileTimeToSystemTime(&ft, &st))) { - *s = 0; + // win10 : that function doesn't work, if bit 63 of 64-bit FILETIME is set. return false; } - unsigned val = st.wYear; - if (val >= 10000) - { - *s++ = (char)('0' + val / 10000); - val %= 10000; - } + { + unsigned val = st.wYear; + if (val >= 10000) + { + *s++ = (char)('0' + val / 10000); + val %= 10000; + } s[3] = (char)('0' + val % 10); val /= 10; s[2] = (char)('0' + val % 10); val /= 10; s[1] = (char)('0' + val % 10); @@ -32,40 +38,91 @@ bool ConvertFileTimeToString(const FILETIME &ft, char *s, bool includeTime, bool } UINT_TO_STR_2('-', st.wMonth); UINT_TO_STR_2('-', st.wDay); - if (includeTime) + + if (level > kTimestampPrintLevel_DAY) { UINT_TO_STR_2(' ', st.wHour); UINT_TO_STR_2(':', st.wMinute); - if (includeSeconds) + + if (level >= kTimestampPrintLevel_SEC) { UINT_TO_STR_2(':', st.wSecond); - /* - *s++ = '.'; - unsigned val = st.wMilliseconds; - s[2] = (char)('0' + val % 10); val /= 10; - s[1] = (char)('0' + val % 10); - s[0] = (char)('0' + val / 10); - s += 3; - */ + + if (level > kTimestampPrintLevel_SEC) + { + *s++ = '.'; + /* + { + unsigned val = st.wMilliseconds; + s[2] = (char)('0' + val % 10); val /= 10; + s[1] = (char)('0' + val % 10); + s[0] = (char)('0' + val / 10); + s += 3; + } + *s++ = ' '; + */ + + { + unsigned numDigits = 7; + UInt32 val = (UInt32)((((UInt64)ft.dwHighDateTime << 32) + ft.dwLowDateTime) % 10000000); + for (unsigned i = numDigits; i != 0;) + { + i--; + s[i] = (char)('0' + val % 10); val /= 10; + } + if (numDigits > (unsigned)level) + numDigits = (unsigned)level; + s += numDigits; + } + if (level >= kTimestampPrintLevel_NTFS + 1) + { + *s++ = (char)('0' + (ns100 / 10)); + if (level >= kTimestampPrintLevel_NTFS + 2) + *s++ = (char)('0' + (ns100 % 10)); + } + } } } + *s = 0; return true; } -void ConvertFileTimeToString(const FILETIME &ft, wchar_t *dest, bool includeTime, bool includeSeconds) throw() + +bool ConvertUtcFileTimeToString(const FILETIME &utc, char *s, int level) throw() +{ + return ConvertUtcFileTimeToString2(utc, 0, s, level); +} + +bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, wchar_t *dest, int level) throw() { char s[32]; - ConvertFileTimeToString(ft, s, includeTime, includeSeconds); + bool res = ConvertUtcFileTimeToString2(ft, ns100, s, level); for (unsigned i = 0;; i++) { - unsigned char c = s[i]; + Byte c = (Byte)s[i]; dest[i] = c; if (c == 0) - return; + break; + } + return res; +} + +bool ConvertUtcFileTimeToString(const FILETIME &ft, wchar_t *dest, int level) throw() +{ + char s[32]; + bool res = ConvertUtcFileTimeToString(ft, s, level); + for (unsigned i = 0;; i++) + { + Byte c = (Byte)s[i]; + dest[i] = c; + if (c == 0) + break; } + return res; } + void ConvertPropVariantToShortString(const PROPVARIANT &prop, char *dest) throw() { *dest = 0; @@ -77,7 +134,19 @@ void ConvertPropVariantToShortString(const PROPVARIANT &prop, char *dest) throw( case VT_UI2: ConvertUInt32ToString(prop.uiVal, dest); return; case VT_UI4: ConvertUInt32ToString(prop.ulVal, dest); return; case VT_UI8: ConvertUInt64ToString(prop.uhVal.QuadPart, dest); return; - case VT_FILETIME: ConvertFileTimeToString(prop.filetime, dest, true, true); return; + case VT_FILETIME: + { + // const unsigned prec = prop.wReserved1; + int level = 0; + /* + if (prec == 0) + level = 7; + else if (prec > 16 && prec <= 16 + 9) + level = prec - 16; + */ + ConvertUtcFileTimeToString(prop.filetime, dest, level); + return; + } // case VT_I1: return ConvertInt64ToString(prop.cVal, dest); return; case VT_I2: ConvertInt64ToString(prop.iVal, dest); return; case VT_I4: ConvertInt64ToString(prop.lVal, dest); return; @@ -98,7 +167,19 @@ void ConvertPropVariantToShortString(const PROPVARIANT &prop, wchar_t *dest) thr case VT_UI2: ConvertUInt32ToString(prop.uiVal, dest); return; case VT_UI4: ConvertUInt32ToString(prop.ulVal, dest); return; case VT_UI8: ConvertUInt64ToString(prop.uhVal.QuadPart, dest); return; - case VT_FILETIME: ConvertFileTimeToString(prop.filetime, dest, true, true); return; + case VT_FILETIME: + { + // const unsigned prec = prop.wReserved1; + int level = 0; + /* + if (prec == 0) + level = 7; + else if (prec > 16 && prec <= 16 + 9) + level = prec - 16; + */ + ConvertUtcFileTimeToString(prop.filetime, dest, level); + return; + } // case VT_I1: return ConvertInt64ToString(prop.cVal, dest); return; case VT_I2: ConvertInt64ToString(prop.iVal, dest); return; case VT_I4: ConvertInt64ToString(prop.lVal, dest); return; diff --git a/3rdparty/lzma/CPP/Windows/PropVariantConv.h b/3rdparty/lzma/CPP/Windows/PropVariantConv.h index 5d26357f08d..606778406f4 100644 --- a/3rdparty/lzma/CPP/Windows/PropVariantConv.h +++ b/3rdparty/lzma/CPP/Windows/PropVariantConv.h @@ -6,8 +6,18 @@ #include "../Common/MyTypes.h" // provide at least 32 bytes for buffer including zero-end -bool ConvertFileTimeToString(const FILETIME &ft, char *s, bool includeTime = true, bool includeSeconds = true) throw(); -void ConvertFileTimeToString(const FILETIME &ft, wchar_t *s, bool includeTime = true, bool includeSeconds = true) throw(); + +#define kTimestampPrintLevel_DAY -3 +// #define kTimestampPrintLevel_HOUR -2 +#define kTimestampPrintLevel_MIN -1 +#define kTimestampPrintLevel_SEC 0 +#define kTimestampPrintLevel_NTFS 7 +#define kTimestampPrintLevel_NS 9 + +bool ConvertUtcFileTimeToString(const FILETIME &ft, char *s, int level = kTimestampPrintLevel_SEC) throw(); +bool ConvertUtcFileTimeToString(const FILETIME &ft, wchar_t *s, int level = kTimestampPrintLevel_SEC) throw(); +bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, char *s, int level = kTimestampPrintLevel_SEC) throw(); +bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, wchar_t *s, int level = kTimestampPrintLevel_SEC) throw(); // provide at least 32 bytes for buffer including zero-end // don't send VT_BSTR to these functions diff --git a/3rdparty/lzma/CPP/Windows/Registry.cpp b/3rdparty/lzma/CPP/Windows/Registry.cpp index c000e62897d..b20157d5a2b 100644 --- a/3rdparty/lzma/CPP/Windows/Registry.cpp +++ b/3rdparty/lzma/CPP/Windows/Registry.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include +// #include #ifndef _UNICODE #include "../Common/StringConvert.h" @@ -17,12 +18,27 @@ namespace NWindows { namespace NRegistry { #define MYASSERT(expr) // _ASSERTE(expr) +#define MY_ASSUME(expr) + +/* +static void Error() +{ + #ifdef _CONSOLE + printf("\nregistry error\n"); + #else + MessageBoxW(0, L"registry error", L"", 0); + // exit(1); + #endif +} + +#define MY_ASSUME(expr) { if (!(expr)) Error(); } +*/ LONG CKey::Create(HKEY parentKey, LPCTSTR keyName, LPTSTR keyClass, DWORD options, REGSAM accessMask, LPSECURITY_ATTRIBUTES securityAttributes, LPDWORD disposition) throw() { - MYASSERT(parentKey != NULL); + MY_ASSUME(parentKey != NULL); DWORD dispositionReal; HKEY key = NULL; LONG res = RegCreateKeyEx(parentKey, keyName, 0, keyClass, @@ -39,7 +55,7 @@ LONG CKey::Create(HKEY parentKey, LPCTSTR keyName, LONG CKey::Open(HKEY parentKey, LPCTSTR keyName, REGSAM accessMask) throw() { - MYASSERT(parentKey != NULL); + MY_ASSUME(parentKey != NULL); HKEY key = NULL; LONG res = RegOpenKeyEx(parentKey, keyName, 0, accessMask, &key); if (res == ERROR_SUCCESS) @@ -66,7 +82,7 @@ LONG CKey::Close() throw() // winNT to be deleted must not have subkeys LONG CKey::DeleteSubKey(LPCTSTR subKeyName) throw() { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegDeleteKey(_object, subKeyName); } @@ -101,14 +117,14 @@ static inline bool UINT32ToBool(UInt32 value) { return (value != 0); } LONG CKey::DeleteValue(LPCTSTR name) throw() { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return ::RegDeleteValue(_object, name); } #ifndef _UNICODE LONG CKey::DeleteValue(LPCWSTR name) { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); if (g_IsNT) return ::RegDeleteValueW(_object, name); return DeleteValue(name == 0 ? 0 : (LPCSTR)GetSystemString(name)); @@ -117,9 +133,9 @@ LONG CKey::DeleteValue(LPCWSTR name) LONG CKey::SetValue(LPCTSTR name, UInt32 value) throw() { - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, 0, REG_DWORD, - (BYTE * const)&value, sizeof(UInt32)); + (const BYTE *)&value, sizeof(UInt32)); } LONG CKey::SetValue(LPCTSTR name, bool value) throw() @@ -130,16 +146,16 @@ LONG CKey::SetValue(LPCTSTR name, bool value) throw() LONG CKey::SetValue(LPCTSTR name, LPCTSTR value) throw() { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, 0, REG_SZ, - (const BYTE * )value, (lstrlen(value) + 1) * sizeof(TCHAR)); + (const BYTE *)value, ((DWORD)lstrlen(value) + 1) * sizeof(TCHAR)); } /* LONG CKey::SetValue(LPCTSTR name, const CSysString &value) { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, NULL, REG_SZ, (const BYTE *)(const TCHAR *)value, (value.Len() + 1) * sizeof(TCHAR)); } @@ -150,9 +166,9 @@ LONG CKey::SetValue(LPCTSTR name, const CSysString &value) LONG CKey::SetValue(LPCWSTR name, LPCWSTR value) { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); if (g_IsNT) - return RegSetValueExW(_object, name, NULL, REG_SZ, + return RegSetValueExW(_object, name, 0, REG_SZ, (const BYTE * )value, (DWORD)((wcslen(value) + 1) * sizeof(wchar_t))); return SetValue(name == 0 ? 0 : (LPCSTR)GetSystemString(name), value == 0 ? 0 : (LPCSTR)GetSystemString(value)); @@ -164,7 +180,7 @@ LONG CKey::SetValue(LPCWSTR name, LPCWSTR value) LONG CKey::SetValue(LPCTSTR name, const void *value, UInt32 size) throw() { MYASSERT(value != NULL); - MYASSERT(_object != NULL); + MY_ASSUME(_object != NULL); return RegSetValueEx(_object, name, 0, REG_BINARY, (const BYTE *)value, size); } @@ -193,7 +209,7 @@ LONG CKey::QueryValue(LPCTSTR name, UInt32 &value) throw() { DWORD type = 0; DWORD count = sizeof(DWORD); - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, + LONG res = RegQueryValueEx(_object, name, NULL, &type, (LPBYTE)&value, &count); MYASSERT((res != ERROR_SUCCESS) || (type == REG_DWORD)); MYASSERT((res != ERROR_SUCCESS) || (count == sizeof(UInt32))); @@ -219,7 +235,7 @@ LONG CKey::GetValue_IfOk(LPCTSTR name, UInt32 &value) throw() LONG CKey::GetValue_IfOk(LPCTSTR name, bool &value) throw() { - bool newVal; + bool newVal = false; LONG res = QueryValue(name, newVal); if (res == ERROR_SUCCESS) value = newVal; @@ -229,7 +245,7 @@ LONG CKey::GetValue_IfOk(LPCTSTR name, bool &value) throw() LONG CKey::QueryValue(LPCTSTR name, LPTSTR value, UInt32 &count) throw() { DWORD type = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, (LPBYTE)value, (DWORD *)&count); + LONG res = RegQueryValueEx(_object, name, NULL, &type, (LPBYTE)value, (DWORD *)&count); MYASSERT((res != ERROR_SUCCESS) || (type == REG_SZ) || (type == REG_MULTI_SZ) || (type == REG_EXPAND_SZ)); return res; } @@ -239,7 +255,7 @@ LONG CKey::QueryValue(LPCTSTR name, CSysString &value) value.Empty(); DWORD type = 0; UInt32 curSize = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, NULL, (DWORD *)&curSize); + LONG res = RegQueryValueEx(_object, name, NULL, &type, NULL, (DWORD *)&curSize); if (res != ERROR_SUCCESS && res != ERROR_MORE_DATA) return res; UInt32 curSize2 = curSize; @@ -296,7 +312,7 @@ LONG CKey::QueryValue(LPCWSTR name, UString &value) LONG CKey::QueryValue(LPCTSTR name, void *value, UInt32 &count) throw() { DWORD type = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, (LPBYTE)value, (DWORD *)&count); + LONG res = RegQueryValueEx(_object, name, NULL, &type, (LPBYTE)value, (DWORD *)&count); MYASSERT((res != ERROR_SUCCESS) || (type == REG_BINARY)); return res; } @@ -306,7 +322,7 @@ LONG CKey::QueryValue(LPCTSTR name, CByteBuffer &value, UInt32 &dataSize) { DWORD type = 0; dataSize = 0; - LONG res = RegQueryValueEx(_object, (LPTSTR)name, NULL, &type, NULL, (DWORD *)&dataSize); + LONG res = RegQueryValueEx(_object, name, NULL, &type, NULL, (DWORD *)&dataSize); if (res != ERROR_SUCCESS && res != ERROR_MORE_DATA) return res; value.Alloc(dataSize); @@ -369,7 +385,7 @@ LONG CKey::GetValue_Strings(LPCTSTR valueName, UStringVector &strings) if (dataSize % sizeof(wchar_t) != 0) return E_FAIL; - const wchar_t *data = (const wchar_t *)(const Byte *)buffer; + const wchar_t *data = (const wchar_t *)(const void *)(const Byte *)buffer; size_t numChars = dataSize / sizeof(wchar_t); size_t prev = 0; UString s; diff --git a/3rdparty/lzma/CPP/Windows/ResourceString.cpp b/3rdparty/lzma/CPP/Windows/ResourceString.cpp index cc8b964a213..ae8182ed4e4 100644 --- a/3rdparty/lzma/CPP/Windows/ResourceString.cpp +++ b/3rdparty/lzma/CPP/Windows/ResourceString.cpp @@ -25,10 +25,10 @@ static CSysString MyLoadStringA(HINSTANCE hInstance, UINT resourceID) do { size <<= 1; - len = ::LoadString(hInstance, resourceID, s.GetBuf(size - 1), size); + len = ::LoadString(hInstance, resourceID, s.GetBuf((unsigned)size - 1), size); } while (size - len <= 1); - s.ReleaseBuf_CalcLen(len); + s.ReleaseBuf_CalcLen((unsigned)len); return s; } @@ -43,10 +43,10 @@ static void MyLoadString2(HINSTANCE hInstance, UINT resourceID, UString &s) do { size <<= 1; - len = ::LoadStringW(hInstance, resourceID, s.GetBuf(size - 1), size); + len = ::LoadStringW(hInstance, resourceID, s.GetBuf((unsigned)size - 1), size); } while (size - len <= 1); - s.ReleaseBuf_CalcLen(len); + s.ReleaseBuf_CalcLen((unsigned)len); } // NT4 doesn't support LoadStringW(,,, 0) to get pointer to resource string. So we don't use it. diff --git a/3rdparty/lzma/CPP/Windows/SecurityUtils.cpp b/3rdparty/lzma/CPP/Windows/SecurityUtils.cpp index 67a9d7fdcb3..ede83faf565 100644 --- a/3rdparty/lzma/CPP/Windows/SecurityUtils.cpp +++ b/3rdparty/lzma/CPP/Windows/SecurityUtils.cpp @@ -2,10 +2,11 @@ #include "StdAfx.h" -#include "../Common/MyString.h" - #include "SecurityUtils.h" +#define MY_CAST_FUNC (void(*)()) +// #define MY_CAST_FUNC + namespace NWindows { namespace NSecurity { @@ -34,7 +35,7 @@ bool MyLookupAccountSid(LPCTSTR systemName, PSID sid, static void SetLsaString(LPWSTR src, PLSA_UNICODE_STRING dest) { - int len = (int)wcslen(src); + size_t len = (size_t)wcslen(src); dest->Length = (USHORT)(len * sizeof(WCHAR)); dest->MaximumLength = (USHORT)((len + 1) * sizeof(WCHAR)); dest->Buffer = src; @@ -52,8 +53,10 @@ static void MyLookupSids(CPolicy &policy, PSID ps) } */ +extern "C" { + #ifndef _UNICODE -typedef BOOL (WINAPI * LookupAccountNameWP)( +typedef BOOL (WINAPI * Func_LookupAccountNameW)( LPCWSTR lpSystemName, LPCWSTR lpAccountName, PSID Sid, @@ -64,13 +67,17 @@ typedef BOOL (WINAPI * LookupAccountNameWP)( ); #endif +} + static PSID GetSid(LPWSTR accountName) { #ifndef _UNICODE HMODULE hModule = GetModuleHandle(TEXT("Advapi32.dll")); if (hModule == NULL) return NULL; - LookupAccountNameWP lookupAccountNameW = (LookupAccountNameWP)GetProcAddress(hModule, "LookupAccountNameW"); + Func_LookupAccountNameW lookupAccountNameW = (Func_LookupAccountNameW) + MY_CAST_FUNC + GetProcAddress(hModule, "LookupAccountNameW"); if (lookupAccountNameW == NULL) return NULL; #endif diff --git a/3rdparty/lzma/CPP/Windows/SecurityUtils.h b/3rdparty/lzma/CPP/Windows/SecurityUtils.h index 8966dfd3b7d..c0d7b127a0f 100644 --- a/3rdparty/lzma/CPP/Windows/SecurityUtils.h +++ b/3rdparty/lzma/CPP/Windows/SecurityUtils.h @@ -7,6 +7,31 @@ #include "Defs.h" +#ifndef _UNICODE + +extern "C" { +typedef NTSTATUS (NTAPI *Func_LsaOpenPolicy)(PLSA_UNICODE_STRING SystemName, + PLSA_OBJECT_ATTRIBUTES ObjectAttributes, ACCESS_MASK DesiredAccess, PLSA_HANDLE PolicyHandle); +typedef NTSTATUS (NTAPI *Func_LsaClose)(LSA_HANDLE ObjectHandle); +typedef NTSTATUS (NTAPI *Func_LsaAddAccountRights)(LSA_HANDLE PolicyHandle, + PSID AccountSid, PLSA_UNICODE_STRING UserRights, ULONG CountOfRights ); +#define MY_STATUS_NOT_IMPLEMENTED ((NTSTATUS)0xC0000002L) +} + +#define POLICY_FUNC_CALL(fff, str) \ + if (hModule == NULL) return MY_STATUS_NOT_IMPLEMENTED; \ + Func_ ## fff v = (Func_ ## fff) (void(*)()) GetProcAddress(hModule, str); \ + if (!v) return MY_STATUS_NOT_IMPLEMENTED; \ + const NTSTATUS res = v + +#else + +#define POLICY_FUNC_CALL(fff, str) \ + const NTSTATUS res = ::fff + +#endif + + namespace NWindows { namespace NSecurity { @@ -53,15 +78,9 @@ public: }; -#ifndef _UNICODE -typedef NTSTATUS (NTAPI *LsaOpenPolicyP)(PLSA_UNICODE_STRING SystemName, - PLSA_OBJECT_ATTRIBUTES ObjectAttributes, ACCESS_MASK DesiredAccess, PLSA_HANDLE PolicyHandle); -typedef NTSTATUS (NTAPI *LsaCloseP)(LSA_HANDLE ObjectHandle); -typedef NTSTATUS (NTAPI *LsaAddAccountRightsP)(LSA_HANDLE PolicyHandle, - PSID AccountSid, PLSA_UNICODE_STRING UserRights, ULONG CountOfRights ); -#define MY_STATUS_NOT_IMPLEMENTED ((NTSTATUS)0xC0000002L) -#endif + + struct CPolicy { protected: @@ -82,43 +101,17 @@ public: NTSTATUS Open(PLSA_UNICODE_STRING systemName, PLSA_OBJECT_ATTRIBUTES objectAttributes, ACCESS_MASK desiredAccess) { - #ifndef _UNICODE - if (hModule == NULL) - return MY_STATUS_NOT_IMPLEMENTED; - LsaOpenPolicyP lsaOpenPolicy = (LsaOpenPolicyP)GetProcAddress(hModule, "LsaOpenPolicy"); - if (lsaOpenPolicy == NULL) - return MY_STATUS_NOT_IMPLEMENTED; - #endif - Close(); - return - #ifdef _UNICODE - ::LsaOpenPolicy - #else - lsaOpenPolicy - #endif + POLICY_FUNC_CALL (LsaOpenPolicy, "LsaOpenPolicy") (systemName, objectAttributes, desiredAccess, &_handle); + return res; } NTSTATUS Close() { if (_handle == NULL) return 0; - - #ifndef _UNICODE - if (hModule == NULL) - return MY_STATUS_NOT_IMPLEMENTED; - LsaCloseP lsaClose = (LsaCloseP)GetProcAddress(hModule, "LsaClose"); - if (lsaClose == NULL) - return MY_STATUS_NOT_IMPLEMENTED; - #endif - - NTSTATUS res = - #ifdef _UNICODE - ::LsaClose - #else - lsaClose - #endif + POLICY_FUNC_CALL (LsaClose, "LsaClose") (_handle); _handle = NULL; return res; @@ -137,21 +130,9 @@ public: NTSTATUS AddAccountRights(PSID accountSid, PLSA_UNICODE_STRING userRights, ULONG countOfRights) { - #ifndef _UNICODE - if (hModule == NULL) - return MY_STATUS_NOT_IMPLEMENTED; - LsaAddAccountRightsP lsaAddAccountRights = (LsaAddAccountRightsP)GetProcAddress(hModule, "LsaAddAccountRights"); - if (lsaAddAccountRights == NULL) - return MY_STATUS_NOT_IMPLEMENTED; - #endif - - return - #ifdef _UNICODE - ::LsaAddAccountRights - #else - lsaAddAccountRights - #endif + POLICY_FUNC_CALL (LsaAddAccountRights, "LsaAddAccountRights") (_handle, accountSid, userRights, countOfRights); + return res; } NTSTATUS AddAccountRights(PSID accountSid, PLSA_UNICODE_STRING userRights) { return AddAccountRights(accountSid, userRights, 1); } diff --git a/3rdparty/lzma/CPP/Windows/Shell.cpp b/3rdparty/lzma/CPP/Windows/Shell.cpp index 8e82c14a726..071833cfa8e 100644 --- a/3rdparty/lzma/CPP/Windows/Shell.cpp +++ b/3rdparty/lzma/CPP/Windows/Shell.cpp @@ -2,6 +2,11 @@ #include "StdAfx.h" +/* +#include +#include +*/ + #include "../Common/MyCom.h" #ifndef _UNICODE #include "../Common/StringConvert.h" @@ -114,9 +119,22 @@ UString CDrop::QueryFileName(UINT fileIndex) void CDrop::QueryFileNames(UStringVector &fileNames) { UINT numFiles = QueryCountOfFiles(); + /* + char s[100]; + sprintf(s, "QueryFileNames: %d files", numFiles); + OutputDebugStringA(s); + */ fileNames.ClearAndReserve(numFiles); for (UINT i = 0; i < numFiles; i++) - fileNames.AddInReserved(QueryFileName(i)); + { + const UString s2 = QueryFileName(i); + if (!s2.IsEmpty()) + fileNames.AddInReserved(s2); + /* + OutputDebugStringW(L"file ---"); + OutputDebugStringW(s2); + */ + } } @@ -153,8 +171,8 @@ bool BrowseForFolder(HWND /* owner */, LPCTSTR /* title */, MessageBoxW(0, L"yes", L"", 0); */ /* - UString s = L"all files"; - s += L" (*.*)"; + UString s = "all files"; + s += " (*.*)"; return MyGetOpenFileName(owner, title, initialFolder, s, resultPath, true); */ return false; @@ -174,7 +192,7 @@ bool BrowseForFolder(LPBROWSEINFO browseInfo, CSysString &resultPath) } -int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM data) +static int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM data) { #ifndef UNDER_CE switch (uMsg) @@ -203,7 +221,7 @@ int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM da } -bool BrowseForFolder(HWND owner, LPCTSTR title, UINT ulFlags, +static bool BrowseForFolder(HWND owner, LPCTSTR title, UINT ulFlags, LPCTSTR initialFolder, CSysString &resultPath) { CSysString displayName; @@ -240,14 +258,21 @@ bool BrowseForFolder(HWND owner, LPCTSTR title, #ifndef _UNICODE -typedef BOOL (WINAPI * SHGetPathFromIDListWP)(LPCITEMIDLIST pidl, LPWSTR pszPath); +extern "C" { +typedef BOOL (WINAPI * Func_SHGetPathFromIDListW)(LPCITEMIDLIST pidl, LPWSTR pszPath); +typedef LPITEMIDLIST (WINAPI * Func_SHBrowseForFolderW)(LPBROWSEINFOW lpbi); +} + +#define MY_CAST_FUNC (void(*)()) +// #define MY_CAST_FUNC bool GetPathFromIDList(LPCITEMIDLIST itemIDList, UString &path) { path.Empty(); - SHGetPathFromIDListWP shGetPathFromIDListW = (SHGetPathFromIDListWP) + Func_SHGetPathFromIDListW shGetPathFromIDListW = (Func_SHGetPathFromIDListW) + MY_CAST_FUNC ::GetProcAddress(::GetModuleHandleW(L"shell32.dll"), "SHGetPathFromIDListW"); - if (shGetPathFromIDListW == 0) + if (!shGetPathFromIDListW) return false; const unsigned len = MAX_PATH * 2; bool result = BOOLToBool(shGetPathFromIDListW(itemIDList, path.GetBuf(len))); @@ -255,14 +280,14 @@ bool GetPathFromIDList(LPCITEMIDLIST itemIDList, UString &path) return result; } -typedef LPITEMIDLIST (WINAPI * SHBrowseForFolderWP)(LPBROWSEINFOW lpbi); -bool BrowseForFolder(LPBROWSEINFOW browseInfo, UString &resultPath) +static bool BrowseForFolder(LPBROWSEINFOW browseInfo, UString &resultPath) { NWindows::NCOM::CComInitializer comInitializer; - SHBrowseForFolderWP shBrowseForFolderW = (SHBrowseForFolderWP) + Func_SHBrowseForFolderW shBrowseForFolderW = (Func_SHBrowseForFolderW) + MY_CAST_FUNC ::GetProcAddress(::GetModuleHandleW(L"shell32.dll"), "SHBrowseForFolderW"); - if (shBrowseForFolderW == 0) + if (!shBrowseForFolderW) return false; LPITEMIDLIST itemIDList = shBrowseForFolderW(browseInfo); if (itemIDList == NULL) @@ -272,7 +297,7 @@ bool BrowseForFolder(LPBROWSEINFOW browseInfo, UString &resultPath) return GetPathFromIDList(itemIDList, resultPath); } - +static int CALLBACK BrowseCallbackProc2(HWND hwnd, UINT uMsg, LPARAM /* lp */, LPARAM data) { switch (uMsg) diff --git a/3rdparty/lzma/CPP/Windows/Shell.h b/3rdparty/lzma/CPP/Windows/Shell.h index 4bff18cf99a..de91d3f1142 100644 --- a/3rdparty/lzma/CPP/Windows/Shell.h +++ b/3rdparty/lzma/CPP/Windows/Shell.h @@ -3,8 +3,8 @@ #ifndef __WINDOWS_SHELL_H #define __WINDOWS_SHELL_H -#include -#include +#include "../Common/MyWindows.h" +#include #include "../Common/MyString.h" diff --git a/3rdparty/lzma/CPP/Windows/Synchronization.cpp b/3rdparty/lzma/CPP/Windows/Synchronization.cpp index 5f86d1eb170..fbf919dcbda 100644 --- a/3rdparty/lzma/CPP/Windows/Synchronization.cpp +++ b/3rdparty/lzma/CPP/Windows/Synchronization.cpp @@ -2,9 +2,62 @@ #include "StdAfx.h" +#ifndef _WIN32 + #include "Synchronization.h" namespace NWindows { namespace NSynchronization { +/* +#define INFINITE 0xFFFFFFFF +#define MAXIMUM_WAIT_OBJECTS 64 +#define STATUS_ABANDONED_WAIT_0 ((NTSTATUS)0x00000080L) +#define WAIT_ABANDONED ((STATUS_ABANDONED_WAIT_0 ) + 0 ) +#define WAIT_ABANDONED_0 ((STATUS_ABANDONED_WAIT_0 ) + 0 ) +// WINAPI +DWORD WaitForMultipleObjects(DWORD count, const HANDLE *handles, BOOL wait_all, DWORD timeout); +*/ + +DWORD WINAPI WaitForMultiObj_Any_Infinite(DWORD count, const CHandle_WFMO *handles) +{ + if (count < 1) + { + // abort(); + SetLastError(EINVAL); + return WAIT_FAILED; + } + + CSynchro *synchro = handles[0]->_sync; + synchro->Enter(); + + // #ifdef DEBUG_SYNCHRO + for (DWORD i = 1; i < count; i++) + { + if (synchro != handles[i]->_sync) + { + // abort(); + synchro->Leave(); + SetLastError(EINVAL); + return WAIT_FAILED; + } + } + // #endif + + for (;;) + { + for (DWORD i = 0; i < count; i++) + { + if (handles[i]->IsSignaledAndUpdate()) + { + synchro->Leave(); + return WAIT_OBJECT_0 + i; + } + } + synchro->WaitCond(); + } +} + }} + +#endif diff --git a/3rdparty/lzma/CPP/Windows/Synchronization.h b/3rdparty/lzma/CPP/Windows/Synchronization.h index dc695f6f388..7d2e8d2aef2 100644 --- a/3rdparty/lzma/CPP/Windows/Synchronization.h +++ b/3rdparty/lzma/CPP/Windows/Synchronization.h @@ -5,6 +5,8 @@ #include "../../C/Threads.h" +#include "../Common/MyTypes.h" + #include "Defs.h" #ifdef _WIN32 @@ -14,17 +16,19 @@ namespace NWindows { namespace NSynchronization { -class CBaseEvent +class CBaseEvent MY_UNCOPYABLE { protected: ::CEvent _object; public: bool IsCreated() { return Event_IsCreated(&_object) != 0; } - operator HANDLE() { return _object; } + CBaseEvent() { Event_Construct(&_object); } ~CBaseEvent() { Close(); } WRes Close() { return Event_Close(&_object); } + #ifdef _WIN32 + operator HANDLE() { return _object; } WRes Create(bool manualReset, bool initiallyOwn, LPCTSTR name = NULL, LPSECURITY_ATTRIBUTES sa = NULL) { _object = ::CreateEvent(sa, BoolToBOOL(manualReset), BoolToBOOL(initiallyOwn), name); @@ -54,10 +58,10 @@ public: { return ManualResetEvent_Create(&_object, initiallyOwn ? 1: 0); } - WRes CreateIfNotCreated() + WRes CreateIfNotCreated_Reset() { if (IsCreated()) - return 0; + return Reset(); return ManualResetEvent_CreateNotSignaled(&_object); } #ifdef _WIN32 @@ -75,21 +79,25 @@ public: { return AutoResetEvent_CreateNotSignaled(&_object); } - WRes CreateIfNotCreated() + WRes CreateIfNotCreated_Reset() { if (IsCreated()) - return 0; + return Reset(); return AutoResetEvent_CreateNotSignaled(&_object); } }; + +/* #ifdef _WIN32 + class CObject: public CHandle { public: WRes Lock(DWORD timeoutInterval = INFINITE) { return (::WaitForSingleObject(_handle, timeoutInterval) == WAIT_OBJECT_0 ? 0 : ::GetLastError()); } }; + class CMutex: public CObject { public: @@ -114,33 +122,47 @@ public: return ::ReleaseMutex(_handle) ? 0 : ::GetLastError(); } }; -class CMutexLock + +class CMutexLock MY_UNCOPYABLE { CMutex *_object; public: CMutexLock(CMutex &object): _object(&object) { _object->Lock(); } ~CMutexLock() { _object->Release(); } }; -#endif -class CSemaphore +#endif // _WIN32 +*/ + + +class CSemaphore MY_UNCOPYABLE { ::CSemaphore _object; public: CSemaphore() { Semaphore_Construct(&_object); } ~CSemaphore() { Close(); } - WRes Close() { return Semaphore_Close(&_object); } + WRes Close() { return Semaphore_Close(&_object); } + + #ifdef _WIN32 operator HANDLE() { return _object; } - WRes Create(UInt32 initiallyCount, UInt32 maxCount) + #endif + + // bool IsCreated() const { return Semaphore_IsCreated(&_object) != 0; } + + WRes Create(UInt32 initCount, UInt32 maxCount) + { + return Semaphore_Create(&_object, initCount, maxCount); + } + WRes OptCreateInit(UInt32 initCount, UInt32 maxCount) { - return Semaphore_Create(&_object, initiallyCount, maxCount); + return Semaphore_OptCreateInit(&_object, initCount, maxCount); } WRes Release() { return Semaphore_Release1(&_object); } WRes Release(UInt32 releaseCount) { return Semaphore_ReleaseN(&_object, releaseCount); } WRes Lock() { return Semaphore_Wait(&_object); } }; -class CCriticalSection +class CCriticalSection MY_UNCOPYABLE { ::CCriticalSection _object; public: @@ -150,7 +172,7 @@ public: void Leave() { CriticalSection_Leave(&_object); } }; -class CCriticalSectionLock +class CCriticalSectionLock MY_UNCOPYABLE { CCriticalSection *_object; void Unlock() { _object->Leave(); } @@ -159,6 +181,213 @@ public: ~CCriticalSectionLock() { Unlock(); } }; + +#ifdef _WIN32 + +typedef HANDLE CHandle_WFMO; +typedef CSemaphore CSemaphore_WFMO; +typedef CAutoResetEvent CAutoResetEvent_WFMO; +typedef CManualResetEvent CManualResetEvent_WFMO; + +inline DWORD WINAPI WaitForMultiObj_Any_Infinite(DWORD count, const CHandle_WFMO *handles) +{ + return ::WaitForMultipleObjects(count, handles, FALSE, INFINITE); +} + +#define SYNC_OBJ_DECL(obj) +#define SYNC_WFMO(x) +#define SYNC_PARAM(x) +#define SYNC_PARAM_DECL(x) + +#else // _WIN32 + +// POSIX sync objects for WaitForMultipleObjects + +#define SYNC_WFMO(x) x +#define SYNC_PARAM(x) x, +#define SYNC_PARAM_DECL(x) NWindows::NSynchronization::CSynchro *x +#define SYNC_OBJ_DECL(x) NWindows::NSynchronization::CSynchro x; + +class CSynchro MY_UNCOPYABLE +{ + pthread_mutex_t _mutex; + pthread_cond_t _cond; + bool _isValid; + +public: + CSynchro() { _isValid = false; } + ~CSynchro() + { + if (_isValid) + { + ::pthread_mutex_destroy(&_mutex); + ::pthread_cond_destroy(&_cond); + } + _isValid = false; + } + WRes Create() + { + RINOK(::pthread_mutex_init(&_mutex, 0)); + WRes ret = ::pthread_cond_init(&_cond, 0); + _isValid = 1; + return ret; + } + WRes Enter() + { + return ::pthread_mutex_lock(&_mutex); + } + WRes Leave() + { + return ::pthread_mutex_unlock(&_mutex); + } + WRes WaitCond() + { + return ::pthread_cond_wait(&_cond, &_mutex); + } + WRes LeaveAndSignal() + { + WRes res1 = ::pthread_cond_broadcast(&_cond); + WRes res2 = ::pthread_mutex_unlock(&_mutex); + return (res2 ? res2 : res1); + } +}; + + +struct CBaseHandle_WFMO; +typedef NWindows::NSynchronization::CBaseHandle_WFMO *CHandle_WFMO; + +// these constants are from Windows +#define WAIT_OBJECT_0 0 +#define WAIT_FAILED ((DWORD)0xFFFFFFFF) + +DWORD WINAPI WaitForMultiObj_Any_Infinite(DWORD count, const CHandle_WFMO *handles); + + +struct CBaseHandle_WFMO MY_UNCOPYABLE +{ + CSynchro *_sync; + + CBaseHandle_WFMO(): _sync(NULL) {} + + operator CHandle_WFMO() { return this; } + virtual bool IsSignaledAndUpdate() = 0; +}; + + +class CBaseEvent_WFMO : public CBaseHandle_WFMO +{ + bool _manual_reset; + bool _state; + +public: + + // bool IsCreated() { return (this->_sync != NULL); } + // CBaseEvent_WFMO() { ; } + ~CBaseEvent_WFMO() { Close(); } + + WRes Close() { this->_sync = NULL; return 0; } + + WRes Create( + CSynchro *sync, + bool manualReset, bool initiallyOwn) + { + this->_sync = sync; + this->_manual_reset = manualReset; + this->_state = initiallyOwn; + return 0; + } + + WRes Set() + { + RINOK(this->_sync->Enter()); + this->_state = true; + return this->_sync->LeaveAndSignal(); + } + + WRes Reset() + { + RINOK(this->_sync->Enter()); + this->_state = false; + return this->_sync->Leave(); + } + + virtual bool IsSignaledAndUpdate() + { + if (this->_state == false) + return false; + if (this->_manual_reset == false) + this->_state = false; + return true; + } +}; + + +class CManualResetEvent_WFMO: public CBaseEvent_WFMO +{ +public: + WRes Create(CSynchro *sync, bool initiallyOwn = false) { return CBaseEvent_WFMO::Create(sync, true, initiallyOwn); } +}; + + +class CAutoResetEvent_WFMO: public CBaseEvent_WFMO +{ +public: + WRes Create(CSynchro *sync) { return CBaseEvent_WFMO::Create(sync, false, false); } + WRes CreateIfNotCreated_Reset(CSynchro *sync) + { + return Create(sync); + } +}; + + +class CSemaphore_WFMO : public CBaseHandle_WFMO +{ + UInt32 _count; + UInt32 _maxCount; + +public: + CSemaphore_WFMO() : _count(0), _maxCount(0) {} + + WRes Close() { this->_sync = NULL; return 0; } + + WRes Create(CSynchro *sync, UInt32 initCount, UInt32 maxCount) + { + if (initCount > maxCount || maxCount < 1) + return EINVAL; + this->_sync = sync; + this->_count = initCount; + this->_maxCount = maxCount; + return 0; + } + + WRes Release(UInt32 releaseCount = 1) + { + if (releaseCount < 1) + return EINVAL; + + RINOK(this->_sync->Enter()); + UInt32 newCount = this->_count + releaseCount; + if (newCount > this->_maxCount) + { + RINOK(this->_sync->Leave()); + return ERROR_TOO_MANY_POSTS; // EINVAL + } + this->_count = newCount; + + return this->_sync->LeaveAndSignal(); + } + + virtual bool IsSignaledAndUpdate() + { + if (this->_count == 0) + return false; + this->_count--; + return true; + } +}; + +#endif // _WIN32 + }} #endif diff --git a/3rdparty/lzma/CPP/Windows/System.cpp b/3rdparty/lzma/CPP/Windows/System.cpp index 59cb7c0b4e8..3a14b77ddd2 100644 --- a/3rdparty/lzma/CPP/Windows/System.cpp +++ b/3rdparty/lzma/CPP/Windows/System.cpp @@ -2,9 +2,19 @@ #include "StdAfx.h" -#include "../Common/MyWindows.h" +#ifndef _WIN32 +#include +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif #include "../Common/Defs.h" +// #include "../Common/MyWindows.h" + +// #include "../../C/CpuArch.h" #include "System.h" @@ -13,18 +23,84 @@ namespace NSystem { #ifdef _WIN32 +UInt32 CountAffinity(DWORD_PTR mask) +{ + UInt32 num = 0; + for (unsigned i = 0; i < sizeof(mask) * 8; i++) + num += (UInt32)((mask >> i) & 1); + return num; +} + +BOOL CProcessAffinity::Get() +{ + #ifndef UNDER_CE + return GetProcessAffinityMask(GetCurrentProcess(), &processAffinityMask, &systemAffinityMask); + #else + return FALSE; + #endif +} + + UInt32 GetNumberOfProcessors() { + // We need to know how many threads we can use. + // By default the process is assigned to one group. + // So we get the number of logical processors (threads) + // assigned to current process in the current group. + // Group size can be smaller than total number logical processors, for exammple, 2x36 + + CProcessAffinity pa; + + if (pa.Get() && pa.processAffinityMask != 0) + return pa.GetNumProcessThreads(); + SYSTEM_INFO systemInfo; GetSystemInfo(&systemInfo); + // the number of logical processors in the current group return (UInt32)systemInfo.dwNumberOfProcessors; } #else + +BOOL CProcessAffinity::Get() +{ + numSysThreads = GetNumberOfProcessors(); + + /* + numSysThreads = 8; + for (unsigned i = 0; i < numSysThreads; i++) + CpuSet_Set(&cpu_set, i); + return TRUE; + */ + + #ifdef _7ZIP_AFFINITY_SUPPORTED + + // numSysThreads = sysconf(_SC_NPROCESSORS_ONLN); // The number of processors currently online + if (sched_getaffinity(0, sizeof(cpu_set), &cpu_set) != 0) + return FALSE; + return TRUE; + + #else + + // cpu_set = ((CCpuSet)1 << (numSysThreads)) - 1; + return TRUE; + // errno = ENOSYS; + // return FALSE; + + #endif +} + UInt32 GetNumberOfProcessors() { + #ifndef _7ZIP_ST + long n = sysconf(_SC_NPROCESSORS_CONF); // The number of processors configured + if (n < 1) + n = 1; + return (UInt32)n; + #else return 1; + #endif } #endif @@ -57,17 +133,13 @@ typedef struct _MY_MEMORYSTATUSEX { typedef BOOL (WINAPI *GlobalMemoryStatusExP)(MY_LPMEMORYSTATUSEX lpBuffer); -#endif - -#endif - +#endif // !UNDER_CE + bool GetRamSize(UInt64 &size) { size = (UInt64)(sizeof(size_t)) << 29; - #ifdef _WIN32 - #ifndef UNDER_CE MY_MEMORYSTATUSEX stat; stat.dwLength = sizeof(stat); @@ -84,7 +156,7 @@ bool GetRamSize(UInt64 &size) #ifndef UNDER_CE GlobalMemoryStatusExP globalMemoryStatusEx = (GlobalMemoryStatusExP) - ::GetProcAddress(::GetModuleHandle(TEXT("kernel32.dll")), "GlobalMemoryStatusEx"); + (void *)::GetProcAddress(::GetModuleHandleA("kernel32.dll"), "GlobalMemoryStatusEx"); if (globalMemoryStatusEx && globalMemoryStatusEx(&stat)) { size = MyMin(stat.ullTotalVirtual, stat.ullTotalPhys); @@ -99,14 +171,64 @@ bool GetRamSize(UInt64 &size) size = MyMin(stat2.dwTotalVirtual, stat2.dwTotalPhys); return true; } - #endif +} + +#else + +// POSIX +// #include + +bool GetRamSize(UInt64 &size) +{ + size = (UInt64)(sizeof(size_t)) << 29; + #ifdef __APPLE__ + + #ifdef HW_MEMSIZE + uint64_t val = 0; // support 2Gb+ RAM + int mib[2] = { CTL_HW, HW_MEMSIZE }; + #elif defined(HW_PHYSMEM64) + uint64_t val = 0; // support 2Gb+ RAM + int mib[2] = { CTL_HW, HW_PHYSMEM64 }; + #else + unsigned int val = 0; // For old system + int mib[2] = { CTL_HW, HW_PHYSMEM }; + #endif // HW_MEMSIZE + size_t size_sys = sizeof(val); + + sysctl(mib, 2, &val, &size_sys, NULL, 0); + if (val) + size = val; + + #elif defined(_AIX) + // fixme + #elif defined(__gnu_hurd__) + // fixme + #elif defined(__FreeBSD_kernel__) && defined(__GLIBC__) + // GNU/kFreeBSD Debian + // fixme #else - return false; + struct sysinfo info; + if (::sysinfo(&info) != 0) + return false; + size = (UInt64)info.mem_unit * info.totalram; + const UInt64 kLimit = (UInt64)1 << (sizeof(size_t) * 8 - 1); + if (size > kLimit) + size = kLimit; + + /* + printf("\n mem_unit = %lld", (UInt64)info.mem_unit); + printf("\n totalram = %lld", (UInt64)info.totalram); + printf("\n freeram = %lld", (UInt64)info.freeram); + */ #endif + + return true; } +#endif + }} diff --git a/3rdparty/lzma/CPP/Windows/System.h b/3rdparty/lzma/CPP/Windows/System.h index 910bc7fd5f5..23cb0dabcb0 100644 --- a/3rdparty/lzma/CPP/Windows/System.h +++ b/3rdparty/lzma/CPP/Windows/System.h @@ -3,11 +3,123 @@ #ifndef __WINDOWS_SYSTEM_H #define __WINDOWS_SYSTEM_H +#ifndef _WIN32 +// #include +#include "../../C/Threads.h" +#endif + #include "../Common/MyTypes.h" namespace NWindows { namespace NSystem { + +#ifdef _WIN32 + +UInt32 CountAffinity(DWORD_PTR mask); + +struct CProcessAffinity +{ + // UInt32 numProcessThreads; + // UInt32 numSysThreads; + DWORD_PTR processAffinityMask; + DWORD_PTR systemAffinityMask; + + void InitST() + { + // numProcessThreads = 1; + // numSysThreads = 1; + processAffinityMask = 1; + systemAffinityMask = 1; + } + + void CpuZero() + { + processAffinityMask = 0; + } + + void CpuSet(unsigned cpuIndex) + { + processAffinityMask |= ((DWORD_PTR)1 << cpuIndex); + } + + UInt32 GetNumProcessThreads() const { return CountAffinity(processAffinityMask); } + UInt32 GetNumSystemThreads() const { return CountAffinity(systemAffinityMask); } + + BOOL Get(); + + BOOL SetProcAffinity() const + { + return SetProcessAffinityMask(GetCurrentProcess(), processAffinityMask); + } +}; + + +#else // WIN32 + +struct CProcessAffinity +{ + UInt32 numSysThreads; + + UInt32 GetNumSystemThreads() const { return (UInt32)numSysThreads; } + BOOL Get(); + + #ifdef _7ZIP_AFFINITY_SUPPORTED + + CCpuSet cpu_set; + + void InitST() + { + numSysThreads = 1; + CpuSet_Zero(&cpu_set); + CpuSet_Set(&cpu_set, 0); + } + + UInt32 GetNumProcessThreads() const { return (UInt32)CPU_COUNT(&cpu_set); } + void CpuZero() { CpuSet_Zero(&cpu_set); } + void CpuSet(unsigned cpuIndex) { CpuSet_Set(&cpu_set, cpuIndex); } + int IsCpuSet(unsigned cpuIndex) const { return CpuSet_IsSet(&cpu_set, cpuIndex); } + // void CpuClr(int cpuIndex) { CPU_CLR(cpuIndex, &cpu_set); } + + BOOL SetProcAffinity() const + { + return sched_setaffinity(0, sizeof(cpu_set), &cpu_set) == 0; + } + + #else + + void InitST() + { + numSysThreads = 1; + } + + UInt32 GetNumProcessThreads() const + { + return numSysThreads; + /* + UInt32 num = 0; + for (unsigned i = 0; i < sizeof(cpu_set) * 8; i++) + num += (UInt32)((cpu_set >> i) & 1); + return num; + */ + } + + void CpuZero() { } + void CpuSet(unsigned cpuIndex) { UNUSED_VAR(cpuIndex); } + int IsCpuSet(unsigned cpuIndex) const { return (cpuIndex < numSysThreads) ? 1 : 0; } + + BOOL SetProcAffinity() const + { + errno = ENOSYS; + return FALSE; + } + + #endif +}; + +#endif + + UInt32 GetNumberOfProcessors(); bool GetRamSize(UInt64 &size); // returns false, if unknown ram size diff --git a/3rdparty/lzma/CPP/Windows/SystemInfo.cpp b/3rdparty/lzma/CPP/Windows/SystemInfo.cpp new file mode 100644 index 00000000000..d882a8ee404 --- /dev/null +++ b/3rdparty/lzma/CPP/Windows/SystemInfo.cpp @@ -0,0 +1,917 @@ +// Windows/SystemInfo.cpp + +#include "StdAfx.h" + +#include "../../C/CpuArch.h" + +#include "../Common/IntToString.h" + +#ifdef _WIN32 + +#include "Registry.h" + +#else + +#include +#include +#ifdef __APPLE__ +#include +#elif !defined(_AIX) + +#include + +// #undef AT_HWCAP // to debug +// #undef AT_HWCAP2 // to debug + +/* the following patch for some debian systems. + Is it OK to define AT_HWCAP and AT_HWCAP2 here with these constant numbers? */ +/* +#if defined(__FreeBSD_kernel__) && defined(__GLIBC__) + #ifndef AT_HWCAP + #define AT_HWCAP 16 + #endif + #ifndef AT_HWCAP2 + #define AT_HWCAP2 26 + #endif +#endif +*/ + +#ifdef MY_CPU_ARM_OR_ARM64 +#include +#endif +#endif + +#ifdef __linux__ +#include "../Windows/FileIO.h" +#endif + +#endif // WIN32 + +#include "SystemInfo.h" +#include "System.h" + +using namespace NWindows; + +#ifdef __linux__ + +static bool ReadFile_to_Buffer(CFSTR fileName, CByteBuffer &buf) +{ + NWindows::NFile::NIO::CInFile file; + if (!file.Open(fileName)) + return false; + /* + UInt64 size; + if (!file.GetLength(size)) + { + // GetLength() doesn't work "/proc/cpuinfo" + return false; + } + if (size >= ((UInt32)1 << 29)) + return false; + */ + size_t size = 0; + size_t addSize = ((size_t)1 << 12); + for (;;) + { + // printf("\nsize = %d\n", (unsigned)size); + buf.ChangeSize_KeepData(size + addSize, size); + size_t processed; + if (!file.ReadFull(buf + size, addSize, processed)) + return false; + if (processed == 0) + { + buf.ChangeSize_KeepData(size, size); + return true; + } + size += processed; + addSize *= 2; + } +} + +#endif + + +#if defined(_WIN32) || defined(AT_HWCAP) || defined(AT_HWCAP2) +static void PrintHex(AString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} +#endif + +#ifdef MY_CPU_X86_OR_AMD64 + +static void PrintCpuChars(AString &s, UInt32 v) +{ + for (int j = 0; j < 4; j++) + { + Byte b = (Byte)(v & 0xFF); + v >>= 8; + if (b == 0) + break; + s += (char)b; + } +} + + +static void x86cpuid_to_String(const Cx86cpuid &c, AString &s, AString &ver) +{ + s.Empty(); + + UInt32 maxFunc2 = 0; + UInt32 t[3]; + + MyCPUID(0x80000000, &maxFunc2, &t[0], &t[1], &t[2]); + + bool fullNameIsAvail = (maxFunc2 >= 0x80000004); + + if (fullNameIsAvail) + { + for (unsigned i = 0; i < 3; i++) + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000002 + i, &d[0], &d[1], &d[2], &d[3]); + for (unsigned j = 0; j < 4; j++) + PrintCpuChars(s, d[j]); + } + } + + s.Trim(); + + if (s.IsEmpty()) + { + for (int i = 0; i < 3; i++) + PrintCpuChars(s, c.vendor[i]); + s.Trim(); + } + + { + char temp[32]; + ConvertUInt32ToHex(c.ver, temp); + ver += temp; + } +} + +/* +static void x86cpuid_all_to_String(AString &s) +{ + Cx86cpuid p; + if (!x86cpuid_CheckAndRead(&p)) + return; + s += "x86cpuid maxFunc = "; + s.Add_UInt32(p.maxFunc); + for (unsigned j = 0; j <= p.maxFunc; j++) + { + s.Add_LF(); + // s.Add_UInt32(j); // align + { + char temp[32]; + ConvertUInt32ToString(j, temp); + unsigned len = (unsigned)strlen(temp); + while (len < 8) + { + len++; + s.Add_Space(); + } + s += temp; + } + + s += ":"; + UInt32 d[4] = { 0 }; + MyCPUID(j, &d[0], &d[1], &d[2], &d[3]); + for (unsigned i = 0; i < 4; i++) + { + char temp[32]; + ConvertUInt32ToHex8Digits(d[i], temp); + s += " "; + s += temp; + } + } +} +*/ + +#endif + + + +#ifdef _WIN32 + +static const char * const k_PROCESSOR_ARCHITECTURE[] = +{ + "x86" // "INTEL" + , "MIPS" + , "ALPHA" + , "PPC" + , "SHX" + , "ARM" + , "IA64" + , "ALPHA64" + , "MSIL" + , "x64" // "AMD64" + , "IA32_ON_WIN64" + , "NEUTRAL" + , "ARM64" + , "ARM32_ON_WIN64" +}; + +#define MY__PROCESSOR_ARCHITECTURE_INTEL 0 +#define MY__PROCESSOR_ARCHITECTURE_AMD64 9 + + +#define MY__PROCESSOR_INTEL_PENTIUM 586 +#define MY__PROCESSOR_AMD_X8664 8664 + +/* +static const CUInt32PCharPair k_PROCESSOR[] = +{ + { 2200, "IA64" }, + { 8664, "x64" } +}; + +#define PROCESSOR_INTEL_386 386 +#define PROCESSOR_INTEL_486 486 +#define PROCESSOR_INTEL_PENTIUM 586 +#define PROCESSOR_INTEL_860 860 +#define PROCESSOR_INTEL_IA64 2200 +#define PROCESSOR_AMD_X8664 8664 +#define PROCESSOR_MIPS_R2000 2000 +#define PROCESSOR_MIPS_R3000 3000 +#define PROCESSOR_MIPS_R4000 4000 +#define PROCESSOR_ALPHA_21064 21064 +#define PROCESSOR_PPC_601 601 +#define PROCESSOR_PPC_603 603 +#define PROCESSOR_PPC_604 604 +#define PROCESSOR_PPC_620 620 +#define PROCESSOR_HITACHI_SH3 10003 +#define PROCESSOR_HITACHI_SH3E 10004 +#define PROCESSOR_HITACHI_SH4 10005 +#define PROCESSOR_MOTOROLA_821 821 +#define PROCESSOR_SHx_SH3 103 +#define PROCESSOR_SHx_SH4 104 +#define PROCESSOR_STRONGARM 2577 // 0xA11 +#define PROCESSOR_ARM720 1824 // 0x720 +#define PROCESSOR_ARM820 2080 // 0x820 +#define PROCESSOR_ARM920 2336 // 0x920 +#define PROCESSOR_ARM_7TDMI 70001 +#define PROCESSOR_OPTIL 18767 // 0x494f +*/ + + +/* +static const char * const k_PF[] = +{ + "FP_ERRATA" + , "FP_EMU" + , "CMPXCHG" + , "MMX" + , "PPC_MOVEMEM_64BIT" + , "ALPHA_BYTE" + , "SSE" + , "3DNOW" + , "RDTSC" + , "PAE" + , "SSE2" + , "SSE_DAZ" + , "NX" + , "SSE3" + , "CMPXCHG16B" + , "CMP8XCHG16" + , "CHANNELS" + , "XSAVE" + , "ARM_VFP_32" + , "ARM_NEON" + , "L2AT" + , "VIRT_FIRMWARE" + , "RDWRFSGSBASE" + , "FASTFAIL" + , "ARM_DIVIDE" + , "ARM_64BIT_LOADSTORE_ATOMIC" + , "ARM_EXTERNAL_CACHE" + , "ARM_FMAC" + , "RDRAND" + , "ARM_V8" + , "ARM_V8_CRYPTO" + , "ARM_V8_CRC32" + , "RDTSCP" + , "RDPID" + , "ARM_V81_ATOMIC" + , "MONITORX" +}; +*/ + +#endif + + +#ifdef _WIN32 + +static void PrintPage(AString &s, UInt32 v) +{ + if ((v & 0x3FF) == 0) + { + s.Add_UInt32(v >> 10); + s += "K"; + } + else + s.Add_UInt32(v >> 10); +} + +static AString TypeToString2(const char * const table[], unsigned num, UInt32 value) +{ + char sz[16]; + const char *p = NULL; + if (value < num) + p = table[value]; + if (!p) + { + ConvertUInt32ToString(value, sz); + p = sz; + } + return (AString)p; +} + +// #if defined(_7ZIP_LARGE_PAGES) || defined(_WIN32) +// #ifdef _WIN32 +void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v) +{ + char c = 0; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; + }}}} + else + { + PrintHex(s, v); + return; + } + char temp[32]; + ConvertUInt64ToString(v, temp); + s += temp; + if (c) + s += c; +} +// #endif +// #endif + +static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) +{ + s += TypeToString2(k_PROCESSOR_ARCHITECTURE, ARRAY_SIZE(k_PROCESSOR_ARCHITECTURE), si.wProcessorArchitecture); + + if (!( (si.wProcessorArchitecture == MY__PROCESSOR_ARCHITECTURE_INTEL && si.dwProcessorType == MY__PROCESSOR_INTEL_PENTIUM) + || (si.wProcessorArchitecture == MY__PROCESSOR_ARCHITECTURE_AMD64 && si.dwProcessorType == MY__PROCESSOR_AMD_X8664))) + { + s += " "; + // s += TypePairToString(k_PROCESSOR, ARRAY_SIZE(k_PROCESSOR), si.dwProcessorType); + s.Add_UInt32(si.dwProcessorType); + } + s += " "; + PrintHex(s, si.wProcessorLevel); + s += "."; + PrintHex(s, si.wProcessorRevision); + if ((UInt64)si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != 0 || si.dwNumberOfProcessors != sizeof(UInt64) * 8) + { + s += " act:"; + PrintHex(s, si.dwActiveProcessorMask); + } + s += " cpus:"; + s.Add_UInt32(si.dwNumberOfProcessors); + if (si.dwPageSize != 1 << 12) + { + s += " page:"; + PrintPage(s, si.dwPageSize); + } + if (si.dwAllocationGranularity != 1 << 16) + { + s += " gran:"; + PrintPage(s, si.dwAllocationGranularity); + } + s += " "; + + DWORD_PTR minAdd = (DWORD_PTR)si.lpMinimumApplicationAddress; + UInt64 maxSize = (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1; + const UInt32 kReserveSize = ((UInt32)1 << 16); + if (minAdd != kReserveSize) + { + PrintSize_KMGT_Or_Hex(s, minAdd); + s += "-"; + } + else + { + if ((maxSize & (kReserveSize - 1)) == 0) + maxSize += kReserveSize; + } + PrintSize_KMGT_Or_Hex(s, maxSize); +} + +#ifndef _WIN64 +EXTERN_C_BEGIN +typedef VOID (WINAPI *Func_GetNativeSystemInfo)(LPSYSTEM_INFO lpSystemInfo); +EXTERN_C_END +#endif + +#endif + +#ifdef __APPLE__ +#ifndef MY_CPU_X86_OR_AMD64 +static void Add_sysctlbyname_to_String(const char *name, AString &s) +{ + size_t bufSize = 256; + char buf[256]; + if (My_sysctlbyname_Get(name, &buf, &bufSize) == 0) + s += buf; +} +#endif +#endif + +void GetSysInfo(AString &s1, AString &s2); +void GetSysInfo(AString &s1, AString &s2) +{ + s1.Empty(); + s2.Empty(); + + #ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + { + SysInfo_To_String(s1, si); + // s += " : "; + } + + #if !defined(_WIN64) && !defined(UNDER_CE) + Func_GetNativeSystemInfo fn_GetNativeSystemInfo = (Func_GetNativeSystemInfo)(void *)GetProcAddress( + GetModuleHandleA("kernel32.dll"), "GetNativeSystemInfo"); + if (fn_GetNativeSystemInfo) + { + SYSTEM_INFO si2; + fn_GetNativeSystemInfo(&si2); + // if (memcmp(&si, &si2, sizeof(si)) != 0) + { + // s += " - "; + SysInfo_To_String(s2, si2); + } + } + #endif + #endif +} + + +void GetCpuName(AString &s); + +static void AddBracedString(AString &dest, AString &src) +{ + if (!src.IsEmpty()) + { + AString s; + s += '('; + s += src; + s += ')'; + dest.Add_OptSpaced(s); + } +} + +struct CCpuName +{ + AString CpuName; + AString Revision; + AString Microcode; + AString LargePages; + + void Fill(); + + void Get_Revision_Microcode_LargePages(AString &s) + { + s.Empty(); + AddBracedString(s, Revision); + AddBracedString(s, Microcode); + s.Add_OptSpaced(LargePages); + } +}; + +void CCpuName::Fill() +{ + CpuName.Empty(); + Revision.Empty(); + Microcode.Empty(); + LargePages.Empty(); + + AString &s = CpuName; + + #ifdef MY_CPU_X86_OR_AMD64 + { + Cx86cpuid cpuid; + if (x86cpuid_CheckAndRead(&cpuid)) + { + x86cpuid_to_String(cpuid, s, Revision); + } + else + { + #ifdef MY_CPU_AMD64 + s += "x64"; + #else + s += "x86"; + #endif + } + } + #elif defined(__APPLE__) + { + Add_sysctlbyname_to_String("machdep.cpu.brand_string", s); + } + #endif + + + if (s.IsEmpty()) + { + #ifdef MY_CPU_LE + s += "LE"; + #elif defined(MY_CPU_BE) + s += "BE"; + #endif + } + + #ifdef __APPLE__ + { + AString s2; + UInt32 v = 0; + if (My_sysctlbyname_Get_UInt32("machdep.cpu.core_count", &v) == 0) + { + s2.Add_UInt32(v); + s2 += 'C'; + } + if (My_sysctlbyname_Get_UInt32("machdep.cpu.thread_count", &v) == 0) + { + s2.Add_UInt32(v); + s2 += 'T'; + } + if (!s2.IsEmpty()) + { + s.Add_Space_if_NotEmpty(); + s += s2; + } + } + #endif + + + #ifdef _WIN32 + { + NRegistry::CKey key; + if (key.Open(HKEY_LOCAL_MACHINE, TEXT("HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"), KEY_READ) == ERROR_SUCCESS) + { + LONG res[2]; + CByteBuffer bufs[2]; + { + for (int i = 0; i < 2; i++) + { + UInt32 size = 0; + res[i] = key.QueryValue(i == 0 ? + TEXT("Previous Update Revision") : + TEXT("Update Revision"), bufs[i], size); + if (res[i] == ERROR_SUCCESS) + if (size != bufs[i].Size()) + res[i] = ERROR_SUCCESS + 1; + } + } + if (res[0] == ERROR_SUCCESS || res[1] == ERROR_SUCCESS) + { + for (int i = 0; i < 2; i++) + { + if (i == 1) + Microcode += "->"; + if (res[i] != ERROR_SUCCESS) + continue; + const CByteBuffer &buf = bufs[i]; + if (buf.Size() == 8) + { + UInt32 high = GetUi32(buf); + if (high != 0) + { + PrintHex(Microcode, high); + Microcode += "."; + } + PrintHex(Microcode, GetUi32(buf + 4)); + } + } + } + } + } + #endif + + + #ifdef _7ZIP_LARGE_PAGES + Add_LargePages_String(LargePages); + #endif +} + +void AddCpuFeatures(AString &s); +void AddCpuFeatures(AString &s) +{ + #ifdef _WIN32 + // const unsigned kNumFeatures_Extra = 32; // we check also for unknown features + // const unsigned kNumFeatures = ARRAY_SIZE(k_PF) + kNumFeatures_Extra; + const unsigned kNumFeatures = 64; + UInt64 flags = 0; + for (unsigned i = 0; i < kNumFeatures; i++) + { + if (IsProcessorFeaturePresent(i)) + { + flags += (UInt64)1 << i; + // s.Add_Space_if_NotEmpty(); + // s += TypeToString2(k_PF, ARRAY_SIZE(k_PF), i); + } + } + s.Add_OptSpaced("f:"); + PrintHex(s, flags); + + #elif defined(__APPLE__) + { + UInt32 v = 0; + if (My_sysctlbyname_Get_UInt32("hw.pagesize", &v) == 0) + { + s += "PageSize:"; + s.Add_UInt32(v >> 10); + s += "KB"; + } + } + + #else + + const long v = sysconf(_SC_PAGESIZE); + if (v != -1) + { + s.Add_Space_if_NotEmpty(); + s += "PageSize:"; + s.Add_UInt32((UInt32)(v >> 10)); + s += "KB"; + } + + #if !defined(_AIX) + + #ifdef __linux__ + + CByteBuffer buf; + if (ReadFile_to_Buffer("/sys/kernel/mm/transparent_hugepage/enabled", buf)) + // if (ReadFile_to_Buffer("/proc/cpuinfo", buf)) + { + s.Add_OptSpaced("THP:"); + AString s2; + s2.SetFrom_CalcLen((const char *)(const void *)(const Byte *)buf, (unsigned)buf.Size()); + const int pos = s2.Find('['); + if (pos >= 0) + { + const int pos2 = s2.Find(']', pos + 1); + if (pos2 >= 0) + { + s2.DeleteFrom(pos2); + s2.DeleteFrontal(pos + 1); + } + } + s += s2; + } + // else throw CSystemException(MY_SRes_HRESULT_FROM_WRes(errno)); + + #endif + + + #ifdef AT_HWCAP + s.Add_OptSpaced("hwcap:"); + { + unsigned long h = getauxval(AT_HWCAP); + PrintHex(s, h); + #ifdef MY_CPU_ARM64 + if (h & HWCAP_CRC32) s += ":CRC32"; + if (h & HWCAP_SHA1) s += ":SHA1"; + if (h & HWCAP_SHA2) s += ":SHA2"; + if (h & HWCAP_AES) s += ":AES"; + if (h & HWCAP_ASIMD) s += ":ASIMD"; + #elif defined(MY_CPU_ARM) + if (h & HWCAP_NEON) s += ":NEON"; + #endif + } + #endif // AT_HWCAP + + #ifdef AT_HWCAP2 + { + unsigned long h = getauxval(AT_HWCAP2); + #ifndef MY_CPU_ARM + if (h != 0) + #endif + { + s += " hwcap2:"; + PrintHex(s, h); + #ifdef MY_CPU_ARM + if (h & HWCAP2_CRC32) s += ":CRC32"; + if (h & HWCAP2_SHA1) s += ":SHA1"; + if (h & HWCAP2_SHA2) s += ":SHA2"; + if (h & HWCAP2_AES) s += ":AES"; + #endif + } + } + #endif // AT_HWCAP2 + #endif // _AIX + #endif // _WIN32 +} + + +#ifdef _WIN32 +#ifndef UNDER_CE + +EXTERN_C_BEGIN +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); +EXTERN_C_END + +static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) +{ + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return FALSE; + Func_RtlGetVersion func = (Func_RtlGetVersion)(void *)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return FALSE; + func(vi); + return TRUE; +} + +#endif +#endif + + +void GetOsInfoText(AString &sRes) +{ + sRes.Empty(); + AString s; + + #ifdef _WIN32 + #ifndef UNDER_CE + // OSVERSIONINFO vi; + OSVERSIONINFOEXW vi; + vi.dwOSVersionInfoSize = sizeof(vi); + // if (::GetVersionEx(&vi)) + if (My_RtlGetVersion(&vi)) + { + s += "Windows"; + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + s.Add_UInt32(vi.dwPlatformId); + s += " "; s.Add_UInt32(vi.dwMajorVersion); + s += "."; s.Add_UInt32(vi.dwMinorVersion); + s += " "; s.Add_UInt32(vi.dwBuildNumber); + + if (vi.wServicePackMajor != 0 || vi.wServicePackMinor != 0) + { + s += " SP:"; s.Add_UInt32(vi.wServicePackMajor); + s += "."; s.Add_UInt32(vi.wServicePackMinor); + } + // s += " Suite:"; PrintHex(s, vi.wSuiteMask); + // s += " Type:"; s.Add_UInt32(vi.wProductType); + // s += " "; s += GetOemString(vi.szCSDVersion); + } + /* + { + s += " OEMCP:"; s.Add_UInt32(GetOEMCP()); + s += " ACP:"; s.Add_UInt32(GetACP()); + } + */ + #endif + #else // _WIN32 + + if (!s.IsEmpty()) + s.Add_LF(); + struct utsname un; + if (uname(&un) == 0) + { + s += un.sysname; + // s += " : "; s += un.nodename; // we don't want to show name of computer + s += " : "; s += un.release; + s += " : "; s += un.version; + s += " : "; s += un.machine; + + #ifdef __APPLE__ + // Add_sysctlbyname_to_String("kern.version", s); + // it's same as "utsname.version" + #endif + } + #endif // _WIN32 + + sRes += s; +} + + + +void GetSystemInfoText(AString &sRes) +{ + GetOsInfoText(sRes); + sRes.Add_LF(); + + { + AString s, s1, s2; + GetSysInfo(s1, s2); + if (!s1.IsEmpty() || !s2.IsEmpty()) + { + s = s1; + if (s1 != s2 && !s2.IsEmpty()) + { + s += " - "; + s += s2; + } + } + { + AddCpuFeatures(s); + if (!s.IsEmpty()) + { + sRes += s; + sRes.Add_LF(); + } + } + } + { + AString s; + GetCpuName(s); + if (!s.IsEmpty()) + { + sRes += s; + sRes.Add_LF(); + } + } + /* + #ifdef MY_CPU_X86_OR_AMD64 + { + AString s; + x86cpuid_all_to_String(s); + if (!s.IsEmpty()) + { + printCallback->Print(s); + printCallback->NewLine(); + } + } + #endif + */ +} + + +void GetCpuName(AString &s); +void GetCpuName(AString &s) +{ + CCpuName cpuName; + cpuName.Fill(); + s = cpuName.CpuName; + AString s2; + cpuName.Get_Revision_Microcode_LargePages(s2); + s.Add_OptSpaced(s2); +} + + +void GetCpuName_MultiLine(AString &s); +void GetCpuName_MultiLine(AString &s) +{ + CCpuName cpuName; + cpuName.Fill(); + s = cpuName.CpuName; + AString s2; + cpuName.Get_Revision_Microcode_LargePages(s2); + if (!s2.IsEmpty()) + { + s.Add_LF(); + s += s2; + } +} + +void GetCompiler(AString &s) +{ + #ifdef __VERSION__ + s += __VERSION__; + #endif + + #ifdef __GNUC__ + s += " GCC "; + s.Add_UInt32(__GNUC__); + s += '.'; + s.Add_UInt32(__GNUC_MINOR__); + s += '.'; + s.Add_UInt32(__GNUC_PATCHLEVEL__); + #endif + + #ifdef __clang__ + s += " CLANG "; + s.Add_UInt32(__clang_major__); + s += '.'; + s.Add_UInt32(__clang_minor__); + #endif + + #ifdef __xlC__ + s += " XLC "; + s.Add_UInt32(__xlC__ >> 8); + s += '.'; + s.Add_UInt32(__xlC__ & 0xFF); + #ifdef __xlC_ver__ + s += '.'; + s.Add_UInt32(__xlC_ver__ >> 8); + s += '.'; + s.Add_UInt32(__xlC_ver__ & 0xFF); + #endif + #endif + + #ifdef _MSC_VER + s += " MSC "; + s.Add_UInt32(_MSC_VER); + #endif +} diff --git a/3rdparty/lzma/CPP/Windows/SystemInfo.h b/3rdparty/lzma/CPP/Windows/SystemInfo.h new file mode 100644 index 00000000000..e941d0aa418 --- /dev/null +++ b/3rdparty/lzma/CPP/Windows/SystemInfo.h @@ -0,0 +1,18 @@ +// Windows/SystemInfo.h + +#ifndef __WINDOWS_SYSTEM_INFO_H +#define __WINDOWS_SYSTEM_INFO_H + +#include "../Common/MyString.h" + + +void GetCpuName_MultiLine(AString &s); + +void GetOsInfoText(AString &sRes); +void GetSystemInfoText(AString &s); +void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v); +void Add_LargePages_String(AString &s); + +void GetCompiler(AString &s); + +#endif diff --git a/3rdparty/lzma/CPP/Windows/Thread.h b/3rdparty/lzma/CPP/Windows/Thread.h index 16a509d4768..5fca173f00d 100644 --- a/3rdparty/lzma/CPP/Windows/Thread.h +++ b/3rdparty/lzma/CPP/Windows/Thread.h @@ -9,7 +9,7 @@ namespace NWindows { -class CThread +class CThread MY_UNCOPYABLE { ::CThread thread; public: @@ -17,9 +17,15 @@ public: ~CThread() { Close(); } bool IsCreated() { return Thread_WasCreated(&thread) != 0; } WRes Close() { return Thread_Close(&thread); } - WRes Create(THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID parameter) - { return Thread_Create(&thread, startAddress, parameter); } - WRes Wait() { return Thread_Wait(&thread); } + // WRes Wait() { return Thread_Wait(&thread); } + WRes Wait_Close() { return Thread_Wait_Close(&thread); } + + WRes Create(THREAD_FUNC_TYPE startAddress, LPVOID param) + { return Thread_Create(&thread, startAddress, param); } + WRes Create_With_Affinity(THREAD_FUNC_TYPE startAddress, LPVOID param, CAffinityMask affinity) + { return Thread_Create_With_Affinity(&thread, startAddress, param, affinity); } + WRes Create_With_CpuSet(THREAD_FUNC_TYPE startAddress, LPVOID param, const CCpuSet *cpuSet) + { return Thread_Create_With_CpuSet(&thread, startAddress, param, cpuSet); } #ifdef _WIN32 operator HANDLE() { return thread; } diff --git a/3rdparty/lzma/CPP/Windows/TimeUtils.cpp b/3rdparty/lzma/CPP/Windows/TimeUtils.cpp index 7ef44d9cc50..77d2c510293 100644 --- a/3rdparty/lzma/CPP/Windows/TimeUtils.cpp +++ b/3rdparty/lzma/CPP/Windows/TimeUtils.cpp @@ -2,6 +2,10 @@ #include "StdAfx.h" +#ifndef _WIN32 +#include +#endif + #include "Defs.h" #include "TimeUtils.h" @@ -10,13 +14,15 @@ namespace NTime { static const UInt32 kNumTimeQuantumsInSecond = 10000000; static const UInt32 kFileTimeStartYear = 1601; +#if !defined(_WIN32) || defined(UNDER_CE) static const UInt32 kDosTimeStartYear = 1980; +#endif static const UInt32 kUnixTimeStartYear = 1970; static const UInt64 kUnixTimeOffset = (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); static const UInt64 kNumSecondsInFileTime = (UInt64)(Int64)-1 / kNumTimeQuantumsInSecond; -bool DosTimeToFileTime(UInt32 dosTime, FILETIME &ft) throw() +bool DosTime_To_FileTime(UInt32 dosTime, FILETIME &ft) throw() { #if defined(_WIN32) && !defined(UNDER_CE) return BOOLToBool(::DosDateTimeToFileTime((UInt16)(dosTime >> 16), (UInt16)(dosTime & 0xFFFF), &ft)); @@ -37,11 +43,7 @@ bool DosTimeToFileTime(UInt32 dosTime, FILETIME &ft) throw() static const UInt32 kHighDosTime = 0xFF9FBF7D; static const UInt32 kLowDosTime = 0x210000; -#define PERIOD_4 (4 * 365 + 1) -#define PERIOD_100 (PERIOD_4 * 25 - 1) -#define PERIOD_400 (PERIOD_100 * 4 + 1) - -bool FileTimeToDosTime(const FILETIME &ft, UInt32 &dosTime) throw() +bool FileTime_To_DosTime(const FILETIME &ft, UInt32 &dosTime) throw() { #if defined(_WIN32) && !defined(UNDER_CE) @@ -55,6 +57,10 @@ bool FileTimeToDosTime(const FILETIME &ft, UInt32 &dosTime) throw() #else +#define PERIOD_4 (4 * 365 + 1) +#define PERIOD_100 (PERIOD_4 * 25 - 1) +#define PERIOD_400 (PERIOD_100 * 4 + 1) + unsigned year, mon, day, hour, min, sec; UInt64 v64 = ft.dwLowDateTime | ((UInt64)ft.dwHighDateTime << 32); Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; @@ -115,39 +121,86 @@ bool FileTimeToDosTime(const FILETIME &ft, UInt32 &dosTime) throw() return true; } -void UnixTimeToFileTime(UInt32 unixTime, FILETIME &ft) throw() + +bool UtcFileTime_To_LocalDosTime(const FILETIME &utc, UInt32 &dosTime) throw() { - UInt64 v = (kUnixTimeOffset + (UInt64)unixTime) * kNumTimeQuantumsInSecond; + FILETIME loc = { 0, 0 }; + const UInt64 u1 = FILETIME_To_UInt64(utc); + const UInt64 kDelta = ((UInt64)1 << 41); // it's larger than quantums in 1 sec. + if (u1 >= kDelta) + { + if (!FileTimeToLocalFileTime(&utc, &loc)) + loc = utc; + else + { + const UInt64 u2 = FILETIME_To_UInt64(loc); + const UInt64 delta = u1 < u2 ? (u2 - u1) : (u1 - u2); + if (delta > kDelta) // if FileTimeToLocalFileTime() overflow, we use UTC time + loc = utc; + } + } + return FileTime_To_DosTime(loc, dosTime); +} + +UInt64 UnixTime_To_FileTime64(UInt32 unixTime) throw() +{ + return (kUnixTimeOffset + (UInt64)unixTime) * kNumTimeQuantumsInSecond; +} + +void UnixTime_To_FileTime(UInt32 unixTime, FILETIME &ft) throw() +{ + const UInt64 v = UnixTime_To_FileTime64(unixTime); ft.dwLowDateTime = (DWORD)v; ft.dwHighDateTime = (DWORD)(v >> 32); } -bool UnixTime64ToFileTime(Int64 unixTime, FILETIME &ft) throw() +UInt64 UnixTime64_To_FileTime64(Int64 unixTime) throw() { - if (unixTime > kNumSecondsInFileTime - kUnixTimeOffset) + return (UInt64)((Int64)kUnixTimeOffset + unixTime) * kNumTimeQuantumsInSecond; +} + + +bool UnixTime64_To_FileTime64(Int64 unixTime, UInt64 &fileTime) throw() +{ + if (unixTime > (Int64)(kNumSecondsInFileTime - kUnixTimeOffset)) { - ft.dwLowDateTime = ft.dwHighDateTime = (UInt32)(Int32)-1; + fileTime = (UInt64)(Int64)-1; return false; } - Int64 v = (Int64)kUnixTimeOffset + unixTime; - if (v < 0) + if (unixTime < -(Int64)kUnixTimeOffset) { - ft.dwLowDateTime = ft.dwHighDateTime = 0; + fileTime = 0; return false; } - UInt64 v2 = (UInt64)v * kNumTimeQuantumsInSecond; - ft.dwLowDateTime = (DWORD)v2; - ft.dwHighDateTime = (DWORD)(v2 >> 32); + fileTime = UnixTime64_To_FileTime64(unixTime); return true; } -Int64 FileTimeToUnixTime64(const FILETIME &ft) throw() + +bool UnixTime64_To_FileTime(Int64 unixTime, FILETIME &ft) throw() +{ + UInt64 v; + const bool res = UnixTime64_To_FileTime64(unixTime, v); + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); + return res; +} + + +Int64 FileTime_To_UnixTime64(const FILETIME &ft) throw() +{ + const UInt64 winTime = (((UInt64)ft.dwHighDateTime) << 32) + ft.dwLowDateTime; + return (Int64)(winTime / kNumTimeQuantumsInSecond) - (Int64)kUnixTimeOffset; +} + +Int64 FileTime_To_UnixTime64_and_Quantums(const FILETIME &ft, UInt32 &quantums) throw() { - UInt64 winTime = (((UInt64)ft.dwHighDateTime) << 32) + ft.dwLowDateTime; - return (Int64)(winTime / kNumTimeQuantumsInSecond) - kUnixTimeOffset; + const UInt64 winTime = (((UInt64)ft.dwHighDateTime) << 32) + ft.dwLowDateTime; + quantums = (UInt32)(winTime % kNumTimeQuantumsInSecond); + return (Int64)(winTime / kNumTimeQuantumsInSecond) - (Int64)kUnixTimeOffset; } -bool FileTimeToUnixTime(const FILETIME &ft, UInt32 &unixTime) throw() +bool FileTime_To_UnixTime(const FILETIME &ft, UInt32 &unixTime) throw() { UInt64 winTime = (((UInt64)ft.dwHighDateTime) << 32) + ft.dwLowDateTime; winTime /= kNumTimeQuantumsInSecond; @@ -157,9 +210,9 @@ bool FileTimeToUnixTime(const FILETIME &ft, UInt32 &unixTime) throw() return false; } winTime -= kUnixTimeOffset; - if (winTime > 0xFFFFFFFF) + if (winTime > (UInt32)0xFFFFFFFF) { - unixTime = 0xFFFFFFFF; + unixTime = (UInt32)0xFFFFFFFF; return false; } unixTime = (UInt32)winTime; @@ -186,11 +239,13 @@ bool GetSecondsSince1601(unsigned year, unsigned month, unsigned day, return true; } -void GetCurUtcFileTime(FILETIME &ft) throw() + +void GetCurUtc_FiTime(CFiTime &ft) throw() { + #ifdef _WIN32 + // Both variants provide same low resolution on WinXP: about 15 ms. // But GetSystemTimeAsFileTime is much faster. - #ifdef UNDER_CE SYSTEMTIME st; GetSystemTime(&st); @@ -198,6 +253,152 @@ void GetCurUtcFileTime(FILETIME &ft) throw() #else GetSystemTimeAsFileTime(&ft); #endif + + #else + + FiTime_Clear(ft); + struct timeval now; + if (gettimeofday(&now, 0 ) == 0) + { + ft.tv_sec = now.tv_sec; + ft.tv_nsec = now.tv_usec * 1000; + } + + #endif } +#ifndef _WIN32 +void GetCurUtcFileTime(FILETIME &ft) throw() +{ + UInt64 v = 0; + struct timeval now; + if (gettimeofday(&now, 0 ) == 0) + { + v = ((UInt64)now.tv_sec + kUnixTimeOffset) * + kNumTimeQuantumsInSecond + (UInt64)now.tv_usec * 10; + } + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); +} +#endif + + }} + + +#ifdef _WIN32 + +/* +void FiTime_Normalize_With_Prec(CFiTime &ft, unsigned prec) +{ + if (prec == k_PropVar_TimePrec_0 + || prec == k_PropVar_TimePrec_HighPrec + || prec >= k_PropVar_TimePrec_100ns) + return; + UInt64 v = (((UInt64)ft.dwHighDateTime) << 32) + ft.dwLowDateTime; + + int numDigits = (int)prec - (int)k_PropVar_TimePrec_Base; + UInt32 d; + if (prec == k_PropVar_TimePrec_DOS) + { + // we round up as windows DosDateTimeToFileTime() + v += NWindows::NTime::kNumTimeQuantumsInSecond * 2 - 1; + d = NWindows::NTime::kNumTimeQuantumsInSecond * 2; + } + else + { + if (prec == k_PropVar_TimePrec_Unix) + numDigits = 0; + else if (numDigits < 0) + return; + d = 1; + for (unsigned k = numDigits; k < 7; k++) + d *= 10; + } + v /= d; + v *= d; + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); +} +*/ + +#else + +/* +void FiTime_Normalize_With_Prec(CFiTime &ft, unsigned prec) +{ + if (prec >= k_PropVar_TimePrec_1ns + || prec == k_PropVar_TimePrec_HighPrec) + return; + + int numDigits = (int)prec - (int)k_PropVar_TimePrec_Base; + UInt32 d; + if (prec == k_PropVar_TimePrec_Unix || + prec == (int)k_PropVar_TimePrec_Base) + { + ft.tv_nsec = 0; + return; + } + if (prec == k_PropVar_TimePrec_DOS) + { + // we round up as windows DosDateTimeToFileTime() + const unsigned sec1 = (ft.tv_sec & 1); + if (ft.tv_nsec == 0 && sec1 == 0) + return; + ft.tv_nsec = 0; + ft.tv_sec += 2 - sec1; + return; + } + { + if (prec == k_PropVar_TimePrec_0 + || numDigits < 0) + numDigits = 7; + d = 1; + for (unsigned k = numDigits; k < 9; k++) + d *= 10; + ft.tv_nsec /= d; + ft.tv_nsec *= d; + } +} +*/ + +int Compare_FiTime(const CFiTime *a1, const CFiTime *a2) +{ + if (a1->tv_sec < a2->tv_sec) return -1; + if (a1->tv_sec > a2->tv_sec) return 1; + if (a1->tv_nsec < a2->tv_nsec) return -1; + if (a1->tv_nsec > a2->tv_nsec) return 1; + return 0; +} + +bool FILETIME_To_timespec(const FILETIME &ft, timespec &ts) +{ + UInt32 quantums; + const Int64 sec = NWindows::NTime::FileTime_To_UnixTime64_and_Quantums(ft, quantums); + // time_t is long + const time_t sec2 = (time_t)sec; + if (sec2 == sec) + { + ts.tv_sec = sec2; + ts.tv_nsec = (long)(quantums * 100); + return true; + } + return false; +} + +void FiTime_To_FILETIME_ns100(const CFiTime &ts, FILETIME &ft, unsigned &ns100) +{ + const UInt64 v = NWindows::NTime::UnixTime64_To_FileTime64(ts.tv_sec) + ((UInt64)ts.tv_nsec / 100); + ns100 = (unsigned)((UInt64)ts.tv_nsec % 100); + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); +} + +void FiTime_To_FILETIME(const CFiTime &ts, FILETIME &ft) +{ + const UInt64 v = NWindows::NTime::UnixTime64_To_FileTime64(ts.tv_sec) + ((UInt64)ts.tv_nsec / 100); + ft.dwLowDateTime = (DWORD)v; + ft.dwHighDateTime = (DWORD)(v >> 32); +} + +#endif diff --git a/3rdparty/lzma/CPP/Windows/TimeUtils.h b/3rdparty/lzma/CPP/Windows/TimeUtils.h index d3033ede3ef..60ee739f221 100644 --- a/3rdparty/lzma/CPP/Windows/TimeUtils.h +++ b/3rdparty/lzma/CPP/Windows/TimeUtils.h @@ -5,20 +5,142 @@ #include "../Common/MyTypes.h" #include "../Common/MyWindows.h" +#include "PropVariant.h" + +inline UInt64 FILETIME_To_UInt64(const FILETIME &ft) +{ + return (((UInt64)ft.dwHighDateTime) << 32) + ft.dwLowDateTime; +} + +inline void FILETIME_Clear(FILETIME &ft) +{ + ft.dwLowDateTime = 0; + ft.dwHighDateTime = 0; +} + +inline bool FILETIME_IsZero(const FILETIME &ft) +{ + return (ft.dwHighDateTime == 0 && ft.dwLowDateTime == 0); +} + + +#ifdef _WIN32 + #define CFiTime FILETIME + #define Compare_FiTime ::CompareFileTime + inline void FiTime_To_FILETIME(const CFiTime &ts, FILETIME &ft) + { + ft = ts; + } + /* + inline void FILETIME_To_FiTime(const FILETIME &ft, CFiTime &ts) + { + ts = ft; + } + */ + inline void FiTime_Clear(CFiTime &ft) + { + ft.dwLowDateTime = 0; + ft.dwHighDateTime = 0; + } +#else + + #include + + #if defined(_AIX) + #define CFiTime st_timespec + #else + #define CFiTime timespec + #endif + int Compare_FiTime(const CFiTime *a1, const CFiTime *a2); + bool FILETIME_To_timespec(const FILETIME &ft, CFiTime &ts); + void FiTime_To_FILETIME(const CFiTime &ts, FILETIME &ft); + void FiTime_To_FILETIME_ns100(const CFiTime &ts, FILETIME &ft, unsigned &ns100); + inline void FiTime_Clear(CFiTime &ft) + { + ft.tv_sec = 0; + ft.tv_nsec = 0; + } + + #ifdef __APPLE__ + #define ST_MTIME(st) st.st_mtimespec + #define ST_ATIME(st) st.st_atimespec + #define ST_CTIME(st) st.st_ctimespec + #else + #define ST_MTIME(st) st.st_mtim + #define ST_ATIME(st) st.st_atim + #define ST_CTIME(st) st.st_ctim + #endif + +#endif + +// void FiTime_Normalize_With_Prec(CFiTime &ft, unsigned prec); namespace NWindows { namespace NTime { -bool DosTimeToFileTime(UInt32 dosTime, FILETIME &fileTime) throw(); -bool FileTimeToDosTime(const FILETIME &fileTime, UInt32 &dosTime) throw(); -void UnixTimeToFileTime(UInt32 unixTime, FILETIME &fileTime) throw(); -bool UnixTime64ToFileTime(Int64 unixTime, FILETIME &fileTime) throw(); -bool FileTimeToUnixTime(const FILETIME &fileTime, UInt32 &unixTime) throw(); -Int64 FileTimeToUnixTime64(const FILETIME &ft) throw(); +bool DosTime_To_FileTime(UInt32 dosTime, FILETIME &fileTime) throw(); +bool UtcFileTime_To_LocalDosTime(const FILETIME &utc, UInt32 &dosTime) throw(); +bool FileTime_To_DosTime(const FILETIME &fileTime, UInt32 &dosTime) throw(); + +// UInt32 Unix Time : for dates 1970-2106 +UInt64 UnixTime_To_FileTime64(UInt32 unixTime) throw(); +void UnixTime_To_FileTime(UInt32 unixTime, FILETIME &fileTime) throw(); + +// Int64 Unix Time : negative values for dates before 1970 +UInt64 UnixTime64_To_FileTime64(Int64 unixTime) throw(); // no check +bool UnixTime64_To_FileTime64(Int64 unixTime, UInt64 &fileTime) throw(); +bool UnixTime64_To_FileTime(Int64 unixTime, FILETIME &fileTime) throw(); + +Int64 FileTime64_To_UnixTime64(UInt64 ft64) throw(); +bool FileTime_To_UnixTime(const FILETIME &fileTime, UInt32 &unixTime) throw(); +Int64 FileTime_To_UnixTime64(const FILETIME &ft) throw(); +Int64 FileTime_To_UnixTime64_and_Quantums(const FILETIME &ft, UInt32 &quantums) throw(); + bool GetSecondsSince1601(unsigned year, unsigned month, unsigned day, unsigned hour, unsigned min, unsigned sec, UInt64 &resSeconds) throw(); + +void GetCurUtc_FiTime(CFiTime &ft) throw(); +#ifdef _WIN32 +#define GetCurUtcFileTime GetCurUtc_FiTime +#else void GetCurUtcFileTime(FILETIME &ft) throw(); +#endif }} +inline void PropVariant_SetFrom_UnixTime(NWindows::NCOM::CPropVariant &prop, UInt32 unixTime) +{ + FILETIME ft; + NWindows::NTime::UnixTime_To_FileTime(unixTime, ft); + prop.SetAsTimeFrom_FT_Prec(ft, k_PropVar_TimePrec_Unix); +} + +inline void PropVariant_SetFrom_NtfsTime(NWindows::NCOM::CPropVariant &prop, const FILETIME &ft) +{ + prop.SetAsTimeFrom_FT_Prec(ft, k_PropVar_TimePrec_100ns); +} + +inline void PropVariant_SetFrom_FiTime(NWindows::NCOM::CPropVariant &prop, const CFiTime &fts) +{ + #ifdef _WIN32 + PropVariant_SetFrom_NtfsTime(prop, fts); + #else + unsigned ns100; + FILETIME ft; + FiTime_To_FILETIME_ns100(fts, ft, ns100); + prop.SetAsTimeFrom_FT_Prec_Ns100(ft, k_PropVar_TimePrec_1ns, ns100); + #endif +} + +inline bool PropVariant_SetFrom_DosTime(NWindows::NCOM::CPropVariant &prop, UInt32 dosTime) +{ + FILETIME localFileTime, utc; + if (!NWindows::NTime::DosTime_To_FileTime(dosTime, localFileTime)) + return false; + if (!LocalFileTimeToFileTime(&localFileTime, &utc)) + return false; + prop.SetAsTimeFrom_FT_Prec(utc, k_PropVar_TimePrec_DOS); + return true; +} + #endif diff --git a/3rdparty/lzma/CPP/Windows/Window.cpp b/3rdparty/lzma/CPP/Windows/Window.cpp index 365850223ce..32af4aabcfa 100644 --- a/3rdparty/lzma/CPP/Windows/Window.cpp +++ b/3rdparty/lzma/CPP/Windows/Window.cpp @@ -114,12 +114,12 @@ bool MySetWindowText(HWND wnd, LPCWSTR s) bool CWindow::GetText(CSysString &s) { s.Empty(); - int len = GetTextLength(); + unsigned len = (unsigned)GetTextLength(); if (len == 0) return (::GetLastError() == ERROR_SUCCESS); TCHAR *p = s.GetBuf(len); { - int len2 = GetText(p, len + 1); + unsigned len2 = (unsigned)GetText(p, (int)(len + 1)); if (len > len2) len = len2; } @@ -135,12 +135,12 @@ bool CWindow::GetText(UString &s) if (g_IsNT) { s.Empty(); - int len = GetWindowTextLengthW(_window); + unsigned len = (unsigned)GetWindowTextLengthW(_window); if (len == 0) return (::GetLastError() == ERROR_SUCCESS); wchar_t *p = s.GetBuf(len); { - int len2 = GetWindowTextW(_window, p, len + 1); + unsigned len2 = (unsigned)GetWindowTextW(_window, p, (int)(len + 1)); if (len > len2) len = len2; } diff --git a/3rdparty/lzma/CPP/Windows/Window.h b/3rdparty/lzma/CPP/Windows/Window.h index 3bda6795d0c..83726c7ad7d 100644 --- a/3rdparty/lzma/CPP/Windows/Window.h +++ b/3rdparty/lzma/CPP/Windows/Window.h @@ -171,7 +171,7 @@ public: bool Update() { return BOOLToBool(::UpdateWindow(_window)); } bool InvalidateRect(LPCRECT rect, bool backgroundErase = true) { return BOOLToBool(::InvalidateRect(_window, rect, BoolToBOOL(backgroundErase))); } - void SetRedraw(bool redraw = true) { SendMsg(WM_SETREDRAW, BoolToBOOL(redraw), 0); } + void SetRedraw(bool redraw = true) { SendMsg(WM_SETREDRAW, (WPARAM)BoolToBOOL(redraw), 0); } LONG_PTR SetStyle(LONG_PTR style) { return SetLongPtr(GWL_STYLE, style); } LONG_PTR GetStyle() const { return GetLongPtr(GWL_STYLE); } @@ -244,7 +244,7 @@ public: int GetTextLength() const { return GetWindowTextLength(_window); } - UINT GetText(LPTSTR string, int maxCount) const + int GetText(LPTSTR string, int maxCount) const { return GetWindowText(_window, string, maxCount); } bool GetText(CSysString &s); #ifndef _UNICODE diff --git a/3rdparty/lzma/CS/7zip/Compress/LzmaAlone/LzmaAlone.csproj b/3rdparty/lzma/CS/7zip/Compress/LzmaAlone/LzmaAlone.csproj deleted file mode 100644 index 6d87b61ff67..00000000000 --- a/3rdparty/lzma/CS/7zip/Compress/LzmaAlone/LzmaAlone.csproj +++ /dev/null @@ -1,90 +0,0 @@ - - - Debug - AnyCPU - 8.0.50727 - 2.0 - {CE33DF18-F9C8-4D6F-9057-DBB4DB96E973} - Exe - LzmaAlone - Lzma# - 4 - - - true - full - false - .\bin\Debug\ - DEBUG;TRACE - - - false - true - .\bin\Release\ - TRACE - AnyCPU - - - - - - - - - Common\CommandLineParser.cs - - - Common\CRC.cs - - - ICoder.cs - - - LZ\IMatchFinder.cs - - - LZ\LzBinTree.cs - - - LZ\LzInWindow.cs - - - LZ\LzOutWindow.cs - - - LZMA\LzmaBase.cs - - - LZMA\LzmaDecoder.cs - - - LZMA\LzmaEncoder.cs - - - RangeCoder\RangeCoder.cs - - - RangeCoder\RangeCoderBit.cs - - - RangeCoder\RangeCoderBitTree.cs - - - Code - - - Code - - - - True - Settings.settings - - - SettingsSingleFileGenerator - Settings.cs - - - - - \ No newline at end of file diff --git a/3rdparty/lzma/DOC/7zFormat.txt b/3rdparty/lzma/DOC/7zFormat.txt index 55b52b1b9ca..74cdfa4181e 100644 --- a/3rdparty/lzma/DOC/7zFormat.txt +++ b/3rdparty/lzma/DOC/7zFormat.txt @@ -1,4 +1,4 @@ -7z Format description (4.59) +7z Format description (18.06) ---------------------------- This file contains description of 7z archive format. @@ -175,7 +175,7 @@ SignatureHeader ArchiveVersion { BYTE Major; // now = 0 - BYTE Minor; // now = 2 + BYTE Minor; // now = 4 }; UINT32 StartHeaderCRC; @@ -399,7 +399,7 @@ FilesInfo UINT64 DataIndex [] for(Definded Items) - UINT64 Time + REAL_UINT64 Time [] kNames: (0x11) diff --git a/3rdparty/lzma/DOC/Methods.txt b/3rdparty/lzma/DOC/Methods.txt index f6cda7d53d2..d4a1b1dd428 100644 --- a/3rdparty/lzma/DOC/Methods.txt +++ b/3rdparty/lzma/DOC/Methods.txt @@ -1,8 +1,8 @@ 7-Zip method IDs for 7z and xz archives --------------------------------------- -Version: 16.03 -Date: 2016-09-27 +Version: 18.06 +Date: 2018-06-30 Each compression or crypto method in 7z is associated with unique binary value (ID). The length of ID in bytes is arbitrary but it can not exceed 63 bits (8 bytes). @@ -122,9 +122,23 @@ List of defined IDs F7 - External codecs (that are not included to 7-Zip) 0x xx - reserved - 10 xx - reserved (LZHAM) - 11 xx - reserved (Zstd) + 10 xx - reserved (LZHAM) + 01 - LZHAM + + 11 xx - reserved (Tino Reichardt) + 01 - ZSTD + 02 - BROTLI + 04 - LZ4 + 05 - LZ5 + 06 - LIZARD + + 12 xx - reserverd (Denis Anisimov) + + 01 - WavPack2 + FE - eSplitter + FF - RawSplitter + 06.. - Crypto diff --git a/3rdparty/lzma/DOC/lzma-history.txt b/3rdparty/lzma/DOC/lzma-history.txt index cce580645c4..0963c7bf799 100644 --- a/3rdparty/lzma/DOC/lzma-history.txt +++ b/3rdparty/lzma/DOC/lzma-history.txt @@ -1,6 +1,163 @@ HISTORY of the LZMA SDK ----------------------- +21.07 2021-12-26 +------------------------- +- New switches: -spm and -im!{file_path} to exclude directories from processing + for specified paths that don't contain path separator character at the end of path. +- The sorting order of files in archives was slightly changed to be more consistent + for cases where the name of some directory is the same as the prefix part of the name + of another directory or file. + + +21.06 2021-11-24 +------------------------- +- Bug in LZMA encoder in file LzmaEnc.c was fixed: + LzmaEnc_MemEncode(), LzmaEncode() and LzmaCompress() could work incorrectly, + if size value for output buffer is smaller than size required for all compressed data. + LzmaEnc_Encode() could work incorrectly, + if callback ISeqOutStream::Write() doesn't write all compressed data. + NCompress::NLzma::CEncoder::Code() could work incorrectly, + if callback ISequentialOutStream::Write() returns error code. +- Bug in versions 21.00-21.05 was fixed: + 7-Zip didn't set attributes of directories during archive extracting. + + +21.04 beta 2021-11-02 +------------------------- +- 7-Zip now reduces the number of working CPU threads for compression, + if RAM size is not enough for compression with big LZMA2 dictionary. +- 7-Zip now can create and check "file.sha256" text files that contain the list + of file names and SHA-256 checksums in format compatible with sha256sum program. + + +21.03 beta 2021-07-20 +------------------------- +- The maximum dictionary size for LZMA/LZMA2 compressing was increased to 4 GB (3840 MiB). +- Minor speed optimizations in LZMA/LZMA2 compressing. + + +21.02 alpha 2021-05-06 +------------------------- +- The command line version of 7-Zip for macOS was released. +- The speed for LZMA and LZMA2 decompression in arm64 versions for macOS and Linux + was increased by 20%-60%. + + +21.01 alpha 2021-03-09 +------------------------- +- The command line version of 7-Zip for Linux was released. +- The improvements for speed of ARM64 version using hardware CPU instructions + for AES, CRC-32, SHA-1 and SHA-256. +- Some bugs were fixed. + + +20.02 alpha 2020-08-08 +------------------------- +- The default number of LZMA2 chunks per solid block in 7z archive was increased to 64. + It allows to increase the compression speed for big 7z archives, if there is a big number + of CPU cores and threads. +- The speed of PPMd compressing/decompressing was increased for 7z archives. +- The new -ssp switch. If the switch -ssp is specified, 7-Zip doesn't allow the system + to modify "Last Access Time" property of source files for archiving and hashing operations. +- Some bugs were fixed. + + +20.00 alpha 2020-02-06 +------------------------- +- 7-Zip now supports new optional match finders for LZMA/LZMA2 compression: bt5 and hc5, + that can work faster than bt4 and hc4 match finders for the data with big redundancy. +- The compression ratio was improved for Fast and Fastest compression levels with the + following default settings: + - Fastest level (-mx1) : hc5 match finder with 256 KB dictionary. + - Fast level (-mx3) : hc5 match finder with 4 MB dictionary. +- Minor speed optimizations in multithreaded LZMA/LZMA2 compression for Normal/Maximum/Ultra + compression levels. + + +19.00 2019-02-21 +------------------------- +- Encryption strength for 7z archives was increased: + the size of random initialization vector was increased from 64-bit to 128-bit, + and the pseudo-random number generator was improved. +- The bug in 7zIn.c code was fixed. + + +18.06 2018-12-30 +------------------------- +- The speed for LZMA/LZMA2 compressing was increased by 3-10%, + and there are minor changes in compression ratio. +- Some bugs were fixed. +- The bug in 7-Zip 18.02-18.05 was fixed: + There was memory leak in multithreading xz decoder - XzDecMt_Decode(), + if xz stream contains only one block. +- The changes for MSVS compiler makefiles: + - the makefiles now use "PLATFORM" macroname with values (x64, x86, arm64) + instead of "CPU" macroname with values (AMD64, ARM64). + - the makefiles by default now use static version of the run-time library. + + +18.05 2018-04-30 +------------------------- +- The speed for LZMA/LZMA2 compressing was increased + by 8% for fastest/fast compression levels and + by 3% for normal/maximum compression levels. +- Previous versions of 7-Zip could work incorrectly in "Large memory pages" mode in + Windows 10 because of some BUG with "Large Pages" in Windows 10. + Now 7-Zip doesn't use "Large Pages" on Windows 10 up to revision 1709 (16299). +- The BUG was fixed in Lzma2Enc.c + Lzma2Enc_Encode2() function worked incorretly, + if (inStream == NULL) and the number of block threads is more than 1. + + +18.03 beta 2018-03-04 +------------------------- +- Asm\x86\LzmaDecOpt.asm: new optimized LZMA decoder written in asm + for x64 with about 30% higher speed than main version of LZMA decoder written in C. +- The speed for single-thread LZMA/LZMA2 decoder written in C was increased by 3%. +- 7-Zip now can use multi-threading for 7z/LZMA2 decoding, + if there are multiple independent data chunks in LZMA2 stream. +- 7-Zip now can use multi-threading for xz decoding, + if there are multiple blocks in xz stream. + + +18.01 2019-01-28 +------------------------- +- The BUG in 17.01 - 18.00 beta was fixed: + XzDec.c : random block unpacking and XzUnpacker_IsBlockFinished() + didn't work correctly for xz archives without checksum (CRC). + + +18.00 beta 2019-01-10 +------------------------- +- The BUG in xz encoder was fixed: + There was memory leak of 16 KB for each file compressed with + xz compression method, if additional filter was used. + + +17.01 beta 2017-08-28 +------------------------- +- Minor speed optimization for LZMA2 (xz and 7z) multi-threading compression. + 7-Zip now uses additional memory buffers for multi-block LZMA2 compression. + CPU utilization was slightly improved. +- 7-zip now creates multi-block xz archives by default. Block size can be + specified with -ms[Size]{m|g} switch. +- xz decoder now can unpack random block from multi-block xz archives. +- 7-Zip command line: @listfile now doesn't work after -- switch. + Use -i@listfile before -- switch instead. +- The BUGs were fixed: + 7-Zip 17.00 beta crashed for commands that write anti-item to 7z archive. + + +17.00 beta 2017-04-29 +------------------------- +- NewHandler.h / NewHandler.cpp: + now it redefines operator new() only for old MSVC compilers (_MSC_VER < 1900). +- C/7zTypes.h : the names of variables in interface structures were changed (vt). +- Some bugs were fixed. 7-Zip could crash in some cases. +- Some internal changes in code. + + 16.04 2016-10-04 ------------------------- - The bug was fixed in DllSecur.c. diff --git a/3rdparty/lzma/DOC/lzma-sdk.txt b/3rdparty/lzma/DOC/lzma-sdk.txt index b8fce262002..ed1a431ee86 100644 --- a/3rdparty/lzma/DOC/lzma-sdk.txt +++ b/3rdparty/lzma/DOC/lzma-sdk.txt @@ -1,4 +1,4 @@ -LZMA SDK 16.04 +LZMA SDK 22.01 -------------- LZMA SDK provides the documentation, samples, header files, @@ -62,14 +62,61 @@ LZMA SDK Contents UNIX/Linux version ------------------ -To compile C++ version of file->file LZMA encoding, go to directory -CPP/7zip/Bundles/LzmaCon -and call make to recompile it: - make -f makefile.gcc clean all - -In some UNIX/Linux versions you must compile LZMA with static libraries. -To compile with static libraries, you can use -LIB = -lm -static +There are several otpions to compile 7-Zip with different compilers: gcc and clang. +Also 7-Zip code contains two versions for some critical parts of code: in C and in Assembeler. +So if you compile the version with Assembeler code, you will get faster 7-Zip binary. + +7-Zip's assembler code uses the following syntax for different platforms: + +1) x86 and x86-64 (AMD64): MASM syntax. + There are 2 programs that supports MASM syntax in Linux. +' 'Asmc Macro Assembler and JWasm. But JWasm now doesn't support some + cpu instructions used in 7-Zip. + So you must install Asmc Macro Assembler in Linux, if you want to compile fastest version + of 7-Zip x86 and x86-64: + https://github.com/nidud/asmc + +2) arm64: GNU assembler for ARM64 with preprocessor. + That systax of that arm64 assembler code in 7-Zip is supported by GCC and CLANG for ARM64. + +There are different binaries that can be compiled from 7-Zip source. +There are 2 main files in folder for compiling: + makefile - that can be used for compiling Windows version of 7-Zip with nmake command + makefile.gcc - that can be used for compiling Linux/macOS versions of 7-Zip with make command + +At first you must change the current folder to folder that contains `makefile.gcc`: + + cd CPP/7zip/Bundles/Alone7z + +Then you can compile `makefile.gcc` with the command: + + make -j -f makefile.gcc + +Also there are additional "*.mak" files in folder "CPP/7zip/" that can be used to compile +7-Zip binaries with optimized code and optimzing options. + +To compile with GCC without assembler: + cd CPP/7zip/Bundles/Alone7z + make -j -f ../../cmpl_gcc.mak + +To compile with CLANG without assembler: + make -j -f ../../cmpl_clang.mak + +To compile 7-Zip for x86-64 with asmc assembler: + make -j -f ../../cmpl_gcc_x64.mak + +To compile 7-Zip for arm64 with assembler: + make -j -f ../../cmpl_gcc_arm64.mak + +To compile 7-Zip for arm64 for macOS: + make -j -f ../../cmpl_mac_arm64.mak + +Also you can change some compiler options in the mak files: + cmpl_gcc.mak + var_gcc.mak + warn_gcc.mak + + Also you can use p7zip (port of 7-Zip for POSIX systems like Unix or Linux): diff --git a/3rdparty/lzma/DOC/lzma.txt b/3rdparty/lzma/DOC/lzma.txt index 87d727b55b5..a65988fe069 100644 --- a/3rdparty/lzma/DOC/lzma.txt +++ b/3rdparty/lzma/DOC/lzma.txt @@ -33,7 +33,7 @@ If you want to use old interfaces you can download previous version of LZMA SDK from sourceforge.net site. To use ANSI-C LZMA Decoder you need the following files: -1) LzmaDec.h + LzmaDec.c + Types.h +1) LzmaDec.h + LzmaDec.c + 7zTypes.h + Precomp.h + Compiler.h Look example code: C/Util/Lzma/LzmaUtil.c @@ -70,7 +70,7 @@ You can use p = p; operator to disable compiler warnings. Single-call Decompressing ------------------------- When to use: RAM->RAM decompressing -Compile files: LzmaDec.h + LzmaDec.c + Types.h +Compile files: LzmaDec.h + LzmaDec.c + 7zTypes.h Compile defines: no defines Memory Requirements: - Input buffer: compressed size @@ -124,7 +124,7 @@ Multi-call State Decompressing (zlib-like interface) ---------------------------------------------------- When to use: file->file decompressing -Compile files: LzmaDec.h + LzmaDec.c + Types.h +Compile files: LzmaDec.h + LzmaDec.c + 7zTypes.h Memory Requirements: - Buffer for input stream: any size (for example, 16 KB) @@ -167,7 +167,7 @@ How To compress data -------------------- Compile files: - Types.h + 7zTypes.h Threads.h LzmaEnc.h LzmaEnc.c diff --git a/scripts/genie.lua b/scripts/genie.lua index 727be423d2a..9a1f3140bfe 100644 --- a/scripts/genie.lua +++ b/scripts/genie.lua @@ -1105,12 +1105,7 @@ end } end if version >= 130000 then - buildoptions_c { - "-Wno-error=dangling-pointer", - } buildoptions_cpp { - "-Wno-error=dangling-reference", - "-Wno-error=dangling-pointer", "-Wno-xor-used-as-pow", } end diff --git a/scripts/src/3rdparty.lua b/scripts/src/3rdparty.lua index 1b8aa7f7e7f..6731f6c8a10 100755 --- a/scripts/src/3rdparty.lua +++ b/scripts/src/3rdparty.lua @@ -1,3 +1,4 @@ +-- -- license:BSD-3-Clause -- copyright-holders:MAMEdev Team @@ -745,10 +746,20 @@ project "7z" "-Wno-strict-prototypes", "-Wno-undef", } -if _OPTIONS["gcc"]~=nil and string.find(_OPTIONS["gcc"], "clang") and str_to_version(_OPTIONS["gcc_version"]) >= 100000 then - buildoptions_c { - "-Wno-misleading-indentation", - } +if _OPTIONS["gcc"]~=nil then + if string.find(_OPTIONS["gcc"], "clang") then + if str_to_version(_OPTIONS["gcc_version"]) >= 100000 then + buildoptions_c { + "-Wno-misleading-indentation", + } + end + else + if str_to_version(_OPTIONS["gcc_version"]) >= 130000 then + buildoptions_c { + "-Wno-error=dangling-pointer", + } + end + end end configuration { "android-*" } @@ -811,18 +822,22 @@ end -- MAME_DIR .. "3rdparty/lzma/C/DllSecur.c", MAME_DIR .. "3rdparty/lzma/C/LzFind.c", -- MAME_DIR .. "3rdparty/lzma/C/LzFindMt.c", + MAME_DIR .. "3rdparty/lzma/C/LzFindOpt.c", MAME_DIR .. "3rdparty/lzma/C/Lzma2Dec.c", - MAME_DIR .. "3rdparty/lzma/C/Lzma2Enc.c", - MAME_DIR .. "3rdparty/lzma/C/Lzma86Dec.c", - MAME_DIR .. "3rdparty/lzma/C/Lzma86Enc.c", + -- MAME_DIR .. "3rdparty/lzma/C/Lzma2DecMt.c", + -- MAME_DIR .. "3rdparty/lzma/C/Lzma2Enc.c", + -- MAME_DIR .. "3rdparty/lzma/C/Lzma86Dec.c", + -- MAME_DIR .. "3rdparty/lzma/C/Lzma86Enc.c", MAME_DIR .. "3rdparty/lzma/C/LzmaDec.c", MAME_DIR .. "3rdparty/lzma/C/LzmaEnc.c", -- MAME_DIR .. "3rdparty/lzma/C/LzmaLib.c", -- MAME_DIR .. "3rdparty/lzma/C/MtCoder.c", + -- MAME_DIR .. "3rdparty/lzma/C/MtDec.c", MAME_DIR .. "3rdparty/lzma/C/Ppmd7.c", MAME_DIR .. "3rdparty/lzma/C/Ppmd7Dec.c", - MAME_DIR .. "3rdparty/lzma/C/Ppmd7Enc.c", + -- MAME_DIR .. "3rdparty/lzma/C/Ppmd7Enc.c", MAME_DIR .. "3rdparty/lzma/C/Sha256.c", + MAME_DIR .. "3rdparty/lzma/C/Sha256Opt.c", MAME_DIR .. "3rdparty/lzma/C/Sort.c", -- MAME_DIR .. "3rdparty/lzma/C/Threads.c", -- MAME_DIR .. "3rdparty/lzma/C/Xz.c", diff --git a/src/lib/util/chdcodec.cpp b/src/lib/util/chdcodec.cpp index 7d3f3185212..be48cd55ec9 100644 --- a/src/lib/util/chdcodec.cpp +++ b/src/lib/util/chdcodec.cpp @@ -114,11 +114,11 @@ public: private: // internal helpers - static void *fast_alloc(void *p, size_t size); - static void fast_free(void *p, void *address); + static void *fast_alloc(ISzAllocPtr p, size_t size); + static void fast_free(ISzAllocPtr p, void *address); static constexpr int MAX_LZMA_ALLOCS = 64; - uint32_t * m_allocptr[MAX_LZMA_ALLOCS]; + uint32_t *m_allocptr[MAX_LZMA_ALLOCS]; }; @@ -1020,9 +1020,9 @@ chd_lzma_allocator::~chd_lzma_allocator() // allocates and frees memory frequently //------------------------------------------------- -void *chd_lzma_allocator::fast_alloc(void *p, size_t size) +void *chd_lzma_allocator::fast_alloc(ISzAllocPtr p, size_t size) { - auto *codec = reinterpret_cast(p); + auto *const codec = static_cast(const_cast(p)); // compute the size, rounding to the nearest 1k size = (size + 0x3ff) & ~0x3ff; @@ -1059,12 +1059,12 @@ void *chd_lzma_allocator::fast_alloc(void *p, size_t size) // allocates and frees memory frequently //------------------------------------------------- -void chd_lzma_allocator::fast_free(void *p, void *address) +void chd_lzma_allocator::fast_free(ISzAllocPtr p, void *address) { if (address == nullptr) return; - auto *codec = reinterpret_cast(p); + auto *const codec = static_cast(const_cast(p)); // find the hunk uint32_t *ptr = reinterpret_cast(address) - 1; @@ -1149,7 +1149,7 @@ uint32_t chd_lzma_compressor::compress(const uint8_t *src, uint32_t srclen, uint void chd_lzma_compressor::configure_properties(CLzmaEncProps &props, uint32_t hunkbytes) { LzmaEncProps_Init(&props); - props.level = 9; + props.level = 8; props.reduceSize = hunkbytes; LzmaEncProps_Normalize(&props); } diff --git a/src/lib/util/un7z.cpp b/src/lib/util/un7z.cpp index b35af479d5d..896818a45e3 100644 --- a/src/lib/util/un7z.cpp +++ b/src/lib/util/un7z.cpp @@ -51,8 +51,8 @@ struct CFileInStream : public ISeekInStream { CFileInStream() noexcept { - Read = [] (void *pp, void *buf, size_t *size) { return reinterpret_cast(pp)->read(buf, *size); }; - Seek = [] (void *pp, Int64 *pos, ESzSeek origin) { return reinterpret_cast(pp)->seek(*pos, origin); }; + Read = &CFileInStream::read_static; + Seek = &CFileInStream::seek_static; } random_read::ptr file; @@ -109,6 +109,16 @@ private: pos = currfpos; return SZ_OK; } + + static SRes read_static(ISeekInStream const *pp, void *buf, size_t *size) noexcept + { + return static_cast(const_cast(pp))->read(buf, *size); + } + + static SRes seek_static(ISeekInStream const *pp, Int64 *pos, ESzSeek origin) noexcept + { + return static_cast(const_cast(pp))->seek(*pos, origin); + } }; @@ -128,7 +138,7 @@ public: virtual ~m7z_file_impl() { if (m_out_buffer) - IAlloc_Free(&m_alloc_imp, m_out_buffer); + ISzAlloc_Free(&m_alloc_imp, m_out_buffer); if (m_inited) SzArEx_Free(&m_db, &m_alloc_imp); } @@ -230,7 +240,7 @@ private: std::vector m_utf8_buf; CFileInStream m_archive_stream; - CLookToRead m_look_stream; + CLookToRead2 m_look_stream; CSzArEx m_db; ISzAlloc m_alloc_imp; ISzAlloc m_alloc_temp_imp; @@ -240,6 +250,7 @@ private: UInt32 m_block_index; Byte * m_out_buffer; std::size_t m_out_buffer_size; + Byte m_look_stream_buf[65'536]; }; @@ -314,9 +325,11 @@ m7z_file_impl::m7z_file_impl(std::string &&filename) noexcept m_alloc_temp_imp.Alloc = &SzAllocTemp; m_alloc_temp_imp.Free = &SzFreeTemp; - LookToRead_CreateVTable(&m_look_stream, False); + LookToRead2_CreateVTable(&m_look_stream, False); m_look_stream.realStream = &m_archive_stream; - LookToRead_Init(&m_look_stream); + m_look_stream.buf = m_look_stream_buf; + m_look_stream.bufSize = std::size(m_look_stream_buf); + LookToRead2_Init(&m_look_stream); } @@ -362,7 +375,7 @@ std::error_condition m7z_file_impl::initialize() noexcept SzArEx_Init(&m_db); m_inited = true; - SRes const res = SzArEx_Open(&m_db, &m_look_stream.s, &m_alloc_imp, &m_alloc_temp_imp); + SRes const res = SzArEx_Open(&m_db, &m_look_stream.vt, &m_alloc_imp, &m_alloc_temp_imp); if (res != SZ_OK) { osd_printf_error("un7z: error opening %s as 7z archive (%d)\n", m_filename, int(res)); @@ -458,7 +471,7 @@ std::error_condition m7z_file_impl::decompress(void *buffer, std::size_t length) std::size_t offset(0); std::size_t out_size_processed(0); SRes const res = SzArEx_Extract( - &m_db, &m_look_stream.s, m_curr_file_idx, // requested file + &m_db, &m_look_stream.vt, m_curr_file_idx, // requested file &m_block_index, &m_out_buffer, &m_out_buffer_size, // solid block caching &offset, &out_size_processed, // data size/offset &m_alloc_imp, &m_alloc_temp_imp); // allocator helpers -- cgit v1.2.3