From ba8d2cf04d08226bb125546cbec119ff6b924ae6 Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Fri, 1 Nov 2024 07:17:19 +0100 Subject: [PATCH 01/22] =?UTF-8?q?=D0=A4=D0=B8=D0=BA=D1=81=20=D0=B4=D0=BB?= =?UTF-8?q?=D1=8F=20=D0=B2=D0=B2=D0=BE=D0=B4=D0=B0=20=D0=BA=D0=BE=D0=B4?= =?UTF-8?q?=D0=B0=20=D1=81=D0=B8=D0=BC=D0=B2=D0=BE=D0=BB=D0=B0=20=D1=87?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B7=20Alt+=D1=86=D0=B8=D1=84=D1=80=D1=8B?= =?UTF-8?q?=20=D1=82=D0=B5=D0=BF=D0=B5=D1=80=D1=8C=20=D0=BF=D1=80=D0=B8?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=8F=D0=B5=D1=82=D1=81=D1=8F=20=D1=82=D0=BE?= =?UTF-8?q?=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=BA=20=D1=86=D0=B8=D1=84=D1=80?= =?UTF-8?q?=D0=B0=D0=BC=20NumPad'=D0=B0=20=E2=80=94=20=D0=B4=D0=BE=D0=BB?= =?UTF-8?q?=D0=B6=D0=BD=D0=BE=20=D1=83=D0=B1=D1=80=D0=B0=D1=82=D1=8C=20?= =?UTF-8?q?=D1=80=D0=B8=D1=81=D0=BA=20=D0=B3=D0=BB=D1=8E=D0=BA=D0=BE=D0=B2?= =?UTF-8?q?=20=D0=B2=20=D0=B4=D1=80=D1=83=D0=B3=D0=B8=D1=85=20=D0=BC=D0=B5?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D1=85.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit И заодно поправил 5 на NumPad'е в kitty — без NumLock эта кнопка не работала на ввод кода символа. --- .../Backend/TTY/TTYInputSequenceParser.cpp | 2 +- .../TTY/TTYInputSequenceParserExts.cpp | 2 +- WinPort/src/Backend/WX/wxMain.cpp | 81 ++++++++++++++++--- WinPort/src/Backend/WX/wxMain.h | 1 + 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp b/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp index a2d4532ce..18a43f2ae 100644 --- a/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp +++ b/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp @@ -179,7 +179,7 @@ TTYInputSequenceParser::TTYInputSequenceParser(ITTYInputSpecialSequenceHandler * AddStrF1F5(VK_F2, "Q"); AddStr(VK_F2, 0, "[[B"); AddStrF1F5(VK_F3, "R"); AddStr(VK_F3, 0, "[[C"); AddStrF1F5(VK_F4, "S"); AddStr(VK_F4, 0, "[[D"); - AddStrF1F5(VK_F5, "E"); AddStr(VK_F5, 0, "[[E"); + AddStrF1F5(VK_CLEAR, "E"); AddStr(VK_CLEAR, 0, "[[E"); // NumPad center (5) AddStrTilde(VK_HOME, 1); AddStrTilde(VK_INSERT, 2); diff --git a/WinPort/src/Backend/TTY/TTYInputSequenceParserExts.cpp b/WinPort/src/Backend/TTY/TTYInputSequenceParserExts.cpp index 2e6332103..4938d50c3 100644 --- a/WinPort/src/Backend/TTY/TTYInputSequenceParserExts.cpp +++ b/WinPort/src/Backend/TTY/TTYInputSequenceParserExts.cpp @@ -345,7 +345,7 @@ size_t TTYInputSequenceParser::TryParseAsKittyEscapeSequence(const char *s, size ir.Event.KeyEvent.dwControlKeyState |= ENHANCED_KEY; break; case 'D': ir.Event.KeyEvent.wVirtualKeyCode = VK_LEFT; ir.Event.KeyEvent.dwControlKeyState |= ENHANCED_KEY; break; - case 'E': ir.Event.KeyEvent.wVirtualKeyCode = VK_NUMPAD5; break; + case 'E': ir.Event.KeyEvent.wVirtualKeyCode = VK_CLEAR; break; // NumPad center (5) case 'H': ir.Event.KeyEvent.wVirtualKeyCode = VK_HOME; ir.Event.KeyEvent.dwControlKeyState |= ENHANCED_KEY; break; case 'F': ir.Event.KeyEvent.wVirtualKeyCode = VK_END; diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index 229038487..8ae9e4a77 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -1242,6 +1242,50 @@ char* FormatWxKeyState(uint16_t state) { return buffer; } +bool isNumpadNumericKey(int keycode) +{ + switch (keycode) { + case WXK_NUMPAD0: + case WXK_NUMPAD1: + case WXK_NUMPAD2: + case WXK_NUMPAD3: + case WXK_NUMPAD4: + case WXK_NUMPAD5: + case WXK_NUMPAD6: + case WXK_NUMPAD7: + case WXK_NUMPAD8: + case WXK_NUMPAD9: + case WXK_NUMPAD_INSERT: + case WXK_NUMPAD_END: + case WXK_NUMPAD_DOWN: + case WXK_NUMPAD_PAGEDOWN: + case WXK_NUMPAD_LEFT: + case WXK_NUMPAD_BEGIN: // NumPad center (5) + case WXK_NUMPAD_RIGHT: + case WXK_NUMPAD_HOME: + case WXK_NUMPAD_UP: + case WXK_NUMPAD_PAGEUP: + return true; + default: + return false; + } +} + +bool isLayoutDependentKey( wxKeyEvent& event ) { + switch (event.GetKeyCode()) { + // Those keys generate Unicode key codes, but they are not keyboard layout-dependent keys + case WXK_ESCAPE: + case WXK_DELETE: + case WXK_BACK: + case WXK_TAB: + case WXK_RETURN: + case WXK_SPACE: + return false; + default: + return event.GetUnicodeKey() > 0; + } +} + void WinPortPanel::OnKeyDown( wxKeyEvent& event ) { ResetTimerIdling(); @@ -1304,9 +1348,15 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) // also it didnt cause problems yet if ( (_key_tracker.Shift() && !event.ShiftDown()) || ((_key_tracker.LeftControl() || _key_tracker.RightControl()) && !event.ControlDown())) { - if ((!_key_tracker.Alt() || _key_tracker.Shift() || g_wayland) && // workaround for #2294, 2464 - _key_tracker.CheckForSuddenModifiersUp()) { - _exclusive_hotkeys.Reset(); + + if ( +#ifndef __WXOSX__ + (!_key_tracker.Alt() || _key_tracker.Shift() || _key_tracker.LeftControl() || _key_tracker.RightControl() + || !isNumpadNumericKey(event.GetKeyCode()) || g_wayland) && // workaround for #2294, 2464 +#endif + + _key_tracker.CheckForSuddenModifiersUp()) { + _exclusive_hotkeys.Reset(); } } @@ -1343,7 +1393,7 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) if ( (dwMods != 0 && event.GetUnicodeKey() < 32) || ((dwMods & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED | LEFT_ALT_PRESSED)) #ifndef __WXOSX__ - && (g_wayland || !event.AltDown() || !event.GetUnicodeKey()) // workaround for wx issue #23421 + && (g_wayland || !event.AltDown() || !isLayoutDependentKey(event)) // workaround for wx issue #23421 #endif ) || event.GetKeyCode() == WXK_DELETE || event.GetKeyCode() == WXK_RETURN @@ -1369,6 +1419,8 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) } #endif + _enqueued_in_onchar = false; + event.Skip(); } @@ -1384,6 +1436,11 @@ void WinPortPanel::OnKeyUp( wxKeyEvent& event ) _exclusive_hotkeys.OnKeyUp(event); + if (_enqueued_in_onchar) { + _enqueued_in_onchar = false; + return; + } + if (event.GetSkipped()) { fprintf(stderr, " SKIPPED\n"); return; @@ -1446,7 +1503,7 @@ void WinPortPanel::OnKeyUp( wxKeyEvent& event ) #endif #ifndef __WXOSX__ - if (g_wayland || !event.AltDown() || !event.GetUnicodeKey()) { // workaround for wx issue #23421 + if (g_wayland || !event.AltDown() || !isLayoutDependentKey(event)) { // workaround for wx issue #23421 #else { #endif @@ -1454,9 +1511,14 @@ void WinPortPanel::OnKeyUp( wxKeyEvent& event ) } } - if ((!_key_tracker.Alt() || _key_tracker.Shift() || g_wayland) && // workaround for #2294, 2464 - _key_tracker.CheckForSuddenModifiersUp()) { - _exclusive_hotkeys.Reset(); + if ( +#ifndef __WXOSX__ + (!_key_tracker.Alt() || _key_tracker.Shift() || _key_tracker.LeftControl() || _key_tracker.RightControl() + || !isNumpadNumericKey(event.GetKeyCode()) || g_wayland) && // workaround for #2294, 2464 +#endif + + _key_tracker.CheckForSuddenModifiersUp()) { + _exclusive_hotkeys.Reset(); } //event.Skip(); } @@ -1521,7 +1583,8 @@ void WinPortPanel::OnChar( wxKeyEvent& event ) ir.Event.KeyEvent.bKeyDown = FALSE; wxConsoleInputShim::Enqueue(&ir, 1); - + + _enqueued_in_onchar = true; } //event.Skip(); } diff --git a/WinPort/src/Backend/WX/wxMain.h b/WinPort/src/Backend/WX/wxMain.h index c4b9748cc..d40e96541 100644 --- a/WinPort/src/Backend/WX/wxMain.h +++ b/WinPort/src/Backend/WX/wxMain.h @@ -97,6 +97,7 @@ class WinPortPanel: public wxPanel, protected IConsoleOutputBackend unsigned char _force_size_on_paint_state{0}; bool _extra_refresh{false}; bool _last_keydown_enqueued{false}; + bool _enqueued_in_onchar{false}; bool _app_entry_started{false}; bool _adhoc_quickedit{false}; enum From 6f235622c6dc23ee7d0d0861451d6c5b54c2bd49 Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Fri, 1 Nov 2024 12:06:52 +0100 Subject: [PATCH 02/22] fix #2482 --- WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp b/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp index a2d4532ce..f79f66469 100644 --- a/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp +++ b/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp @@ -5,6 +5,7 @@ #include "TTYInputSequenceParser.h" #include "Backend.h" +bool bracketed_paste_mode = 0; //See: // http://www.manmrk.net/tutorials/ISPF/XE/xehelp/html/HID00000579.htm @@ -420,6 +421,13 @@ size_t TTYInputSequenceParser::ParseIntoPending(const char *s, size_t l) case 0x01: case 0x02: case 0x03: case 0x04: case 0x05: case 0x06: case 0x07: case 0x08: case 0x0a: case 0x0b: case 0x0c: case 0x0e: case 0x0f: case 0x10: case 0x11: case 0x12: case 0x13: case 0x14: case 0x15: case 0x16: case 0x17: case 0x18: case 0x19: case 0x1a: + + // workaround for \x0a received instead of \x0d in kitty and wezterm in bracketed paste mode + if (bracketed_paste_mode && *s == 0x0a) { + AddPendingKeyEvent(TTYInputKey{VK_RETURN, 0}); + return 1; + } + AddPendingKeyEvent(TTYInputKey{WORD('A' + (*s - 0x01)), LEFT_CTRL_PRESSED}); return 1; @@ -661,6 +669,8 @@ void TTYInputSequenceParser::OnBracketedPaste(bool start) ir.EventType = BRACKETED_PASTE_EVENT; ir.Event.BracketedPaste.bStartPaste = start ? TRUE : FALSE; _ir_pending.emplace_back(ir); + + bracketed_paste_mode = start ? TRUE : FALSE; } //work-around for double encoded events in win32-input mode From 24bcaf4154c2b4c585b4c7f2cdeac88fbb759b39 Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Fri, 1 Nov 2024 13:47:48 +0100 Subject: [PATCH 03/22] looks like wx issue #23421 also reproduces on wayland system, so removed "do not apply on wayland" condition --- WinPort/src/Backend/WX/wxMain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index 8ae9e4a77..49e616030 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -1393,7 +1393,7 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) if ( (dwMods != 0 && event.GetUnicodeKey() < 32) || ((dwMods & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED | LEFT_ALT_PRESSED)) #ifndef __WXOSX__ - && (g_wayland || !event.AltDown() || !isLayoutDependentKey(event)) // workaround for wx issue #23421 + && (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) // workaround for wx issue #23421 #endif ) || event.GetKeyCode() == WXK_DELETE || event.GetKeyCode() == WXK_RETURN @@ -1503,7 +1503,7 @@ void WinPortPanel::OnKeyUp( wxKeyEvent& event ) #endif #ifndef __WXOSX__ - if (g_wayland || !event.AltDown() || !isLayoutDependentKey(event)) { // workaround for wx issue #23421 + if (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) { // workaround for wx issue #23421 #else { #endif From ec4999f7f2cb31520d4d4fac7351f259990f76ee Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 1 Nov 2024 22:06:11 +0400 Subject: [PATCH 04/22] fix 2479: keep directory selection after its renaming --- far2l/src/copy.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/far2l/src/copy.cpp b/far2l/src/copy.cpp index a4f21c6b1..043a90a33 100644 --- a/far2l/src/copy.cpp +++ b/far2l/src/copy.cpp @@ -2232,6 +2232,8 @@ COPY_CODES ShellCopy::ShellCopyOneFileNoRetry(const wchar_t *Src, const FAR_FIND } if (SrcData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) TreeList::AddTreeName(strDestPath); + if (Flags.MOVE && PointToName(strDestPath) == strDestPath.CPtr()) + strRenamedName = strDestPath; return COPY_SUCCESS; } From acf701941e20a644ff44a9be5df178ebdf077473 Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Sat, 2 Nov 2024 01:45:29 +0100 Subject: [PATCH 05/22] workaround for wx issue #23421 is not needed under wx 3.2.6+ also it is not working properly in wx 3.2.2- due to absence of wx commit 2c0f6a2 added required conditional compilation directives --- WinPort/src/Backend/WX/wxMain.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index 49e616030..14f2b735d 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -1392,7 +1392,8 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) if ( (dwMods != 0 && event.GetUnicodeKey() < 32) || ((dwMods & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED | LEFT_ALT_PRESSED)) -#ifndef __WXOSX__ +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && !wxCHECK_VERSION(3, 2 ,6) + && (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) // workaround for wx issue #23421 #endif ) @@ -1502,7 +1503,7 @@ void WinPortPanel::OnKeyUp( wxKeyEvent& event ) } #endif -#ifndef __WXOSX__ +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && !wxCHECK_VERSION(3, 2 ,6) if (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) { // workaround for wx issue #23421 #else { @@ -1564,6 +1565,7 @@ void WinPortPanel::OnChar( wxKeyEvent& event ) } ir.Event.KeyEvent.uChar.UnicodeChar = event.GetUnicodeKey(); +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && !wxCHECK_VERSION(3, 2 ,6) if (event.AltDown()) { // workaround for wx issue #23421 @@ -1577,6 +1579,7 @@ void WinPortPanel::OnChar( wxKeyEvent& event ) ir.Event.KeyEvent.dwControlKeyState |= LEFT_ALT_PRESSED; } +#endif ir.Event.KeyEvent.bKeyDown = TRUE; wxConsoleInputShim::Enqueue(&ir, 1); From 62c3230a70878f594df57b89f2c3b7f0857ddf6e Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Sat, 2 Nov 2024 11:41:14 +0100 Subject: [PATCH 06/22] https://github.com/wxWidgets/wxWidgets/issues/24772 still reproduces in wx master! --- WinPort/src/Backend/WX/wxMain.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index 14f2b735d..fefc3e378 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -1392,7 +1392,7 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) if ( (dwMods != 0 && event.GetUnicodeKey() < 32) || ((dwMods & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED | LEFT_ALT_PRESSED)) -#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && !wxCHECK_VERSION(3, 2 ,6) +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) // workaround for wx issue #23421 #endif @@ -1503,7 +1503,7 @@ void WinPortPanel::OnKeyUp( wxKeyEvent& event ) } #endif -#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && !wxCHECK_VERSION(3, 2 ,6) +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) if (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) { // workaround for wx issue #23421 #else { @@ -1565,7 +1565,7 @@ void WinPortPanel::OnChar( wxKeyEvent& event ) } ir.Event.KeyEvent.uChar.UnicodeChar = event.GetUnicodeKey(); -#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) && !wxCHECK_VERSION(3, 2 ,6) +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) if (event.AltDown()) { // workaround for wx issue #23421 From 8224217ea11d484bfd364ada10763b5764a3dfb2 Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Sat, 2 Nov 2024 13:41:57 +0100 Subject: [PATCH 07/22] comment --- WinPort/src/Backend/WX/wxMain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index fefc3e378..63dbd378d 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -1392,7 +1392,7 @@ void WinPortPanel::OnKeyDown( wxKeyEvent& event ) if ( (dwMods != 0 && event.GetUnicodeKey() < 32) || ((dwMods & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED | LEFT_ALT_PRESSED)) -#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) +#if !defined(__WXOSX__) && wxCHECK_VERSION(3, 2, 3) // workaround is still needed at least in wx 3.2.6, see wx issue #24772 && (/*g_wayland ||*/ !event.AltDown() || !isLayoutDependentKey(event)) // workaround for wx issue #23421 #endif From 7f89badc080deb87be8305f6525ce53cc9f99c7d Mon Sep 17 00:00:00 2001 From: elfmz Date: Sat, 2 Nov 2024 23:10:59 +0300 Subject: [PATCH 08/22] cosmetic --- WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp | 6 ++---- WinPort/src/Backend/TTY/TTYInputSequenceParser.h | 1 + WinPort/src/Backend/WX/wxMain.cpp | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp b/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp index 1e4af4f8f..4b84e405f 100644 --- a/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp +++ b/WinPort/src/Backend/TTY/TTYInputSequenceParser.cpp @@ -5,8 +5,6 @@ #include "TTYInputSequenceParser.h" #include "Backend.h" -bool bracketed_paste_mode = 0; - //See: // http://www.manmrk.net/tutorials/ISPF/XE/xehelp/html/HID00000579.htm // http://www.leonerd.org.uk/hacks/fixterms/ @@ -423,7 +421,7 @@ size_t TTYInputSequenceParser::ParseIntoPending(const char *s, size_t l) case 0x13: case 0x14: case 0x15: case 0x16: case 0x17: case 0x18: case 0x19: case 0x1a: // workaround for \x0a received instead of \x0d in kitty and wezterm in bracketed paste mode - if (bracketed_paste_mode && *s == 0x0a) { + if (_bracketed_paste_mode && *s == 0x0a) { AddPendingKeyEvent(TTYInputKey{VK_RETURN, 0}); return 1; } @@ -670,7 +668,7 @@ void TTYInputSequenceParser::OnBracketedPaste(bool start) ir.Event.BracketedPaste.bStartPaste = start ? TRUE : FALSE; _ir_pending.emplace_back(ir); - bracketed_paste_mode = start ? TRUE : FALSE; + _bracketed_paste_mode = start; } //work-around for double encoded events in win32-input mode diff --git a/WinPort/src/Backend/TTY/TTYInputSequenceParser.h b/WinPort/src/Backend/TTY/TTYInputSequenceParser.h index 07517f5d6..184390318 100644 --- a/WinPort/src/Backend/TTY/TTYInputSequenceParser.h +++ b/WinPort/src/Backend/TTY/TTYInputSequenceParser.h @@ -110,6 +110,7 @@ class TTYInputSequenceParser StackSerializer _tmp_stk_ser; DWORD _extra_control_keys = 0; std::vector _ir_pending; + bool _bracketed_paste_mode = false; bool _kitty_right_ctrl_down = false; int _iterm_last_flags = 0; char _using_extension = 0; diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index 63dbd378d..f37990fa7 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -1242,7 +1242,7 @@ char* FormatWxKeyState(uint16_t state) { return buffer; } -bool isNumpadNumericKey(int keycode) +static bool isNumpadNumericKey(int keycode) { switch (keycode) { case WXK_NUMPAD0: From d08a7aecaba7ce0e5260f6033ea1643f15a39b56 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sat, 2 Nov 2024 23:16:51 +0300 Subject: [PATCH 09/22] FindFile: avoid Stop button reaction delay due to text search in huge file(s) (touch #2485) --- far2l/src/findfile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/far2l/src/findfile.cpp b/far2l/src/findfile.cpp index 6f49ad23a..0a3153cec 100644 --- a/far2l/src/findfile.cpp +++ b/far2l/src/findfile.cpp @@ -1032,7 +1032,7 @@ static bool ScanFileByMapping(const char *Name) const void *View = smm.View(); size_t Length = smm.Length(); - for (UINT LastPercents = 0;;) { + for (UINT LastPercents = 0;!StopFlag;) { const bool FirstFragment = (FilePos == 0); const bool LastFragment = (FilePos + off_t(smm.Length()) >= FileSize); const size_t AnalyzeLength = LastFragment From 78b481cc61d73402457891ac53dbb9b5c7f9e18f Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 08:51:22 +0300 Subject: [PATCH 10/22] cosmetic (touch #2472) --- far2l/bootstrap/notify.sh | 18 ++++++++++-------- far2l/bootstrap/open.sh | 18 ++++++++++-------- far2l/bootstrap/ps.sh | 6 ++++-- far2l/bootstrap/trash.sh | 12 +++++++----- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/far2l/bootstrap/notify.sh b/far2l/bootstrap/notify.sh index f81c8b282..e7f69acd1 100755 --- a/far2l/bootstrap/notify.sh +++ b/far2l/bootstrap/notify.sh @@ -1,13 +1,15 @@ #!/bin/sh -############################################################ -#This script used by FAR to display desktop notifcations -############################################################ -#$1= action -#$2= object -############################################################ -#For per user customization - create: +################################################################ +# This script used by FAR to display desktop notifcations +################################################################ +# $1= action +# $2= object +################################################################ +# For per user customization - create: #~/.config/far2l/notify.sh -############################################################ +# note that per-user script must do 'exit 0' at the end if +# no need to continue default implementation of main notify.sh +################################################################ action=$1 object=$2 diff --git a/far2l/bootstrap/open.sh b/far2l/bootstrap/open.sh index 89489814f..403ab2770 100755 --- a/far2l/bootstrap/open.sh +++ b/far2l/bootstrap/open.sh @@ -1,16 +1,18 @@ #!/bin/sh -############################################################ -#This script used by FAR to open files by other applications -############################################################ -#$1= [exec|dir|other] where: +################################################################ +# This script used by FAR to open files by other applications +################################################################ +# $1= [exec|dir|other] where: # exec - execute given command in other terminal # dir - open given directory with GUI # other - open given file with GUI -#Other arguments - actual command/file to be executed/opened -############################################################ -#For per user customization - create: +# Other arguments - actual command/file to be executed/opened +################################################################ +# For per user customization - create: #~/.config/far2l/open.sh -############################################################ +# note that per-user script must do 'exit 0' at the end if +# no need to continue default implementation of main open.sh +################################################################ what=$1 shift diff --git a/far2l/bootstrap/ps.sh b/far2l/bootstrap/ps.sh index 6aeb76260..84dc968f2 100755 --- a/far2l/bootstrap/ps.sh +++ b/far2l/bootstrap/ps.sh @@ -1,10 +1,12 @@ #!/bin/sh ############################################################ -#This script used by FAR to show processes list +# This script used by FAR to show processes list ############################################################ ############################################################ -#For per user customization - create: +# For per user customization - create: #~/.config/far2l/ps.sh +# note that per-user script must do 'exit 0' at the end if +# no need to continue default implementation in main ps.sh ############################################################ if [ -x ~/.config/far2l/ps.sh ]; then diff --git a/far2l/bootstrap/trash.sh b/far2l/bootstrap/trash.sh index e2af561f4..6e67ce2b2 100755 --- a/far2l/bootstrap/trash.sh +++ b/far2l/bootstrap/trash.sh @@ -1,10 +1,12 @@ #!/bin/sh -########################################################## -#This script used by FAR to move files to Trash -########################################################## -#For per user customization - create: +################################################################ +# This script used by FAR to move files to Trash +################################################################ +# For per user customization - create: #~/.config/far2l/trash.sh -########################################################## +# note that per-user script must do 'exit 0' at the end if +# no need to continue default implementation of main trash.sh +################################################################ set -e From beb3cdf25eae16f423daa3c355a53ca5c70f0556 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 10:44:04 +0300 Subject: [PATCH 11/22] VT: reset special terminal modes to avoid messing up of terminal if application that used them exited abnormally --- far2l/src/vt/vtshell.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/far2l/src/vt/vtshell.cpp b/far2l/src/vt/vtshell.cpp index aaaf00a6b..35d7ba21c 100644 --- a/far2l/src/vt/vtshell.cpp +++ b/far2l/src/vt/vtshell.cpp @@ -1076,10 +1076,16 @@ class VTShell : VTOutputReader::IProcessor, VTInputReader::IProcessor, IVTShell } std::lock_guard lock(_read_state_mutex); + // reset special terminal modes to avoid messing up of + // terminal if application that used them exited abnormally + _bracketed_paste_expected = false; + _win32_input_mode_expected = false; + _kitty_kb_flags = 0; + _mouse_expectations = 0; _far2l_exts.reset(); - _host_id.clear(); _mouse.reset(); - _mouse_expectations = 0; + // cleanup also NetRocks per-session identifier + _host_id.clear(); return true; } From c1e7ff38a6cb9f8205d4a2fc3935c01f34170be0 Mon Sep 17 00:00:00 2001 From: Aleksey Dobrunov Date: Sun, 3 Nov 2024 14:25:17 +0500 Subject: [PATCH 12/22] update colorer_lib to 1.4.1 --- .../src/colorer/common/Logger.h | 23 ++++++++++++------- .../src/colorer/strings/icu/Encodings.cpp | 8 +++++-- .../src/colorer/strings/icu/Encodings.h | 5 ++-- .../src/colorer/strings/icu/UStr.cpp | 5 ++++ .../src/colorer/strings/icu/UStr.h | 1 + .../src/colorer/strings/legacy/Encodings.cpp | 8 ++++++- .../src/colorer/strings/legacy/Encodings.h | 1 + .../src/colorer/strings/legacy/UStr.cpp | 10 +++++--- .../src/colorer/strings/legacy/UStr.h | 1 + .../src/colorer/utils/Environment.cpp | 13 ++++++----- .../src/Colorer-library/src/colorer/version.h | 2 +- .../src/colorer/xml/libxml2/LibXmlReader.cpp | 18 ++++++++++++--- .../xml/libxml2/SharedXmlInputSource.cpp | 2 +- colorer/src/pcolorer2/CerrLogger.h | 2 ++ 14 files changed, 72 insertions(+), 27 deletions(-) diff --git a/colorer/src/Colorer-library/src/colorer/common/Logger.h b/colorer/src/Colorer-library/src/colorer/common/Logger.h index 28d19034c..ae444ef1a 100644 --- a/colorer/src/Colorer-library/src/colorer/common/Logger.h +++ b/colorer/src/Colorer-library/src/colorer/common/Logger.h @@ -55,8 +55,8 @@ void format_log_string(std::ostream& out, std::string_view format, Args&&... arg #include "unicode/unistr.h" namespace details { -template <> inline -void ArgumentT::print(std::ostream& out) const +template <> +inline void ArgumentT::print(std::ostream& out) const { std::string result8; mData.toUTF8String(result8); @@ -66,13 +66,13 @@ void ArgumentT::print(std::ostream& out) const #else #include "colorer/strings/legacy/strings.h" namespace details { -template <> inline -void details::ArgumentT::print(std::ostream& out) const +template <> +inline void details::ArgumentT::print(std::ostream& out) const { std::string const result8 = mData.getChars(); out << result8; } -} +} // namespace details #endif class Logger @@ -83,13 +83,20 @@ class Logger virtual ~Logger() = default; virtual void log(LogLevel level, const char* filename_in, int line_in, const char* funcname_in, const char* message) = 0; + virtual void flush() = 0; }; class Log { public: - static void registerLogger(Logger& logger_) { logger = &logger_; } + static void removeLogger() { logger = nullptr; } + static void flush() + { + if (logger) { + logger->flush(); + } + } template static void log(const Logger::LogLevel level, const char* filename_in, const int line_in, const char* funcname_in, @@ -106,7 +113,8 @@ class Log static Logger* logger; }; -#define COLORER_LOGGER_PRINTF(level, ...) Log::log(level, __FILE__, __LINE__, static_cast(__FUNCTION__), __VA_ARGS__) +#define COLORER_LOGGER_PRINTF(level, ...) \ + Log::log(level, __FILE__, __LINE__, static_cast(__FUNCTION__), __VA_ARGS__) #define COLORER_LOG_ERROR(...) COLORER_LOGGER_PRINTF(Logger::LogLevel::LOG_ERROR, __VA_ARGS__) #define COLORER_LOG_WARN(...) COLORER_LOGGER_PRINTF(Logger::LogLevel::LOG_WARN, __VA_ARGS__) #define COLORER_LOG_INFO(...) COLORER_LOGGER_PRINTF(Logger::LogLevel::LOG_INFO, __VA_ARGS__) @@ -119,5 +127,4 @@ class Log #define COLORER_LOG_DEEPTRACE(...) #endif - #endif // COLORER_LOGGER_H diff --git a/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.cpp b/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.cpp index ae2ec7fb0..340d5b73f 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.cpp +++ b/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.cpp @@ -12,8 +12,7 @@ uUnicodeString Encodings::toUnicodeString(char* data, int32_t len) int32_t signatureLength; encoding = ucnv_detectUnicodeSignature(data, len, &signatureLength, &status); if (U_FAILURE(status)) { - COLORER_LOG_ERROR("Encodings: Error \"%\" from ucnv_detectUnicodeSignature()\n", - u_errorName(status)); + COLORER_LOG_ERROR("Encodings: Error \"%\" from ucnv_detectUnicodeSignature()\n", u_errorName(status)); throw Exception("Error from ucnv_detectUnicodeSignature"); } if (encoding == nullptr) { @@ -23,6 +22,11 @@ uUnicodeString Encodings::toUnicodeString(char* data, int32_t len) return std::make_unique(data + signatureLength, len - signatureLength, encoding); } +uUnicodeString Encodings::fromUTF8(char* data, int32_t len) +{ + return std::make_unique(data , len , ENC_UTF8); +} + int Encodings::toUTF8Bytes(UChar wc, byte* dest) { int32_t len = 0; diff --git a/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.h b/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.h index ef16de6aa..b79abc4db 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.h +++ b/colorer/src/Colorer-library/src/colorer/strings/icu/Encodings.h @@ -3,12 +3,13 @@ #include "colorer/strings/icu/common_icu.h" -constexpr char ENC_UTF8[] = "UTF-8"; - class Encodings { public: + static constexpr char ENC_UTF8[] = "UTF-8"; + static uUnicodeString toUnicodeString(char* data, int32_t len); + static uUnicodeString fromUTF8(char* data, int32_t len); static int toUTF8Bytes(UChar, byte*); }; diff --git a/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.cpp b/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.cpp index ebc001856..12a900135 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.cpp +++ b/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.cpp @@ -6,6 +6,11 @@ UnicodeString UStr::to_unistr(const int number) return {std::to_string(number).c_str()}; } +UnicodeString UStr::to_unistr(const std::string& str) +{ + return {str.c_str(), static_cast(str.length()), Encodings::ENC_UTF8}; +} + std::string UStr::to_stdstr(const UnicodeString* str) { std::string out_str; diff --git a/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.h b/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.h index c8f60d9f6..115b3a177 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.h +++ b/colorer/src/Colorer-library/src/colorer/strings/icu/UStr.h @@ -8,6 +8,7 @@ class UStr { public: [[nodiscard]] static UnicodeString to_unistr(int number); + [[nodiscard]] static UnicodeString to_unistr(const std::string& str); [[nodiscard]] static std::string to_stdstr(const UnicodeString* str); [[nodiscard]] static std::string to_stdstr(const uUnicodeString& str); #ifdef _WINDOWS diff --git a/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.cpp b/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.cpp index b30505532..23d2bc1ff 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.cpp +++ b/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.cpp @@ -168,7 +168,13 @@ wchar Encodings::toWChar(int eidx, char c) uUnicodeString Encodings::toUnicodeString(char* data, int32_t len) { - return std::make_unique(data , len , -2); + // TODO change ENC_UTF8 to detect + return std::make_unique(data, len, Encodings::ENC_UTF8); +} + +uUnicodeString Encodings::fromUTF8(char* data, int32_t len) +{ + return std::make_unique(data, len, Encodings::ENC_UTF8); } diff --git a/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.h b/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.h index bfb3251c1..ca9c0d99e 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.h +++ b/colorer/src/Colorer-library/src/colorer/strings/legacy/Encodings.h @@ -45,6 +45,7 @@ class Encodings static const char* getEncodingName(int enc); static uUnicodeString toUnicodeString(char* data, int32_t len); + static uUnicodeString fromUTF8(char* data, int32_t len); }; #endif diff --git a/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.cpp b/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.cpp index 927682a78..0f204e04c 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.cpp +++ b/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.cpp @@ -14,7 +14,7 @@ std::string UStr::to_stdstr(const uUnicodeString& str) std::wstring UStr::to_stdwstr(const UnicodeString* str) { - std::wstring out_string (str->getWChars()); + std::wstring out_string(str->getWChars()); return out_string; } @@ -23,8 +23,7 @@ std::wstring UStr::to_stdwstr(const UnicodeString& str) return to_stdwstr(&str); } -std::unique_ptr UStr::createCharClass(const UnicodeString& ccs, int pos, - int* retPos, bool ignore_case) +std::unique_ptr UStr::createCharClass(const UnicodeString& ccs, int pos, int* retPos, bool ignore_case) { return CharacterClass::createCharClass(ccs, pos, retPos, ignore_case); } @@ -39,6 +38,11 @@ UnicodeString UStr::to_unistr(int number) return {number}; } +UnicodeString UStr::to_unistr(const std::string& str) +{ + return {str.c_str(), static_cast(str.length()), Encodings::ENC_UTF8}; +} + bool UStr::HexToUInt(const UnicodeString& str_hex, unsigned int* result) { UnicodeString s; diff --git a/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.h b/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.h index 817a2a3f7..8a1a61e4c 100644 --- a/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.h +++ b/colorer/src/Colorer-library/src/colorer/strings/legacy/UStr.h @@ -10,6 +10,7 @@ class UStr public: [[nodiscard]] static UnicodeString to_unistr(int number); + [[nodiscard]] static UnicodeString to_unistr(const std::string& str); [[nodiscard]] static std::string to_stdstr(const UnicodeString* str); [[nodiscard]] static std::string to_stdstr(const uUnicodeString& str); [[nodiscard]] static std::wstring to_stdwstr(const UnicodeString* str); diff --git a/colorer/src/Colorer-library/src/colorer/utils/Environment.cpp b/colorer/src/Colorer-library/src/colorer/utils/Environment.cpp index 7b04c4ed1..a1121124c 100644 --- a/colorer/src/Colorer-library/src/colorer/utils/Environment.cpp +++ b/colorer/src/Colorer-library/src/colorer/utils/Environment.cpp @@ -127,7 +127,7 @@ bool Environment::isRegularFile(const UnicodeString* basePath, const UnicodeStri bool Environment::isRegularFile(const UnicodeString& path) { - return fs::is_regular_file(UStr::to_stdstr(&path)); + return fs::is_regular_file(to_filepath(&path)); } UnicodeString Environment::getAbsolutePath(const UnicodeString& basePath, const UnicodeString& relPath) @@ -156,7 +156,7 @@ UnicodeString Environment::expandSpecialEnvironment(const UnicodeString& path) auto result = expandEnvByRegexp(text, std::regex(R"--(\$([[:alpha:]]\w*)\b)--")); COLORER_LOG_DEBUG("result of expand '%'", result); - return {result.c_str()}; + return UStr::to_unistr(result); } UnicodeString Environment::expandEnvironment(const UnicodeString& path) @@ -172,20 +172,21 @@ UnicodeString Environment::expandEnvironment(const UnicodeString& path) size_t i = ExpandEnvironmentStringsW(path_ws.c_str(), nullptr, 0); auto temp = std::make_unique(i); ExpandEnvironmentStringsW(path_ws.c_str(), temp.get(), static_cast(i)); - COLORER_LOG_DEBUG("result of expand '%'", temp.get()); - return {temp.get()}; + UnicodeString result(temp.get()); + COLORER_LOG_DEBUG("result of expand '%'", result); + return result; #else const auto text = UStr::to_stdstr(&path); auto res = expandEnvByRegexp(text, std::regex(R"--(\$\{([[:alpha:]]\w*)\})--")); res = expandEnvByRegexp(res, std::regex(R"--(\$([[:alpha:]]\w*)\b)--")); COLORER_LOG_DEBUG("result of expand '%'", res); - return {res.c_str()}; + return UStr::to_unistr(res); #endif } uintmax_t Environment::getFileSize(const UnicodeString& path) { - return fs::file_size(UStr::to_stdstr(&path)); + return fs::file_size( to_filepath(&path)); } std::string Environment::expandEnvByRegexp(const std::string& path, const std::regex& regex) diff --git a/colorer/src/Colorer-library/src/colorer/version.h b/colorer/src/Colorer-library/src/colorer/version.h index 4cd8aa527..732a716da 100644 --- a/colorer/src/Colorer-library/src/colorer/version.h +++ b/colorer/src/Colorer-library/src/colorer/version.h @@ -3,7 +3,7 @@ #define COLORER_VER_MAJOR 1 #define COLORER_VER_MINOR 4 -#define COLORER_VER_PATCH 0 +#define COLORER_VER_PATCH 1 #define COLORER_COPYRIGHT "(c) 1999-2009 Igor Russkih, (c) 2009-2024 Aleksey Dobrunov" diff --git a/colorer/src/Colorer-library/src/colorer/xml/libxml2/LibXmlReader.cpp b/colorer/src/Colorer-library/src/colorer/xml/libxml2/LibXmlReader.cpp index 1c287e5f3..ca54463e6 100644 --- a/colorer/src/Colorer-library/src/colorer/xml/libxml2/LibXmlReader.cpp +++ b/colorer/src/Colorer-library/src/colorer/xml/libxml2/LibXmlReader.cpp @@ -138,13 +138,15 @@ xmlParserInputPtr LibXmlReader::xmlMyExternalEntityLoader(const char* URL, const /* * The function is called before each opening of a file within libxml, whether it is an xmlReadFile, * or opening a file for an external entity. - * I.e., the path to the file can be checked or modified here. If the external entity specifies a path similar + * I.e., the path to the file can be checked or modified here. If the external entity specifies a path similar * to the file path, then libxml itself forms the full path to the entity file by gluing the path from the current - * file and the one specified in the external entity. + * file and the one specified in the external entity. But sometimes it fails, for example, on non-Latin letters in + * the path to the main file. * At the same time, there is no path to the source file or to the file in the entity in the function parameters. */ - UnicodeString string_url(URL); + auto filename = Encodings::fromUTF8(const_cast(URL), static_cast(strlen(URL))); + UnicodeString string_url(*filename.get()); // read entity string like "env:$FAR_HOME/hrd/catalog-console.xml" static const UnicodeString env(u"env:"); @@ -152,6 +154,7 @@ xmlParserInputPtr LibXmlReader::xmlMyExternalEntityLoader(const char* URL, const const auto exp = colorer::Environment::expandSpecialEnvironment(string_url); string_url = UnicodeString(exp, env.length()); } + #ifdef COLORER_FEATURE_ZIPINPUTSOURCE if (string_url.startsWith(jar) || current_file->startsWith(jar)) { const auto paths = LibXmlInputSource::getFullPathsToZip(string_url, is_first_call ? nullptr : current_file.get()); @@ -164,6 +167,15 @@ xmlParserInputPtr LibXmlReader::xmlMyExternalEntityLoader(const char* URL, const return ret; } #endif + // We check if the file exists after all the conversions. If not, then we check the file relative to the one being processed. + // This is relevant for the case of non-Latin letters in the path to the main file and entity on linux. There is no such problem on windows. + if (!is_first_call && !colorer::Environment::isRegularFile(string_url)) { + auto new_string_url = colorer::Environment::getAbsolutePath(*current_file.get(), string_url); + if (colorer::Environment::isRegularFile(new_string_url)) { + string_url = std::move(new_string_url); + } + } + is_first_call = false; // read it as a regular file xmlParserInputPtr ret = xmlNewInputFromFile(ctxt, UStr::to_stdstr(&string_url).c_str()); diff --git a/colorer/src/Colorer-library/src/colorer/xml/libxml2/SharedXmlInputSource.cpp b/colorer/src/Colorer-library/src/colorer/xml/libxml2/SharedXmlInputSource.cpp index 68f6f602c..da029ee26 100644 --- a/colorer/src/Colorer-library/src/colorer/xml/libxml2/SharedXmlInputSource.cpp +++ b/colorer/src/Colorer-library/src/colorer/xml/libxml2/SharedXmlInputSource.cpp @@ -67,7 +67,7 @@ byte* SharedXmlInputSource::getSrc() const void SharedXmlInputSource::open() { if (!is_open) { - std::ifstream f(UStr::to_stdstr(&source_path), std::ios::in | std::ios::binary); + std::ifstream f(colorer::Environment::to_filepath(&source_path), std::ios::in | std::ios::binary); if (!f.is_open()) { COLORER_LOG_ERROR("failed to open %", source_path); throw InputSourceException("failed to open " + source_path); diff --git a/colorer/src/pcolorer2/CerrLogger.h b/colorer/src/pcolorer2/CerrLogger.h index 7ab9f96d3..bb8783f3f 100644 --- a/colorer/src/pcolorer2/CerrLogger.h +++ b/colorer/src/pcolorer2/CerrLogger.h @@ -49,6 +49,8 @@ class CerrLogger : public Logger return Logger::LOG_OFF; } + void flush() override {}; + private: Logger::LogLevel current_level = Logger::LOG_OFF; }; From acbbc5713f638b641cd998fa2af6a311c079ca80 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 14:04:19 +0300 Subject: [PATCH 13/22] ma: updated bundled unrar sources to 7.0.9 --- multiarc/src/formats/rar/unrar/arccmt.cpp | 65 +- multiarc/src/formats/rar/unrar/archive.cpp | 33 +- multiarc/src/formats/rar/unrar/archive.hpp | 41 +- multiarc/src/formats/rar/unrar/arcread.cpp | 306 +++--- multiarc/src/formats/rar/unrar/array.hpp | 2 +- multiarc/src/formats/rar/unrar/blake2s.cpp | 32 +- .../src/formats/rar/unrar/blake2s_sse.cpp | 30 +- multiarc/src/formats/rar/unrar/blake2sp.cpp | 4 +- multiarc/src/formats/rar/unrar/cmddata.cpp | 380 ++++--- multiarc/src/formats/rar/unrar/cmddata.hpp | 38 +- multiarc/src/formats/rar/unrar/cmdfilter.cpp | 82 +- multiarc/src/formats/rar/unrar/cmdmix.cpp | 2 +- multiarc/src/formats/rar/unrar/coder.cpp | 8 +- multiarc/src/formats/rar/unrar/coder.hpp | 2 +- multiarc/src/formats/rar/unrar/compress.hpp | 15 +- multiarc/src/formats/rar/unrar/consio.cpp | 162 ++- multiarc/src/formats/rar/unrar/consio.hpp | 8 +- multiarc/src/formats/rar/unrar/crc.cpp | 187 ++-- multiarc/src/formats/rar/unrar/crc.hpp | 4 - multiarc/src/formats/rar/unrar/crypt.cpp | 8 +- multiarc/src/formats/rar/unrar/crypt.hpp | 3 +- multiarc/src/formats/rar/unrar/crypt1.cpp | 17 +- multiarc/src/formats/rar/unrar/crypt5.cpp | 6 +- multiarc/src/formats/rar/unrar/dll.cpp | 86 +- multiarc/src/formats/rar/unrar/dll.hpp | 11 +- multiarc/src/formats/rar/unrar/dll.rc | 28 - multiarc/src/formats/rar/unrar/encname.cpp | 22 +- multiarc/src/formats/rar/unrar/encname.hpp | 6 +- multiarc/src/formats/rar/unrar/errhnd.cpp | 109 ++- multiarc/src/formats/rar/unrar/errhnd.hpp | 38 +- multiarc/src/formats/rar/unrar/extinfo.cpp | 145 +-- multiarc/src/formats/rar/unrar/extinfo.hpp | 17 +- multiarc/src/formats/rar/unrar/extract.cpp | 563 ++++++----- multiarc/src/formats/rar/unrar/extract.hpp | 33 +- multiarc/src/formats/rar/unrar/filcreat.cpp | 83 +- multiarc/src/formats/rar/unrar/filcreat.hpp | 6 +- multiarc/src/formats/rar/unrar/file.cpp | 87 +- multiarc/src/formats/rar/unrar/file.hpp | 34 +- multiarc/src/formats/rar/unrar/filefn.cpp | 340 ++++--- multiarc/src/formats/rar/unrar/filefn.hpp | 41 +- multiarc/src/formats/rar/unrar/filestr.cpp | 57 +- multiarc/src/formats/rar/unrar/filestr.hpp | 2 +- multiarc/src/formats/rar/unrar/find.cpp | 70 +- multiarc/src/formats/rar/unrar/find.hpp | 10 +- multiarc/src/formats/rar/unrar/getbits.cpp | 14 +- multiarc/src/formats/rar/unrar/getbits.hpp | 23 +- multiarc/src/formats/rar/unrar/hardlinks.cpp | 12 +- multiarc/src/formats/rar/unrar/hash.cpp | 152 ++- multiarc/src/formats/rar/unrar/hash.hpp | 19 +- multiarc/src/formats/rar/unrar/headers.cpp | 4 +- multiarc/src/formats/rar/unrar/headers.hpp | 37 +- multiarc/src/formats/rar/unrar/headers5.hpp | 35 +- multiarc/src/formats/rar/unrar/list.cpp | 111 +-- multiarc/src/formats/rar/unrar/loclang.hpp | 37 +- multiarc/src/formats/rar/unrar/log.cpp | 25 +- multiarc/src/formats/rar/unrar/log.hpp | 3 +- multiarc/src/formats/rar/unrar/makefile | 103 +- multiarc/src/formats/rar/unrar/match.cpp | 90 +- multiarc/src/formats/rar/unrar/match.hpp | 7 +- multiarc/src/formats/rar/unrar/model.cpp | 8 +- multiarc/src/formats/rar/unrar/model.hpp | 2 +- multiarc/src/formats/rar/unrar/options.cpp | 1 + multiarc/src/formats/rar/unrar/options.hpp | 30 +- multiarc/src/formats/rar/unrar/os.hpp | 104 +- multiarc/src/formats/rar/unrar/pathfn.cpp | 924 ++++++++++-------- multiarc/src/formats/rar/unrar/pathfn.hpp | 108 +- multiarc/src/formats/rar/unrar/qopen.cpp | 12 +- multiarc/src/formats/rar/unrar/qopen.hpp | 2 +- multiarc/src/formats/rar/unrar/rar.cpp | 17 +- multiarc/src/formats/rar/unrar/rar.hpp | 4 - multiarc/src/formats/rar/unrar/rardefs.hpp | 15 + multiarc/src/formats/rar/unrar/raros.hpp | 11 +- multiarc/src/formats/rar/unrar/rartypes.hpp | 2 +- multiarc/src/formats/rar/unrar/rarvm.cpp | 2 +- multiarc/src/formats/rar/unrar/rawint.hpp | 71 +- multiarc/src/formats/rar/unrar/rawread.cpp | 15 +- multiarc/src/formats/rar/unrar/rawread.hpp | 4 +- multiarc/src/formats/rar/unrar/rdwrfn.cpp | 32 +- multiarc/src/formats/rar/unrar/rdwrfn.hpp | 7 +- multiarc/src/formats/rar/unrar/recvol.cpp | 44 +- multiarc/src/formats/rar/unrar/recvol.hpp | 20 +- multiarc/src/formats/rar/unrar/recvol3.cpp | 128 +-- multiarc/src/formats/rar/unrar/recvol5.cpp | 92 +- multiarc/src/formats/rar/unrar/resource.cpp | 9 - multiarc/src/formats/rar/unrar/resource.hpp | 2 - multiarc/src/formats/rar/unrar/rijndael.cpp | 47 +- multiarc/src/formats/rar/unrar/rijndael.hpp | 17 +- multiarc/src/formats/rar/unrar/rs16.cpp | 16 +- multiarc/src/formats/rar/unrar/rs16.hpp | 3 + multiarc/src/formats/rar/unrar/scantree.cpp | 169 ++-- multiarc/src/formats/rar/unrar/scantree.hpp | 23 +- .../src/formats/rar/unrar/secpassword.cpp | 20 +- .../src/formats/rar/unrar/secpassword.hpp | 2 + multiarc/src/formats/rar/unrar/sha256.cpp | 14 +- multiarc/src/formats/rar/unrar/sha256.hpp | 1 + multiarc/src/formats/rar/unrar/strfn.cpp | 276 ++++-- multiarc/src/formats/rar/unrar/strfn.hpp | 29 +- multiarc/src/formats/rar/unrar/strlist.cpp | 62 +- multiarc/src/formats/rar/unrar/strlist.hpp | 13 +- multiarc/src/formats/rar/unrar/system.cpp | 25 +- multiarc/src/formats/rar/unrar/system.hpp | 3 +- multiarc/src/formats/rar/unrar/timefn.cpp | 22 +- multiarc/src/formats/rar/unrar/timefn.hpp | 4 +- multiarc/src/formats/rar/unrar/ui.hpp | 58 +- multiarc/src/formats/rar/unrar/uicommon.cpp | 58 +- multiarc/src/formats/rar/unrar/uiconsole.cpp | 65 +- multiarc/src/formats/rar/unrar/uisilent.cpp | 21 +- multiarc/src/formats/rar/unrar/ulinks.cpp | 64 +- multiarc/src/formats/rar/unrar/unicode.cpp | 272 +++++- multiarc/src/formats/rar/unrar/unicode.hpp | 16 +- multiarc/src/formats/rar/unrar/unpack.cpp | 132 +-- multiarc/src/formats/rar/unrar/unpack.hpp | 79 +- multiarc/src/formats/rar/unrar/unpack15.cpp | 107 +- multiarc/src/formats/rar/unrar/unpack20.cpp | 8 +- multiarc/src/formats/rar/unrar/unpack30.cpp | 65 +- multiarc/src/formats/rar/unrar/unpack50.cpp | 119 ++- .../src/formats/rar/unrar/unpack50frag.cpp | 30 +- multiarc/src/formats/rar/unrar/unpack50mt.cpp | 68 +- .../src/formats/rar/unrar/unpackinline.cpp | 59 +- multiarc/src/formats/rar/unrar/uowners.cpp | 87 +- multiarc/src/formats/rar/unrar/version.hpp | 10 +- multiarc/src/formats/rar/unrar/volume.cpp | 89 +- multiarc/src/formats/rar/unrar/win32acl.cpp | 18 +- multiarc/src/formats/rar/unrar/win32lnk.cpp | 54 +- multiarc/src/formats/rar/unrar/win32stm.cpp | 115 ++- 125 files changed, 4551 insertions(+), 3350 deletions(-) delete mode 100644 multiarc/src/formats/rar/unrar/dll.rc diff --git a/multiarc/src/formats/rar/unrar/arccmt.cpp b/multiarc/src/formats/rar/unrar/arccmt.cpp index 8b7e498f4..f23a7dd70 100644 --- a/multiarc/src/formats/rar/unrar/arccmt.cpp +++ b/multiarc/src/formats/rar/unrar/arccmt.cpp @@ -1,6 +1,6 @@ static bool IsAnsiEscComment(const wchar *Data,size_t Size); -bool Archive::GetComment(Array *CmtData) +bool Archive::GetComment(std::wstring &CmtData) { if (!MainComment) return false; @@ -11,7 +11,7 @@ bool Archive::GetComment(Array *CmtData) } -bool Archive::DoGetComment(Array *CmtData) +bool Archive::DoGetComment(std::wstring &CmtData) { #ifndef SFX_MODULE uint CmtLength; @@ -101,10 +101,8 @@ bool Archive::DoGetComment(Array *CmtData) // 4x memory for OEM to UTF-8 output here. OemToCharBuffA((char *)UnpData,(char *)UnpData,(DWORD)UnpDataSize); #endif - CmtData->Alloc(UnpDataSize+1); - memset(CmtData->Addr(0),0,CmtData->Size()*sizeof(wchar)); - CharToWide((char *)UnpData,CmtData->Addr(0),CmtData->Size()); - CmtData->Alloc(wcslen(CmtData->Addr(0))); + std::string UnpStr((char*)UnpData,UnpDataSize); + CharToWide(UnpStr,CmtData); } } } @@ -112,12 +110,12 @@ bool Archive::DoGetComment(Array *CmtData) { if (CmtLength==0) return false; - Array CmtRaw(CmtLength); - int ReadSize=Read(&CmtRaw[0],CmtLength); + std::vector CmtRaw(CmtLength); + int ReadSize=Read(CmtRaw.data(),CmtLength); if (ReadSize>=0 && (uint)ReadSize *CmtData) uiMsg(UIERROR_CMTBROKEN,FileName); return false; } - CmtData->Alloc(CmtLength+1); - CmtRaw.Push(0); +// CmtData.resize(CmtLength+1); + CmtRaw.push_back(0); #ifdef _WIN_ALL // If we ever decide to extend it to Android, we'll need to alloc // 4x memory for OEM to UTF-8 output here. - OemToCharA((char *)&CmtRaw[0],(char *)&CmtRaw[0]); + OemToCharA((char *)CmtRaw.data(),(char *)CmtRaw.data()); #endif - CharToWide((char *)&CmtRaw[0],CmtData->Addr(0),CmtData->Size()); - CmtData->Alloc(wcslen(CmtData->Addr(0))); + CharToWide((const char *)CmtRaw.data(),CmtData); +// CmtData->resize(wcslen(CmtData->data())); } #endif - return CmtData->Size() > 0; + return CmtData.size() > 0; } -bool Archive::ReadCommentData(Array *CmtData) +bool Archive::ReadCommentData(std::wstring &CmtData) { - Array CmtRaw; + std::vector CmtRaw; if (!ReadSubData(&CmtRaw,NULL,false)) return false; - size_t CmtSize=CmtRaw.Size(); - CmtRaw.Push(0); - CmtData->Alloc(CmtSize+1); + size_t CmtSize=CmtRaw.size(); + CmtRaw.push_back(0); +// CmtData->resize(CmtSize+1); if (Format==RARFMT50) - UtfToWide((char *)&CmtRaw[0],CmtData->Addr(0),CmtData->Size()); + UtfToWide((char *)CmtRaw.data(),CmtData); else if ((SubHead.SubFlags & SUBHEAD_FLAGS_CMT_UNICODE)!=0) { - RawToWide(&CmtRaw[0],CmtData->Addr(0),CmtSize/2); - (*CmtData)[CmtSize/2]=0; - + CmtData=RawToWide(CmtRaw); } else { - CharToWide((char *)&CmtRaw[0],CmtData->Addr(0),CmtData->Size()); + CharToWide((const char *)CmtRaw.data(),CmtData); } - CmtData->Alloc(wcslen(CmtData->Addr(0))); // Set buffer size to actual comment length. +// CmtData->resize(wcslen(CmtData->data())); // Set buffer size to actual comment length. return true; } @@ -170,15 +166,16 @@ void Archive::ViewComment() { if (Cmd->DisableComment) return; - Array CmtBuf; - if (GetComment(&CmtBuf)) // In GUI too, so "Test" command detects broken comments. + std::wstring CmtBuf; + if (GetComment(CmtBuf)) // In GUI too, so "Test" command detects broken comments. { - size_t CmtSize=CmtBuf.Size(); - wchar *ChPtr=wcschr(&CmtBuf[0],0x1A); - if (ChPtr!=NULL) - CmtSize=ChPtr-&CmtBuf[0]; - mprintf(L"\n"); - OutComment(&CmtBuf[0],CmtSize); + size_t CmtSize=CmtBuf.size(); + auto EndPos=CmtBuf.find(0x1A); + if (EndPos!=std::wstring::npos) + CmtSize=EndPos; + mprintf(St(MArcComment)); + mprintf(L":\n"); + OutComment(CmtBuf); } } diff --git a/multiarc/src/formats/rar/unrar/archive.cpp b/multiarc/src/formats/rar/unrar/archive.cpp index 25f0c3b79..c2293de7c 100644 --- a/multiarc/src/formats/rar/unrar/archive.cpp +++ b/multiarc/src/formats/rar/unrar/archive.cpp @@ -39,7 +39,6 @@ Archive::Archive(CommandData *InitCmd) VolWrite=0; AddingFilesSize=0; AddingHeadersSize=0; - *FirstVolumeName=0; Splitting=false; NewArchive=false; @@ -74,7 +73,7 @@ void Archive::CheckArc(bool EnableBroken) #if !defined(SFX_MODULE) -void Archive::CheckOpen(const wchar *Name) +void Archive::CheckOpen(const std::wstring &Name) { TOpen(Name); CheckArc(false); @@ -82,7 +81,7 @@ void Archive::CheckOpen(const wchar *Name) #endif -bool Archive::WCheckOpen(const wchar *Name) +bool Archive::WCheckOpen(const std::wstring &Name) { if (!WOpen(Name)) return false; @@ -148,9 +147,9 @@ bool Archive::IsArchive(bool EnableBroken) } else { - Array Buffer(MAXSFXSIZE); + std::vector Buffer(MAXSFXSIZE); long CurPos=(long)Tell(); - int ReadSize=Read(&Buffer[0],Buffer.Size()-16); + int ReadSize=Read(Buffer.data(),Buffer.size()-16); for (int I=0;I0x10000000000ULL) + return 0; + uint64 Pow2=0x20000; // Power of 2 dictionary size. + for (;2*Pow2<=Size;Pow2*=2) + Flags+=FCI_DICT_BIT0; + if (Size==Pow2) + return Size; // If 'Size' is the power of 2, return it as is. + + // Get the number of Pow2/32 to add to Pow2 for nearest value not exceeding 'Size'. + uint64 Fraction=(Size-Pow2)/(Pow2/32); + Flags+=(uint)Fraction*FCI_DICT_FRACT0; + return Pow2+Fraction*(Pow2/32); +} diff --git a/multiarc/src/formats/rar/unrar/archive.hpp b/multiarc/src/formats/rar/unrar/archive.hpp index 35a399bca..58bdafaa1 100644 --- a/multiarc/src/formats/rar/unrar/archive.hpp +++ b/multiarc/src/formats/rar/unrar/archive.hpp @@ -27,7 +27,7 @@ class Archive:public File { private: void UpdateLatestTime(FileHeader *CurBlock); - void ConvertNameCase(wchar *Name); + void ConvertNameCase(std::wstring &Name); void ConvertFileHeader(FileHeader *hd); size_t ReadHeader14(); size_t ReadHeader15(); @@ -36,9 +36,9 @@ class Archive:public File void RequestArcPassword(RarCheckPassword *SelPwd); void UnexpEndArcMsg(); void BrokenHeaderMsg(); - void UnkEncVerMsg(const wchar *Name,const wchar *Info); - bool DoGetComment(Array *CmtData); - bool ReadCommentData(Array *CmtData); + void UnkEncVerMsg(const std::wstring &Name,const std::wstring &Info); + bool DoGetComment(std::wstring &CmtData); + bool ReadCommentData(std::wstring &CmtData); #if !defined(RAR_NOCRYPT) CryptData HeadersCrypt; @@ -67,9 +67,9 @@ class Archive:public File size_t SearchRR(); size_t ReadHeader(); void CheckArc(bool EnableBroken); - void CheckOpen(const wchar *Name); - bool WCheckOpen(const wchar *Name); - bool GetComment(Array *CmtData); + void CheckOpen(const std::wstring &Name); + bool WCheckOpen(const std::wstring &Name); + bool GetComment(std::wstring &CmtData); void ViewComment(); void SetLatestTime(RarTime *NewTime); void SeekToNext(); @@ -79,23 +79,25 @@ class Archive:public File void VolSubtractHeaderSize(size_t SubSize); uint FullHeaderSize(size_t Size); int64 GetStartPos(); - void AddSubData(byte *SrcData,uint64 DataSize,File *SrcFile, + void AddSubData(const byte *SrcData,uint64 DataSize,File *SrcFile, const wchar *Name,uint Flags); - bool ReadSubData(Array *UnpData,File *DestFile,bool TestMode); + bool ReadSubData(std::vector *UnpData,File *DestFile,bool TestMode); HEADER_TYPE GetHeaderType() {return CurHeaderType;} CommandData* GetCommandData() {return Cmd;} void SetSilentOpen(bool Mode) {SilentOpen=Mode;} -#if 0 - void GetRecoveryInfo(bool Required,int64 *Size,int *Percent); -#endif #ifdef USE_QOPEN - bool Open(const wchar *Name,uint Mode=FMF_READ); - int Read(void *Data,size_t Size); - void Seek(int64 Offset,int Method); - int64 Tell(); + bool Open(const std::wstring &Name,uint Mode=FMF_READ) override; + int Read(void *Data,size_t Size) override; + void Seek(int64 Offset,int Method) override; + int64 Tell() override; void QOpenUnload() {QOpen.Unload();} void SetProhibitQOpen(bool Mode) {ProhibitQOpen=Mode;} #endif + static uint64 GetWinSize(uint64 Size,uint &Flags); + + // Needed to see wstring based Open from File. Otherwise compiler finds + // Open in Archive and doesn't check the base class overloads. + using File::Open; BaseBlock ShortBlock; MarkHeader MarkHead; @@ -107,7 +109,6 @@ class Archive:public File FileHeader SubHead; CommentHeader CommHead; ProtectHeader ProtectHead; - UnixOwnersHeader UOHead; EAHeader EAHead; StreamHeader StreamHead; @@ -136,12 +137,16 @@ class Archive:public File uint VolNumber; int64 VolWrite; + + // Total size of files adding to archive. Might also include the size of + // files repacked in solid archive. uint64 AddingFilesSize; + uint64 AddingHeadersSize; bool NewArchive; - wchar FirstVolumeName[NM]; + std::wstring FirstVolumeName; }; diff --git a/multiarc/src/formats/rar/unrar/arcread.cpp b/multiarc/src/formats/rar/unrar/arcread.cpp index 97d118490..825da7266 100644 --- a/multiarc/src/formats/rar/unrar/arcread.cpp +++ b/multiarc/src/formats/rar/unrar/arcread.cpp @@ -119,10 +119,10 @@ void Archive::BrokenHeaderMsg() } -void Archive::UnkEncVerMsg(const wchar *Name,const wchar *Info) +void Archive::UnkEncVerMsg(const std::wstring &Name,const std::wstring &Info) { uiMsg(UIERROR_UNKNOWNENCMETHOD,FileName,Name,Info); - ErrHandler.SetErrorCode(RARX_WARNING); + ErrHandler.SetErrorCode(RARX_FATAL); } @@ -222,7 +222,7 @@ size_t Archive::ReadHeader15() { case HEAD_MAIN: MainHead.Reset(); - *(BaseBlock *)&MainHead=ShortBlock; + MainHead.SetBaseBlock(ShortBlock); MainHead.HighPosAV=Raw.Get2(); MainHead.PosAV=Raw.Get4(); @@ -248,13 +248,17 @@ size_t Archive::ReadHeader15() FileHeader *hd=FileBlock ? &FileHead:&SubHead; hd->Reset(); - *(BaseBlock *)hd=ShortBlock; + hd->SetBaseBlock(ShortBlock); hd->SplitBefore=(hd->Flags & LHD_SPLIT_BEFORE)!=0; hd->SplitAfter=(hd->Flags & LHD_SPLIT_AFTER)!=0; hd->Encrypted=(hd->Flags & LHD_PASSWORD)!=0; hd->SaltSet=(hd->Flags & LHD_SALT)!=0; + + // RAR versions earlier than 2.0 do not set the solid flag + // in file header. They use only a global solid archive flag. hd->Solid=FileBlock && (hd->Flags & LHD_SOLID)!=0; + hd->SubBlock=!FileBlock && (hd->Flags & LHD_SOLID)!=0; hd->Dir=(hd->Flags & LHD_WINDOWMASK)==LHD_DIRECTORY; hd->WinSize=hd->Dir ? 0:0x10000<<((hd->Flags & LHD_WINDOWMASK)>>5); @@ -303,7 +307,7 @@ size_t Archive::ReadHeader15() if (hd->HostOS==HOST_UNIX && (hd->FileAttr & 0xF000)==0xA000) { hd->RedirType=FSREDIR_UNIXSYMLINK; - *hd->RedirName=0; + hd->RedirName.clear(); } hd->Inherited=!FileBlock && (hd->SubFlags & SUBHEAD_FLAGS_INHERITED)!=0; @@ -330,27 +334,26 @@ size_t Archive::ReadHeader15() if (hd->UnknownUnpSize) hd->UnpSize=INT64NDF; - char FileName[NM*4]; - size_t ReadNameSize=Min(NameSize,ASIZE(FileName)-1); - Raw.GetB((byte *)FileName,ReadNameSize); - FileName[ReadNameSize]=0; + size_t ReadNameSize=Min(NameSize,MAXPATHSIZE); + std::string FileName(ReadNameSize,0); + Raw.GetB((byte *)&FileName[0],ReadNameSize); if (FileBlock) { - *hd->FileName=0; + hd->FileName.clear(); if ((hd->Flags & LHD_UNICODE)!=0) { EncodeFileName NameCoder; - size_t Length=strlen(FileName); + size_t Length=strlen(FileName.data()); Length++; if (ReadNameSize>Length) - NameCoder.Decode(FileName,ReadNameSize,(byte *)FileName+Length, - ReadNameSize-Length,hd->FileName, - ASIZE(hd->FileName)); + NameCoder.Decode(FileName.data(),ReadNameSize, + (byte *)&FileName[Length], + ReadNameSize-Length,hd->FileName); } - if (*hd->FileName==0) - ArcCharToWide(FileName,hd->FileName,ASIZE(hd->FileName),ACTW_OEM); + if (hd->FileName.empty()) + ArcCharToWide(FileName.data(),hd->FileName,ACTW_OEM); #ifndef SFX_MODULE ConvertNameCase(hd->FileName); @@ -359,7 +362,7 @@ size_t Archive::ReadHeader15() } else { - CharToWide(FileName,hd->FileName,ASIZE(hd->FileName)); + CharToWide(FileName.data(),hd->FileName); // Calculate the size of optional data. int DataSize=int(hd->HeadSize-NameSize-SIZEOF_FILEHEAD3); @@ -370,14 +373,15 @@ size_t Archive::ReadHeader15() { // Here we read optional additional fields for subheaders. // They are stored after the file name and before salt. - hd->SubData.Alloc(DataSize); - Raw.GetB(&hd->SubData[0],DataSize); + hd->SubData.resize(DataSize); + Raw.GetB(hd->SubData.data(),DataSize); } if (hd->CmpName(SUBHEAD_TYPE_CMT)) MainComment=true; } + if ((hd->Flags & LHD_SALT)!=0) Raw.GetB(hd->Salt,SIZE_SALT30); hd->mtime.SetDos(FileTime); @@ -420,7 +424,7 @@ size_t Archive::ReadHeader15() NextBlockPos=SafeAdd(NextBlockPos,hd->PackSize,0); bool CRCProcessedOnly=hd->CommentInHeader; - ushort HeaderCRC=Raw.GetCRC15(CRCProcessedOnly); + uint HeaderCRC=Raw.GetCRC15(CRCProcessedOnly); if (hd->HeadCRC!=HeaderCRC) { BrokenHeader=true; @@ -437,7 +441,7 @@ size_t Archive::ReadHeader15() } break; case HEAD_ENDARC: - *(BaseBlock *)&EndArcHead=ShortBlock; + EndArcHead.SetBaseBlock(ShortBlock); EndArcHead.NextVolume=(EndArcHead.Flags & EARC_NEXT_VOLUME)!=0; EndArcHead.DataCRC=(EndArcHead.Flags & EARC_DATACRC)!=0; EndArcHead.RevSpace=(EndArcHead.Flags & EARC_REVSPACE)!=0; @@ -449,14 +453,14 @@ size_t Archive::ReadHeader15() break; #ifndef SFX_MODULE case HEAD3_CMT: - *(BaseBlock *)&CommHead=ShortBlock; + CommHead.SetBaseBlock(ShortBlock); CommHead.UnpSize=Raw.Get2(); CommHead.UnpVer=Raw.Get1(); CommHead.Method=Raw.Get1(); CommHead.CommCRC=Raw.Get2(); break; case HEAD3_PROTECT: - *(BaseBlock *)&ProtectHead=ShortBlock; + ProtectHead.SetBaseBlock(ShortBlock); ProtectHead.DataSize=Raw.Get4(); ProtectHead.Version=Raw.Get1(); ProtectHead.RecSectors=Raw.Get2(); @@ -465,26 +469,13 @@ size_t Archive::ReadHeader15() NextBlockPos+=ProtectHead.DataSize; break; case HEAD3_OLDSERVICE: // RAR 2.9 and earlier. - *(BaseBlock *)&SubBlockHead=ShortBlock; + SubBlockHead.SetBaseBlock(ShortBlock); SubBlockHead.DataSize=Raw.Get4(); NextBlockPos+=SubBlockHead.DataSize; SubBlockHead.SubType=Raw.Get2(); SubBlockHead.Level=Raw.Get1(); switch(SubBlockHead.SubType) { - case UO_HEAD: - *(SubBlockHeader *)&UOHead=SubBlockHead; - UOHead.OwnerNameSize=Raw.Get2(); - UOHead.GroupNameSize=Raw.Get2(); - if (UOHead.OwnerNameSize>=ASIZE(UOHead.OwnerName)) - UOHead.OwnerNameSize=ASIZE(UOHead.OwnerName)-1; - if (UOHead.GroupNameSize>=ASIZE(UOHead.GroupName)) - UOHead.GroupNameSize=ASIZE(UOHead.GroupName)-1; - Raw.GetB(UOHead.OwnerName,UOHead.OwnerNameSize); - Raw.GetB(UOHead.GroupName,UOHead.GroupNameSize); - UOHead.OwnerName[UOHead.OwnerNameSize]=0; - UOHead.GroupName[UOHead.GroupNameSize]=0; - break; case NTACL_HEAD: *(SubBlockHeader *)&EAHead=SubBlockHead; EAHead.UnpSize=Raw.Get4(); @@ -499,10 +490,13 @@ size_t Archive::ReadHeader15() StreamHead.Method=Raw.Get1(); StreamHead.StreamCRC=Raw.Get4(); StreamHead.StreamNameSize=Raw.Get2(); - if (StreamHead.StreamNameSize>=ASIZE(StreamHead.StreamName)) - StreamHead.StreamNameSize=ASIZE(StreamHead.StreamName)-1; - Raw.GetB(StreamHead.StreamName,StreamHead.StreamNameSize); - StreamHead.StreamName[StreamHead.StreamNameSize]=0; + + const size_t MaxStreamName20=260; // Maximum allowed stream name in RAR 2.x format. + if (StreamHead.StreamNameSize>MaxStreamName20) + StreamHead.StreamNameSize=MaxStreamName20; + + StreamHead.StreamName.resize(StreamHead.StreamNameSize); + Raw.GetB(&StreamHead.StreamName[0],StreamHead.StreamNameSize); break; } break; @@ -513,11 +507,15 @@ size_t Archive::ReadHeader15() break; } - ushort HeaderCRC=Raw.GetCRC15(false); + uint HeaderCRC=Raw.GetCRC15(false); // Old AV header does not have header CRC properly set. + // Old Unix owners header didn't include string fields into header size, + // but included them into CRC, so it couldn't be verified with generic + // approach here. if (ShortBlock.HeadCRC!=HeaderCRC && ShortBlock.HeaderType!=HEAD3_SIGN && - ShortBlock.HeaderType!=HEAD3_AV) + ShortBlock.HeaderType!=HEAD3_AV && + (ShortBlock.HeaderType!=HEAD3_OLDSERVICE || SubBlockHead.SubType!=UO_HEAD)) { bool Recovered=false; if (ShortBlock.HeaderType==HEAD_ENDARC && EndArcHead.RevSpace) @@ -589,10 +587,10 @@ size_t Archive::ReadHeader50() RequestArcPassword(CheckPwd.IsSet() ? &CheckPwd:NULL); byte PswCheck[SIZE_PSWCHECK]; - HeadersCrypt.SetCryptKeys(false,CRYPT_RAR50,&Cmd->Password,CryptHead.Salt,HeadersInitV,CryptHead.Lg2Count,NULL,PswCheck); + bool EncSet=HeadersCrypt.SetCryptKeys(false,CRYPT_RAR50,&Cmd->Password,CryptHead.Salt,HeadersInitV,CryptHead.Lg2Count,NULL,PswCheck); // Verify password validity. If header is damaged, we cannot rely on // password check value, because it can be damaged too. - if (CryptHead.UsePswCheck && !BrokenHeader && + if (EncSet && CryptHead.UsePswCheck && !BrokenHeader && memcmp(PswCheck,CryptHead.PswCheck,SIZE_PSWCHECK)!=0) { if (GlobalPassword) // For -p or Ctrl+P. @@ -651,7 +649,7 @@ size_t Archive::ReadHeader50() } int SizeToRead=int(BlockSize); - SizeToRead-=FirstReadSize-SizeBytes-4; // Adjust overread size bytes if any. + SizeToRead-=int(FirstReadSize-SizeBytes-4); // Adjust overread size bytes if any. uint HeaderSize=4+SizeBytes+(uint)BlockSize; if (SizeToRead<0 || HeaderSizeCRYPT_VERSION) { - wchar Info[20]; - swprintf(Info,ASIZE(Info),L"h%u",CryptVersion); - UnkEncVerMsg(FileName,Info); + UnkEncVerMsg(FileName,L"h" + std::to_wstring(CryptVersion)); + FailedHeaderDecryption=true; return 0; } uint EncFlags=(uint)Raw.GetV(); @@ -730,9 +727,8 @@ size_t Archive::ReadHeader50() CryptHead.Lg2Count=Raw.Get1(); if (CryptHead.Lg2Count>CRYPT5_KDF_LG2_COUNT_MAX) { - wchar Info[20]; - swprintf(Info,ASIZE(Info),L"hc%u",CryptHead.Lg2Count); - UnkEncVerMsg(FileName,Info); + UnkEncVerMsg(FileName,L"hc" + std::to_wstring(CryptHead.Lg2Count)); + FailedHeaderDecryption=true; return 0; } @@ -744,12 +740,8 @@ size_t Archive::ReadHeader50() byte csum[SIZE_PSWCHECK_CSUM]; Raw.GetB(csum,SIZE_PSWCHECK_CSUM); - sha256_context ctx; - sha256_init(&ctx); - sha256_process(&ctx, CryptHead.PswCheck, SIZE_PSWCHECK); - byte Digest[SHA256_DIGEST_SIZE]; - sha256_done(&ctx, Digest); + sha256_get(CryptHead.PswCheck, SIZE_PSWCHECK, Digest); CryptHead.UsePswCheck=memcmp(csum,Digest,SIZE_PSWCHECK_CSUM)==0; } @@ -759,7 +751,7 @@ size_t Archive::ReadHeader50() case HEAD_MAIN: { MainHead.Reset(); - *(BaseBlock *)&MainHead=ShortBlock; + MainHead.SetBaseBlock(ShortBlock); uint ArcFlags=(uint)Raw.GetV(); Volume=(ArcFlags & MHFL_VOLUME)!=0; @@ -837,9 +829,14 @@ size_t Archive::ReadHeader50() // we may need to use the compression algorithm 15 in the future, // but it was already used in RAR 1.5 and Unpack needs to distinguish // them. - hd->UnpVer=(CompInfo & 0x3f) + 50; - if (hd->UnpVer!=50) // Only 5.0 compression is known now. - hd->UnpVer=VER_UNKNOWN; + uint UnpVer=(CompInfo & 0x3f); + if (UnpVer==0) + hd->UnpVer=VER_PACK5; + else + if (UnpVer==1) + hd->UnpVer=VER_PACK7; + else + hd->UnpVer=VER_UNKNOWN; hd->HostOS=(byte)Raw.GetV(); size_t NameSize=(size_t)Raw.GetV(); @@ -857,14 +854,29 @@ size_t Archive::ReadHeader50() hd->SubBlock=(hd->Flags & HFL_CHILD)!=0; hd->Solid=FileBlock && (CompInfo & FCI_SOLID)!=0; hd->Dir=(hd->FileFlags & FHFL_DIRECTORY)!=0; - hd->WinSize=hd->Dir ? 0:size_t(0x20000)<<((CompInfo>>10)&0xf); + if (hd->Dir || UnpVer>1) + hd->WinSize=0; + else + { + hd->WinSize=0x20000ULL<<((CompInfo>>10)&(UnpVer==0 ? 0x0f:0x1f)); + if (UnpVer==1) + { + hd->WinSize+=hd->WinSize/32*((CompInfo>>15)&0x1f); + + // RAR7 header with RAR5 compression. Needed to append RAR7 files + // to RAR5 solid stream if new dictionary is larger than existing. + if ((CompInfo & FCI_RAR5_COMPAT)!=0) + hd->UnpVer=VER_PACK5; + if (hd->WinSize>UNPACK_MAX_DICT) + hd->UnpVer=VER_UNKNOWN; + } + } - char FileName[NM*4]; - size_t ReadNameSize=Min(NameSize,ASIZE(FileName)-1); - Raw.GetB((byte *)FileName,ReadNameSize); - FileName[ReadNameSize]=0; + size_t ReadNameSize=Min(NameSize,MAXPATHSIZE); + std::string FileName(ReadNameSize,0); + Raw.GetB((byte *)&FileName[0],ReadNameSize); - UtfToWide(FileName,hd->FileName,ASIZE(hd->FileName)); + UtfToWide(FileName.data(),hd->FileName); // Should do it before converting names, because extra fields can // affect name processing, like in case of NTFS streams. @@ -889,7 +901,7 @@ size_t Archive::ReadHeader50() break; case HEAD_ENDARC: { - *(BaseBlock *)&EndArcHead=ShortBlock; + EndArcHead.SetBaseBlock(ShortBlock); uint ArcFlags=(uint)Raw.GetV(); EndArcHead.NextVolume=(ArcFlags & EHFL_NEXTVOLUME)!=0; EndArcHead.StoreVolNumber=false; @@ -921,7 +933,7 @@ void Archive::RequestArcPassword(RarCheckPassword *CheckPwd) *PasswordA=0; if (Cmd->Callback(UCM_NEEDPASSWORD,Cmd->UserData,(LPARAM)PasswordA,ASIZE(PasswordA))==-1) *PasswordA=0; - GetWideName(PasswordA,NULL,PasswordW,ASIZE(PasswordW)); + CharToWide(PasswordA,PasswordW,ASIZE(PasswordW)); cleandata(PasswordA,sizeof(PasswordA)); } Cmd->Password.Set(PasswordW); @@ -996,19 +1008,14 @@ void Archive::ProcessExtra50(RawRead *Raw,size_t ExtraSize,const BaseBlock *bb) if ((Flags & MHEXTRA_METADATA_NAME)!=0) { uint64 NameSize=Raw->GetV(); - if (NameSize<0x10000) // Prevent excessive allocation. + if (NameSize>0 && NameSize NameU((size_t)NameSize); // UTF-8 name. + std::string NameU((size_t)NameSize,0); // UTF-8 name. Raw->GetB(&NameU[0],(size_t)NameSize); // If starts from 0, the name was longer than reserved space // when saving this extra field. if (NameU[0]!=0) - { - NameU.push_back(0); - std::vector NameW(NameU.size()*4); - UtfToWide(&NameU[0],&NameW[0],NameW.size()); - hd->OrigName.assign(&NameW[0]); - } + UtfToWide(&NameU[0],hd->OrigName); } } if ((Flags & MHEXTRA_METADATA_CTIME)!=0) @@ -1034,11 +1041,7 @@ void Archive::ProcessExtra50(RawRead *Raw,size_t ExtraSize,const BaseBlock *bb) FileHeader *hd=(FileHeader *)bb; uint EncVersion=(uint)Raw->GetV(); if (EncVersion>CRYPT_VERSION) - { - wchar Info[20]; - swprintf(Info,ASIZE(Info),L"x%u",EncVersion); - UnkEncVerMsg(hd->FileName,Info); - } + UnkEncVerMsg(hd->FileName,L"x" + std::to_wstring(EncVersion)); else { uint Flags=(uint)Raw->GetV(); @@ -1046,44 +1049,39 @@ void Archive::ProcessExtra50(RawRead *Raw,size_t ExtraSize,const BaseBlock *bb) hd->UseHashKey=(Flags & FHEXTRA_CRYPT_HASHMAC)!=0; hd->Lg2Count=Raw->Get1(); if (hd->Lg2Count>CRYPT5_KDF_LG2_COUNT_MAX) + UnkEncVerMsg(hd->FileName,L"xc" + std::to_wstring(hd->Lg2Count)); + else { - wchar Info[20]; - swprintf(Info,ASIZE(Info),L"xc%u",hd->Lg2Count); - UnkEncVerMsg(hd->FileName,Info); - } - Raw->GetB(hd->Salt,SIZE_SALT50); - Raw->GetB(hd->InitV,SIZE_INITV); - if (hd->UsePswCheck) - { - Raw->GetB(hd->PswCheck,SIZE_PSWCHECK); - - // It is important to know if password check data is valid. - // If it is damaged and header CRC32 fails to detect it, - // archiver would refuse to decompress a possibly valid file. - // Since we want to be sure distinguishing a wrong password - // or corrupt file data, we use 64-bit password check data - // and to control its validity we use 32 bits of password - // check data SHA-256 additionally to 32-bit header CRC32. - byte csum[SIZE_PSWCHECK_CSUM]; - Raw->GetB(csum,SIZE_PSWCHECK_CSUM); - - sha256_context ctx; - sha256_init(&ctx); - sha256_process(&ctx, hd->PswCheck, SIZE_PSWCHECK); - - byte Digest[SHA256_DIGEST_SIZE]; - sha256_done(&ctx, Digest); - - hd->UsePswCheck=memcmp(csum,Digest,SIZE_PSWCHECK_CSUM)==0; - - // RAR 5.21 and earlier set PswCheck field in service records to 0 - // even if UsePswCheck was present. - if (bb->HeaderType==HEAD_SERVICE && memcmp(hd->PswCheck,"\0\0\0\0\0\0\0\0",SIZE_PSWCHECK)==0) - hd->UsePswCheck=0; + Raw->GetB(hd->Salt,SIZE_SALT50); + Raw->GetB(hd->InitV,SIZE_INITV); + if (hd->UsePswCheck) + { + Raw->GetB(hd->PswCheck,SIZE_PSWCHECK); + + // It is important to know if password check data is valid. + // If it is damaged and header CRC32 fails to detect it, + // archiver would refuse to decompress a possibly valid file. + // Since we want to be sure distinguishing a wrong password + // or corrupt file data, we use 64-bit password check data + // and to control its validity we use 32 bits of password + // check data SHA-256 additionally to 32-bit header CRC32. + byte csum[SIZE_PSWCHECK_CSUM]; + Raw->GetB(csum,SIZE_PSWCHECK_CSUM); + + byte Digest[SHA256_DIGEST_SIZE]; + sha256_get(hd->PswCheck, SIZE_PSWCHECK, Digest); + + hd->UsePswCheck=memcmp(csum,Digest,SIZE_PSWCHECK_CSUM)==0; + + // RAR 5.21 and earlier set PswCheck field in service records to 0 + // even if UsePswCheck was present. + if (bb->HeaderType==HEAD_SERVICE && memcmp(hd->PswCheck,"\0\0\0\0\0\0\0\0",SIZE_PSWCHECK)==0) + hd->UsePswCheck=0; + } + hd->SaltSet=true; + hd->CryptMethod=CRYPT_RAR50; + hd->Encrypted=true; } - hd->SaltSet=true; - hd->CryptMethod=CRYPT_RAR50; - hd->Encrypted=true; } } break; @@ -1138,31 +1136,27 @@ void Archive::ProcessExtra50(RawRead *Raw,size_t ExtraSize,const BaseBlock *bb) if (Version!=0) { hd->Version=true; - - wchar VerText[20]; - swprintf(VerText,ASIZE(VerText),L";%u",Version); - wcsncatz(hd->FileName,VerText,ASIZE(hd->FileName)); + hd->FileName += L';' + std::to_wstring(Version); } } break; case FHEXTRA_REDIR: { - hd->RedirType=(FILE_SYSTEM_REDIRECT)Raw->GetV(); + FILE_SYSTEM_REDIRECT RedirType=(FILE_SYSTEM_REDIRECT)Raw->GetV(); uint Flags=(uint)Raw->GetV(); - hd->DirTarget=(Flags & FHEXTRA_REDIR_DIR)!=0; size_t NameSize=(size_t)Raw->GetV(); - char UtfName[NM*4]; - *UtfName=0; - if (NameSize0 && NameSizeGetB(UtfName,NameSize); - UtfName[NameSize]=0; - } + std::string UtfName(NameSize,0); + hd->RedirType=RedirType; + hd->DirTarget=(Flags & FHEXTRA_REDIR_DIR)!=0; + Raw->GetB(&UtfName[0],NameSize); + UtfToWide(&UtfName[0],hd->RedirName); #ifdef _WIN_ALL - UnixSlashToDos(UtfName,UtfName,ASIZE(UtfName)); + UnixSlashToDos(hd->RedirName,hd->RedirName); #endif - UtfToWide(UtfName,hd->RedirName,ASIZE(hd->RedirName)); + } } break; case FHEXTRA_UOWNER: @@ -1219,8 +1213,8 @@ void Archive::ProcessExtra50(RawRead *Raw,size_t ExtraSize,const BaseBlock *bb) // We cannot allocate too much memory here, because above // we check FieldSize againt Raw size and we control that Raw size // is sensible when reading headers. - hd->SubData.Alloc((size_t)FieldSize); - Raw->GetB(hd->SubData.Addr(0),(size_t)FieldSize); + hd->SubData.resize((size_t)FieldSize); + Raw->GetB(hd->SubData.data(),(size_t)FieldSize); } break; } @@ -1243,7 +1237,7 @@ size_t Archive::ReadHeader14() Raw.GetB(Mark,4); uint HeadSize=Raw.Get2(); if (HeadSize<7) - return false; + return 0; byte Flags=Raw.Get1(); NextBlockPos=CurBlockPos+HeadSize; CurHeaderType=HEAD_MAIN; @@ -1266,7 +1260,7 @@ size_t Archive::ReadHeader14() FileHead.FileHash.CRC32=Raw.Get2(); FileHead.HeadSize=Raw.Get2(); if (FileHead.HeadSize<21) - return false; + return 0; uint FileTime=Raw.Get4(); FileHead.FileAttr=Raw.Get1(); FileHead.Flags=Raw.Get1()|LONG_BLOCK; @@ -1290,12 +1284,13 @@ size_t Archive::ReadHeader14() Raw.Read(NameSize); - char FileName[NM]; - size_t ReadNameSize=Min(NameSize,ASIZE(FileName)-1); - Raw.GetB((byte *)FileName,ReadNameSize); - FileName[ReadNameSize]=0; - IntToExt(FileName,FileName,ASIZE(FileName)); - CharToWide(FileName,FileHead.FileName,ASIZE(FileHead.FileName)); + // RAR 1.4 name size is stored in a single byte field and it can't + // exceed 255, so additional checks are not needed. + std::string FileName(NameSize,0); + Raw.GetB((byte *)&FileName[0],NameSize); + std::string NameA; + IntToExt(FileName,NameA); + CharToWide(NameA,FileHead.FileName); ConvertNameCase(FileHead.FileName); ConvertFileHeader(&FileHead); @@ -1309,7 +1304,7 @@ size_t Archive::ReadHeader14() #ifndef SFX_MODULE -void Archive::ConvertNameCase(wchar *Name) +void Archive::ConvertNameCase(std::wstring &Name) { if (Cmd->ConvertNames==NAMES_UPPERCASE) wcsupper(Name); @@ -1327,7 +1322,7 @@ bool Archive::IsArcDir() void Archive::ConvertAttributes() { -#if defined(_WIN_ALL) || defined(_EMX) +#ifdef _WIN_ALL if (FileHead.HSType!=HSYS_WINDOWS) FileHead.FileAttr=FileHead.Dir ? 0x10 : 0x20; #endif @@ -1392,19 +1387,23 @@ void Archive::ConvertAttributes() void Archive::ConvertFileHeader(FileHeader *hd) { +/* if (hd->HSType==HSYS_UNKNOWN) if (hd->Dir) hd->FileAttr=0x10; else hd->FileAttr=0x20; +*/ #ifdef _WIN_ALL if (hd->HSType==HSYS_UNIX) // Convert Unix, OS X and Android decomposed chracters to Windows precomposed. - ConvertToPrecomposed(hd->FileName,ASIZE(hd->FileName)); + ConvertToPrecomposed(hd->FileName); #endif - for (wchar *s=hd->FileName;*s!=0;s++) + for (uint I=0;IFileName.size();I++) { + wchar *s=&hd->FileName[I]; + #ifdef _UNIX // Backslash is the invalid character for Windows file headers, // but it can present in Unix file names extracted in Unix. @@ -1412,7 +1411,7 @@ void Archive::ConvertFileHeader(FileHeader *hd) *s='_'; #endif -#if defined(_WIN_ALL) || defined(_EMX) +#ifdef _WIN_ALL // RAR 5.0 archives do not use '\' as path separator, so if we see it, // it means that it is a part of Unix file name, which we cannot // extract in Windows. @@ -1437,6 +1436,9 @@ void Archive::ConvertFileHeader(FileHeader *hd) if (*s=='/' || *s=='\\' && Format!=RARFMT50) *s=CPATHDIVIDER; } + + // Zeroes inside might be possible in broken Unicode names decoded with EncodeFileName::Decode. + TruncateAtZero(hd->FileName); // Ensure there are no zeroes inside of string. } @@ -1451,7 +1453,7 @@ int64 Archive::GetStartPos() } -bool Archive::ReadSubData(Array *UnpData,File *DestFile,bool TestMode) +bool Archive::ReadSubData(std::vector *UnpData,File *DestFile,bool TestMode) { if (BrokenHeader) { @@ -1459,7 +1461,7 @@ bool Archive::ReadSubData(Array *UnpData,File *DestFile,bool TestMode) ErrHandler.SetErrorCode(RARX_CRC); return false; } - if (SubHead.Method>5 || SubHead.UnpVer>(Format==RARFMT50 ? VER_UNPACK5:VER_UNPACK)) + if (SubHead.Method>5 || SubHead.UnpVer>(Format==RARFMT50 ? VER_UNPACK7:VER_UNPACK)) { uiMsg(UIERROR_SUBHEADERUNKNOWN,FileName); return false; @@ -1486,7 +1488,7 @@ bool Archive::ReadSubData(Array *UnpData,File *DestFile,bool TestMode) SubDataIO.SetTestMode(true); else { - UnpData->Alloc((size_t)SubHead.UnpSize); + UnpData->resize((size_t)SubHead.UnpSize); SubDataIO.SetUnpackToMemory(&(*UnpData)[0],(uint)SubHead.UnpSize); } } @@ -1515,7 +1517,7 @@ bool Archive::ReadSubData(Array *UnpData,File *DestFile,bool TestMode) uiMsg(UIERROR_SUBHEADERDATABROKEN,FileName,SubHead.FileName); ErrHandler.SetErrorCode(RARX_CRC); if (UnpData!=NULL) - UnpData->Reset(); + UnpData->clear(); return false; } return true; diff --git a/multiarc/src/formats/rar/unrar/array.hpp b/multiarc/src/formats/rar/unrar/array.hpp index 18aacc969..7b34de44d 100644 --- a/multiarc/src/formats/rar/unrar/array.hpp +++ b/multiarc/src/formats/rar/unrar/array.hpp @@ -97,7 +97,7 @@ template void Array::Add(size_t Items) { if (MaxSize!=0 && BufSize>MaxSize) { - ErrHandler.GeneralErrMsg(L"Maximum allowed array size (%lu) is exceeded",MaxSize); + ErrHandler.GeneralErrMsg(L"Maximum allowed array size (%lu) is exceeded",(unsigned long)MaxSize); ErrHandler.MemoryError(); } diff --git a/multiarc/src/formats/rar/unrar/blake2s.cpp b/multiarc/src/formats/rar/unrar/blake2s.cpp index 317603dac..ae9d4e02a 100644 --- a/multiarc/src/formats/rar/unrar/blake2s.cpp +++ b/multiarc/src/formats/rar/unrar/blake2s.cpp @@ -2,6 +2,20 @@ #include "rar.hpp" +static const byte blake2s_sigma[10][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , +}; + #ifdef USE_SSE #include "blake2s_sse.cpp" #endif @@ -18,20 +32,6 @@ static const uint32 blake2s_IV[8] = 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL }; -static const byte blake2s_sigma[10][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; - static inline void blake2s_set_lastnode( blake2s_state *S ) { S->f[1] = ~0U; @@ -134,11 +134,7 @@ void blake2s_update( blake2s_state *S, const byte *in, size_t inlen ) blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); #ifdef USE_SSE -#ifdef _WIN_32 // We use SSSE3 _mm_shuffle_epi8 only in x64 mode. - if (_SSE_Version>=SSE_SSE2) -#else if (_SSE_Version>=SSE_SSSE3) -#endif blake2s_compress_sse( S, S->buf ); else blake2s_compress( S, S->buf ); // Compress diff --git a/multiarc/src/formats/rar/unrar/blake2s_sse.cpp b/multiarc/src/formats/rar/unrar/blake2s_sse.cpp index 1a02f2106..933fcad8c 100644 --- a/multiarc/src/formats/rar/unrar/blake2s_sse.cpp +++ b/multiarc/src/formats/rar/unrar/blake2s_sse.cpp @@ -1,15 +1,14 @@ // Based on public domain code written in 2012 by Samuel Neves -extern const byte blake2s_sigma[10][16]; - // Initialization vector. static __m128i blake2s_IV_0_3, blake2s_IV_4_7; -#ifdef _WIN_64 -// Constants for cyclic rotation. Used in 64-bit mode in mm_rotr_epi32 macro. +// Constants for cyclic rotation. static __m128i crotr8, crotr16; -#endif +#ifdef __GNUC__ +__attribute__((target("sse2"))) +#endif static void blake2s_init_sse() { // We cannot initialize these 128 bit variables in place when declaring @@ -24,28 +23,18 @@ static void blake2s_init_sse() blake2s_IV_0_3 = _mm_setr_epi32( 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A ); blake2s_IV_4_7 = _mm_setr_epi32( 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 ); -#ifdef _WIN_64 crotr8 = _mm_set_epi8( 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1 ); crotr16 = _mm_set_epi8( 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 ); -#endif } #define LOAD(p) _mm_load_si128( (__m128i *)(p) ) #define STORE(p,r) _mm_store_si128((__m128i *)(p), r) -#ifdef _WIN_32 -// 32-bit mode has less SSE2 registers and in MSVC2008 it is more efficient -// to not use _mm_shuffle_epi8 here. -#define mm_rotr_epi32(r, c) ( \ - _mm_xor_si128(_mm_srli_epi32( (r), c ),_mm_slli_epi32( (r), 32-c )) ) -#else #define mm_rotr_epi32(r, c) ( \ c==8 ? _mm_shuffle_epi8(r,crotr8) \ : c==16 ? _mm_shuffle_epi8(r,crotr16) \ : _mm_xor_si128(_mm_srli_epi32( (r), c ),_mm_slli_epi32( (r), 32-c )) ) -#endif - #define G1(row1,row2,row3,row4,buf) \ row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \ @@ -73,14 +62,6 @@ static void blake2s_init_sse() row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \ row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(2,1,0,3) ); -#ifdef _WIN_64 - // MSVC 2008 in x64 mode expands _mm_set_epi32 to store to stack and load - // from stack operations, which are slower than this code. - #define _mm_set_epi32(i3,i2,i1,i0) \ - _mm_unpacklo_epi32(_mm_unpacklo_epi32(_mm_cvtsi32_si128(i0),_mm_cvtsi32_si128(i2)), \ - _mm_unpacklo_epi32(_mm_cvtsi32_si128(i1),_mm_cvtsi32_si128(i3))) -#endif - // Original BLAKE2 SSE4.1 message loading code was a little slower in x86 mode // and about the same in x64 mode in our test. Perhaps depends on compiler. // We also tried _mm_i32gather_epi32 and _mm256_i32gather_epi32 AVX2 gather @@ -101,6 +82,9 @@ static void blake2s_init_sse() } +#ifdef __GNUC__ +__attribute__((target("ssse3"))) +#endif static int blake2s_compress_sse( blake2s_state *S, const byte block[BLAKE2S_BLOCKBYTES] ) { __m128i row[4]; diff --git a/multiarc/src/formats/rar/unrar/blake2sp.cpp b/multiarc/src/formats/rar/unrar/blake2sp.cpp index da645883b..91dbd1448 100644 --- a/multiarc/src/formats/rar/unrar/blake2sp.cpp +++ b/multiarc/src/formats/rar/unrar/blake2sp.cpp @@ -20,7 +20,7 @@ void blake2sp_init( blake2sp_state *S ) blake2s_init_param( &S->R, 0, 1 ); // Init root. - for( uint i = 0; i < PARALLELISM_DEGREE; ++i ) + for( uint32 i = 0; i < PARALLELISM_DEGREE; ++i ) blake2s_init_param( &S->S[i], i, 0 ); // Init leaf. S->R.last_node = 1; @@ -49,6 +49,8 @@ void Blake2ThreadData::Update() if (_SSE_Version>=SSE_SSE && inlen__ >= 2 * PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES) _mm_prefetch((char*)(in__ + PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES), _MM_HINT_T0); #endif + // We tried to _forceinline blake2s_update and blake2s_compress_sse, + // but it didn't improve performance. blake2s_update( S, in__, BLAKE2S_BLOCKBYTES ); in__ += PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES; inlen__ -= PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES; diff --git a/multiarc/src/formats/rar/unrar/cmddata.cpp b/multiarc/src/formats/rar/unrar/cmddata.cpp index 3a4ab99eb..37bcf5b0f 100644 --- a/multiarc/src/formats/rar/unrar/cmddata.cpp +++ b/multiarc/src/formats/rar/unrar/cmddata.cpp @@ -13,8 +13,18 @@ void CommandData::Init() { RAROptions::Init(); - *Command=0; - *ArcName=0; + Command.clear(); + ArcName.clear(); + ExtrPath.clear(); + TempPath.clear(); + SFXModule.clear(); + CommentFile.clear(); + ArcPath.clear(); + ExclArcPath.clear(); + LogName.clear(); + EmailTo.clear(); + UseStdin.clear(); + FileLists=false; NoMoreSwitches=false; @@ -30,57 +40,45 @@ void CommandData::Init() StoreArgs.Reset(); Password.Clean(); NextVolSizes.clear(); -} - - -// Return the pointer to next position in the string and store dynamically -// allocated command line parameter in Par. -static const wchar *AllocCmdParam(const wchar *CmdLine,wchar **Par) -{ - const wchar *NextCmd=GetCmdParam(CmdLine,NULL,0); - if (NextCmd==NULL) - return NULL; - size_t ParSize=NextCmd-CmdLine+2; // Parameter size including the trailing zero. - *Par=(wchar *)malloc(ParSize*sizeof(wchar)); - if (*Par==NULL) - return NULL; - return GetCmdParam(CmdLine,*Par,ParSize); +#ifdef RARDLL + DllDestName.clear(); +#endif } #if !defined(SFX_MODULE) void CommandData::ParseCommandLine(bool Preprocess,int argc, char *argv[]) { - *Command=0; + Command.clear(); NoMoreSwitches=false; #ifdef CUSTOM_CMDLINE_PARSER // In Windows we may prefer to implement our own command line parser // to avoid replacing \" by " in standard parser. Such replacing corrupts // destination paths like "dest path\" in extraction commands. - const wchar *CmdLine=GetCommandLine(); + std::wstring CmdLine=GetCommandLine(); + + std::wstring Param; + std::wstring::size_type Pos=0; - wchar *Par; for (bool FirstParam=true;;FirstParam=false) { - if ((CmdLine=AllocCmdParam(CmdLine,&Par))==NULL) + if (!GetCmdParam(CmdLine,Pos,Param)) break; if (!FirstParam) // First parameter is the executable name. if (Preprocess) - PreprocessArg(Par); + PreprocessArg(Param.data()); else - ParseArg(Par); - free(Par); + ParseArg(Param.data()); } #else - Array Arg; for (int I=1;I0 && Arg[L-1]=='.' && (L==1 || L>=2 && (IsPathDiv(Arg[L-2]) || + Arg[L-2]=='.' && (L==2 || L>=3 && IsPathDiv(Arg[L-3]))))) + FolderArg=true; + + wchar CmdChar=toupperw(Command[0]); bool Add=wcschr(L"AFUM",CmdChar)!=NULL; bool Extract=CmdChar=='X' || CmdChar=='E'; bool Repair=CmdChar=='R' && Command[1]==0; - if (EndSeparator && !Add) - wcsncpyz(ExtrPath,Arg,ASIZE(ExtrPath)); + if (FolderArg && !Add) + ExtrPath=Arg; else if ((Add || CmdChar=='T') && (*Arg!='@' || ListMode==RCLM_REJECT_LISTS)) FileArgs.AddString(Arg); @@ -147,10 +158,10 @@ void CommandData::ParseArg(wchar *Arg) } else // We use 'destpath\' when extracting and reparing. - if (Found && FileData.IsDir && (Extract || Repair) && *ExtrPath==0) + if (Found && FileData.IsDir && (Extract || Repair) && ExtrPath.empty()) { - wcsncpyz(ExtrPath,Arg,ASIZE(ExtrPath)); - AddEndSlash(ExtrPath,ASIZE(ExtrPath)); + ExtrPath=Arg; + AddEndSlash(ExtrPath); } else FileArgs.AddString(Arg); @@ -178,12 +189,12 @@ void CommandData::ParseDone() #if !defined(SFX_MODULE) void CommandData::ParseEnvVar() { - char *EnvStr=getenv("RAR"); - if (EnvStr!=NULL) + char *EnvVar=getenv("RAR"); + if (EnvVar!=NULL) { - Array EnvStrW(strlen(EnvStr)+1); - CharToWide(EnvStr,&EnvStrW[0],EnvStrW.Size()); - ProcessSwitchesString(&EnvStrW[0]); + std::wstring EnvStr; + CharToWide(EnvVar,EnvStr); + ProcessSwitchesString(EnvStr); } } #endif @@ -192,7 +203,7 @@ void CommandData::ParseEnvVar() #if !defined(SFX_MODULE) // Preprocess those parameters, which must be processed before the rest of -// command line. Return 'false' to stop further processing. +// command line. void CommandData::PreprocessArg(const wchar *Arg) { if (IsSwitch(Arg[0]) && !NoMoreSwitches) @@ -201,7 +212,7 @@ void CommandData::PreprocessArg(const wchar *Arg) if (Arg[0]=='-' && Arg[1]==0) // Switch "--". NoMoreSwitches=true; if (wcsicomp(Arg,L"cfg-")==0) - ConfigDisabled=true; + ProcessSwitch(Arg); if (wcsnicomp(Arg,L"ilog",4)==0) { // Ensure that correct log file name is already set @@ -213,13 +224,13 @@ void CommandData::PreprocessArg(const wchar *Arg) { // Process -sc before reading any file lists. ProcessSwitch(Arg); - if (*LogName!=0) + if (!LogName.empty()) InitLogOptions(LogName,ErrlogCharset); } } else - if (*Command==0) - wcsncpy(Command,Arg,ASIZE(Command)); // Need for rar.ini. + if (Command.empty()) + Command=Arg; // Need for rar.ini. } #endif @@ -237,10 +248,10 @@ void CommandData::ReadConfig() Str++; if (wcsnicomp(Str,L"switches=",9)==0) ProcessSwitchesString(Str+9); - if (*Command!=0) + if (!Command.empty()) { wchar Cmd[16]; - wcsncpyz(Cmd,Command,ASIZE(Cmd)); + wcsncpyz(Cmd,Command.c_str(),ASIZE(Cmd)); wchar C0=toupperw(Cmd[0]); wchar C1=toupperw(Cmd[1]); if (C0=='I' || C0=='L' || C0=='M' || C0=='S' || C0=='V') @@ -260,14 +271,19 @@ void CommandData::ReadConfig() #if !defined(SFX_MODULE) -void CommandData::ProcessSwitchesString(const wchar *Str) +void CommandData::ProcessSwitchesString(const std::wstring &Str) { - wchar *Par; - while ((Str=AllocCmdParam(Str,&Par))!=NULL) + std::wstring Par; + std::wstring::size_type Pos=0; + while (GetCmdParam(Str,Pos,Par)) { - if (IsSwitch(*Par)) - ProcessSwitch(Par+1); - free(Par); + if (IsSwitch(Par[0])) + ProcessSwitch(&Par[1]); + else + { + mprintf(St(MSwSyntaxError),Par.c_str()); + ErrHandler.Exit(RARX_USERERROR); + } } } #endif @@ -330,13 +346,12 @@ void CommandData::ProcessSwitch(const wchar *Switch) break; } break; - case 'N': // Reserved for archive name. - break; case 'O': AddArcOnly=true; break; case 'P': - wcsncpyz(ArcPath,Switch+2,ASIZE(ArcPath)); + // Convert slashes here than before every comparison. + SlashToNative(Switch+2,ArcPath); break; case 'S': SyncFiles=true; @@ -347,7 +362,14 @@ void CommandData::ProcessSwitch(const wchar *Switch) } break; case 'C': - if (Switch[2]==0) + if (Switch[2]!=0) + { + if (wcsicomp(Switch+1,L"FG-")==0) + ConfigDisabled=true; + else + BadSwitch(Switch); + } + else switch(toupperw(Switch[1])) { case '-': @@ -359,10 +381,15 @@ void CommandData::ProcessSwitch(const wchar *Switch) case 'L': ConvertNames=NAMES_LOWERCASE; break; + default: + BadSwitch(Switch); + break; } break; case 'D': - if (Switch[2]==0) + if (Switch[2]!=0) + BadSwitch(Switch); + else switch(toupperw(Switch[1])) { case 'S': @@ -374,6 +401,9 @@ void CommandData::ProcessSwitch(const wchar *Switch) case 'F': DeleteFiles=true; break; + default: + BadSwitch(Switch); + break; } break; case 'E': @@ -395,7 +425,11 @@ void CommandData::ProcessSwitch(const wchar *Switch) ExclPath=EXCL_ABSPATH; break; case '4': - wcsncpyz(ExclArcPath,Switch+3,ASIZE(ExclArcPath)); + // Convert slashes here than before every comparison. + SlashToNative(Switch+3,ExclArcPath); + break; + default: + BadSwitch(Switch); break; } break; @@ -431,7 +465,7 @@ void CommandData::ProcessSwitch(const wchar *Switch) else if (!Password.IsSet()) { - uiGetPassword(UIPASSWORD_GLOBAL,NULL,&Password,NULL); + uiGetPassword(UIPASSWORD_GLOBAL,L"",&Password,NULL); eprintf(L"\n"); } break; @@ -443,7 +477,7 @@ void CommandData::ProcessSwitch(const wchar *Switch) case 'I': if (wcsnicomp(Switch+1,L"LOG",3)==0) { - wcsncpyz(LogName,Switch[4]!=0 ? Switch+4:DefLogName,ASIZE(LogName)); + LogName=Switch[4]!=0 ? Switch+4:DefLogName; break; } if (wcsnicomp(Switch+1,L"SND",3)==0) @@ -461,7 +495,7 @@ void CommandData::ProcessSwitch(const wchar *Switch) } if (wcsnicomp(Switch+1,L"EML",3)==0) { - wcsncpyz(EmailTo,Switch[4]!=0 ? Switch+4:L"@",ASIZE(EmailTo)); + EmailTo=Switch[4]!=0 ? Switch+4:L"@"; break; } if (wcsicomp(Switch+1,L"M")==0) // For compatibility with pre-WinRAR 6.0 -im syntax. Replaced with -idv. @@ -568,12 +602,14 @@ void CommandData::ProcessSwitch(const wchar *Switch) } switch(toupperw(*(Str++))) { - case 'T': Type=FILTER_PPM; break; +// case 'T': Type=FILTER_TEXT; break; case 'E': Type=FILTER_E8; break; case 'D': Type=FILTER_DELTA; break; - case 'A': Type=FILTER_AUDIO; break; - case 'C': Type=FILTER_RGB; break; - case 'R': Type=FILTER_ARM; break; +// case 'A': Type=FILTER_AUDIO; break; +// case 'C': Type=FILTER_RGB; break; +// case 'R': Type=FILTER_ARM; break; + case 'L': Type=FILTER_LONGRANGE; break; + case 'X': Type=FILTER_EXHAUSTIVE; break; } if (*Str=='+' || *Str=='-') State=*(Str++)=='+' ? FILTER_FORCE:FILTER_DISABLE; @@ -586,6 +622,44 @@ void CommandData::ProcessSwitch(const wchar *Switch) case 'M': break; case 'D': + { + bool SetDictLimit=toupperw(Switch[2])=='X'; + + uint64 Size=atoiw(Switch+(SetDictLimit ? 3 : 2)); + wchar LastChar=toupperw(Switch[wcslen(Switch)-1]); + if (IsDigit(LastChar)) + LastChar=SetDictLimit ? 'G':'M'; // Treat -md128 as -md128m and -mdx32 as -mdx32g. + switch(LastChar) + { + case 'K': + Size*=1024; + break; + case 'M': + Size*=1024*1024; + break; + case 'G': + Size*=1024*1024*1024; + break; + default: + BadSwitch(Switch); + } + + // 2023.07.22: For 4 GB and less we also check that it is power of 2, + // so archives are compatible with RAR 5.0+. + // We allow Size>PACK_MAX_DICT here, so we can use -md[x] to unpack + // archives created by future versions with higher PACK_MAX_DICT + uint Flags; + if ((Size=Archive::GetWinSize(Size,Flags))==0 || + Size<=0x100000000ULL && !IsPow2(Size)) + BadSwitch(Switch); + else + if (SetDictLimit) + WinSizeLimit=Size; + else + { + WinSize=Size; + } + } break; case 'E': if (toupperw(Switch[2])=='S' && Switch[3]==0) @@ -593,25 +667,20 @@ void CommandData::ProcessSwitch(const wchar *Switch) break; case 'S': { - wchar StoreNames[1024]; - wcsncpyz(StoreNames,(Switch[2]==0 ? DefaultStoreList:Switch+2),ASIZE(StoreNames)); - wchar *Names=StoreNames; - while (*Names!=0) + std::wstring StoreNames=(Switch[2]==0 ? DefaultStoreList:Switch+2); + size_t Pos=0; + while (Pos. + std::wstring ExclArcPath; // For -ep4 switch. + std::wstring LogName; + std::wstring EmailTo; + + // Read data from stdin and store in archive under a name specified here + // when archiving. Read an archive from stdin if any non-empty string + // is specified here when extracting. + std::wstring UseStdin; StringList FileArgs; StringList ExclArgs; @@ -69,6 +82,11 @@ class CommandData:public RAROptions SecPassword Password; std::vector NextVolSizes; + + +#ifdef RARDLL + std::wstring DllDestName; +#endif }; #endif diff --git a/multiarc/src/formats/rar/unrar/cmdfilter.cpp b/multiarc/src/formats/rar/unrar/cmdfilter.cpp index e0add14bf..c20d8d6b9 100644 --- a/multiarc/src/formats/rar/unrar/cmdfilter.cpp +++ b/multiarc/src/formats/rar/unrar/cmdfilter.cpp @@ -1,7 +1,7 @@ // Return 'true' if we need to exclude the file from processing as result // of -x switch. If CheckInclList is true, we also check the file against // the include list created with -n switch. -bool CommandData::ExclCheck(const wchar *CheckName,bool Dir,bool CheckFullPath,bool CheckInclList) +bool CommandData::ExclCheck(const std::wstring &CheckName,bool Dir,bool CheckFullPath,bool CheckInclList) { if (CheckArgs(&ExclArgs,Dir,CheckName,CheckFullPath,MATCH_WILDSUBPATH)) return true; @@ -13,17 +13,15 @@ bool CommandData::ExclCheck(const wchar *CheckName,bool Dir,bool CheckFullPath,b } -bool CommandData::CheckArgs(StringList *Args,bool Dir,const wchar *CheckName,bool CheckFullPath,int MatchMode) +bool CommandData::CheckArgs(StringList *Args,bool Dir,const std::wstring &CheckName,bool CheckFullPath,int MatchMode) { - wchar *Name=ConvertPath(CheckName,NULL,0); - wchar FullName[NM]; - wchar CurMask[NM]; - *FullName=0; + std::wstring Name,FullName,CurMask; + ConvertPath(&CheckName,&Name); Args->Rewind(); - while (Args->GetString(CurMask,ASIZE(CurMask))) + while (Args->GetString(CurMask)) { - wchar *LastMaskChar=PointToLastChar(CurMask); - bool DirMask=IsPathDiv(*LastMaskChar); // Mask for directories only. + wchar LastMaskChar=GetLastChar(CurMask); + bool DirMask=IsPathDiv(LastMaskChar); // Mask for directories only. if (Dir) { @@ -33,16 +31,33 @@ bool CommandData::CheckArgs(StringList *Args,bool Dir,const wchar *CheckName,boo // We process the directory and have the directory exclusion mask. // So let's convert "mask\" to "mask" and process it normally. - *LastMaskChar=0; + CurMask.pop_back(); } else { - // REMOVED, we want -npath\* to match empty folders too. - // If mask has wildcards in name part and does not have the trailing - // '\' character, we cannot use it for directories. - - // if (IsWildcard(PointToName(CurMask))) - // continue; + // This code doesn't allow to apply -n and -x wildcard masks without + // trailing slash to folders unless these masks are * and *.*. + // See the changes history below. + // 2023.03.26: Previously we removed this code completely to let + // 'rar a arc dir -ndir\path\*' include empty folders in 'path' too. + // But then we received an email from user not willing -x*.avi to + // exclude folders like dir.avi with non-avi files. Also rar.txt + // mentions that masks like *.avi exclude only files. Initially + // we wanted masks like -npath\* or -xpath\* to match the entire + // contents of path including empty folders and added the special + // check for "*" and "*.*". But this is not very straightforward, + // when *.* and *.avi are processed differently, especially taking + // into account that we can specify the exact folder name without + // wildcards to process it and masks like 'dir*\' can be used to + // exclude folders. So we decided to skip all usual wildcard masks + // for folders. + // 2023.11.22: We returned the special check for "*" and "*.*", + // because users expected 'rar a arc dir -xdir\*' to exclude + // everything including subfolders in 'dir'. For now we returned it + // both for -n and -x, but we can limit it to -x only if needed. + std::wstring Name=PointToName(CurMask); + if (IsWildcard(Name) && Name!=L"*" && Name!=L"*.*") + continue; } } else @@ -54,7 +69,7 @@ bool CommandData::CheckArgs(StringList *Args,bool Dir,const wchar *CheckName,boo // is excluded from further scanning. if (DirMask) - wcsncatz(CurMask,L"*",ASIZE(CurMask)); + CurMask+=L"*"; } #ifndef SFX_MODULE @@ -66,19 +81,20 @@ bool CommandData::CheckArgs(StringList *Args,bool Dir,const wchar *CheckName,boo // correctly. Moreover, removing "*\" from mask would break // the comparison, because now all names have the path. - if (*FullName==0) - ConvertNameToFull(CheckName,FullName,ASIZE(FullName)); + if (FullName.empty()) + ConvertNameToFull(CheckName,FullName); if (CmpName(CurMask,FullName,MatchMode)) return true; } else #endif { - wchar NewName[NM+2],*CurName=Name; + std::wstring CurName=Name; // Important to convert before "*\" check below, so masks like // d:*\something are processed properly. - wchar *CmpMask=ConvertPath(CurMask,NULL,0); + size_t MaskOffset=ConvertPath(&CurMask,nullptr); + std::wstring CmpMask=CurMask.substr(MaskOffset); if (CmpMask[0]=='*' && IsPathDiv(CmpMask[1])) { @@ -86,10 +102,9 @@ bool CommandData::CheckArgs(StringList *Args,bool Dir,const wchar *CheckName,boo // but also in the current directory. We convert the name // from 'name' to '.\name' to be matched by "*\" part even if it is // in current directory. - NewName[0]='.'; - NewName[1]=CPATHDIVIDER; - wcsncpyz(NewName+2,Name,ASIZE(NewName)-2); - CurName=NewName; + CurName=L'.'; + CurName+=CPATHDIVIDER; + CurName+=Name; } if (CmpName(CmpMask,CurName,MatchMode)) @@ -277,10 +292,10 @@ bool CommandData::SizeCheck(int64 Size) // Return 0 if file must not be processed or a number of matched parameter otherwise. int CommandData::IsProcessFile(FileHeader &FileHead,bool *ExactMatch,int MatchType, - bool Flags,wchar *MatchedArg,uint MatchedArgSize) + bool Flags,std::wstring *MatchedArg) { - if (MatchedArg!=NULL && MatchedArgSize>0) - *MatchedArg=0; + if (MatchedArg!=NULL) + MatchedArg->clear(); bool Dir=FileHead.Dir; if (ExclCheck(FileHead.FileName,Dir,false,true)) return 0; @@ -295,17 +310,22 @@ int CommandData::IsProcessFile(FileHeader &FileHead,bool *ExactMatch,int MatchTy if (!Dir && SizeCheck(FileHead.UnpSize)) return 0; #endif - wchar *ArgName; + std::wstring ArgName; FileArgs.Rewind(); - for (int StringCount=1;(ArgName=FileArgs.GetString())!=NULL;StringCount++) + for (int StringCount=1;FileArgs.GetString(ArgName);StringCount++) + { + // Ensure that both parameters of CmpName are either C++ strings or + // pointers, so we avoid time consuming string construction for one of + // parameters in this expensive loop. if (CmpName(ArgName,FileHead.FileName,MatchType)) { if (ExactMatch!=NULL) *ExactMatch=wcsicompc(ArgName,FileHead.FileName)==0; if (MatchedArg!=NULL) - wcsncpyz(MatchedArg,ArgName,MatchedArgSize); + *MatchedArg=ArgName; return StringCount; } + } return 0; } diff --git a/multiarc/src/formats/rar/unrar/cmdmix.cpp b/multiarc/src/formats/rar/unrar/cmdmix.cpp index 6bd1e1ac0..e70b7d001 100644 --- a/multiarc/src/formats/rar/unrar/cmdmix.cpp +++ b/multiarc/src/formats/rar/unrar/cmdmix.cpp @@ -103,7 +103,7 @@ void CommandData::OutHelp(RAR_EXIT ExitCode) if (CmpMSGID(Help[I],MCHelpSwOW)) continue; #endif -#if !defined(_WIN_ALL) && !defined(_EMX) +#ifndef _WIN_ALL if (CmpMSGID(Help[I],MCHelpSwAC)) continue; #endif diff --git a/multiarc/src/formats/rar/unrar/coder.cpp b/multiarc/src/formats/rar/unrar/coder.cpp index 9d971a8ad..6282371cd 100644 --- a/multiarc/src/formats/rar/unrar/coder.cpp +++ b/multiarc/src/formats/rar/unrar/coder.cpp @@ -1,8 +1,8 @@ -inline unsigned int RangeCoder::GetChar() +inline byte RangeCoder::GetChar() { - return(UnpackRead->GetChar()); + return UnpackRead->GetChar(); } @@ -11,8 +11,8 @@ void RangeCoder::InitDecoder(Unpack *UnpackRead) RangeCoder::UnpackRead=UnpackRead; low=code=0; - range=uint(-1); - for (int i=0;i < 4;i++) + range=0xffffffff; + for (uint i = 0; i < 4; i++) code=(code << 8) | GetChar(); } diff --git a/multiarc/src/formats/rar/unrar/coder.hpp b/multiarc/src/formats/rar/unrar/coder.hpp index 7b36ff218..42c3f3384 100644 --- a/multiarc/src/formats/rar/unrar/coder.hpp +++ b/multiarc/src/formats/rar/unrar/coder.hpp @@ -11,7 +11,7 @@ class RangeCoder inline uint GetCurrentShiftCount(uint SHIFT); inline void Decode(); inline void PutChar(unsigned int c); - inline unsigned int GetChar(); + inline byte GetChar(); uint low, code, range; struct SUBRANGE diff --git a/multiarc/src/formats/rar/unrar/compress.hpp b/multiarc/src/formats/rar/unrar/compress.hpp index 4ef8570f4..018909a01 100644 --- a/multiarc/src/formats/rar/unrar/compress.hpp +++ b/multiarc/src/formats/rar/unrar/compress.hpp @@ -21,10 +21,12 @@ class PackDef static const uint LOW_DIST_REP_COUNT = 16; static const uint NC = 306; /* alphabet = {0, 1, 2, ..., NC - 1} */ - static const uint DC = 64; + static const uint DCB = 64; // Base distance codes up to 4 GB. + static const uint DCX = 80; // Extended distance codes up to 1 TB. static const uint LDC = 16; static const uint RC = 44; - static const uint HUFF_TABLE_SIZE = NC + DC + RC + LDC; + static const uint HUFF_TABLE_SIZEB = NC + DCB + RC + LDC; + static const uint HUFF_TABLE_SIZEX = NC + DCX + RC + LDC; static const uint BC = 20; static const uint NC30 = 299; /* alphabet = {0, 1, 2, ..., NC - 1} */ @@ -43,10 +45,6 @@ class PackDef // Largest alphabet size among all values listed above. static const uint LARGEST_TABLE_SIZE = 306; - enum { - CODE_HUFFMAN, CODE_LZ, CODE_REPEATLZ, CODE_CACHELZ, CODE_STARTFILE, - CODE_ENDFILE, CODE_FILTER, CODE_FILTERDATA - }; }; @@ -54,7 +52,10 @@ enum FilterType { // These values must not be changed, because we use them directly // in RAR5 compression and decompression code. FILTER_DELTA=0, FILTER_E8, FILTER_E8E9, FILTER_ARM, - FILTER_AUDIO, FILTER_RGB, FILTER_ITANIUM, FILTER_PPM, FILTER_NONE + FILTER_AUDIO, FILTER_RGB, FILTER_ITANIUM, FILTER_TEXT, + + // These values can be changed. + FILTER_LONGRANGE,FILTER_EXHAUSTIVE,FILTER_NONE }; #endif diff --git a/multiarc/src/formats/rar/unrar/consio.cpp b/multiarc/src/formats/rar/unrar/consio.cpp index fa35d6146..023c3f3c8 100644 --- a/multiarc/src/formats/rar/unrar/consio.cpp +++ b/multiarc/src/formats/rar/unrar/consio.cpp @@ -5,8 +5,6 @@ static MESSAGE_TYPE MsgStream=MSG_STDOUT; static RAR_CHARSET RedirectCharset=RCH_DEFAULT; static bool ProhibitInput=false; -const int MaxMsgSize=2*NM+2048; - static bool StdoutRedirected=false,StderrRedirected=false,StdinRedirected=false; #ifdef _WIN_ALL @@ -71,44 +69,42 @@ void ProhibitConsoleInput() #ifndef SILENT static void cvt_wprintf(FILE *dest,const wchar *fmt,va_list arglist) { - // This buffer is for format string only, not for entire output, - // so it can be short enough. - wchar fmtw[1024]; - PrintfPrepareFmt(fmt,fmtw,ASIZE(fmtw)); + // No need for PrintfPrepareFmt here, vwstrprintf calls it. + std::wstring s=vwstrprintf(fmt,arglist); + + ReplaceEsc(s); + #ifdef _WIN_ALL - safebuf wchar Msg[MaxMsgSize]; if (dest==stdout && StdoutRedirected || dest==stderr && StderrRedirected) { HANDLE hOut=GetStdHandle(dest==stdout ? STD_OUTPUT_HANDLE:STD_ERROR_HANDLE); - vswprintf(Msg,ASIZE(Msg),fmtw,arglist); DWORD Written; if (RedirectCharset==RCH_UNICODE) - WriteFile(hOut,Msg,(DWORD)wcslen(Msg)*sizeof(*Msg),&Written,NULL); + WriteFile(hOut,s.data(),(DWORD)s.size()*sizeof(s[0]),&Written,NULL); else { // Avoid Unicode for redirect in Windows, it does not work with pipes. - safebuf char MsgA[MaxMsgSize]; + std::string MsgA; if (RedirectCharset==RCH_UTF8) - WideToUtf(Msg,MsgA,ASIZE(MsgA)); + WideToUtf(s,MsgA); else - WideToChar(Msg,MsgA,ASIZE(MsgA)); + WideToChar(s,MsgA); if (RedirectCharset==RCH_DEFAULT || RedirectCharset==RCH_OEM) - CharToOemA(MsgA,MsgA); // Console tools like 'more' expect OEM encoding. + CharToOemA(&MsgA[0],&MsgA[0]); // Console tools like 'more' expect OEM encoding. // We already converted \n to \r\n above, so we use WriteFile instead // of C library to avoid unnecessary additional conversion. - WriteFile(hOut,MsgA,(DWORD)strlen(MsgA),&Written,NULL); + WriteFile(hOut,MsgA.data(),(DWORD)MsgA.size(),&Written,NULL); } return; } // MSVC2008 vfwprintf writes every character to console separately // and it is too slow. We use direct WriteConsole call instead. - vswprintf(Msg,ASIZE(Msg),fmtw,arglist); HANDLE hOut=GetStdHandle(dest==stderr ? STD_ERROR_HANDLE:STD_OUTPUT_HANDLE); DWORD Written; - WriteConsole(hOut,Msg,(DWORD)wcslen(Msg),&Written,NULL); + WriteConsole(hOut,s.data(),(DWORD)s.size(),&Written,NULL); #else - vfwprintf(dest,fmtw,arglist); + fputws(s.c_str(),dest); // We do not use setbuf(NULL) in Unix (see comments in InitConsole). fflush(dest); #endif @@ -160,13 +156,11 @@ static void QuitIfInputProhibited() } -static void GetPasswordText(wchar *Str,uint MaxLength) +static void GetPasswordText(std::wstring &Str) { - if (MaxLength==0) - return; QuitIfInputProhibited(); if (StdinRedirected) - getwstr(Str,MaxLength); // Read from pipe or redirected file. + getwstr(Str); // Read from pipe or redirected file. else { #ifdef _WIN_ALL @@ -175,50 +169,47 @@ static void GetPasswordText(wchar *Str,uint MaxLength) GetConsoleMode(hConIn,&ConInMode); SetConsoleMode(hConIn,ENABLE_LINE_INPUT); // Remove ENABLE_ECHO_INPUT. + std::vector Buf(MAXPASSWORD); + // We prefer ReadConsole to ReadFile, so we can read Unicode input. DWORD Read=0; - ReadConsole(hConIn,Str,MaxLength-1,&Read,NULL); - Str[Read]=0; + ReadConsole(hConIn,Buf.data(),(DWORD)Buf.size()-1,&Read,NULL); + Buf[Read]=0; + Str=Buf.data(); + cleandata(Buf.data(),Buf.size()*sizeof(Buf[0])); + SetConsoleMode(hConIn,ConInMode); - // If entered password is longer than MAXPASSWORD and truncated, - // read its unread part anyway, so it isn't read later as the second - // password for -p switch. Low level FlushConsoleInputBuffer doesn't help + // 2023.03.12: Previously we checked for presence of "\n" in entered + // passwords, supposing that truncated strings do not include it. + // We did it to read the rest of excessively long string, so it is not + // read later as the second password for -p switch. But this "\n" check + // doesn't seem to work in Windows 10 anymore and "\r" is present even + // in truncated strings. Also we increased MAXPASSWORD, so it is larger + // than MAXPASSWORD_RAR. Thus we removed this check as not working + // and not that necessary. Low level FlushConsoleInputBuffer doesn't help // for high level ReadConsole, which in line input mode seems to store // the rest of string in its own internal buffer. - if (wcschr(Str,'\r')==NULL) // If '\r' is missing, the password was truncated. - while (true) - { - wchar Trail[64]; - DWORD TrailRead=0; - // Use ASIZE(Trail)-1 to reserve the space for trailing 0. - ReadConsole(hConIn,Trail,ASIZE(Trail)-1,&TrailRead,NULL); - Trail[TrailRead]=0; - if (TrailRead==0 || wcschr(Trail,'\r')!=NULL) - break; - } - #else - char StrA[MAXPASSWORD*4]; // "*4" for multibyte UTF-8 characters. -#if defined(_EMX) || defined (__VMS) - fgets(StrA,ASIZE(StrA)-1,stdin); + std::vector StrA(MAXPASSWORD*4); // "*4" for multibyte UTF-8 characters. +#ifdef __VMS + fgets(StrA.data(),StrA.size()-1,stdin); #elif defined(__sun) - strncpyz(StrA,getpassphrase(""),ASIZE(StrA)); + strncpyz(StrA.data(),getpassphrase(""),StrA.size()); #else - strncpyz(StrA,getpass(""),ASIZE(StrA)); + strncpyz(StrA.data(),getpass(""),StrA.size()); #endif - CharToWide(StrA,Str,MaxLength); - cleandata(StrA,sizeof(StrA)); + CharToWide(StrA.data(),Str); + cleandata(StrA.data(),StrA.size()*sizeof(StrA[0])); #endif } - Str[MaxLength-1]=0; RemoveLF(Str); } #endif #ifndef SILENT -bool GetConsolePassword(UIPASSWORD_TYPE Type,const wchar *FileName,SecPassword *Password) +bool GetConsolePassword(UIPASSWORD_TYPE Type,const std::wstring &FileName,SecPassword *Password) { if (!StdinRedirected) uiAlarm(UIALARM_QUESTION); @@ -229,33 +220,33 @@ bool GetConsolePassword(UIPASSWORD_TYPE Type,const wchar *FileName,SecPassword * if (Type==UIPASSWORD_GLOBAL) eprintf(L"\n%s: ",St(MAskPsw)); else - eprintf(St(MAskPswFor),FileName); + eprintf(St(MAskPswFor),FileName.c_str()); - wchar PlainPsw[MAXPASSWORD+1]; - GetPasswordText(PlainPsw,ASIZE(PlainPsw)); - if (*PlainPsw==0 && Type==UIPASSWORD_GLOBAL) + std::wstring PlainPsw; + GetPasswordText(PlainPsw); + if (PlainPsw.empty() && Type==UIPASSWORD_GLOBAL) return false; - if (wcslen(PlainPsw)>=MAXPASSWORD) + if (PlainPsw.size()>=MAXPASSWORD) { - PlainPsw[MAXPASSWORD-1]=0; + PlainPsw.erase(MAXPASSWORD-1); uiMsg(UIERROR_TRUNCPSW,MAXPASSWORD-1); } if (!StdinRedirected && Type==UIPASSWORD_GLOBAL) { eprintf(St(MReAskPsw)); - wchar CmpStr[MAXPASSWORD]; - GetPasswordText(CmpStr,ASIZE(CmpStr)); - if (*CmpStr==0 || wcscmp(PlainPsw,CmpStr)!=0) + std::wstring CmpStr; + GetPasswordText(CmpStr); + if (CmpStr.empty() || PlainPsw!=CmpStr) { eprintf(St(MNotMatchPsw)); - cleandata(PlainPsw,sizeof(PlainPsw)); - cleandata(CmpStr,sizeof(CmpStr)); + cleandata(&PlainPsw[0],PlainPsw.size()*sizeof(PlainPsw[0])); + cleandata(&CmpStr[0],CmpStr.size()*sizeof(CmpStr[0])); continue; } - cleandata(CmpStr,sizeof(CmpStr)); + cleandata(&CmpStr[0],CmpStr.size()*sizeof(CmpStr[0])); } - Password->Set(PlainPsw); - cleandata(PlainPsw,sizeof(PlainPsw)); + Password->Set(PlainPsw.c_str()); + cleandata(&PlainPsw[0],PlainPsw.size()*sizeof(PlainPsw[0])); break; } return true; @@ -264,14 +255,17 @@ bool GetConsolePassword(UIPASSWORD_TYPE Type,const wchar *FileName,SecPassword * #ifndef SILENT -bool getwstr(wchar *str,size_t n) +bool getwstr(std::wstring &str) { // Print buffered prompt title function before waiting for input. fflush(stderr); QuitIfInputProhibited(); - *str=0; + str.clear(); + + const size_t MaxRead=MAXPATHSIZE; // Large enough to read a file name. + #if defined(_WIN_ALL) // fgetws does not work well with non-English text in Windows, // so we do not use it. @@ -279,11 +273,11 @@ bool getwstr(wchar *str,size_t n) { // fgets does not work well with pipes in Windows in our test. // Let's use files. - Array StrA(n*4); // Up to 4 UTF-8 characters per wchar_t. + std::vector StrA(MaxRead*4); // Up to 4 UTF-8 characters per wchar_t. File SrcFile; SrcFile.SetHandleType(FILE_HANDLESTD); SrcFile.SetLineInputMode(true); - int ReadSize=SrcFile.Read(&StrA[0],StrA.Size()-1); + int ReadSize=SrcFile.Read(&StrA[0],StrA.size()-1); if (ReadSize<=0) { // Looks like stdin is a null device. We can enter to infinite loop @@ -297,19 +291,23 @@ bool getwstr(wchar *str,size_t n) // use "chcp" in console. But we avoid OEM to ANSI conversion, // because we also want to handle ANSI files redirection correctly, // like "rar ... < ansifile.txt". - CharToWide(&StrA[0],str,n); - cleandata(&StrA[0],StrA.Size()); // We can use this function to enter passwords. + CharToWide(&StrA[0],str); + cleandata(&StrA[0],StrA.size()); // We can use this function to enter passwords. } else { + std::vector Buf(MaxRead); // Up to 4 UTF-8 characters per wchar_t. DWORD ReadSize=0; - if (ReadConsole(GetStdHandle(STD_INPUT_HANDLE),str,DWORD(n-1),&ReadSize,NULL)==0) + if (ReadConsole(GetStdHandle(STD_INPUT_HANDLE),&Buf[0],(DWORD)Buf.size()-1,&ReadSize,NULL)==0) return false; - str[ReadSize]=0; + Buf[ReadSize]=0; + str=Buf.data(); } #else - if (fgetws(str,n,stdin)==NULL) + std::vector Buf(MaxRead); // Up to 4 UTF-8 characters per wchar_t. + if (fgetws(&Buf[0],Buf.size(),stdin)==NULL) ErrHandler.Exit(RARX_USERBREAK); // Avoid infinite Ask() loop. + str=Buf.data(); #endif RemoveLF(str); return true; @@ -361,8 +359,8 @@ int Ask(const wchar *AskStr) eprintf(L"[%c]%ls",Item[I][KeyPos],&Item[I][KeyPos+1]); } eprintf(L" "); - wchar Str[50]; - getwstr(Str,ASIZE(Str)); + std::wstring Str; + getwstr(Str); wchar Ch=toupperw(Str[0]); for (int I=0;I[{key};"{string}"p used to redefine // a keyboard key on some terminals. @@ -389,18 +387,16 @@ static bool IsCommentUnsafe(const wchar *Data,size_t Size) } -void OutComment(const wchar *Comment,size_t Size) +void OutComment(const std::wstring &Comment) { - if (IsCommentUnsafe(Comment,Size)) + if (IsCommentUnsafe(Comment)) return; const size_t MaxOutSize=0x400; - for (size_t I=0;I>1)^0xEDB88320 : (C>>1); CRCTab[I]=C; } + +#ifdef USE_NEON_CRC32 + #ifdef _APPLE + // getauxval isn't available in OS X + uint Value=0; + size_t Size=sizeof(Value); + int RetCode=sysctlbyname("hw.optional.armv8_crc32",&Value,&Size,NULL,0); + CRC_Neon=RetCode==0 && Value!=0; + #else + CRC_Neon=(getauxval(AT_HWCAP) & HWCAP_CRC32)!=0; + #endif +#endif + } @@ -47,7 +64,7 @@ static void InitTables() for (uint I=0;I<256;I++) // Build additional lookup tables. { uint C=crc_tables[0][I]; - for (uint J=1;J<8;J++) + for (uint J=1;J<16;J++) { C=crc_tables[0][(byte)C]^(C>>8); crc_tables[J][I]=C; @@ -63,28 +80,66 @@ uint CRC32(uint StartCRC,const void *Addr,size_t Size) { byte *Data=(byte *)Addr; +#ifdef USE_NEON_CRC32 + if (CRC_Neon) + { + for (;Size>=8;Size-=8,Data+=8) +#ifdef __clang__ + StartCRC = __builtin_arm_crc32d(StartCRC, RawGet8(Data)); +#else + StartCRC = __builtin_aarch64_crc32x(StartCRC, RawGet8(Data)); +#endif + for (;Size>0;Size--,Data++) // Process left data. +#ifdef __clang__ + StartCRC = __builtin_arm_crc32b(StartCRC, *Data); +#else + StartCRC = __builtin_aarch64_crc32b(StartCRC, *Data); +#endif + return StartCRC; + } +#endif + #ifdef USE_SLICING - // Align Data to 8 for better performance. - for (;Size>0 && ((size_t)Data & 7);Size--,Data++) + // Align Data to 16 for better performance and to avoid ALLOW_MISALIGNED + // check below. + for (;Size>0 && ((size_t)Data & 15)!=0;Size--,Data++) StartCRC=crc_tables[0][(byte)(StartCRC^Data[0])]^(StartCRC>>8); - for (;Size>=8;Size-=8,Data+=8) + // 2023.12.06: We switched to slicing-by-16, which seems to be faster than + // slicing-by-8 on modern CPUs. Slicing-by-32 would require 32 KB for tables + // and could be limited by L1 cache size on some CPUs. + for (;Size>=16;Size-=16,Data+=16) { #ifdef BIG_ENDIAN - StartCRC ^= Data[0]|(Data[1] << 8)|(Data[2] << 16)|(Data[3] << 24); - uint NextData = Data[4]|(Data[5] << 8)|(Data[6] << 16)|(Data[7] << 24); + StartCRC ^= RawGet4(Data); + uint D1 = RawGet4(Data+4); + uint D2 = RawGet4(Data+8); + uint D3 = RawGet4(Data+12); #else + // We avoid RawGet4 here for performance reason, to access uint32 + // directly even if ALLOW_MISALIGNED isn't defined. We can do it, + // because we aligned 'Data' above. StartCRC ^= *(uint32 *) Data; - uint NextData = *(uint32 *) (Data+4); + uint D1 = *(uint32 *) (Data+4); + uint D2 = *(uint32 *) (Data+8); + uint D3 = *(uint32 *) (Data+12); #endif - StartCRC = crc_tables[7][(byte) StartCRC ] ^ - crc_tables[6][(byte)(StartCRC >> 8) ] ^ - crc_tables[5][(byte)(StartCRC >> 16)] ^ - crc_tables[4][(byte)(StartCRC >> 24)] ^ - crc_tables[3][(byte) NextData ] ^ - crc_tables[2][(byte)(NextData >> 8) ] ^ - crc_tables[1][(byte)(NextData >> 16)] ^ - crc_tables[0][(byte)(NextData >> 24)]; + StartCRC = crc_tables[15][(byte) StartCRC ] ^ + crc_tables[14][(byte)(StartCRC >> 8) ] ^ + crc_tables[13][(byte)(StartCRC >> 16)] ^ + crc_tables[12][(byte)(StartCRC >> 24)] ^ + crc_tables[11][(byte) D1 ] ^ + crc_tables[10][(byte)(D1 >> 8) ] ^ + crc_tables[ 9][(byte)(D1 >> 16)] ^ + crc_tables[ 8][(byte)(D1 >> 24)] ^ + crc_tables[ 7][(byte) D2 ] ^ + crc_tables[ 6][(byte)(D2 >> 8)] ^ + crc_tables[ 5][(byte)(D2 >> 16)] ^ + crc_tables[ 4][(byte)(D2 >> 24)] ^ + crc_tables[ 3][(byte) D3 ] ^ + crc_tables[ 2][(byte)(D3 >> 8)] ^ + crc_tables[ 1][(byte)(D3 >> 16)] ^ + crc_tables[ 0][(byte)(D3 >> 24)]; } #endif @@ -110,74 +165,6 @@ ushort Checksum14(ushort StartCRC,const void *Addr,size_t Size) #endif -#if 0 -static uint64 crc64_tables[8][256]; // Tables for Slicing-by-8 for CRC64. - -void InitCRC64(uint64 *CRCTab) -{ - const uint64 poly=INT32TO64(0xC96C5795, 0xD7870F42); // 0xC96C5795D7870F42; - for (uint I=0;I<256;I++) - { - uint64 C=I; - for (uint J=0;J<8;J++) - C=(C & 1) ? (C>>1)^poly: (C>>1); - CRCTab[I]=C; - } -} - - -static void InitTables64() -{ - InitCRC64(crc64_tables[0]); - - for (uint I=0;I<256;I++) // Build additional lookup tables. - { - uint64 C=crc64_tables[0][I]; - for (uint J=1;J<8;J++) - { - C=crc64_tables[0][(byte)C]^(C>>8); - crc64_tables[J][I]=C; - } - } -} - - -// We cannot place the intialization to CRC64(), because we use this function -// in multithreaded mode and it conflicts with multithreading. -struct CallInitCRC64 {CallInitCRC64() {InitTables64();}} static CallInit64; - -uint64 CRC64(uint64 StartCRC,const void *Addr,size_t Size) -{ - byte *Data=(byte *)Addr; - - // Align Data to 8 for better performance. - for (;Size>0 && ((size_t)Data & 7)!=0;Size--,Data++) - StartCRC=crc64_tables[0][(byte)(StartCRC^Data[0])]^(StartCRC>>8); - - for (byte *DataEnd=Data+Size/8*8; Data> 8 ) ] ^ - crc64_tables[ 5 ] [ ( byte ) (Index >> 16 ) ] ^ - crc64_tables[ 4 ] [ ( byte ) (Index >> 24 ) ] ^ - crc64_tables[ 3 ] [ ( byte ) (Index >> 32 ) ] ^ - crc64_tables[ 2 ] [ ( byte ) (Index >> 40 ) ] ^ - crc64_tables[ 1 ] [ ( byte ) (Index >> 48 ) ] ^ - crc64_tables[ 0 ] [ ( byte ) (Index >> 56 ) ] ; - } - - for (Size%=8;Size>0;Size--,Data++) // Process left data. - StartCRC=crc64_tables[0][(byte)(StartCRC^Data[0])]^(StartCRC>>8); - - return StartCRC; -} #if 0 @@ -186,6 +173,11 @@ struct TestCRCStruct {TestCRCStruct() {TestCRC();exit(0);}} GlobalTesCRC; void TestCRC() { + // This function is invoked from global object and _SSE_Version is global + // and can be initialized after this function. So we explicitly initialize + // it here to enable SSE support in Blake2sp. + _SSE_Version=GetSSEVersion(); + const uint FirstSize=300; byte b[FirstSize]; @@ -251,23 +243,38 @@ void TestCRC() const size_t BufSize=0x100000; byte *Buf=new byte[BufSize]; - memset(Buf,0,BufSize); + GetRnd(Buf,BufSize); clock_t StartTime=clock(); r32=0xffffffff; - const uint BufCount=5000; + const uint64 BufCount=5000; for (uint I=0;I>16); + Key15[3]+=ushort(P+(CRCTab[P]>>16)); } } -void CryptData::SetAV15Encryption() -{ - InitCRC32(CRCTab); - Method=CRYPT_RAR15; - Key15[0]=0x4765; - Key15[1]=0x9021; - Key15[2]=0x7382; - Key15[3]=0x5215; -} - - void CryptData::SetCmt13Encryption() { Method=CRYPT_RAR13; @@ -68,7 +55,7 @@ void CryptData::Crypt15(byte *Data,size_t Count) { Key15[0]+=0x1234; Key15[1]^=CRCTab[(Key15[0] & 0x1fe)>>1]; - Key15[2]-=CRCTab[(Key15[0] & 0x1fe)>>1]>>16; + Key15[2]-=ushort(CRCTab[(Key15[0] & 0x1fe)>>1]>>16); Key15[0]^=Key15[2]; Key15[3]=rotrs(Key15[3]&0xffff,1,16)^Key15[1]; Key15[3]=rotrs(Key15[3]&0xffff,1,16); diff --git a/multiarc/src/formats/rar/unrar/crypt5.cpp b/multiarc/src/formats/rar/unrar/crypt5.cpp index 5ed65af81..2183c8f08 100644 --- a/multiarc/src/formats/rar/unrar/crypt5.cpp +++ b/multiarc/src/formats/rar/unrar/crypt5.cpp @@ -128,12 +128,12 @@ void pbkdf2(const byte *Pwd, size_t PwdLength, } -void CryptData::SetKey50(bool Encrypt,SecPassword *Password,const wchar *PwdW, +bool CryptData::SetKey50(bool Encrypt,SecPassword *Password,const wchar *PwdW, const byte *Salt,const byte *InitV,uint Lg2Cnt,byte *HashKey, byte *PswCheck) { if (Lg2Cnt>CRYPT5_KDF_LG2_COUNT_MAX) - return; + return false; byte Key[32],PswCheckValue[SHA256_DIGEST_SIZE],HashKeyValue[SHA256_DIGEST_SIZE]; bool Found=false; @@ -186,6 +186,7 @@ void CryptData::SetKey50(bool Encrypt,SecPassword *Password,const wchar *PwdW, rin.Init(Encrypt, Key, 256, InitV); cleandata(Key,sizeof(Key)); + return true; } @@ -200,6 +201,7 @@ void ConvertHashToMAC(HashValue *Value,byte *Key) Value->CRC32=0; for (uint I=0;ICRC32^=Digest[I] << ((I & 3) * 8); + Value->CRC32&=0xffffffff; // In case the variable size is larger than 32-bit. } if (Value->Type==HASH_BLAKE2) { diff --git a/multiarc/src/formats/rar/unrar/dll.cpp b/multiarc/src/formats/rar/unrar/dll.cpp index 31818e498..de8796dc2 100644 --- a/multiarc/src/formats/rar/unrar/dll.cpp +++ b/multiarc/src/formats/rar/unrar/dll.cpp @@ -44,22 +44,21 @@ HANDLE PASCAL RAROpenArchiveEx(struct RAROpenArchiveDataEx *r) Data->Cmd.FileArgs.AddString(L"*"); Data->Cmd.KeepBroken=(r->OpFlags&ROADOF_KEEPBROKEN)!=0; - char AnsiArcName[NM]; - *AnsiArcName=0; - if (r->ArcName!=NULL) + std::string AnsiArcName; + if (r->ArcName!=nullptr) { - strncpyz(AnsiArcName,r->ArcName,ASIZE(AnsiArcName)); + AnsiArcName=r->ArcName; #ifdef _WIN_ALL if (!AreFileApisANSI()) - { - OemToCharBuffA(r->ArcName,AnsiArcName,ASIZE(AnsiArcName)); - AnsiArcName[ASIZE(AnsiArcName)-1]=0; - } + IntToExt(r->ArcName,AnsiArcName); #endif } - wchar ArcName[NM]; - GetWideName(AnsiArcName,r->ArcNameW,ArcName,ASIZE(ArcName)); + std::wstring ArcName; + if (r->ArcNameW!=nullptr && *r->ArcNameW!=0) + ArcName=r->ArcNameW; + else + CharToWide(AnsiArcName,ArcName); Data->Cmd.AddArcName(ArcName); Data->Cmd.Overwrite=OVERWRITE_ALL; @@ -113,35 +112,35 @@ HANDLE PASCAL RAROpenArchiveEx(struct RAROpenArchiveDataEx *r) if (Data->Arc.FirstVolume) r->Flags|=ROADF_FIRSTVOLUME; - Array CmtDataW; - if (r->CmtBufSize!=0 && Data->Arc.GetComment(&CmtDataW)) + std::wstring CmtDataW; + if (r->CmtBufSize!=0 && Data->Arc.GetComment(CmtDataW)) { if (r->CmtBufW!=NULL) { - CmtDataW.Push(0); - size_t Size=wcslen(&CmtDataW[0])+1; +// CmtDataW.push_back(0); + size_t Size=wcslen(CmtDataW.data())+1; r->CmtState=Size>r->CmtBufSize ? ERAR_SMALL_BUF:1; r->CmtSize=(uint)Min(Size,r->CmtBufSize); - memcpy(r->CmtBufW,&CmtDataW[0],(r->CmtSize-1)*sizeof(*r->CmtBufW)); + memcpy(r->CmtBufW,CmtDataW.data(),(r->CmtSize-1)*sizeof(*r->CmtBufW)); r->CmtBufW[r->CmtSize-1]=0; } else if (r->CmtBuf!=NULL) { - Array CmtData(CmtDataW.Size()*4+1); - memset(&CmtData[0],0,CmtData.Size()); - WideToChar(&CmtDataW[0],&CmtData[0],CmtData.Size()-1); - size_t Size=strlen(&CmtData[0])+1; + std::vector CmtData(CmtDataW.size()*4+1); + WideToChar(&CmtDataW[0],&CmtData[0],CmtData.size()-1); + size_t Size=strlen(CmtData.data())+1; r->CmtState=Size>r->CmtBufSize ? ERAR_SMALL_BUF:1; r->CmtSize=(uint)Min(Size,r->CmtBufSize); - memcpy(r->CmtBuf,&CmtData[0],r->CmtSize-1); + memcpy(r->CmtBuf,CmtData.data(),r->CmtSize-1); r->CmtBuf[r->CmtSize-1]=0; } } else r->CmtState=r->CmtSize=0; + Data->Extract.ExtractArchiveInit(Data->Arc); return (HANDLE)Data; } @@ -183,8 +182,7 @@ int PASCAL RARCloseArchive(HANDLE hArcData) int PASCAL RARReadHeader(HANDLE hArcData,struct RARHeaderData *D) { - struct RARHeaderDataEx X; - memset(&X,0,sizeof(X)); + struct RARHeaderDataEx X{}; int Code=RARReadHeaderEx(hArcData,&X); @@ -242,14 +240,18 @@ int PASCAL RARReadHeaderEx(HANDLE hArcData,struct RARHeaderDataEx *D) else return Code; } - wcsncpy(D->ArcNameW,Data->Arc.FileName,ASIZE(D->ArcNameW)); + wcsncpyz(D->ArcNameW,Data->Arc.FileName.c_str(),ASIZE(D->ArcNameW)); WideToChar(D->ArcNameW,D->ArcName,ASIZE(D->ArcName)); + if (D->ArcNameEx!=nullptr) + wcsncpyz(D->ArcNameEx,Data->Arc.FileName.c_str(),D->ArcNameExSize); - wcsncpy(D->FileNameW,hd->FileName,ASIZE(D->FileNameW)); + wcsncpyz(D->FileNameW,hd->FileName.c_str(),ASIZE(D->FileNameW)); WideToChar(D->FileNameW,D->FileName,ASIZE(D->FileName)); #ifdef _WIN_ALL CharToOemA(D->FileName,D->FileName); #endif + if (D->FileNameEx!=nullptr) + wcsncpyz(D->FileNameEx,hd->FileName.c_str(),D->FileNameExSize); D->Flags=0; if (hd->SplitBefore) @@ -311,7 +313,7 @@ int PASCAL RARReadHeaderEx(HANDLE hArcData,struct RARHeaderDataEx *D) // this RedirNameSize check sometimes later. if (hd->RedirType!=FSREDIR_NONE && D->RedirName!=NULL && D->RedirNameSize>0 && D->RedirNameSize<100000) - wcsncpyz(D->RedirName,hd->RedirName,D->RedirNameSize); + wcsncpyz(D->RedirName,hd->RedirName.c_str(),D->RedirNameSize); D->DirTarget=hd->DirTarget; } catch (RAR_EXIT ErrCode) @@ -346,50 +348,52 @@ int PASCAL ProcessFile(HANDLE hArcData,int Operation,char *DestPath,char *DestNa { Data->Cmd.DllOpMode=Operation; - *Data->Cmd.ExtrPath=0; - *Data->Cmd.DllDestName=0; + Data->Cmd.ExtrPath.clear(); + Data->Cmd.DllDestName.clear(); if (DestPath!=NULL) { - char ExtrPathA[NM]; - strncpyz(ExtrPathA,DestPath,ASIZE(ExtrPathA)-2); + std::string ExtrPathA=DestPath; #ifdef _WIN_ALL // We must not apply OemToCharBuffA directly to DestPath, // because we do not know DestPath length and OemToCharBuffA // does not stop at 0. - OemToCharA(ExtrPathA,ExtrPathA); + IntToExt(ExtrPathA,ExtrPathA); #endif - CharToWide(ExtrPathA,Data->Cmd.ExtrPath,ASIZE(Data->Cmd.ExtrPath)); - AddEndSlash(Data->Cmd.ExtrPath,ASIZE(Data->Cmd.ExtrPath)); + CharToWide(ExtrPathA,Data->Cmd.ExtrPath); + AddEndSlash(Data->Cmd.ExtrPath); } if (DestName!=NULL) { - char DestNameA[NM]; - strncpyz(DestNameA,DestName,ASIZE(DestNameA)-2); + std::string DestNameA=DestName; #ifdef _WIN_ALL // We must not apply OemToCharBuffA directly to DestName, // because we do not know DestName length and OemToCharBuffA // does not stop at 0. - OemToCharA(DestNameA,DestNameA); + IntToExt(DestNameA,DestNameA); #endif - CharToWide(DestNameA,Data->Cmd.DllDestName,ASIZE(Data->Cmd.DllDestName)); + CharToWide(DestNameA,Data->Cmd.DllDestName); } if (DestPathW!=NULL) { - wcsncpy(Data->Cmd.ExtrPath,DestPathW,ASIZE(Data->Cmd.ExtrPath)); - AddEndSlash(Data->Cmd.ExtrPath,ASIZE(Data->Cmd.ExtrPath)); + Data->Cmd.ExtrPath=DestPathW; + AddEndSlash(Data->Cmd.ExtrPath); } if (DestNameW!=NULL) - wcsncpyz(Data->Cmd.DllDestName,DestNameW,ASIZE(Data->Cmd.DllDestName)); + Data->Cmd.DllDestName=DestNameW; - wcsncpyz(Data->Cmd.Command,Operation==RAR_EXTRACT ? L"X":L"T",ASIZE(Data->Cmd.Command)); + Data->Cmd.Command=Operation==RAR_EXTRACT ? L"X":L"T"; Data->Cmd.Test=Operation!=RAR_EXTRACT; bool Repeat=false; Data->Extract.ExtractCurrentFile(Data->Arc,Data->HeaderSize,Repeat); // Now we process extra file information if any. + // It is important to do it in the same ProcessFile(), because caller + // app can rely on this behavior, for example, to overwrite + // the extracted Mark of the Web with propagated from archive + // immediately after ProcessFile() call. // // Archive can be closed if we process volumes, next volume is missing // and current one is already removed or deleted. So we need to check @@ -456,7 +460,7 @@ void PASCAL RARSetPassword(HANDLE hArcData,char *Password) #ifndef RAR_NOCRYPT DataSet *Data=(DataSet *)hArcData; wchar PasswordW[MAXPASSWORD]; - GetWideName(Password,NULL,PasswordW,ASIZE(PasswordW)); + CharToWide(Password,PasswordW,ASIZE(PasswordW)); Data->Cmd.Password.Set(PasswordW); cleandata(PasswordW,sizeof(PasswordW)); #endif diff --git a/multiarc/src/formats/rar/unrar/dll.hpp b/multiarc/src/formats/rar/unrar/dll.hpp index c785ff188..ac8d53fac 100644 --- a/multiarc/src/formats/rar/unrar/dll.hpp +++ b/multiarc/src/formats/rar/unrar/dll.hpp @@ -19,6 +19,7 @@ #define ERAR_MISSING_PASSWORD 22 #define ERAR_EREFERENCE 23 #define ERAR_BAD_PASSWORD 24 +#define ERAR_LARGE_DICT 25 #define RAR_OM_LIST 0 #define RAR_OM_EXTRACT 1 @@ -31,7 +32,7 @@ #define RAR_VOL_ASK 0 #define RAR_VOL_NOTIFY 1 -#define RAR_DLL_VERSION 8 +#define RAR_DLL_VERSION 9 #define RAR_HASH_NONE 0 #define RAR_HASH_CRC32 1 @@ -108,7 +109,11 @@ struct RARHeaderDataEx unsigned int CtimeHigh; unsigned int AtimeLow; unsigned int AtimeHigh; - unsigned int Reserved[988]; + wchar_t *ArcNameEx; + unsigned int ArcNameExSize; + wchar_t *FileNameEx; + unsigned int FileNameExSize; + unsigned int Reserved[982]; }; @@ -157,7 +162,7 @@ struct RAROpenArchiveDataEx enum UNRARCALLBACK_MESSAGES { UCM_CHANGEVOLUME,UCM_PROCESSDATA,UCM_NEEDPASSWORD,UCM_CHANGEVOLUMEW, - UCM_NEEDPASSWORDW + UCM_NEEDPASSWORDW,UCM_LARGEDICT }; typedef int (PASCAL *CHANGEVOLPROC)(char *ArcName,int Mode); diff --git a/multiarc/src/formats/rar/unrar/dll.rc b/multiarc/src/formats/rar/unrar/dll.rc deleted file mode 100644 index e0083da16..000000000 --- a/multiarc/src/formats/rar/unrar/dll.rc +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include - -VS_VERSION_INFO VERSIONINFO -FILEVERSION 6, 21, 100, 778 -PRODUCTVERSION 6, 21, 100, 778 -FILEOS VOS__WINDOWS32 -FILETYPE VFT_APP -{ - BLOCK "StringFileInfo" - { - BLOCK "040904E4" - { - VALUE "CompanyName", "Alexander Roshal\0" - VALUE "ProductName", "RAR decompression library\0" - VALUE "FileDescription", "RAR decompression library\0" - VALUE "FileVersion", "6.21.0\0" - VALUE "ProductVersion", "6.21.0\0" - VALUE "LegalCopyright", "Copyright Alexander Roshal 1993-2023\0" - VALUE "OriginalFilename", "Unrar.dll\0" - } - } - BLOCK "VarFileInfo" - { - VALUE "Translation", 0x0409, 0x04E4 - } -} - diff --git a/multiarc/src/formats/rar/unrar/encname.cpp b/multiarc/src/formats/rar/unrar/encname.cpp index e1ba1ed70..d602090ef 100644 --- a/multiarc/src/formats/rar/unrar/encname.cpp +++ b/multiarc/src/formats/rar/unrar/encname.cpp @@ -11,12 +11,13 @@ EncodeFileName::EncodeFileName() -void EncodeFileName::Decode(char *Name,size_t NameSize,byte *EncName,size_t EncSize, - wchar *NameW,size_t MaxDecSize) +void EncodeFileName::Decode(const char *Name,size_t NameSize, + const byte *EncName,size_t EncSize, + std::wstring &NameW) { size_t EncPos=0,DecPos=0; byte HighByte=EncPos=EncSize) break; + // We need DecPos also for ASCII "Name", so resize() instead of push_back(). + NameW.resize(DecPos+1); NameW[DecPos++]=EncName[EncPos++]; break; case 1: if (EncPos>=EncSize) break; + NameW.resize(DecPos+1); NameW[DecPos++]=EncName[EncPos++]+(HighByte<<8); break; case 2: if (EncPos+1>=EncSize) break; + NameW.resize(DecPos+1); NameW[DecPos++]=EncName[EncPos]+(EncName[EncPos+1]<<8); EncPos+=2; break; @@ -51,17 +56,22 @@ void EncodeFileName::Decode(char *Name,size_t NameSize,byte *EncName,size_t EncS if (EncPos>=EncSize) break; byte Correction=EncName[EncPos++]; - for (Length=(Length&0x7f)+2;Length>0 && DecPos0 && DecPos0 && DecPos0 && DecPos &EncName); byte Flags; uint FlagBits; @@ -12,8 +12,8 @@ class EncodeFileName size_t DestSize; public: EncodeFileName(); - size_t Encode(char *Name,wchar *NameW,byte *EncName); - void Decode(char *Name,size_t NameSize,byte *EncName,size_t EncSize,wchar *NameW,size_t MaxDecSize); + void Encode(const std::string &Name,const std::wstring &NameW,std::vector &EncName); + void Decode(const char *Name,size_t NameSize,const byte *EncName,size_t EncSize,std::wstring &NameW); }; #endif diff --git a/multiarc/src/formats/rar/unrar/errhnd.cpp b/multiarc/src/formats/rar/unrar/errhnd.cpp index 97193e5ac..b07e2f664 100644 --- a/multiarc/src/formats/rar/unrar/errhnd.cpp +++ b/multiarc/src/formats/rar/unrar/errhnd.cpp @@ -26,7 +26,7 @@ void ErrorHandler::MemoryError() } -void ErrorHandler::OpenError(const wchar *FileName) +void ErrorHandler::OpenError(const std::wstring &FileName) { #ifndef SILENT OpenErrorMsg(FileName); @@ -35,7 +35,7 @@ void ErrorHandler::OpenError(const wchar *FileName) } -void ErrorHandler::CloseError(const wchar *FileName) +void ErrorHandler::CloseError(const std::wstring &FileName) { if (!UserBreak) { @@ -51,7 +51,7 @@ void ErrorHandler::CloseError(const wchar *FileName) } -void ErrorHandler::ReadError(const wchar *FileName) +void ErrorHandler::ReadError(const std::wstring &FileName) { #ifndef SILENT ReadErrorMsg(FileName); @@ -62,13 +62,13 @@ void ErrorHandler::ReadError(const wchar *FileName) } -void ErrorHandler::AskRepeatRead(const wchar *FileName,bool &Ignore,bool &Retry,bool &Quit) +void ErrorHandler::AskRepeatRead(const std::wstring &FileName,bool &Ignore,bool &Retry,bool &Quit) { SetErrorCode(RARX_READ); #if !defined(SILENT) && !defined(SFX_MODULE) if (!Silent) { - uiMsg(UIERROR_FILEREAD,UINULL,FileName); + uiMsg(UIERROR_FILEREAD,L"",FileName); SysErrMsg(); if (ReadErrIgnoreAll) Ignore=true; @@ -88,7 +88,7 @@ void ErrorHandler::AskRepeatRead(const wchar *FileName,bool &Ignore,bool &Retry, } -void ErrorHandler::WriteError(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::WriteError(const std::wstring &ArcName,const std::wstring &FileName) { #ifndef SILENT WriteErrorMsg(ArcName,FileName); @@ -100,7 +100,7 @@ void ErrorHandler::WriteError(const wchar *ArcName,const wchar *FileName) #ifdef _WIN_ALL -void ErrorHandler::WriteErrorFAT(const wchar *FileName) +void ErrorHandler::WriteErrorFAT(const std::wstring &FileName) { SysErrMsg(); uiMsg(UIERROR_NTFSREQUIRED,FileName); @@ -111,7 +111,7 @@ void ErrorHandler::WriteErrorFAT(const wchar *FileName) #endif -bool ErrorHandler::AskRepeatWrite(const wchar *FileName,bool DiskFull) +bool ErrorHandler::AskRepeatWrite(const std::wstring &FileName,bool DiskFull) { #ifndef SILENT if (!Silent) @@ -129,7 +129,7 @@ bool ErrorHandler::AskRepeatWrite(const wchar *FileName,bool DiskFull) } -void ErrorHandler::SeekError(const wchar *FileName) +void ErrorHandler::SeekError(const std::wstring &FileName) { if (!UserBreak) { @@ -144,13 +144,16 @@ void ErrorHandler::SeekError(const wchar *FileName) void ErrorHandler::GeneralErrMsg(const wchar *fmt,...) { +#ifndef RARDLL va_list arglist; va_start(arglist,fmt); - wchar Msg[1024]; - vswprintf(Msg,ASIZE(Msg),fmt,arglist); + + std::wstring Msg=vwstrprintf(fmt,arglist); uiMsg(UIERROR_GENERALERRMSG,Msg); SysErrMsg(); + va_end(arglist); +#endif } @@ -161,13 +164,13 @@ void ErrorHandler::MemoryErrorMsg() } -void ErrorHandler::OpenErrorMsg(const wchar *FileName) +void ErrorHandler::OpenErrorMsg(const std::wstring &FileName) { - OpenErrorMsg(NULL,FileName); + OpenErrorMsg(L"",FileName); } -void ErrorHandler::OpenErrorMsg(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::OpenErrorMsg(const std::wstring &ArcName,const std::wstring &FileName) { uiMsg(UIERROR_FILEOPEN,ArcName,FileName); SysErrMsg(); @@ -179,13 +182,13 @@ void ErrorHandler::OpenErrorMsg(const wchar *ArcName,const wchar *FileName) } -void ErrorHandler::CreateErrorMsg(const wchar *FileName) +void ErrorHandler::CreateErrorMsg(const std::wstring &FileName) { - CreateErrorMsg(NULL,FileName); + CreateErrorMsg(L"",FileName); } -void ErrorHandler::CreateErrorMsg(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::CreateErrorMsg(const std::wstring &ArcName,const std::wstring &FileName) { uiMsg(UIERROR_FILECREATE,ArcName,FileName); SysErrMsg(); @@ -193,13 +196,13 @@ void ErrorHandler::CreateErrorMsg(const wchar *ArcName,const wchar *FileName) } -void ErrorHandler::ReadErrorMsg(const wchar *FileName) +void ErrorHandler::ReadErrorMsg(const std::wstring &FileName) { - ReadErrorMsg(NULL,FileName); + ReadErrorMsg(L"",FileName); } -void ErrorHandler::ReadErrorMsg(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::ReadErrorMsg(const std::wstring &ArcName,const std::wstring &FileName) { uiMsg(UIERROR_FILEREAD,ArcName,FileName); SysErrMsg(); @@ -207,7 +210,7 @@ void ErrorHandler::ReadErrorMsg(const wchar *ArcName,const wchar *FileName) } -void ErrorHandler::WriteErrorMsg(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::WriteErrorMsg(const std::wstring &ArcName,const std::wstring &FileName) { uiMsg(UIERROR_FILEWRITE,ArcName,FileName); SysErrMsg(); @@ -215,21 +218,21 @@ void ErrorHandler::WriteErrorMsg(const wchar *ArcName,const wchar *FileName) } -void ErrorHandler::ArcBrokenMsg(const wchar *ArcName) +void ErrorHandler::ArcBrokenMsg(const std::wstring &ArcName) { uiMsg(UIERROR_ARCBROKEN,ArcName); SetErrorCode(RARX_CRC); } -void ErrorHandler::ChecksumFailedMsg(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::ChecksumFailedMsg(const std::wstring &ArcName,const std::wstring &FileName) { uiMsg(UIERROR_CHECKSUM,ArcName,FileName); SetErrorCode(RARX_CRC); } -void ErrorHandler::UnknownMethodMsg(const wchar *ArcName,const wchar *FileName) +void ErrorHandler::UnknownMethodMsg(const std::wstring &ArcName,const std::wstring &FileName) { uiMsg(UIERROR_UNKNOWNMETHOD,ArcName,FileName); ErrHandler.SetErrorCode(RARX_FATAL); @@ -332,33 +335,44 @@ void ErrorHandler::Throw(RAR_EXIT Code) if (Code==RARX_USERBREAK && !EnableBreak) return; #if !defined(SILENT) - // Do not write "aborted" when just displaying online help. - if (Code!=RARX_SUCCESS && Code!=RARX_USERERROR) - mprintf(L"\n%s\n",St(MProgAborted)); + if (Code!=RARX_SUCCESS) + if (Code==RARX_USERERROR) // Do not write "aborted" when just displaying the online help. + mprintf(L"\n"); // For consistency with other errors, which print the final "\n". + else + mprintf(L"\n%s\n",St(MProgAborted)); #endif SetErrorCode(Code); throw Code; } -bool ErrorHandler::GetSysErrMsg(wchar *Msg,size_t Size) +bool ErrorHandler::GetSysErrMsg(std::wstring &Msg) { #ifndef SILENT #ifdef _WIN_ALL int ErrType=GetLastError(); if (ErrType!=0) - return FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS, - NULL,ErrType,MAKELANGID(LANG_NEUTRAL,SUBLANG_DEFAULT), - Msg,(DWORD)Size,NULL)!=0; + { + wchar *Buf=nullptr; + if (FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM| + FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL,ErrType,MAKELANGID(LANG_NEUTRAL,SUBLANG_DEFAULT), + (LPTSTR)&Buf,0,NULL)!=0) + { + Msg=Buf; + LocalFree(Buf); + return true; + } + } #endif -#if defined(_UNIX) || defined(_EMX) +#ifdef _UNIX if (errno!=0) { char *err=strerror(errno); if (err!=NULL) { - CharToWide(err,Msg,Size); + CharToWide(err,Msg); return true; } } @@ -371,31 +385,26 @@ bool ErrorHandler::GetSysErrMsg(wchar *Msg,size_t Size) void ErrorHandler::SysErrMsg() { #ifndef SILENT - wchar Msg[1024]; - if (!GetSysErrMsg(Msg,ASIZE(Msg))) + std::wstring Msg; + if (!GetSysErrMsg(Msg)) return; #ifdef _WIN_ALL - wchar *CurMsg=Msg; - while (CurMsg!=NULL) // Print string with \r\n as several strings to multiple lines. + // Print string with \r\n as several strings to multiple lines. + size_t Pos=0; + while (Pos!=std::wstring::npos) { - while (*CurMsg=='\r' || *CurMsg=='\n') - CurMsg++; - if (*CurMsg==0) + while (Msg[Pos]=='\r' || Msg[Pos]=='\n') + Pos++; + if (Pos==Msg.size()) break; - wchar *EndMsg=wcschr(CurMsg,'\r'); - if (EndMsg==NULL) - EndMsg=wcschr(CurMsg,'\n'); - if (EndMsg!=NULL) - { - *EndMsg=0; - EndMsg++; - } + size_t EndPos=Msg.find_first_of(L"\r\n",Pos); + std::wstring CurMsg=Msg.substr(Pos,EndPos==std::wstring::npos ? EndPos:EndPos-Pos); uiMsg(UIERROR_SYSERRMSG,CurMsg); - CurMsg=EndMsg; + Pos=EndPos; } #endif -#if defined(_UNIX) || defined(_EMX) +#ifdef _UNIX uiMsg(UIERROR_SYSERRMSG,Msg); #endif diff --git a/multiarc/src/formats/rar/unrar/errhnd.hpp b/multiarc/src/formats/rar/unrar/errhnd.hpp index 06f4f616f..3b558b102 100644 --- a/multiarc/src/formats/rar/unrar/errhnd.hpp +++ b/multiarc/src/formats/rar/unrar/errhnd.hpp @@ -33,26 +33,26 @@ class ErrorHandler ErrorHandler(); void Clean(); void MemoryError(); - void OpenError(const wchar *FileName); - void CloseError(const wchar *FileName); - void ReadError(const wchar *FileName); - void AskRepeatRead(const wchar *FileName,bool &Ignore,bool &Retry,bool &Quit); - void WriteError(const wchar *ArcName,const wchar *FileName); - void WriteErrorFAT(const wchar *FileName); - bool AskRepeatWrite(const wchar *FileName,bool DiskFull); - void SeekError(const wchar *FileName); + void OpenError(const std::wstring &FileName); + void CloseError(const std::wstring &FileName); + void ReadError(const std::wstring &FileName); + void AskRepeatRead(const std::wstring &FileName,bool &Ignore,bool &Retry,bool &Quit); + void WriteError(const std::wstring &ArcName,const std::wstring &FileName); + void WriteErrorFAT(const std::wstring &FileName); + bool AskRepeatWrite(const std::wstring &FileName,bool DiskFull); + void SeekError(const std::wstring &FileName); void GeneralErrMsg(const wchar *fmt,...); void MemoryErrorMsg(); - void OpenErrorMsg(const wchar *FileName); - void OpenErrorMsg(const wchar *ArcName,const wchar *FileName); - void CreateErrorMsg(const wchar *FileName); - void CreateErrorMsg(const wchar *ArcName,const wchar *FileName); - void ReadErrorMsg(const wchar *FileName); - void ReadErrorMsg(const wchar *ArcName,const wchar *FileName); - void WriteErrorMsg(const wchar *ArcName,const wchar *FileName); - void ArcBrokenMsg(const wchar *ArcName); - void ChecksumFailedMsg(const wchar *ArcName,const wchar *FileName); - void UnknownMethodMsg(const wchar *ArcName,const wchar *FileName); + void OpenErrorMsg(const std::wstring &FileName); + void OpenErrorMsg(const std::wstring &ArcName,const std::wstring &FileName); + void CreateErrorMsg(const std::wstring &FileName); + void CreateErrorMsg(const std::wstring &ArcName,const std::wstring &FileName); + void ReadErrorMsg(const std::wstring &FileName); + void ReadErrorMsg(const std::wstring &ArcName,const std::wstring &FileName); + void WriteErrorMsg(const std::wstring &ArcName,const std::wstring &FileName); + void ArcBrokenMsg(const std::wstring &ArcName); + void ChecksumFailedMsg(const std::wstring &ArcName,const std::wstring &FileName); + void UnknownMethodMsg(const std::wstring &ArcName,const std::wstring &FileName); void Exit(RAR_EXIT ExitCode); void SetErrorCode(RAR_EXIT Code); RAR_EXIT GetErrorCode() {return ExitCode;} @@ -60,7 +60,7 @@ class ErrorHandler void SetSignalHandlers(bool Enable); void Throw(RAR_EXIT Code); void SetSilent(bool Mode) {Silent=Mode;} - bool GetSysErrMsg(wchar *Msg,size_t Size); + bool GetSysErrMsg(std::wstring &Msg); void SysErrMsg(); int GetSystemErrorCode(); void SetSystemErrorCode(int Code); diff --git a/multiarc/src/formats/rar/unrar/extinfo.cpp b/multiarc/src/formats/rar/unrar/extinfo.cpp index 0f25f3124..b3fe7495b 100644 --- a/multiarc/src/formats/rar/unrar/extinfo.cpp +++ b/multiarc/src/formats/rar/unrar/extinfo.cpp @@ -19,19 +19,13 @@ // RAR2 service header extra records. #ifndef SFX_MODULE -void SetExtraInfo20(CommandData *Cmd,Archive &Arc,wchar *Name) +void SetExtraInfo20(CommandData *Cmd,Archive &Arc,const std::wstring &Name) { +#ifdef _WIN_ALL if (Cmd->Test) return; switch(Arc.SubBlockHead.SubType) { -#ifdef _UNIX - case UO_HEAD: - if (Cmd->ProcessOwners) - ExtractUnixOwner20(Arc,Name); - break; -#endif -#ifdef _WIN_ALL case NTACL_HEAD: if (Cmd->ProcessOwners) ExtractACL20(Arc,Name); @@ -39,19 +33,19 @@ void SetExtraInfo20(CommandData *Cmd,Archive &Arc,wchar *Name) case STREAM_HEAD: ExtractStreams20(Arc,Name); break; -#endif } +#endif } #endif // RAR3 and RAR5 service header extra records. -void SetExtraInfo(CommandData *Cmd,Archive &Arc,wchar *Name) +void SetExtraInfo(CommandData *Cmd,Archive &Arc,const std::wstring &Name) { #ifdef _UNIX if (!Cmd->Test && Cmd->ProcessOwners && Arc.Format==RARFMT15 && Arc.SubHead.CmpName(SUBHEAD_TYPE_UOWNER)) - ExtractUnixOwner30(Arc,Name); + ExtractUnixOwner30(Arc,Name.c_str()); #endif #ifdef _WIN_ALL if (!Cmd->Test && Cmd->ProcessOwners && Arc.SubHead.CmpName(SUBHEAD_TYPE_ACL)) @@ -63,7 +57,7 @@ void SetExtraInfo(CommandData *Cmd,Archive &Arc,wchar *Name) // Extra data stored directly in file header. -void SetFileHeaderExtra(CommandData *Cmd,Archive &Arc,wchar *Name) +void SetFileHeaderExtra(CommandData *Cmd,Archive &Arc,const std::wstring &Name) { #ifdef _UNIX if (Cmd->ProcessOwners && Arc.Format==RARFMT50 && Arc.FileHead.UnixOwnerSet) @@ -74,36 +68,34 @@ void SetFileHeaderExtra(CommandData *Cmd,Archive &Arc,wchar *Name) -// Calculate a number of path components except \. and \.. -static int CalcAllowedDepth(const wchar *Name) +// Calculate the number of path components except \. and \.. +static int CalcAllowedDepth(const std::wstring &Name) { int AllowedDepth=0; - while (*Name!=0) - { - if (IsPathDiv(Name[0]) && Name[1]!=0 && !IsPathDiv(Name[1])) + for (size_t I=0;I=ASIZE(Path)) - return true; // It should not be that long, skip. - wcsncpyz(Path,Name,ASIZE(Path)); - for (wchar *s=Path+wcslen(Path)-1;s>Path;s--) - if (IsPathDiv(*s)) + if (Path.empty()) // So we can safely use Path.size()-1 below. + return false; + for (size_t I=Path.size()-1;I>0;I--) + if (IsPathDiv(Path[I])) { - *s=0; + Path.erase(I); FindData FD; if (FindFile::FastFind(Path,&FD,true) && (FD.IsLink || !FD.IsDir)) return true; @@ -112,69 +104,7 @@ static bool LinkInPath(const wchar *Name) } -// Delete symbolic links in file path, if any, and replace them by directories. -// Prevents extracting files outside of destination folder with symlink chains. -bool LinksToDirs(const wchar *SrcName,const wchar *SkipPart,std::wstring &LastChecked) -{ - // Unlike Unix, Windows doesn't expand lnk1 in symlink targets like - // "lnk1/../dir", but converts the path to "dir". In Unix we need to call - // this function to prevent placing unpacked files outside of destination - // folder if previously we unpacked "dir/lnk1" -> "..", - // "dir/lnk2" -> "lnk1/.." and "dir/lnk2/anypath/poc.txt". - // We may still need this function to prevent abusing symlink chains - // in link source path if we remove detection of such chains - // in IsRelativeSymlinkSafe. This function seems to make other symlink - // related safety checks redundant, but for now we prefer to keep them too. - // - // 2022.12.01: the performance impact is minimized after adding the check - // against the previous path and enabling this verification only after - // extracting a symlink with ".." in target. So we enabled it for Windows - // as well for extra safety. -//#ifdef _UNIX - wchar Path[NM]; - if (wcslen(SrcName)>=ASIZE(Path)) - return false; // It should not be that long, skip. - wcsncpyz(Path,SrcName,ASIZE(Path)); - - size_t SkipLength=wcslen(SkipPart); - - if (SkipLength>0 && wcsncmp(Path,SkipPart,SkipLength)!=0) - SkipLength=0; // Parameter validation, not really needed now. - - // Do not check parts already checked in previous path to improve performance. - for (uint I=0;Path[I]!=0 && ISkipLength) - SkipLength=I; - - wchar *Name=Path; - if (SkipLength>0) - { - // Avoid converting symlinks in destination path part specified by user. - Name+=SkipLength; - while (IsPathDiv(*Name)) - Name++; - } - - for (wchar *s=Path+wcslen(Path)-1;s>Name;s--) - if (IsPathDiv(*s)) - { - *s=0; - FindData FD; - if (FindFile::FastFind(Path,&FD,true) && FD.IsLink) -#ifdef _WIN_ALL - if (!DelDir(Path)) -#else - if (!DelFile(Path)) -#endif - return false; // Couldn't delete the symlink to replace it with directory. - } - LastChecked=SrcName; -//#endif - return true; -} - - -bool IsRelativeSymlinkSafe(CommandData *Cmd,const wchar *SrcName,const wchar *PrepSrcName,const wchar *TargetName) +bool IsRelativeSymlinkSafe(CommandData *Cmd,const std::wstring &SrcName,std::wstring PrepSrcName,const std::wstring &TargetName) { // Catch root dir based /path/file paths also as stuff like \\?\. // Do not check PrepSrcName here, it can be root based if destination path @@ -184,14 +114,13 @@ bool IsRelativeSymlinkSafe(CommandData *Cmd,const wchar *SrcName,const wchar *Pr // Number of ".." in link target. int UpLevels=0; - for (int Pos=0;*TargetName!=0;Pos++) + for (uint Pos=0;PosExtrPath); - if (ExtrPathLength>0 && wcsncmp(PrepSrcName,Cmd->ExtrPath,ExtrPathLength)==0) + size_t ExtrPathLength=Cmd->ExtrPath.size(); + if (ExtrPathLength>0 && PrepSrcName.compare(0,ExtrPathLength,Cmd->ExtrPath)==0) { - PrepSrcName+=ExtrPathLength; - while (IsPathDiv(*PrepSrcName)) - PrepSrcName++; + while (IsPathDiv(PrepSrcName[ExtrPathLength])) + ExtrPathLength++; + PrepSrcName.erase(0,ExtrPathLength); } int PrepAllowedDepth=CalcAllowedDepth(PrepSrcName); @@ -226,7 +155,7 @@ bool IsRelativeSymlinkSafe(CommandData *Cmd,const wchar *SrcName,const wchar *Pr } -bool ExtractSymlink(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc,const wchar *LinkName,bool &UpLink) +bool ExtractSymlink(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc,const std::wstring &LinkName,bool &UpLink) { // Returning true in Uplink indicates that link target might include ".." // and enables additional checks. It is ok to falsely return true here, @@ -236,20 +165,20 @@ bool ExtractSymlink(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc,const wcha UpLink=true; // Assume the target might include potentially unsafe "..". #if defined(SAVE_LINKS) && defined(_UNIX) || defined(_WIN_ALL) if (Arc.Format==RARFMT50) // For RAR5 archives we can check RedirName for both Unix and Windows. - UpLink=wcsstr(Arc.FileHead.RedirName,L"..")!=NULL; + UpLink=Arc.FileHead.RedirName.find(L"..")!=std::wstring::npos; #endif #if defined(SAVE_LINKS) && defined(_UNIX) // For RAR 3.x archives we process links even in test mode to skip link data. if (Arc.Format==RARFMT15) - return ExtractUnixLink30(Cmd,DataIO,Arc,LinkName,UpLink); + return ExtractUnixLink30(Cmd,DataIO,Arc,LinkName.c_str(),UpLink); if (Arc.Format==RARFMT50) - return ExtractUnixLink50(Cmd,LinkName,&Arc.FileHead); + return ExtractUnixLink50(Cmd,LinkName.c_str(),&Arc.FileHead); #elif defined(_WIN_ALL) // RAR 5.0 archives store link information in file header, so there is // no need to additionally test it if we do not create a file. if (Arc.Format==RARFMT50) - return CreateReparsePoint(Cmd,LinkName,&Arc.FileHead); + return CreateReparsePoint(Cmd,LinkName.c_str(),&Arc.FileHead); #endif return false; } diff --git a/multiarc/src/formats/rar/unrar/extinfo.hpp b/multiarc/src/formats/rar/unrar/extinfo.hpp index d8551d463..11b89feff 100644 --- a/multiarc/src/formats/rar/unrar/extinfo.hpp +++ b/multiarc/src/formats/rar/unrar/extinfo.hpp @@ -1,24 +1,23 @@ #ifndef _RAR_EXTINFO_ #define _RAR_EXTINFO_ -bool LinksToDirs(const wchar *SrcName,const wchar *SkipPart,std::wstring &LastChecked); -bool IsRelativeSymlinkSafe(CommandData *Cmd,const wchar *SrcName,const wchar *PrepSrcName,const wchar *TargetName); -bool ExtractSymlink(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc,const wchar *LinkName,bool &UpLink); +bool IsRelativeSymlinkSafe(CommandData *Cmd,const std::wstring &SrcName,std::wstring PrepSrcName,const std::wstring &TargetName); +bool ExtractSymlink(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc,const std::wstring &LinkName,bool &UpLink); #ifdef _UNIX -void SetUnixOwner(Archive &Arc,const wchar *FileName); +void SetUnixOwner(Archive &Arc,const std::wstring &FileName); #endif -bool ExtractHardlink(CommandData *Cmd,wchar *NameNew,wchar *NameExisting,size_t NameExistingSize); +bool ExtractHardlink(CommandData *Cmd,const std::wstring &NameNew,const std::wstring &NameExisting); -void GetStreamNameNTFS(Archive &Arc,wchar *StreamName,size_t MaxSize); +std::wstring GetStreamNameNTFS(Archive &Arc); #ifdef _WIN_ALL bool SetPrivilege(LPCTSTR PrivName); #endif -void SetExtraInfo20(CommandData *Cmd,Archive &Arc,wchar *Name); -void SetExtraInfo(CommandData *Cmd,Archive &Arc,wchar *Name); -void SetFileHeaderExtra(CommandData *Cmd,Archive &Arc,wchar *Name); +void SetExtraInfo20(CommandData *Cmd,Archive &Arc,const std::wstring &Name); +void SetExtraInfo(CommandData *Cmd,Archive &Arc,const std::wstring &Name); +void SetFileHeaderExtra(CommandData *Cmd,Archive &Arc,const std::wstring &Name); #endif diff --git a/multiarc/src/formats/rar/unrar/extract.cpp b/multiarc/src/formats/rar/unrar/extract.cpp index 65b507098..f799e6a3e 100644 --- a/multiarc/src/formats/rar/unrar/extract.cpp +++ b/multiarc/src/formats/rar/unrar/extract.cpp @@ -4,30 +4,22 @@ CmdExtract::CmdExtract(CommandData *Cmd) { CmdExtract::Cmd=Cmd; - *ArcName=0; - *DestFileName=0; - ArcAnalyzed=false; - Analyze=new AnalyzeData; - memset(Analyze,0,sizeof(*Analyze)); + Analyze={}; TotalFileCount=0; // Common for all archives involved. Set here instead of DoExtract() - // to use in unrar.dll too. Allows to avoid LinksToDirs() calls - // and save CPU time in no symlinks including ".." in target were extracted. -#if defined(_WIN_ALL) - // We can't expand symlink path components in another symlink target - // in Windows. We can't create symlinks in Android now. Even though we do not - // really need LinksToDirs() calls in these systems, we still call it - // for extra safety, but only if symlink with ".." in target was extracted. - ConvertSymlinkPaths=false; -#else + // to use in unrar.dll too. // We enable it by default in Unix to care about the case when several // archives are unpacked to same directory with several independent RAR runs. // Worst case performance penalty for a lot of small files seems to be ~3%. + // 2023.09.15: Windows performance impact seems to be negligible, + // less than 0.5% when extracting mix of small files and folders. + // So for extra security we enabled it for Windows too, even though + // unlike Unix, Windows doesn't expand lnk1 in symlink targets like + // "lnk1/../dir", but converts such path to "dir". ConvertSymlinkPaths=true; -#endif Unp=new Unpack(&DataIO); #ifdef RAR_SMP @@ -40,24 +32,21 @@ CmdExtract::~CmdExtract() { FreeAnalyzeData(); delete Unp; - delete Analyze; } void CmdExtract::FreeAnalyzeData() { - for (size_t I=0;ICommand[0]); - if (*Cmd->UseStdin==0) + if (Cmd->UseStdin.empty()) { FindData FD; - while (Cmd->GetArcName(ArcName,ASIZE(ArcName))) + while (Cmd->GetArcName(ArcName)) if (FindFile::FastFind(ArcName,&FD)) DataIO.TotalArcSize+=FD.Size; } Cmd->ArcNames.Rewind(); - while (Cmd->GetArcName(ArcName,ASIZE(ArcName))) + while (Cmd->GetArcName(ArcName)) { if (Cmd->ManualPassword) Cmd->Password.Clean(); // Clean user entered password before processing next archive. @@ -101,7 +90,7 @@ void CmdExtract::DoExtract() if (TotalFileCount==0 && Cmd->Command[0]!='I' && ErrHandler.GetErrorCode()!=RARX_BADPWD) // Not in case of wrong archive password. { - if (!PasswordCancelled) + if (!SuppressNoFilesMessage) uiMsg(UIERROR_NOFILESTOEXTRACT,ArcName); // Other error codes may explain a reason of "no files extracted" clearer, @@ -150,7 +139,7 @@ void CmdExtract::ExtractArchiveInit(Archive &Arc) EXTRACT_ARC_CODE CmdExtract::ExtractArchive() { Archive Arc(Cmd); - if (*Cmd->UseStdin!=0) + if (!Cmd->UseStdin.empty()) { Arc.SetHandleType(FILE_HANDLESTD); #ifdef USE_QOPEN @@ -159,8 +148,10 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() } else { -#if defined(_WIN_ALL) && !defined(SFX_MODULE) // WinRAR GUI code also resets the cache. - if (*Cmd->Command=='T' || Cmd->Test) + // We commented out "&& !defined(WINRAR)", because WinRAR GUI code resets + // the cache for usual test command, but not for test after archiving. +#if defined(_WIN_ALL) && !defined(SFX_MODULE) + if (Cmd->Command[0]=='T' || Cmd->Test) ResetFileCache(ArcName); // Reset the file cache when testing an archive. #endif if (!Arc.WOpen(ArcName)) @@ -172,8 +163,8 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() #if !defined(SFX_MODULE) && !defined(RARDLL) if (CmpExt(ArcName,L"rev")) { - wchar FirstVolName[NM]; - VolNameToFirstName(ArcName,FirstVolName,ASIZE(FirstVolName),true); + std::wstring FirstVolName; + VolNameToFirstName(ArcName,FirstVolName,true); // If several volume names from same volume set are specified // and current volume is not first in set and first volume is present @@ -187,7 +178,7 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() } #endif - mprintf(St(MNotRAR),ArcName); + mprintf(St(MNotRAR),ArcName.c_str()); #ifndef SFX_MODULE if (CmpExt(ArcName,L"rar")) @@ -202,8 +193,8 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() #ifndef SFX_MODULE if (Arc.Volume && !Arc.FirstVolume && !UseExactVolName) { - wchar FirstVolName[NM]; - VolNameToFirstName(ArcName,FirstVolName,ASIZE(FirstVolName),Arc.NewNumbering); + std::wstring FirstVolName; + VolNameToFirstName(ArcName,FirstVolName,Arc.NewNumbering); // If several volume names from same volume set are specified // and current volume is not first in set and first volume is present @@ -219,7 +210,7 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() int64 VolumeSetSize=0; // Total size of volumes after the current volume. #ifndef SFX_MODULE - if (!ArcAnalyzed && *Cmd->UseStdin==0) + if (!ArcAnalyzed && Cmd->UseStdin.empty()) { AnalyzeArchive(Arc.FileName,Arc.Volume,Arc.NewNumbering); ArcAnalyzed=true; // Avoid repeated analysis on EXTRACT_ARC_REPEAT. @@ -231,10 +222,10 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() #ifndef SFX_MODULE // Try to speed up extraction for independent solid volumes by starting // extraction from non-first volume if we can. - if (*Analyze->StartName!=0) + if (!Analyze.StartName.empty()) { - wcsncpyz(ArcName,Analyze->StartName,ASIZE(ArcName)); - *Analyze->StartName=0; + ArcName=Analyze.StartName; + Analyze.StartName.clear(); UseExactVolName=true; return EXTRACT_ARC_REPEAT; @@ -244,15 +235,14 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() // Calculate the total size of all accessible volumes. // This size is necessary to display the correct total progress indicator. - wchar NextName[NM]; - wcsncpyz(NextName,Arc.FileName,ASIZE(NextName)); + std::wstring NextName=Arc.FileName; while (true) { // First volume is already added to DataIO.TotalArcSize // in initial TotalArcSize calculation in DoExtract. // So we skip it and start from second volume. - NextVolumeName(NextName,ASIZE(NextName),!Arc.NewNumbering); + NextVolumeName(NextName,!Arc.NewNumbering); FindData FD; if (FindFile::FastFind(NextName,&FD)) VolumeSetSize+=FD.Size; @@ -264,11 +254,11 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() ExtractArchiveInit(Arc); - if (*Cmd->Command=='T' || *Cmd->Command=='I') + if (Cmd->Command[0]=='T' || Cmd->Command[0]=='I') Cmd->Test=true; - if (*Cmd->Command=='I') + if (Cmd->Command[0]=='I') { Cmd->DisablePercentage=true; } @@ -276,10 +266,10 @@ EXTRACT_ARC_CODE CmdExtract::ExtractArchive() uiStartArchiveExtract(!Cmd->Test,ArcName); #ifndef SFX_MODULE - if (Analyze->StartPos!=0) + if (Analyze.StartPos!=0) { - Arc.Seek(Analyze->StartPos,SEEK_SET); - Analyze->StartPos=0; + Arc.Seek(Analyze.StartPos,SEEK_SET); + Analyze.StartPos=0; } #endif @@ -340,8 +330,8 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) if (HeaderType==HEAD_FILE) { // Unlike Arc.FileName, ArcName might store an old volume name here. - if (Analyze->EndPos!=0 && Analyze->EndPos==Arc.CurBlockPos && - (*Analyze->EndName==0 || wcscmp(Analyze->EndName,Arc.FileName)==0)) + if (Analyze.EndPos!=0 && Analyze.EndPos==Arc.CurBlockPos && + (Analyze.EndName.empty() || Analyze.EndName==Arc.FileName)) return false; } else @@ -380,32 +370,30 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) // when reading an archive. But we prefer to do it here, because this // function is called directly in unrar.dll, so we fix bad parameters // passed to dll. Also we want to see real negative sizes in the listing - // of corrupt archive. To prevent uninitialized data access perform + // of corrupt archive. To prevent the uninitialized data access, perform // these checks after rejecting zero length and non-file headers above. if (Arc.FileHead.PackSize<0) Arc.FileHead.PackSize=0; if (Arc.FileHead.UnpSize<0) Arc.FileHead.UnpSize=0; - // 2022.03.20: We might remove this check in the future. - // It duplicates Analyze->EndPos and Analyze->EndName in all cases except - // volumes on removable media. + // This check duplicates Analyze.EndPos and Analyze.EndName + // in all cases except volumes on removable media. if (!Cmd->Recurse && MatchedArgs>=Cmd->FileArgs.ItemsCount() && AllMatchesExact) return false; int MatchType=MATCH_WILDSUBPATH; bool EqualNames=false; - wchar MatchedArg[NM]; - int MatchNumber=Cmd->IsProcessFile(Arc.FileHead,&EqualNames,MatchType,0,MatchedArg,ASIZE(MatchedArg)); - bool MatchFound=MatchNumber!=0; + std::wstring MatchedArg; + bool MatchFound=Cmd->IsProcessFile(Arc.FileHead,&EqualNames,MatchType,0,&MatchedArg)!=0; #ifndef SFX_MODULE if (Cmd->ExclPath==EXCL_BASEPATH) { - wcsncpyz(Cmd->ArcPath,MatchedArg,ASIZE(Cmd->ArcPath)); - *PointToName(Cmd->ArcPath)=0; + Cmd->ArcPath=MatchedArg; + GetPathWithSep(Cmd->ArcPath,Cmd->ArcPath); if (IsWildcard(Cmd->ArcPath)) // Cannot correctly process path*\* masks here. - *Cmd->ArcPath=0; + Cmd->ArcPath.clear(); } #endif if (MatchFound && !EqualNames) @@ -416,13 +404,13 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) #if !defined(SFX_MODULE) && !defined(RARDLL) if (Arc.FileHead.SplitBefore && FirstFile && !UseExactVolName) { - wchar CurVolName[NM]; - wcsncpyz(CurVolName,ArcName,ASIZE(CurVolName)); - GetFirstVolIfFullSet(ArcName,Arc.NewNumbering,ArcName,ASIZE(ArcName)); + std::wstring StartVolName; + GetFirstVolIfFullSet(ArcName,Arc.NewNumbering,StartVolName); - if (wcsicomp(ArcName,CurVolName)!=0 && FileExist(ArcName)) + if (StartVolName!=ArcName && FileExist(StartVolName)) { - wcsncpyz(Cmd->ArcName,ArcName,ASIZE(ArcName)); // For GUI "Delete archive after extraction". + ArcName=StartVolName; + Cmd->ArcName=ArcName; // For GUI "Delete archive after extraction". // If first volume name does not match the current name and if such // volume name really exists, let's unpack from this first volume. Repeat=true; @@ -439,12 +427,11 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) } } #endif - wcsncpyz(ArcName,CurVolName,ASIZE(ArcName)); } #endif - wchar ArcFileName[NM]; - ConvertPath(Arc.FileHead.FileName,ArcFileName,ASIZE(ArcFileName)); + std::wstring ArcFileName; + ConvertPath(&Arc.FileHead.FileName,&ArcFileName); if (Arc.FileHead.Version) { @@ -490,10 +477,10 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) bool RefTarget=false; if (!MatchFound) - for (size_t I=0;ITest) // While harmless, it is useless for 't'. { @@ -504,11 +491,11 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) // targets and it is possible that first target isn't unpacked // for some reason. Also targets might have associated service blocks // like ACLs. All this would complicate processing a lot. - wcsncpyz(DestFileName,*Cmd->TempPath!=0 ? Cmd->TempPath:Cmd->ExtrPath,ASIZE(DestFileName)); - AddEndSlash(DestFileName,ASIZE(DestFileName)); - wcsncatz(DestFileName,L"__tmp_reference_source_",ASIZE(DestFileName)); - MkTemp(DestFileName,ASIZE(DestFileName)); - MatchedRef->TmpName=wcsdup(DestFileName); + DestFileName=!Cmd->TempPath.empty() ? Cmd->TempPath:Cmd->ExtrPath; + AddEndSlash(DestFileName); + DestFileName+=L"__tmp_reference_source_"; + MkTemp(DestFileName); + MatchedRef.TmpName=DestFileName; } RefTarget=true; // Need it even for 't' to test the reference source. break; @@ -520,7 +507,7 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) else MatchFound=false; // Skip only the current file for non-solid archive. - if (MatchFound || RefTarget || (SkipSolid=Arc.Solid)!=0) + if (MatchFound || RefTarget || (SkipSolid=Arc.Solid)!=false) { // First common call of uiStartFileExtract. It is done before overwrite // prompts, so if SkipSolid state is changed below, we'll need to make @@ -529,10 +516,10 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) return false; if (!RefTarget) - ExtrPrepareName(Arc,ArcFileName,DestFileName,ASIZE(DestFileName)); + ExtrPrepareName(Arc,ArcFileName,DestFileName); // DestFileName can be set empty in case of excessive -ap switch. - ExtrFile=!SkipSolid && *DestFileName!=0 && !Arc.FileHead.SplitBefore; + ExtrFile=!SkipSolid && !DestFileName.empty() && !Arc.FileHead.SplitBefore; if ((Cmd->FreshFiles || Cmd->UpdateFiles) && (Command=='E' || Command=='X')) { @@ -584,7 +571,7 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) #else if (!ExtrGetPassword(Arc,ArcFileName,CheckPwd.IsSet() ? &CheckPwd:NULL)) { - PasswordCancelled=true; + SuppressNoFilesMessage=true; return false; } #endif @@ -597,14 +584,14 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) #endif byte PswCheck[SIZE_PSWCHECK]; - DataIO.SetEncryption(false,Arc.FileHead.CryptMethod,&FilePassword, - Arc.FileHead.SaltSet ? Arc.FileHead.Salt:NULL, + bool EncSet=DataIO.SetEncryption(false,Arc.FileHead.CryptMethod, + &FilePassword,Arc.FileHead.SaltSet ? Arc.FileHead.Salt:nullptr, Arc.FileHead.InitV,Arc.FileHead.Lg2Count, Arc.FileHead.HashKey,PswCheck); // If header is damaged, we cannot rely on password check value, // because it can be damaged too. - if (Arc.FileHead.UsePswCheck && !Arc.BrokenHeader && + if (EncSet && Arc.FileHead.UsePswCheck && !Arc.BrokenHeader && memcmp(Arc.FileHead.PswCheck,PswCheck,SIZE_PSWCHECK)!=0) { if (GlobalPassword) // For -p or Ctrl+P to avoid the infinite loop. @@ -641,13 +628,25 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) else DataIO.SetEncryption(false,CRYPT_NONE,NULL,NULL,NULL,0,NULL,NULL); + // Per file symlink conversion flag. Can be turned off in unrar.dll. + bool CurConvertSymlinkPaths=ConvertSymlinkPaths; + #ifdef RARDLL - if (*Cmd->DllDestName!=0) - wcsncpyz(DestFileName,Cmd->DllDestName,ASIZE(DestFileName)); + if (!Cmd->DllDestName.empty()) + { + DestFileName=Cmd->DllDestName; + + // If unrar.dll sets the entire destination pathname, there is no + // destination path and we can't convert symlinks, because we would + // risk converting important user or system symlinks in this case. + // If DllDestName is set, it turns off our path processing and app + // invoking the library cares about everything including safety. + CurConvertSymlinkPaths=false; + } #endif if (ExtrFile && Command!='P' && !Cmd->Test && !Cmd->AbsoluteLinks && - ConvertSymlinkPaths) + CurConvertSymlinkPaths) ExtrFile=LinksToDirs(DestFileName,Cmd->ExtrPath,LastCheckedSymlink); File CurFile; @@ -655,13 +654,17 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) bool LinkEntry=Arc.FileHead.RedirType!=FSREDIR_NONE; if (LinkEntry && (Arc.FileHead.RedirType!=FSREDIR_FILECOPY)) { + if (Cmd->SkipSymLinks && (Arc.FileHead.RedirType==FSREDIR_UNIXSYMLINK || + Arc.FileHead.RedirType==FSREDIR_WINSYMLINK || Arc.FileHead.RedirType==FSREDIR_JUNCTION)) + ExtrFile=false; + if (ExtrFile && Command!='P' && !Cmd->Test) { // Overwrite prompt for symbolic and hard links and when we move // a temporary file to the file reference instead of copying it. bool UserReject=false; - if (FileExist(DestFileName) && !UserReject) - FileCreate(Cmd,NULL,DestFileName,ASIZE(DestFileName),&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime); + if (FileExist(DestFileName)) + FileCreate(Cmd,NULL,DestFileName,&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime); if (UserReject) ExtrFile=false; } @@ -680,7 +683,13 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) } else if (ExtrFile) // Create files and file copies (FSREDIR_FILECOPY). + { + // Check the dictionary size before creating a file and issuing + // any overwrite prompts. + if (!CheckWinLimit(Arc,ArcFileName)) + return false; ExtrFile=ExtrCreateFile(Arc,CurFile); + } if (!ExtrFile && Arc.Solid) { @@ -694,6 +703,9 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) // a solid archive. if (!uiStartFileExtract(ArcFileName,false,false,true)) return false; + // Check the dictionary size also for skipping files. + if (!CheckWinLimit(Arc,ArcFileName)) + return false; } if (ExtrFile) { @@ -716,21 +728,21 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) FileCount++; if (Command!='I' && !Cmd->DisableNames) if (SkipSolid) - mprintf(St(MExtrSkipFile),ArcFileName); + mprintf(St(MExtrSkipFile),ArcFileName.c_str()); else switch(Cmd->Test ? 'T':Command) // "Test" can be also enabled by -t switch. { case 'T': - mprintf(St(MExtrTestFile),ArcFileName); + mprintf(St(MExtrTestFile),ArcFileName.c_str()); break; #ifndef SFX_MODULE case 'P': - mprintf(St(MExtrPrinting),ArcFileName); + mprintf(St(MExtrPrinting),ArcFileName.c_str()); break; #endif case 'X': case 'E': - mprintf(St(MExtrFile),DestFileName); + mprintf(St(MExtrFile),DestFileName.c_str()); break; } if (!Cmd->DisablePercentage && !Cmd->DisableNames) @@ -778,7 +790,7 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) if (Type==FSREDIR_HARDLINK || Type==FSREDIR_FILECOPY) { - wchar RedirName[NM]; + std::wstring RedirName; // 2022.11.15: Might be needed when unpacking WinRAR 5.0 links with // Unix RAR. WinRAR 5.0 used \ path separators here, when beginning @@ -787,17 +799,17 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) // We must perform this conversion before ConvertPath call, // so paths mixing different slashes like \dir1/dir2\file are // processed correctly. - SlashToNative(Arc.FileHead.RedirName,RedirName,ASIZE(RedirName)); + SlashToNative(Arc.FileHead.RedirName,RedirName); - ConvertPath(RedirName,RedirName,ASIZE(RedirName)); + ConvertPath(&RedirName,&RedirName); - wchar NameExisting[NM]; - ExtrPrepareName(Arc,RedirName,NameExisting,ASIZE(NameExisting)); - if (FileCreateMode && *NameExisting!=0) // *NameExisting can be 0 in case of excessive -ap switch. + std::wstring NameExisting; + ExtrPrepareName(Arc,RedirName,NameExisting); + if (FileCreateMode && !NameExisting.empty()) // *NameExisting can be empty in case of excessive -ap switch. if (Type==FSREDIR_HARDLINK) - LinkSuccess=ExtractHardlink(Cmd,DestFileName,NameExisting,ASIZE(NameExisting)); + LinkSuccess=ExtractHardlink(Cmd,DestFileName,NameExisting); else - LinkSuccess=ExtractFileCopy(CurFile,Arc.FileName,RedirName,DestFileName,NameExisting,ASIZE(NameExisting),Arc.FileHead.UnpSize); + LinkSuccess=ExtractFileCopy(CurFile,Arc.FileName,RedirName,DestFileName,NameExisting,Arc.FileHead.UnpSize); } else if (Type==FSREDIR_UNIXSYMLINK || Type==FSREDIR_WINSYMLINK || Type==FSREDIR_JUNCTION) @@ -806,6 +818,7 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) { bool UpLink; LinkSuccess=ExtractSymlink(Cmd,DataIO,Arc,DestFileName,UpLink); + ConvertSymlinkPaths|=LinkSuccess && UpLink; // We do not actually need to reset the cache here if we cache @@ -843,9 +856,20 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) UnstoreFile(DataIO,Arc.FileHead.UnpSize); else { - Unp->Init(Arc.FileHead.WinSize,Arc.FileHead.Solid); + try + { + Unp->Init(Arc.FileHead.WinSize,Arc.FileHead.Solid); + } + catch (std::bad_alloc) + { + if (Arc.FileHead.WinSize>=0x40000000) + uiMsg(UIERROR_EXTRDICTOUTMEM,Arc.FileName,uint(Arc.FileHead.WinSize/0x40000000+(Arc.FileHead.WinSize%0x40000000!=0 ? 1 : 0))); + throw; + } + Unp->SetDestSize(Arc.FileHead.UnpSize); #ifndef SFX_MODULE + // RAR 1.3 - 1.5 archives do not set per file solid flag. if (Arc.Format!=RARFMT50 && Arc.FileHead.UnpVer<=15) Unp->DoUnpack(15,FileCount>1 && Arc.Solid); else @@ -910,26 +934,36 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) if (SkipSolid) mprintf(L"\b\b\b\b\b "); } - - // If we successfully unpacked a hard link, we wish to set its file - // attributes. Hard link shares file metadata with link target, - // so we do not need to set link time or owner. But when we overwrite - // an existing link, we can call PrepareToDelete(), which affects - // link target attributes as well. So we set link attributes to restore - // both target and link attributes if PrepareToDelete() changed them. - bool SetAttrOnly=LinkEntry && Arc.FileHead.RedirType==FSREDIR_HARDLINK && LinkSuccess; - if (!TestMode && (Command=='X' || Command=='E') && - (!LinkEntry || SetAttrOnly || Arc.FileHead.RedirType==FSREDIR_FILECOPY && LinkSuccess) && - (!BrokenFile || Cmd->KeepBroken)) + (!LinkEntry || LinkSuccess) && (!BrokenFile || Cmd->KeepBroken)) { + // Set everything for usual files and file references. + bool SetAll=!LinkEntry || Arc.FileHead.RedirType==FSREDIR_FILECOPY; + + // Set time and adjust size for usual files and references. + // Symlink time requires the special treatment and it is set directly + // after creating a symlink. + bool SetTimeAndSize=SetAll; + + // Set file attributes for usual files, references and hard links. + // Hard link shares the file metadata with link target, so we do not + // need to set link time or owner. But when we overwrite an existing + // link, we can call PrepareToDelete(), which affects link target + // attributes too. So we set link attributes to restore both target + // and link attributes if PrepareToDelete() has changed them. + bool SetAttr=SetAll || Arc.FileHead.RedirType==FSREDIR_HARDLINK; + + // Call SetFileHeaderExtra to set Unix user and group for usual files, + // references and symlinks. Unix symlink can have its own owner data. + bool SetExtra=SetAll || Arc.FileHead.RedirType==FSREDIR_UNIXSYMLINK; + // Below we use DestFileName instead of CurFile.FileName, // so we can set file attributes also for hard links, which do not // have the open CurFile. These strings are the same for other items. - if (!SetAttrOnly) + if (SetTimeAndSize) { - // We could preallocate more space that really written to broken file + // We could preallocate more space than really written to broken file // or file with crafted header. if (Preallocated>0 && (BrokenFile || DataIO.CurUnpWrite!=Preallocated)) CurFile.Truncate(); @@ -940,28 +974,33 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) Cmd->xctime==EXTTIME_NONE ? NULL:&Arc.FileHead.ctime, Cmd->xatime==EXTTIME_NONE ? NULL:&Arc.FileHead.atime); CurFile.Close(); + } + if (SetExtra) SetFileHeaderExtra(Cmd,Arc,DestFileName); + if (SetTimeAndSize) CurFile.SetCloseFileTime( Cmd->xmtime==EXTTIME_NONE ? NULL:&Arc.FileHead.mtime, Cmd->xatime==EXTTIME_NONE ? NULL:&Arc.FileHead.atime); - } + if (SetAttr) + { #if defined(_WIN_ALL) && !defined(SFX_MODULE) - if (Cmd->SetCompressedAttr && - (Arc.FileHead.FileAttr & FILE_ATTRIBUTE_COMPRESSED)!=0) - SetFileCompression(DestFileName,true); - if (Cmd->ClearArc) - Arc.FileHead.FileAttr&=~FILE_ATTRIBUTE_ARCHIVE; + if (Cmd->SetCompressedAttr && + (Arc.FileHead.FileAttr & FILE_ATTRIBUTE_COMPRESSED)!=0) + SetFileCompression(DestFileName,true); + if (Cmd->ClearArc) + Arc.FileHead.FileAttr&=~FILE_ATTRIBUTE_ARCHIVE; #endif - if (!Cmd->IgnoreGeneralAttr && !SetFileAttr(DestFileName,Arc.FileHead.FileAttr)) - { - uiMsg(UIERROR_FILEATTR,Arc.FileName,DestFileName); - // Android cannot set file attributes and while UIERROR_FILEATTR - // above is handled by Android RAR silently, this call would cause - // "Operation not permitted" message for every unpacked file. - ErrHandler.SysErrMsg(); + if (!Cmd->IgnoreGeneralAttr && !SetFileAttr(DestFileName,Arc.FileHead.FileAttr)) + { + uiMsg(UIERROR_FILEATTR,Arc.FileName,DestFileName); + // Android cannot set file attributes and while UIERROR_FILEATTR + // above is handled by Android RAR silently, this call would cause + // "Operation not permitted" message for every unpacked file. + ErrHandler.SysErrMsg(); + } } PrevProcessed=true; @@ -987,42 +1026,44 @@ bool CmdExtract::ExtractCurrentFile(Archive &Arc,size_t HeaderSize,bool &Repeat) void CmdExtract::UnstoreFile(ComprDataIO &DataIO,int64 DestUnpSize) { - Array Buffer(File::CopyBufferSize()); + std::vector Buffer(File::CopyBufferSize()); while (true) { - int ReadSize=DataIO.UnpRead(&Buffer[0],Buffer.Size()); + int ReadSize=DataIO.UnpRead(Buffer.data(),Buffer.size()); if (ReadSize<=0) break; int WriteSize=ReadSize0) { - DataIO.UnpWrite(&Buffer[0],WriteSize); + DataIO.UnpWrite(Buffer.data(),WriteSize); DestUnpSize-=WriteSize; } } } -bool CmdExtract::ExtractFileCopy(File &New,wchar *ArcName,const wchar *RedirName,wchar *NameNew,wchar *NameExisting,size_t NameExistingSize,int64 UnpSize) +bool CmdExtract::ExtractFileCopy(File &New,const std::wstring &ArcName,const std::wstring &RedirName,const std::wstring &NameNew,const std::wstring &NameExisting,int64 UnpSize) { File Existing; if (!Existing.Open(NameExisting)) { + std::wstring TmpExisting=NameExisting; // NameExisting is 'const', so copy it here. + bool OpenFailed=true; // If we couldn't find the existing file, check if match is present // in temporary reference sources list. - for (size_t I=0;IDllError=ERAR_EREFERENCE; @@ -1060,20 +1100,20 @@ bool CmdExtract::ExtractFileCopy(File &New,wchar *ArcName,const wchar *RedirName } } - Array Buffer(0x100000); + std::vector Buffer(0x100000); int64 CopySize=0; while (true) { Wait(); - int ReadSize=Existing.Read(&Buffer[0],Buffer.Size()); + int ReadSize=Existing.Read(Buffer.data(),Buffer.size()); if (ReadSize==0) break; // Update only the current file progress in WinRAR, set the total to 0 // to keep it as is. It looks better for WinRAR. uiExtractProgress(CopySize,UnpSize,0,0); - New.Write(&Buffer[0],ReadSize); + New.Write(Buffer.data(),ReadSize); CopySize+=ReadSize; } @@ -1081,7 +1121,7 @@ bool CmdExtract::ExtractFileCopy(File &New,wchar *ArcName,const wchar *RedirName } -void CmdExtract::ExtrPrepareName(Archive &Arc,const wchar *ArcFileName,wchar *DestName,size_t DestSize) +void CmdExtract::ExtrPrepareName(Archive &Arc,const std::wstring &ArcFileName,std::wstring &DestName) { if (Cmd->Test) { @@ -1089,15 +1129,15 @@ void CmdExtract::ExtrPrepareName(Archive &Arc,const wchar *ArcFileName,wchar *De // This check also allows to avoid issuing "Attempting to correct... // Renaming..." messages in MakeNameCompatible() below for problematic // names like aux.txt when testing an archive. - wcsncpyz(DestName,ArcFileName,DestSize); + DestName=ArcFileName; return; } - wcsncpyz(DestName,Cmd->ExtrPath,DestSize); + DestName=Cmd->ExtrPath; - if (*Cmd->ExtrPath!=0) + if (!Cmd->ExtrPath.empty()) { - wchar LastChar=*PointToLastChar(Cmd->ExtrPath); + wchar LastChar=GetLastChar(Cmd->ExtrPath); // We need IsPathDiv check here to correctly handle Unix forward slash // in the end of destination path in Windows: rar x arc dest/ // so we call IsPathDiv first instead of just calling AddEndSlash, @@ -1106,7 +1146,7 @@ void CmdExtract::ExtrPrepareName(Archive &Arc,const wchar *ArcFileName,wchar *De if (!IsPathDiv(LastChar) && !IsDriveDiv(LastChar)) { // Destination path can be without trailing slash if it come from GUI shell. - AddEndSlash(DestName,DestSize); + AddEndSlash(DestName); } } @@ -1116,38 +1156,40 @@ void CmdExtract::ExtrPrepareName(Archive &Arc,const wchar *ArcFileName,wchar *De switch(Cmd->AppendArcNameToPath) { case APPENDARCNAME_DESTPATH: // To subdir of destination path. - wcsncatz(DestName,PointToName(Arc.FirstVolumeName),DestSize); - SetExt(DestName,NULL,DestSize); + DestName+=PointToName(Arc.FirstVolumeName); + RemoveExt(DestName); break; case APPENDARCNAME_OWNSUBDIR: // To subdir of archive own dir. - wcsncpyz(DestName,Arc.FirstVolumeName,DestSize); - SetExt(DestName,NULL,DestSize); + DestName=Arc.FirstVolumeName; + RemoveExt(DestName); break; case APPENDARCNAME_OWNDIR: // To archive own dir. - wcsncpyz(DestName,Arc.FirstVolumeName,DestSize); + DestName=Arc.FirstVolumeName; RemoveNameFromPath(DestName); break; } - AddEndSlash(DestName,DestSize); + AddEndSlash(DestName); } #endif - + // We need to modify the name below and ArcFileName is const. + std::wstring CurName=ArcFileName; #ifndef SFX_MODULE - wchar *ArcPath=*Cmd->ExclArcPath!=0 ? Cmd->ExclArcPath:Cmd->ArcPath; - size_t ArcPathLength=wcslen(ArcPath); + std::wstring &ArcPath=!Cmd->ExclArcPath.empty() ? Cmd->ExclArcPath:Cmd->ArcPath; + size_t ArcPathLength=ArcPath.size(); if (ArcPathLength>0) { - size_t NameLength=wcslen(ArcFileName); - if (NameLength>=ArcPathLength && wcsnicompc(ArcPath,ArcFileName,ArcPathLength)==0 && + size_t NameLength=CurName.size(); + if (NameLength>=ArcPathLength && wcsnicompc(ArcPath,CurName,ArcPathLength)==0 && (IsPathDiv(ArcPath[ArcPathLength-1]) || - IsPathDiv(ArcFileName[ArcPathLength]) || ArcFileName[ArcPathLength]==0)) + IsPathDiv(CurName[ArcPathLength]) || CurName[ArcPathLength]==0)) { - ArcFileName+=Min(ArcPathLength,NameLength); - while (IsPathDiv(*ArcFileName)) - ArcFileName++; - if (*ArcFileName==0) // Excessive -ap switch. + size_t Pos=Min(ArcPathLength,NameLength); + while (PosExclPath==EXCL_ABSPATH && Command=='X' && IsDriveDiv(':'); - // We do not use any user specified destination paths when extracting - // absolute paths in -ep3 mode. if (AbsPaths) - *DestName=0; + { + // We do not use a user specified destination path when extracting + // absolute paths in -ep3 mode. + wchar DiskLetter=toupperw(CurName[0]); + if (CurName[1]=='_' && IsPathDiv(CurName[2]) && DiskLetter>='A' && DiskLetter<='Z') + DestName=CurName.substr(0,1) + L':' + CurName.substr(2); + else + if (CurName[0]=='_' && CurName[1]=='_') + DestName=std::wstring(2,CPATHDIVIDER) + CurName.substr(2); + else + AbsPaths=false; // Apply the destination path even with -ep3 for not absolute path. + } if (Command=='E' || Cmd->ExclPath==EXCL_SKIPWHOLEPATH) - wcsncatz(DestName,PointToName(ArcFileName),DestSize); - else - wcsncatz(DestName,ArcFileName,DestSize); + CurName=PointToName(CurName); + if (!AbsPaths) + DestName+=CurName; #ifdef _WIN_ALL // Must do after Cmd->ArcPath processing above, so file name and arc path // trailing spaces are in sync. if (!Cmd->AllowIncompatNames) - MakeNameCompatible(DestName,DestSize); + MakeNameCompatible(DestName); #endif - - wchar DiskLetter=toupperw(DestName[0]); - - if (AbsPaths) - { - if (DestName[1]=='_' && IsPathDiv(DestName[2]) && - DiskLetter>='A' && DiskLetter<='Z') - DestName[1]=':'; - else - if (DestName[0]=='_' && DestName[1]=='_') - { - // Convert __server\share to \\server\share. - DestName[0]=CPATHDIVIDER; - DestName[1]=CPATHDIVIDER; - } - } } @@ -1210,7 +1245,7 @@ bool CmdExtract::ExtrDllGetPassword() *PasswordA=0; if (Cmd->Callback(UCM_NEEDPASSWORD,Cmd->UserData,(LPARAM)PasswordA,ASIZE(PasswordA))==-1) *PasswordA=0; - GetWideName(PasswordA,NULL,PasswordW,ASIZE(PasswordW)); + CharToWide(PasswordA,PasswordW,ASIZE(PasswordW)); cleandata(PasswordA,sizeof(PasswordA)); } Cmd->Password.Set(PasswordW); @@ -1226,7 +1261,7 @@ bool CmdExtract::ExtrDllGetPassword() #ifndef RARDLL -bool CmdExtract::ExtrGetPassword(Archive &Arc,const wchar *ArcFileName,RarCheckPassword *CheckPwd) +bool CmdExtract::ExtrGetPassword(Archive &Arc,const std::wstring &ArcFileName,RarCheckPassword *CheckPwd) { if (!Cmd->Password.IsSet()) { @@ -1242,7 +1277,7 @@ bool CmdExtract::ExtrGetPassword(Archive &Arc,const wchar *ArcFileName,RarCheckP else if (!GlobalPassword && !Arc.FileHead.Solid) { - eprintf(St(MUseCurPsw),ArcFileName); + eprintf(St(MUseCurPsw),ArcFileName.c_str()); switch(Cmd->AllYes ? 1 : Ask(St(MYesNoAll))) { case -1: @@ -1283,13 +1318,13 @@ void CmdExtract::ConvertDosPassword(Archive &Arc,SecPassword &DestPwd) #endif -void CmdExtract::ExtrCreateDir(Archive &Arc,const wchar *ArcFileName) +void CmdExtract::ExtrCreateDir(Archive &Arc,const std::wstring &ArcFileName) { if (Cmd->Test) { if (!Cmd->DisableNames) { - mprintf(St(MExtrTestFile),ArcFileName); + mprintf(St(MExtrTestFile),ArcFileName.c_str()); mprintf(L" %s",St(MOk)); } return; @@ -1305,7 +1340,7 @@ void CmdExtract::ExtrCreateDir(Archive &Arc,const wchar *ArcFileName) // File with name same as this directory exists. Propose user // to overwrite it. bool UserReject; - FileCreate(Cmd,NULL,DestFileName,ASIZE(DestFileName),&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime); + FileCreate(Cmd,NULL,DestFileName,&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime); DirExist=false; } if (!DirExist) @@ -1315,17 +1350,15 @@ void CmdExtract::ExtrCreateDir(Archive &Arc,const wchar *ArcFileName) if (MDCode!=MKDIR_SUCCESS && !IsNameUsable(DestFileName)) { uiMsg(UIMSG_CORRECTINGNAME,Arc.FileName); - wchar OrigName[ASIZE(DestFileName)]; - wcsncpyz(OrigName,DestFileName,ASIZE(OrigName)); + std::wstring OrigName=DestFileName; MakeNameUsable(DestFileName,true); #ifndef SFX_MODULE uiMsg(UIERROR_RENAMING,Arc.FileName,OrigName,DestFileName); #endif DirExist=FileExist(DestFileName) && IsDir(GetFileAttr(DestFileName)); - if (!DirExist) + if (!DirExist && (Cmd->AbsoluteLinks || !ConvertSymlinkPaths || + LinksToDirs(DestFileName,Cmd->ExtrPath,LastCheckedSymlink))) { - if (!Cmd->AbsoluteLinks && ConvertSymlinkPaths) - LinksToDirs(DestFileName,Cmd->ExtrPath,LastCheckedSymlink); CreatePath(DestFileName,true,Cmd->DisableNames); MDCode=MakeDir(DestFileName,!Cmd->IgnoreGeneralAttr,Arc.FileHead.FileAttr); } @@ -1336,7 +1369,7 @@ void CmdExtract::ExtrCreateDir(Archive &Arc,const wchar *ArcFileName) { if (!Cmd->DisableNames) { - mprintf(St(MCreatDir),DestFileName); + mprintf(St(MCreatDir),DestFileName.c_str()); mprintf(L" %s",St(MOk)); } PrevProcessed=true; @@ -1386,7 +1419,7 @@ bool CmdExtract::ExtrCreateFile(Archive &Arc,File &CurFile) bool UserReject; // Specify "write only" mode to avoid OpenIndiana NAS problems // with SetFileTime and read+write files. - if (!FileCreate(Cmd,&CurFile,DestFileName,ASIZE(DestFileName),&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime,true)) + if (!FileCreate(Cmd,&CurFile,DestFileName,&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime,true)) { Success=false; if (!UserReject) @@ -1402,23 +1435,24 @@ bool CmdExtract::ExtrCreateFile(Archive &Arc,File &CurFile) { uiMsg(UIMSG_CORRECTINGNAME,Arc.FileName); - wchar OrigName[ASIZE(DestFileName)]; - wcsncpyz(OrigName,DestFileName,ASIZE(OrigName)); + std::wstring OrigName=DestFileName; MakeNameUsable(DestFileName,true); - if (!Cmd->AbsoluteLinks && ConvertSymlinkPaths) - LinksToDirs(DestFileName,Cmd->ExtrPath,LastCheckedSymlink); - CreatePath(DestFileName,true,Cmd->DisableNames); - if (FileCreate(Cmd,&CurFile,DestFileName,ASIZE(DestFileName),&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime,true)) + if (Cmd->AbsoluteLinks || !ConvertSymlinkPaths || + LinksToDirs(DestFileName,Cmd->ExtrPath,LastCheckedSymlink)) { + CreatePath(DestFileName,true,Cmd->DisableNames); + if (FileCreate(Cmd,&CurFile,DestFileName,&UserReject,Arc.FileHead.UnpSize,&Arc.FileHead.mtime,true)) + { #ifndef SFX_MODULE - uiMsg(UIERROR_RENAMING,Arc.FileName,OrigName,DestFileName); + uiMsg(UIERROR_RENAMING,Arc.FileName,OrigName,DestFileName); #endif - Success=true; + Success=true; + } + else + ErrHandler.CreateErrorMsg(Arc.FileName,DestFileName); } - else - ErrHandler.CreateErrorMsg(Arc.FileName,DestFileName); } } } @@ -1427,11 +1461,11 @@ bool CmdExtract::ExtrCreateFile(Archive &Arc,File &CurFile) } -bool CmdExtract::CheckUnpVer(Archive &Arc,const wchar *ArcFileName) +bool CmdExtract::CheckUnpVer(Archive &Arc,const std::wstring &ArcFileName) { bool WrongVer; - if (Arc.Format==RARFMT50) // Both SFX and RAR can unpack RAR 5.0 archives. - WrongVer=Arc.FileHead.UnpVer>VER_UNPACK5; + if (Arc.Format==RARFMT50) // Both SFX and RAR can unpack RAR 5.0 and 7.0 archives. + WrongVer=Arc.FileHead.UnpVer>VER_UNPACK7; else { #ifdef SFX_MODULE // SFX can unpack only RAR 2.9 archives. @@ -1464,7 +1498,7 @@ bool CmdExtract::CheckUnpVer(Archive &Arc,const wchar *ArcFileName) // But it would be slower for solid archives than scaning headers // in first pass and extracting everything in second, as implemented now. // -void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumbering) +void CmdExtract::AnalyzeArchive(const std::wstring &ArcName,bool Volume,bool NewNumbering) { FreeAnalyzeData(); // If processing non-first archive in multiple archives set. @@ -1474,11 +1508,11 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi return; // No need to check further for * and *.* masks. // Start search from first volume if all volumes preceding current are available. - wchar NextName[NM]; + std::wstring NextName; if (Volume) - GetFirstVolIfFullSet(ArcName,NewNumbering,NextName,ASIZE(NextName)); + GetFirstVolIfFullSet(ArcName,NewNumbering,NextName); else - wcsncpyz(NextName,ArcName,ASIZE(NextName)); + NextName=ArcName; bool MatchFound=false; bool PrevMatched=false; @@ -1487,7 +1521,7 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi bool FirstVolume=true; // We shall set FirstFile once for all volumes and not for each volume. - // So we do not reuse the outdated Analyze->StartPos from previous volume + // So we do not reuse the outdated Analyze.StartPos from previous volume // if extracted file resides completely in the beginning of current one. bool FirstFile=true; @@ -1501,8 +1535,8 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi // If we couldn't open trailing volumes, we can't set early exit // parameters. It is possible that some volume are on removable media // and will be provided by user when extracting. - *Analyze->EndName=0; - Analyze->EndPos=0; + Analyze.EndName.clear(); + Analyze.EndPos=0; } break; } @@ -1520,6 +1554,15 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi } if (HeaderType==HEAD_FILE) { + if ((Arc.Format==RARFMT14 || Arc.Format==RARFMT15) && Arc.FileHead.UnpVer<=15) + { + // RAR versions earlier than 2.0 do not set per file solid flag. + // They have only the global archive solid flag, so we can't + // reliably analyze them here. + OpenNext=false; + break; + } + if (!Arc.FileHead.SplitBefore) { if (!MatchFound && !Arc.FileHead.Solid) // Can start extraction from here. @@ -1528,24 +1571,24 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi // if we set StartName for first volume or StartPos for first // archived file. if (!FirstVolume) - wcsncpyz(Analyze->StartName,NextName,ASIZE(Analyze->StartName)); + Analyze.StartName=NextName; // We shall set FirstFile once for all volumes for this code // to work properly. Alternatively we could append - // "|| Analyze->StartPos!=0" to the condition, so we do not reuse - // the outdated Analyze->StartPos value from previous volume. + // "|| Analyze.StartPos!=0" to the condition, so we do not reuse + // the outdated Analyze.StartPos value from previous volume. if (!FirstFile) - Analyze->StartPos=Arc.CurBlockPos; + Analyze.StartPos=Arc.CurBlockPos; } - if (Cmd->IsProcessFile(Arc.FileHead,NULL,MATCH_WILDSUBPATH,0,NULL,0)!=0) + if (Cmd->IsProcessFile(Arc.FileHead,NULL,MATCH_WILDSUBPATH,0,NULL)!=0) { MatchFound = true; PrevMatched = true; // Reset the previously set early exit position, if any, because // we found a new matched file. - Analyze->EndPos=0; + Analyze.EndPos=0; // Matched file reference pointing at maybe non-matched source file. // Even though we know RedirName, we can't check if source file @@ -1554,8 +1597,8 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi if (Arc.FileHead.RedirType==FSREDIR_FILECOPY) { bool AlreadyAdded=false; - for (size_t I=0;IEndName,NextName,ASIZE(Analyze->EndName)); - Analyze->EndPos=Arc.CurBlockPos; + Analyze.EndName=NextName; + Analyze.EndPos=Arc.CurBlockPos; } PrevMatched=false; } @@ -1605,7 +1648,7 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi if (Volume && OpenNext) { - NextVolumeName(NextName,ASIZE(NextName),!Arc.NewNumbering); + NextVolumeName(NextName,!Arc.NewNumbering); FirstVolume=false; // Needed for multivolume archives. Added in case some 'break' @@ -1620,8 +1663,8 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi // If file references are present, we can't reliably skip in semi-solid // archives, because reference source can be present in skipped data. - if (RefList.Size()!=0) - memset(Analyze,0,sizeof(*Analyze)); + if (RefList.size()!=0) + Analyze={}; } #endif @@ -1629,26 +1672,46 @@ void CmdExtract::AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumberi #ifndef SFX_MODULE // Return the first volume name if all volumes preceding the specified // are available. Otherwise return the specified volume name. -void CmdExtract::GetFirstVolIfFullSet(const wchar *SrcName,bool NewNumbering,wchar *DestName,size_t DestSize) +void CmdExtract::GetFirstVolIfFullSet(const std::wstring &SrcName,bool NewNumbering,std::wstring &DestName) { - wchar FirstVolName[NM]; - VolNameToFirstName(SrcName,FirstVolName,ASIZE(FirstVolName),NewNumbering); - wchar NextName[NM]; - wcsncpyz(NextName,FirstVolName,ASIZE(NextName)); - wchar ResultName[NM]; - wcsncpyz(ResultName,SrcName,ASIZE(ResultName)); + std::wstring FirstVolName; + VolNameToFirstName(SrcName,FirstVolName,NewNumbering); + std::wstring NextName=FirstVolName; + std::wstring ResultName=SrcName; while (true) { - if (wcscmp(SrcName,NextName)==0) + if (SrcName==NextName) { - wcsncpyz(ResultName,FirstVolName,DestSize); + ResultName=FirstVolName; // Reached the specified volume starting from the first. break; } if (!FileExist(NextName)) break; - NextVolumeName(NextName,ASIZE(NextName),!NewNumbering); + NextVolumeName(NextName,!NewNumbering); } - wcsncpyz(DestName,ResultName,DestSize); + DestName=ResultName; } +#endif -#endif \ No newline at end of file + +bool CmdExtract::CheckWinLimit(Archive &Arc,std::wstring &ArcFileName) +{ + if (Arc.FileHead.WinSize<=Cmd->WinSizeLimit || Arc.FileHead.WinSize<=Cmd->WinSize) + return true; + if (uiDictLimit(Cmd,ArcFileName,Arc.FileHead.WinSize,Max(Cmd->WinSizeLimit,Cmd->WinSize))) + { + // No more prompts when extracting other files. Important for GUI versions, + // where we might not have [Max]WinSize set permanently when extracting. + Cmd->WinSizeLimit=Arc.FileHead.WinSize; + } + else + { + ErrHandler.SetErrorCode(RARX_FATAL); +#ifdef RARDLL + Cmd->DllError=ERAR_LARGE_DICT; +#endif + Arc.SeekToNext(); + return false; + } + return true; +} diff --git a/multiarc/src/formats/rar/unrar/extract.hpp b/multiarc/src/formats/rar/unrar/extract.hpp index 18396c5b9..4400057ae 100644 --- a/multiarc/src/formats/rar/unrar/extract.hpp +++ b/multiarc/src/formats/rar/unrar/extract.hpp @@ -8,41 +8,42 @@ class CmdExtract private: struct ExtractRef { - wchar *RefName; - wchar *TmpName; + std::wstring RefName; + std::wstring TmpName; uint64 RefCount; }; - Array RefList; + std::vector RefList; struct AnalyzeData { - wchar StartName[NM]; + std::wstring StartName; uint64 StartPos; - wchar EndName[NM]; + std::wstring EndName; uint64 EndPos; - } *Analyze; + } Analyze; bool ArcAnalyzed; void FreeAnalyzeData(); EXTRACT_ARC_CODE ExtractArchive(); - bool ExtractFileCopy(File &New,wchar *ArcName,const wchar *RedirName,wchar *NameNew,wchar *NameExisting,size_t NameExistingSize,int64 UnpSize); - void ExtrPrepareName(Archive &Arc,const wchar *ArcFileName,wchar *DestName,size_t DestSize); + bool ExtractFileCopy(File &New,const std::wstring &ArcName,const std::wstring &RedirName,const std::wstring &NameNew,const std::wstring &NameExisting,int64 UnpSize); + void ExtrPrepareName(Archive &Arc,const std::wstring &ArcFileName,std::wstring &DestName); #ifdef RARDLL bool ExtrDllGetPassword(); #else - bool ExtrGetPassword(Archive &Arc,const wchar *ArcFileName,RarCheckPassword *CheckPwd); + bool ExtrGetPassword(Archive &Arc,const std::wstring &ArcFileName,RarCheckPassword *CheckPwd); #endif #if defined(_WIN_ALL) && !defined(SFX_MODULE) void ConvertDosPassword(Archive &Arc,SecPassword &DestPwd); #endif - void ExtrCreateDir(Archive &Arc,const wchar *ArcFileName); + void ExtrCreateDir(Archive &Arc,const std::wstring &ArcFileName); bool ExtrCreateFile(Archive &Arc,File &CurFile); - bool CheckUnpVer(Archive &Arc,const wchar *ArcFileName); + bool CheckUnpVer(Archive &Arc,const std::wstring &ArcFileName); #ifndef SFX_MODULE - void AnalyzeArchive(const wchar *ArcName,bool Volume,bool NewNumbering); - void GetFirstVolIfFullSet(const wchar *SrcName,bool NewNumbering,wchar *DestName,size_t DestSize); + void AnalyzeArchive(const std::wstring &ArcName,bool Volume,bool NewNumbering); + void GetFirstVolIfFullSet(const std::wstring &SrcName,bool NewNumbering,std::wstring &DestName); #endif + bool CheckWinLimit(Archive &Arc,std::wstring &ArcFileName); RarTime StartTime; // Time when extraction started. @@ -65,12 +66,12 @@ class CmdExtract // any wrong password hints. bool AnySolidDataUnpackedWell; - wchar ArcName[NM]; + std::wstring ArcName; bool GlobalPassword; bool PrevProcessed; // If previous file was successfully extracted or tested. - wchar DestFileName[NM]; - bool PasswordCancelled; + std::wstring DestFileName; + bool SuppressNoFilesMessage; // In Windows it is set to true if at least one symlink with ".." // in target was extracted. diff --git a/multiarc/src/formats/rar/unrar/filcreat.cpp b/multiarc/src/formats/rar/unrar/filcreat.cpp index d58e4f6fe..6b21aa260 100644 --- a/multiarc/src/formats/rar/unrar/filcreat.cpp +++ b/multiarc/src/formats/rar/unrar/filcreat.cpp @@ -3,7 +3,7 @@ // If NewFile==NULL, we delete created file after user confirmation. // It is useful if we need to overwrite an existing folder or file, // but need user confirmation for that. -bool FileCreate(CommandData *Cmd,File *NewFile,wchar *Name,size_t MaxNameSize, +bool FileCreate(CommandData *Cmd,File *NewFile,std::wstring &Name, bool *UserReject,int64 FileSize,RarTime *FileTime,bool WriteOnly) { if (UserReject!=NULL) @@ -29,7 +29,7 @@ bool FileCreate(CommandData *Cmd,File *NewFile,wchar *Name,size_t MaxNameSize, // autorename below can change the name, so we need to check it again. ShortNameChanged=false; #endif - UIASKREP_RESULT Choice=uiAskReplaceEx(Cmd,Name,MaxNameSize,FileSize,FileTime,(NewFile==NULL ? UIASKREP_F_NORENAME:0)); + UIASKREP_RESULT Choice=uiAskReplaceEx(Cmd,Name,FileSize,FileTime,(NewFile==NULL ? UIASKREP_F_NORENAME:0)); if (Choice==UIASKREP_R_REPLACE) break; @@ -56,85 +56,70 @@ bool FileCreate(CommandData *Cmd,File *NewFile,wchar *Name,size_t MaxNameSize, } -bool GetAutoRenamedName(wchar *Name,size_t MaxNameSize) -{ - wchar NewName[NM]; - size_t NameLength=wcslen(Name); - wchar *Ext=GetExt(Name); - if (Ext==NULL) - Ext=Name+NameLength; - for (uint FileVer=1;;FileVer++) - { - swprintf(NewName,ASIZE(NewName),L"%.*ls(%u)%ls",uint(Ext-Name),Name,FileVer,Ext); - if (!FileExist(NewName)) - { - wcsncpyz(Name,NewName,MaxNameSize); - break; - } - if (FileVer>=1000000) - return false; - } - return true; -} - - #if defined(_WIN_ALL) // If we find a file, which short name is equal to 'Name', we try to change // its short name, while preserving the long name. It helps when unpacking // an archived file, which long name is equal to short name of already // existing file. Otherwise we would overwrite the already existing file, // even though its long name does not match the name of unpacking file. -bool UpdateExistingShortName(const wchar *Name) +bool UpdateExistingShortName(const std::wstring &Name) { - wchar LongPathName[NM]; - DWORD Res=GetLongPathName(Name,LongPathName,ASIZE(LongPathName)); - if (Res==0 || Res>=ASIZE(LongPathName)) + DWORD Res=GetLongPathName(Name.c_str(),NULL,0); + if (Res==0) return false; - wchar ShortPathName[NM]; - Res=GetShortPathName(Name,ShortPathName,ASIZE(ShortPathName)); - if (Res==0 || Res>=ASIZE(ShortPathName)) + std::vector LongPathBuf(Res); + Res=GetLongPathName(Name.c_str(),LongPathBuf.data(),(DWORD)LongPathBuf.size()); + if (Res==0 || Res>=LongPathBuf.size()) return false; - wchar *LongName=PointToName(LongPathName); - wchar *ShortName=PointToName(ShortPathName); + Res=GetShortPathName(Name.c_str(),NULL,0); + if (Res==0) + return false; + std::vector ShortPathBuf(Res); + Res=GetShortPathName(Name.c_str(),ShortPathBuf.data(),(DWORD)ShortPathBuf.size()); + if (Res==0 || Res>=ShortPathBuf.size()) + return false; + std::wstring LongPathName=LongPathBuf.data(); + std::wstring ShortPathName=ShortPathBuf.data(); + + std::wstring LongName=PointToName(LongPathName); + std::wstring ShortName=PointToName(ShortPathName); // We continue only if file has a short name, which does not match its // long name, and this short name is equal to name of file which we need // to create. - if (*ShortName==0 || wcsicomp(LongName,ShortName)==0 || + if (ShortName.empty() || wcsicomp(LongName,ShortName)==0 || wcsicomp(PointToName(Name),ShortName)!=0) return false; // Generate the temporary new name for existing file. - wchar NewName[NM]; - *NewName=0; - for (int I=0;I<10000 && *NewName==0;I+=123) + std::wstring NewName; + for (uint I=0;I<10000 && NewName.empty();I+=123) { // Here we copy the path part of file to create. We'll make the temporary // file in the same folder. - wcsncpyz(NewName,Name,ASIZE(NewName)); + NewName=Name; // Here we set the random name part. - swprintf(PointToName(NewName),ASIZE(NewName),L"rtmp%d",I); + SetName(NewName,std::wstring(L"rtmp") + std::to_wstring(I)); // If such file is already exist, try next random name. if (FileExist(NewName)) - *NewName=0; + NewName.clear(); } // If we could not generate the name not used by any other file, we return. - if (*NewName==0) + if (NewName.empty()) return false; // FastFind returns the name without path, but we need the fully qualified // name for renaming, so we use the path from file to create and long name // from existing file. - wchar FullName[NM]; - wcsncpyz(FullName,Name,ASIZE(FullName)); - SetName(FullName,LongName,ASIZE(FullName)); + std::wstring FullName=Name; + SetName(FullName,LongName); // Rename the existing file to randomly generated name. Normally it changes // the short name too. - if (!MoveFile(FullName,NewName)) + if (!MoveFile(FullName.c_str(),NewName.c_str())) return false; // Now we need to create the temporary empty file with same name as @@ -149,7 +134,7 @@ bool UpdateExistingShortName(const wchar *Name) // Now we rename the existing file from temporary name to original long name. // Since its previous short name is occupied by another file, it should // get another short name. - MoveFile(NewName,FullName); + MoveFile(NewName.c_str(),FullName.c_str()); if (Created) { @@ -157,9 +142,9 @@ bool UpdateExistingShortName(const wchar *Name) KeepShortFile.Close(); KeepShortFile.Delete(); } - // We successfully changed the short name. Maybe sometimes we'll simplify - // this function by use of SetFileShortName Windows API call. - // But SetFileShortName is not available in older Windows. + // We successfully changed the short name. We do not use the simpler + // SetFileShortName Windows API call, because it requires SE_RESTORE_NAME + // privilege. return true; } #endif diff --git a/multiarc/src/formats/rar/unrar/filcreat.hpp b/multiarc/src/formats/rar/unrar/filcreat.hpp index 456a4a4a1..ad95feef9 100644 --- a/multiarc/src/formats/rar/unrar/filcreat.hpp +++ b/multiarc/src/formats/rar/unrar/filcreat.hpp @@ -1,14 +1,12 @@ #ifndef _RAR_FILECREATE_ #define _RAR_FILECREATE_ -bool FileCreate(CommandData *Cmd,File *NewFile,wchar *Name,size_t MaxNameSize, +bool FileCreate(CommandData *Cmd,File *NewFile,std::wstring &Name, bool *UserReject,int64 FileSize=INT64NDF, RarTime *FileTime=NULL,bool WriteOnly=false); -bool GetAutoRenamedName(wchar *Name,size_t MaxNameSize); - #if defined(_WIN_ALL) -bool UpdateExistingShortName(const wchar *Name); +bool UpdateExistingShortName(const std::wstring &Name); #endif #endif diff --git a/multiarc/src/formats/rar/unrar/file.cpp b/multiarc/src/formats/rar/unrar/file.cpp index 7bf60fd4e..2d0d785cb 100644 --- a/multiarc/src/formats/rar/unrar/file.cpp +++ b/multiarc/src/formats/rar/unrar/file.cpp @@ -3,7 +3,6 @@ File::File() { hFile=FILE_BAD_HANDLE; - *FileName=0; NewFile=false; LastWrite=false; HandleType=FILE_HANDLENORMAL; @@ -40,12 +39,12 @@ void File::operator = (File &SrcFile) LastWrite=SrcFile.LastWrite; HandleType=SrcFile.HandleType; TruncatedAfterReadError=SrcFile.TruncatedAfterReadError; - wcsncpyz(FileName,SrcFile.FileName,ASIZE(FileName)); + FileName=SrcFile.FileName; SrcFile.SkipClose=true; } -bool File::Open(const wchar *Name,uint Mode) +bool File::Open(const std::wstring &Name,uint Mode) { ErrorType=FILE_SUCCESS; FileHandle hNewFile; @@ -63,17 +62,17 @@ bool File::Open(const wchar *Name,uint Mode) FindData FD; if (PreserveAtime) Access|=FILE_WRITE_ATTRIBUTES; // Needed to preserve atime. - hNewFile=CreateFile(Name,Access,ShareMode,NULL,OPEN_EXISTING,Flags,NULL); + hNewFile=CreateFile(Name.c_str(),Access,ShareMode,NULL,OPEN_EXISTING,Flags,NULL); DWORD LastError; if (hNewFile==FILE_BAD_HANDLE) { LastError=GetLastError(); - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) { - hNewFile=CreateFile(LongName,Access,ShareMode,NULL,OPEN_EXISTING,Flags,NULL); + hNewFile=CreateFile(LongName.c_str(),Access,ShareMode,NULL,OPEN_EXISTING,Flags,NULL); // For archive names longer than 260 characters first CreateFile // (without \\?\) fails and sets LastError to 3 (access denied). @@ -112,10 +111,10 @@ bool File::Open(const wchar *Name,uint Mode) if (PreserveAtime) flags|=O_NOATIME; #endif - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); + std::string NameA; + WideToChar(Name,NameA); - int handle=open(NameA,flags); + int handle=open(NameA.c_str(),flags); #ifdef LOCK_EX #ifdef _OSF_SOURCE @@ -148,7 +147,7 @@ bool File::Open(const wchar *Name,uint Mode) if (Success) { hFile=hNewFile; - wcsncpyz(FileName,Name,ASIZE(FileName)); + FileName=Name; TruncatedAfterReadError=false; } return Success; @@ -156,7 +155,7 @@ bool File::Open(const wchar *Name,uint Mode) #if !defined(SFX_MODULE) -void File::TOpen(const wchar *Name) +void File::TOpen(const std::wstring &Name) { if (!WOpen(Name)) ErrHandler.Exit(RARX_OPEN); @@ -164,7 +163,7 @@ void File::TOpen(const wchar *Name) #endif -bool File::WOpen(const wchar *Name) +bool File::WOpen(const std::wstring &Name) { if (Open(Name)) return true; @@ -173,7 +172,7 @@ bool File::WOpen(const wchar *Name) } -bool File::Create(const wchar *Name,uint Mode) +bool File::Create(const std::wstring &Name,uint Mode) { // OpenIndiana based NAS and CIFS shares fail to set the file time if file // was created in read+write mode and some data was written and not flushed @@ -188,40 +187,40 @@ bool File::Create(const wchar *Name,uint Mode) // Windows automatically removes dots and spaces in the end of file name, // So we detect such names and process them with \\?\ prefix. - wchar *LastChar=PointToLastChar(Name); - bool Special=*LastChar=='.' || *LastChar==' '; + wchar LastChar=GetLastChar(Name); + bool Special=LastChar=='.' || LastChar==' '; if (Special && (Mode & FMF_STANDARDNAMES)==0) hFile=FILE_BAD_HANDLE; else - hFile=CreateFile(Name,Access,ShareMode,NULL,CREATE_ALWAYS,0,NULL); + hFile=CreateFile(Name.c_str(),Access,ShareMode,NULL,CREATE_ALWAYS,0,NULL); if (hFile==FILE_BAD_HANDLE) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - hFile=CreateFile(LongName,Access,ShareMode,NULL,CREATE_ALWAYS,0,NULL); + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + hFile=CreateFile(LongName.c_str(),Access,ShareMode,NULL,CREATE_ALWAYS,0,NULL); } #else - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); #ifdef FILE_USE_OPEN - hFile=open(NameA,(O_CREAT|O_TRUNC) | (WriteMode ? O_WRONLY : O_RDWR),0666); + std::string NameA; + WideToChar(Name,NameA); + hFile=open(NameA.c_str(),(O_CREAT|O_TRUNC) | (WriteMode ? O_WRONLY : O_RDWR),0666); #else - hFile=fopen(NameA,WriteMode ? WRITEBINARY:CREATEBINARY); + hFile=fopen(NameA.c_str(),WriteMode ? WRITEBINARY:CREATEBINARY); #endif #endif NewFile=true; HandleType=FILE_HANDLENORMAL; SkipClose=false; - wcsncpyz(FileName,Name,ASIZE(FileName)); + FileName=Name; return hFile!=FILE_BAD_HANDLE; } #if !defined(SFX_MODULE) -void File::TCreate(const wchar *Name,uint Mode) +void File::TCreate(const std::wstring &Name,uint Mode) { if (!WCreate(Name,Mode)) ErrHandler.Exit(RARX_FATAL); @@ -229,7 +228,7 @@ void File::TCreate(const wchar *Name,uint Mode) #endif -bool File::WCreate(const wchar *Name,uint Mode) +bool File::WCreate(const std::wstring &Name,uint Mode) { if (Create(Name,Mode)) return true; @@ -250,7 +249,7 @@ bool File::Close() // We use the standard system handle for stdout in Windows // and it must not be closed here. if (HandleType==FILE_HANDLENORMAL) - Success=CloseHandle(hFile)==TRUE; + Success=CloseHandle(hFile)!=FALSE; #else #ifdef FILE_USE_OPEN Success=close(hFile)!=-1; @@ -280,16 +279,16 @@ bool File::Delete() } -bool File::Rename(const wchar *NewName) +bool File::Rename(const std::wstring &NewName) { // No need to rename if names are already same. - bool Success=wcscmp(FileName,NewName)==0; + bool Success=(NewName==FileName); if (!Success) Success=RenameFile(FileName,NewName); if (Success) - wcsncpyz(FileName,NewName,ASIZE(FileName)); + FileName=NewName; return Success; } @@ -327,13 +326,13 @@ bool File::Write(const void *Data,size_t Size) const size_t MaxSize=0x4000; for (size_t I=0;IIsSet(); bool seta=fta!=NULL && fta->IsSet(); if (setm || seta) { - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); + std::string NameA; + WideToChar(Name,NameA); #ifdef UNIX_TIME_NS timespec times[2]; @@ -720,7 +719,7 @@ void File::SetCloseFileTimeByName(const wchar *Name,RarTime *ftm,RarTime *fta) times[0].tv_nsec=seta ? long(fta->GetUnixNS()%1000000000) : UTIME_NOW; times[1].tv_sec=setm ? ftm->GetUnix() : 0; times[1].tv_nsec=setm ? long(ftm->GetUnixNS()%1000000000) : UTIME_NOW; - utimensat(AT_FDCWD,NameA,times,0); + utimensat(AT_FDCWD,NameA.c_str(),times,0); #else utimbuf ut; if (setm) @@ -731,7 +730,7 @@ void File::SetCloseFileTimeByName(const wchar *Name,RarTime *ftm,RarTime *fta) ut.actime=fta->GetUnix(); else ut.actime=ut.modtime; // Need to set something, cannot left it 0. - utime(NameA,&ut); + utime(NameA.c_str(),&ut); #endif } #endif @@ -802,15 +801,15 @@ bool File::IsDevice() #ifndef SFX_MODULE int64 File::Copy(File &Dest,int64 Length) { - Array Buffer(File::CopyBufferSize()); + std::vector Buffer(File::CopyBufferSize()); int64 CopySize=0; bool CopyAll=(Length==INT64NDF); while (CopyAll || Length>0) { Wait(); - size_t SizeToRead=(!CopyAll && Length<(int64)Buffer.Size()) ? (size_t)Length:Buffer.Size(); - byte *Buf=&Buffer[0]; + size_t SizeToRead=(!CopyAll && Length<(int64)Buffer.size()) ? (size_t)Length:Buffer.size(); + byte *Buf=Buffer.data(); int ReadSize=Read(Buf,SizeToRead); if (ReadSize==0) break; diff --git a/multiarc/src/formats/rar/unrar/file.hpp b/multiarc/src/formats/rar/unrar/file.hpp index 5f55de960..a95cc398f 100644 --- a/multiarc/src/formats/rar/unrar/file.hpp +++ b/multiarc/src/formats/rar/unrar/file.hpp @@ -83,12 +83,9 @@ class File protected: bool OpenShared; // Set by 'Archive' class. public: - wchar FileName[NM]; + std::wstring FileName; FILE_ERRORTYPE ErrorType; - - byte *SeekBuf; // To read instead of seek for stdin files. - static const size_t SeekBufSize=0x10000; public: File(); virtual ~File(); @@ -96,15 +93,15 @@ class File // Several functions below are 'virtual', because they are redefined // by Archive for QOpen and by MultiFile for split files in WinRAR. - virtual bool Open(const wchar *Name,uint Mode=FMF_READ); - void TOpen(const wchar *Name); - bool WOpen(const wchar *Name); - bool Create(const wchar *Name,uint Mode=FMF_UPDATE|FMF_SHAREREAD); - void TCreate(const wchar *Name,uint Mode=FMF_UPDATE|FMF_SHAREREAD); - bool WCreate(const wchar *Name,uint Mode=FMF_UPDATE|FMF_SHAREREAD); + virtual bool Open(const std::wstring &Name,uint Mode=FMF_READ); + void TOpen(const std::wstring &Name); + bool WOpen(const std::wstring &Name); + bool Create(const std::wstring &Name,uint Mode=FMF_UPDATE|FMF_SHAREREAD); + void TCreate(const std::wstring &Name,uint Mode=FMF_UPDATE|FMF_SHAREREAD); + bool WCreate(const std::wstring &Name,uint Mode=FMF_UPDATE|FMF_SHAREREAD); virtual bool Close(); // 'virtual' for MultiFile class. bool Delete(); - bool Rename(const wchar *NewName); + bool Rename(const std::wstring &NewName); bool Write(const void *Data,size_t Size); virtual int Read(void *Data,size_t Size); int DirectRead(void *Data,size_t Size); @@ -118,13 +115,13 @@ class File void Flush(); void SetOpenFileTime(RarTime *ftm,RarTime *ftc=NULL,RarTime *fta=NULL); void SetCloseFileTime(RarTime *ftm,RarTime *fta=NULL); - static void SetCloseFileTimeByName(const wchar *Name,RarTime *ftm,RarTime *fta); + static void SetCloseFileTimeByName(const std::wstring &Name,RarTime *ftm,RarTime *fta); #ifdef _UNIX static void StatToRarTime(struct stat &st,RarTime *ftm,RarTime *ftc,RarTime *fta); #endif void GetOpenFileTime(RarTime *ftm,RarTime *ftc=NULL,RarTime *fta=NULL); virtual bool IsOpened() {return hFile!=FILE_BAD_HANDLE;} // 'virtual' for MultiFile class. - int64 FileLength(); + virtual int64 FileLength(); // 'virtual' for MultiFile class. void SetHandleType(FILE_HANDLETYPE Type) {HandleType=Type;} void SetLineInputMode(bool Mode) {LineInput=Mode;} FILE_HANDLETYPE GetHandleType() {return HandleType;} @@ -151,14 +148,9 @@ class File #endif static size_t CopyBufferSize() { -#ifdef _WIN_ALL - // USB flash performance is poor with 64 KB buffer, 256+ KB resolved it. - // For copying from HDD to same HDD the best performance was with 256 KB - // buffer in XP and with 1 MB buffer in Win10. - return WinNT()==WNT_WXP ? 0x40000:0x100000; -#else - return 0x100000; -#endif + // Values in 0x100000 - 0x400000 range are ok, but multithreaded CRC32 + // seems to benefit from 0x400000, especially on ARM CPUs. + return 0x400000; } }; diff --git a/multiarc/src/formats/rar/unrar/filefn.cpp b/multiarc/src/formats/rar/unrar/filefn.cpp index aaef305b8..3cb258926 100644 --- a/multiarc/src/formats/rar/unrar/filefn.cpp +++ b/multiarc/src/formats/rar/unrar/filefn.cpp @@ -1,18 +1,18 @@ #include "rar.hpp" -MKDIR_CODE MakeDir(const wchar *Name,bool SetAttr,uint Attr) +MKDIR_CODE MakeDir(const std::wstring &Name,bool SetAttr,uint Attr) { #ifdef _WIN_ALL // Windows automatically removes dots and spaces in the end of directory // name. So we detect such names and process them with \\?\ prefix. - wchar *LastChar=PointToLastChar(Name); - bool Special=*LastChar=='.' || *LastChar==' '; - BOOL RetCode=Special ? FALSE : CreateDirectory(Name,NULL); + wchar LastChar=GetLastChar(Name); + bool Special=LastChar=='.' || LastChar==' '; + BOOL RetCode=Special ? FALSE : CreateDirectory(Name.c_str(),NULL); if (RetCode==0 && !FileExist(Name)) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - RetCode=CreateDirectory(LongName,NULL); + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + RetCode=CreateDirectory(LongName.c_str(),NULL); } if (RetCode!=0) // Non-zero return code means success for CreateDirectory. { @@ -25,10 +25,10 @@ MKDIR_CODE MakeDir(const wchar *Name,bool SetAttr,uint Attr) return MKDIR_BADPATH; return MKDIR_ERROR; #elif defined(_UNIX) - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); + std::string NameA; + WideToChar(Name,NameA); mode_t uattr=SetAttr ? (mode_t)Attr:0777; - int ErrCode=mkdir(NameA,uattr); + int ErrCode=mkdir(NameA.c_str(),uattr); if (ErrCode==-1) return errno==ENOENT ? MKDIR_BADPATH:MKDIR_ERROR; return MKDIR_SUCCESS; @@ -38,12 +38,19 @@ MKDIR_CODE MakeDir(const wchar *Name,bool SetAttr,uint Attr) } -bool CreatePath(const wchar *Path,bool SkipLastName,bool Silent) +// Simplified version of MakeDir(). +bool CreateDir(const std::wstring &Name) { - if (Path==NULL || *Path==0) + return MakeDir(Name,false,0)==MKDIR_SUCCESS; +} + + +bool CreatePath(const std::wstring &Path,bool SkipLastName,bool Silent) +{ + if (Path.empty()) return false; -#if defined(_WIN_ALL) || defined(_EMX) +#ifdef _WIN_ALL uint DirAttr=0; #else uint DirAttr=0777; @@ -51,42 +58,36 @@ bool CreatePath(const wchar *Path,bool SkipLastName,bool Silent) bool Success=true; - for (const wchar *s=Path;*s!=0;s++) + for (size_t I=0;I=ASIZE(DirName)) - break; - // Process all kinds of path separators, so user can enter Unix style - // path in Windows or Windows in Unix. s>Path check avoids attempting + // path in Windows or Windows in Unix. I>0 check avoids attempting // creating an empty directory for paths starting from path separator. - if (IsPathDiv(*s) && s>Path) + if (IsPathDiv(Path[I]) && I>0) { #ifdef _WIN_ALL // We must not attempt to create "D:" directory, because first // CreateDirectory will fail, so we'll use \\?\D:, which forces Wine // to create "D:" directory. - if (s==Path+2 && Path[1]==':') + if (I==2 && Path[1]==':') continue; #endif - wcsncpy(DirName,Path,s-Path); - DirName[s-Path]=0; - + std::wstring DirName=Path.substr(0,I); Success=MakeDir(DirName,true,DirAttr)==MKDIR_SUCCESS; if (Success && !Silent) { - mprintf(St(MCreatDir),DirName); + mprintf(St(MCreatDir),DirName.c_str()); mprintf(L" %s",St(MOk)); } } } - if (!SkipLastName && !IsPathDiv(*PointToLastChar(Path))) + if (!SkipLastName && !IsPathDiv(GetLastChar(Path))) Success=MakeDir(Path,true,DirAttr)==MKDIR_SUCCESS; return Success; } -void SetDirTime(const wchar *Name,RarTime *ftm,RarTime *ftc,RarTime *fta) +void SetDirTime(const std::wstring &Name,RarTime *ftm,RarTime *ftc,RarTime *fta) { #if defined(_WIN_ALL) bool sm=ftm!=NULL && ftm->IsSet(); @@ -98,13 +99,13 @@ void SetDirTime(const wchar *Name,RarTime *ftm,RarTime *ftc,RarTime *fta) if (ResetAttr) SetFileAttr(Name,0); - HANDLE hFile=CreateFile(Name,GENERIC_WRITE,FILE_SHARE_READ|FILE_SHARE_WRITE, + HANDLE hFile=CreateFile(Name.c_str(),GENERIC_WRITE,FILE_SHARE_READ|FILE_SHARE_WRITE, NULL,OPEN_EXISTING,FILE_FLAG_BACKUP_SEMANTICS,NULL); if (hFile==INVALID_HANDLE_VALUE) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - hFile=CreateFile(LongName,GENERIC_WRITE,FILE_SHARE_READ|FILE_SHARE_WRITE, + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + hFile=CreateFile(LongName.c_str(),GENERIC_WRITE,FILE_SHARE_READ|FILE_SHARE_WRITE, NULL,OPEN_EXISTING,FILE_FLAG_BACKUP_SEMANTICS,NULL); } @@ -122,18 +123,18 @@ void SetDirTime(const wchar *Name,RarTime *ftm,RarTime *ftc,RarTime *fta) if (ResetAttr) SetFileAttr(Name,DirAttr); #endif -#if defined(_UNIX) || defined(_EMX) +#ifdef _UNIX File::SetCloseFileTimeByName(Name,ftm,fta); #endif } -bool IsRemovable(const wchar *Name) +bool IsRemovable(const std::wstring &Name) { #if defined(_WIN_ALL) - wchar Root[NM]; - GetPathRoot(Name,Root,ASIZE(Root)); - int Type=GetDriveType(*Root!=0 ? Root:NULL); + std::wstring Root; + GetPathRoot(Name,Root); + int Type=GetDriveType(Root.empty() ? nullptr : Root.c_str()); return Type==DRIVE_REMOVABLE || Type==DRIVE_CDROM; #else return false; @@ -142,25 +143,25 @@ bool IsRemovable(const wchar *Name) #ifndef SFX_MODULE -int64 GetFreeDisk(const wchar *Name) +int64 GetFreeDisk(const std::wstring &Name) { #ifdef _WIN_ALL - wchar Root[NM]; - GetFilePath(Name,Root,ASIZE(Root)); + std::wstring Root; + GetPathWithSep(Name,Root); ULARGE_INTEGER uiTotalSize,uiTotalFree,uiUserFree; uiUserFree.u.LowPart=uiUserFree.u.HighPart=0; - if (GetDiskFreeSpaceEx(*Root!=0 ? Root:NULL,&uiUserFree,&uiTotalSize,&uiTotalFree) && + if (GetDiskFreeSpaceEx(Root.empty() ? NULL:Root.c_str(),&uiUserFree,&uiTotalSize,&uiTotalFree) && uiUserFree.u.HighPart<=uiTotalFree.u.HighPart) return INT32TO64(uiUserFree.u.HighPart,uiUserFree.u.LowPart); return 0; #elif defined(_UNIX) - wchar Root[NM]; - GetFilePath(Name,Root,ASIZE(Root)); - char RootA[NM]; - WideToChar(Root,RootA,ASIZE(RootA)); + std::wstring Root; + GetPathWithSep(Name,Root); + std::string RootA; + WideToChar(Root,RootA); struct statvfs sfs; - if (statvfs(*RootA!=0 ? RootA:".",&sfs)!=0) + if (statvfs(RootA.empty() ? ".":RootA.c_str(),&sfs)!=0) return 0; int64 FreeSize=sfs.f_bsize; FreeSize=FreeSize*sfs.f_bavail; @@ -175,26 +176,27 @@ int64 GetFreeDisk(const wchar *Name) #if defined(_WIN_ALL) && !defined(SFX_MODULE) && !defined(SILENT) // Return 'true' for FAT and FAT32, so we can adjust the maximum supported // file size to 4 GB for these file systems. -bool IsFAT(const wchar *Name) +bool IsFAT(const std::wstring &Name) { - wchar Root[NM]; - GetPathRoot(Name,Root,ASIZE(Root)); + std::wstring Root; + GetPathRoot(Name,Root); wchar FileSystem[MAX_PATH+1]; - if (GetVolumeInformation(Root,NULL,0,NULL,NULL,NULL,FileSystem,ASIZE(FileSystem))) + // Root can be empty, when we create volumes with -v in the current folder. + if (GetVolumeInformation(Root.empty() ? NULL:Root.c_str(),NULL,0,NULL,NULL,NULL,FileSystem,ASIZE(FileSystem))) return wcscmp(FileSystem,L"FAT")==0 || wcscmp(FileSystem,L"FAT32")==0; return false; } #endif -bool FileExist(const wchar *Name) +bool FileExist(const std::wstring &Name) { #ifdef _WIN_ALL return GetFileAttr(Name)!=0xffffffff; #elif defined(ENABLE_ACCESS) - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); - return access(NameA,0)==0; + std::string NameA; + WideToChar(Name,NameA); + return access(NameA.c_str(),0)==0; #else FindData FD; return FindFile::FastFind(Name,&FD); @@ -202,7 +204,7 @@ bool FileExist(const wchar *Name) } -bool WildFileExist(const wchar *Name) +bool WildFileExist(const std::wstring &Name) { if (IsWildcard(Name)) { @@ -230,8 +232,9 @@ bool IsUnreadable(uint Attr) { #if defined(_UNIX) && defined(S_ISFIFO) && defined(S_ISSOCK) && defined(S_ISCHR) return S_ISFIFO(Attr) || S_ISSOCK(Attr) || S_ISCHR(Attr); -#endif +#else return false; +#endif } @@ -261,66 +264,63 @@ bool IsDeleteAllowed(uint FileAttr) } -void PrepareToDelete(const wchar *Name) +void PrepareToDelete(const std::wstring &Name) { -#if defined(_WIN_ALL) || defined(_EMX) +#ifdef _WIN_ALL SetFileAttr(Name,0); #endif #ifdef _UNIX - if (Name!=NULL) - { - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); - chmod(NameA,S_IRUSR|S_IWUSR|S_IXUSR); - } + std::string NameA; + WideToChar(Name,NameA); + chmod(NameA.c_str(),S_IRUSR|S_IWUSR|S_IXUSR); #endif } -uint GetFileAttr(const wchar *Name) +uint GetFileAttr(const std::wstring &Name) { #ifdef _WIN_ALL - DWORD Attr=GetFileAttributes(Name); + DWORD Attr=GetFileAttributes(Name.c_str()); if (Attr==0xffffffff) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - Attr=GetFileAttributes(LongName); + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + Attr=GetFileAttributes(LongName.c_str()); } return Attr; #else - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); + std::string NameA; + WideToChar(Name,NameA); struct stat st; - if (stat(NameA,&st)!=0) + if (stat(NameA.c_str(),&st)!=0) return 0; return st.st_mode; #endif } -bool SetFileAttr(const wchar *Name,uint Attr) +bool SetFileAttr(const std::wstring &Name,uint Attr) { #ifdef _WIN_ALL - bool Success=SetFileAttributes(Name,Attr)!=0; + bool Success=SetFileAttributes(Name.c_str(),Attr)!=0; if (!Success) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - Success=SetFileAttributes(LongName,Attr)!=0; + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + Success=SetFileAttributes(LongName.c_str(),Attr)!=0; } return Success; #elif defined(_UNIX) - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); - return chmod(NameA,(mode_t)Attr)==0; + std::string NameA; + WideToChar(Name,NameA); + return chmod(NameA.c_str(),(mode_t)Attr)==0; #else return false; #endif } -wchar *MkTemp(wchar *Name,size_t MaxSize) +wchar* MkTemp(wchar *Name,size_t MaxSize) { size_t Length=wcslen(Name); @@ -344,7 +344,11 @@ wchar *MkTemp(wchar *Name,size_t MaxSize) { uint Ext=Random%50000+Attempt; wchar RndText[50]; - swprintf(RndText,ASIZE(RndText),L"%u.%03u",PID,Ext); + // User asked to specify the single extension for all temporary files, + // so it can be added to server ransomware protection exceptions. + // He wrote, this protection blocks temporary files when adding + // a file to RAR archive with drag and drop. + swprintf(RndText,ASIZE(RndText),L"%u.%03u.rartemp",PID,Ext); if (Length+wcslen(RndText)>=MaxSize || Attempt==1000) return NULL; wcsncpyz(Name+Length,RndText,MaxSize-Length); @@ -355,6 +359,40 @@ wchar *MkTemp(wchar *Name,size_t MaxSize) } +bool MkTemp(std::wstring &Name) +{ + RarTime CurTime; + CurTime.SetCurrentTime(); + + // We cannot use CurTime.GetWin() as is, because its lowest bits can + // have low informational value, like being a zero or few fixed numbers. + uint Random=(uint)(CurTime.GetWin()/100000); + + // Using PID we guarantee that different RAR copies use different temp names + // even if started in exactly the same time. + uint PID=0; +#ifdef _WIN_ALL + PID=(uint)GetCurrentProcessId(); +#elif defined(_UNIX) + PID=(uint)getpid(); +#endif + + for (uint Attempt=0;;Attempt++) + { + uint Ext=Random%50000+Attempt; + if (Attempt==1000) + return false; + std::wstring NewName=Name + std::to_wstring(PID) + L"." + std::to_wstring(Ext) + L".rartemp"; + if (!FileExist(NewName)) + { + Name=NewName; + break; + } + } + return true; +} + + #if !defined(SFX_MODULE) void CalcFileSum(File *SrcFile,uint *CRC32,byte *Blake2,uint Threads,int64 Size,uint Flags) { @@ -370,8 +408,7 @@ void CalcFileSum(File *SrcFile,uint *CRC32,byte *Blake2,uint Threads,int64 Size, SrcFile->Seek(0,SEEK_SET); const size_t BufSize=0x100000; - Array Data(BufSize); - + std::vector Data(BufSize); DataHash HashCRC,HashBlake2; HashCRC.Init(HASH_CRC32,Threads); @@ -386,7 +423,7 @@ void CalcFileSum(File *SrcFile,uint *CRC32,byte *Blake2,uint Threads,int64 Size, SizeToRead=BufSize; // Then always attempt to read the entire buffer. else SizeToRead=(size_t)Min((int64)BufSize,Size); - int ReadSize=SrcFile->Read(&Data[0],SizeToRead); + int ReadSize=SrcFile->Read(Data.data(),SizeToRead); if (ReadSize==0) break; TotalRead+=ReadSize; @@ -410,9 +447,9 @@ void CalcFileSum(File *SrcFile,uint *CRC32,byte *Blake2,uint Threads,int64 Size, } if (CRC32!=NULL) - HashCRC.Update(&Data[0],ReadSize); + HashCRC.Update(Data.data(),ReadSize); if (Blake2!=NULL) - HashBlake2.Update(&Data[0],ReadSize); + HashBlake2.Update(Data.data(),ReadSize); if (Size!=INT64NDF) Size-=ReadSize; @@ -434,79 +471,78 @@ void CalcFileSum(File *SrcFile,uint *CRC32,byte *Blake2,uint Threads,int64 Size, #endif -bool RenameFile(const wchar *SrcName,const wchar *DestName) +bool RenameFile(const std::wstring &SrcName,const std::wstring &DestName) { #ifdef _WIN_ALL - bool Success=MoveFile(SrcName,DestName)!=0; + bool Success=MoveFile(SrcName.c_str(),DestName.c_str())!=0; if (!Success) { - wchar LongName1[NM],LongName2[NM]; - if (GetWinLongPath(SrcName,LongName1,ASIZE(LongName1)) && - GetWinLongPath(DestName,LongName2,ASIZE(LongName2))) - Success=MoveFile(LongName1,LongName2)!=0; + std::wstring LongName1,LongName2; + if (GetWinLongPath(SrcName,LongName1) && GetWinLongPath(DestName,LongName2)) + Success=MoveFile(LongName1.c_str(),LongName2.c_str())!=0; } return Success; #else - char SrcNameA[NM],DestNameA[NM]; - WideToChar(SrcName,SrcNameA,ASIZE(SrcNameA)); - WideToChar(DestName,DestNameA,ASIZE(DestNameA)); - bool Success=rename(SrcNameA,DestNameA)==0; + std::string SrcNameA,DestNameA; + WideToChar(SrcName,SrcNameA); + WideToChar(DestName,DestNameA); + bool Success=rename(SrcNameA.c_str(),DestNameA.c_str())==0; return Success; #endif } -bool DelFile(const wchar *Name) +bool DelFile(const std::wstring &Name) { #ifdef _WIN_ALL - bool Success=DeleteFile(Name)!=0; + bool Success=DeleteFile(Name.c_str())!=0; if (!Success) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - Success=DeleteFile(LongName)!=0; + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + Success=DeleteFile(LongName.c_str())!=0; } return Success; #else - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); - bool Success=remove(NameA)==0; + std::string NameA; + WideToChar(Name,NameA); + bool Success=remove(NameA.c_str())==0; return Success; #endif } -bool DelDir(const wchar *Name) +bool DelDir(const std::wstring &Name) { #ifdef _WIN_ALL - bool Success=RemoveDirectory(Name)!=0; + bool Success=RemoveDirectory(Name.c_str())!=0; if (!Success) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - Success=RemoveDirectory(LongName)!=0; + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + Success=RemoveDirectory(LongName.c_str())!=0; } return Success; #else - char NameA[NM]; - WideToChar(Name,NameA,ASIZE(NameA)); - bool Success=rmdir(NameA)==0; + std::string NameA; + WideToChar(Name,NameA); + bool Success=rmdir(NameA.c_str())==0; return Success; #endif } #if defined(_WIN_ALL) && !defined(SFX_MODULE) -bool SetFileCompression(const wchar *Name,bool State) +bool SetFileCompression(const std::wstring &Name,bool State) { - HANDLE hFile=CreateFile(Name,FILE_READ_DATA|FILE_WRITE_DATA, + HANDLE hFile=CreateFile(Name.c_str(),FILE_READ_DATA|FILE_WRITE_DATA, FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_SEQUENTIAL_SCAN,NULL); if (hFile==INVALID_HANDLE_VALUE) { - wchar LongName[NM]; - if (GetWinLongPath(Name,LongName,ASIZE(LongName))) - hFile=CreateFile(LongName,FILE_READ_DATA|FILE_WRITE_DATA, + std::wstring LongName; + if (GetWinLongPath(Name,LongName)) + hFile=CreateFile(LongName.c_str(),FILE_READ_DATA|FILE_WRITE_DATA, FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_SEQUENTIAL_SCAN,NULL); } @@ -521,11 +557,11 @@ bool SetFileCompression(const wchar *Name,bool State) } -void ResetFileCache(const wchar *Name) +void ResetFileCache(const std::wstring &Name) { // To reset file cache in Windows it is enough to open it with // FILE_FLAG_NO_BUFFERING and then close it. - HANDLE hSrc=CreateFile(Name,GENERIC_READ, + HANDLE hSrc=CreateFile(Name.c_str(),GENERIC_READ, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL,OPEN_EXISTING,FILE_FLAG_NO_BUFFERING,NULL); if (hSrc!=INVALID_HANDLE_VALUE) @@ -542,3 +578,69 @@ void ResetFileCache(const wchar *Name) + + +// Delete symbolic links in file path, if any, and replace them by directories. +// Prevents extracting files outside of destination folder with symlink chains. +bool LinksToDirs(const std::wstring &SrcName,const std::wstring &SkipPart,std::wstring &LastChecked) +{ + // Unlike Unix, Windows doesn't expand lnk1 in symlink targets like + // "lnk1/../dir", but converts the path to "dir". In Unix we need to call + // this function to prevent placing unpacked files outside of destination + // folder if previously we unpacked "dir/lnk1" -> "..", + // "dir/lnk2" -> "lnk1/.." and "dir/lnk2/anypath/poc.txt". + // We may still need this function to prevent abusing symlink chains + // in link source path if we remove detection of such chains + // in IsRelativeSymlinkSafe. This function seems to make other symlink + // related safety checks redundant, but for now we prefer to keep them too. + // + // 2022.12.01: the performance impact is minimized after adding the check + // against the previous path and enabling this verification only after + // extracting a symlink with ".." in target. So we enabled it for Windows + // as well for extra safety. +//#ifdef _UNIX + std::wstring Path=SrcName; + + size_t SkipLength=SkipPart.size(); + + if (SkipLength>0 && Path.rfind(SkipPart,0)!=0) + SkipLength=0; // Parameter validation, not really needed now. + + // Do not check parts already checked in previous path to improve performance. + for (size_t I=0;ISkipLength) + SkipLength=I; + + // Avoid converting symlinks in destination path part specified by user. + while (SkipLength0) + for (size_t I=Path.size()-1;I>SkipLength;I--) + if (IsPathDiv(Path[I])) + { + Path.erase(I); + FindData FD; + if (FindFile::FastFind(Path,&FD,true) && FD.IsLink) + { +#ifdef _WIN_ALL + // Normally Windows symlinks to directory look like a directory + // and are deleted with DelDir(). It is possible to create + // a file-like symlink pointing at directory, which can be deleted + // only with && DelFile, but such symlink isn't really functional. + // Here we prefer to fail deleting such symlink and skip extracting + // a file. + if (!DelDir(Path)) +#else + if (!DelFile(Path)) +#endif + { + ErrHandler.CreateErrorMsg(SrcName); // Extraction command will skip this file or directory. + return false; // Couldn't delete the symlink to replace it with directory. + } + } + } + LastChecked=SrcName; +//#endif + return true; +} diff --git a/multiarc/src/formats/rar/unrar/filefn.hpp b/multiarc/src/formats/rar/unrar/filefn.hpp index 53d86653f..4e0619739 100644 --- a/multiarc/src/formats/rar/unrar/filefn.hpp +++ b/multiarc/src/formats/rar/unrar/filefn.hpp @@ -3,47 +3,52 @@ enum MKDIR_CODE {MKDIR_SUCCESS,MKDIR_ERROR,MKDIR_BADPATH}; -MKDIR_CODE MakeDir(const wchar *Name,bool SetAttr,uint Attr); -bool CreatePath(const wchar *Path,bool SkipLastName,bool Silent); -void SetDirTime(const wchar *Name,RarTime *ftm,RarTime *ftc,RarTime *fta); -bool IsRemovable(const wchar *Name); +MKDIR_CODE MakeDir(const std::wstring &Name,bool SetAttr,uint Attr); +bool CreateDir(const std::wstring &Name); +bool CreatePath(const std::wstring &Path,bool SkipLastName,bool Silent); +void SetDirTime(const std::wstring &Name,RarTime *ftm,RarTime *ftc,RarTime *fta); +bool IsRemovable(const std::wstring &Name); #ifndef SFX_MODULE -int64 GetFreeDisk(const wchar *Name); +int64 GetFreeDisk(const std::wstring &Name); #endif #if defined(_WIN_ALL) && !defined(SFX_MODULE) && !defined(SILENT) -bool IsFAT(const wchar *Root); +bool IsFAT(const std::wstring &Root); #endif -bool FileExist(const wchar *Name); -bool WildFileExist(const wchar *Name); +bool FileExist(const std::wstring &Name); +bool WildFileExist(const std::wstring &Name); bool IsDir(uint Attr); bool IsUnreadable(uint Attr); bool IsLink(uint Attr); -void SetSFXMode(const wchar *FileName); -void EraseDiskContents(const wchar *FileName); +void SetSFXMode(const std::wstring &FileName); +void EraseDiskContents(const std::wstring &FileName); bool IsDeleteAllowed(uint FileAttr); -void PrepareToDelete(const wchar *Name); -uint GetFileAttr(const wchar *Name); -bool SetFileAttr(const wchar *Name,uint Attr); +void PrepareToDelete(const std::wstring &Name); +uint GetFileAttr(const std::wstring &Name); +bool SetFileAttr(const std::wstring &Name,uint Attr); wchar* MkTemp(wchar *Name,size_t MaxSize); +bool MkTemp(std::wstring &Name); enum CALCFSUM_FLAGS {CALCFSUM_SHOWTEXT=1,CALCFSUM_SHOWPERCENT=2,CALCFSUM_SHOWPROGRESS=4,CALCFSUM_CURPOS=8}; void CalcFileSum(File *SrcFile,uint *CRC32,byte *Blake2,uint Threads,int64 Size=INT64NDF,uint Flags=0); -bool RenameFile(const wchar *SrcName,const wchar *DestName); -bool DelFile(const wchar *Name); -bool DelDir(const wchar *Name); +bool RenameFile(const std::wstring &SrcName,const std::wstring &DestName); +bool DelFile(const std::wstring &Name); +bool DelDir(const std::wstring &Name); #if defined(_WIN_ALL) && !defined(SFX_MODULE) -bool SetFileCompression(const wchar *Name,bool State); -void ResetFileCache(const wchar *Name); +bool SetFileCompression(const std::wstring &Name,bool State); +void ResetFileCache(const std::wstring &Name); #endif +// Keep it here and not in extinfo.cpp, because it is invoked from Zip.SFX too. +bool LinksToDirs(const std::wstring &SrcName,const std::wstring &SkipPart,std::wstring &LastChecked); + #endif diff --git a/multiarc/src/formats/rar/unrar/filestr.cpp b/multiarc/src/formats/rar/unrar/filestr.cpp index a5d29d74b..ec45654d2 100644 --- a/multiarc/src/formats/rar/unrar/filestr.cpp +++ b/multiarc/src/formats/rar/unrar/filestr.cpp @@ -1,7 +1,7 @@ #include "rar.hpp" bool ReadTextFile( - const wchar *Name, + const std::wstring &Name, StringList *List, bool Config, bool AbortOnError, @@ -10,17 +10,15 @@ bool ReadTextFile( bool SkipComments, bool ExpandEnvStr) { - wchar FileName[NM]; - *FileName=0; + std::wstring FileName; - if (Name!=NULL) - if (Config) - GetConfigName(Name,FileName,ASIZE(FileName),true,false); - else - wcsncpyz(FileName,Name,ASIZE(FileName)); + if (Config) + GetConfigName(Name,FileName,true,false); + else + FileName=Name; File SrcFile; - if (*FileName!=0) + if (!FileName.empty()) { bool OpenCode=AbortOnError ? SrcFile.WOpen(FileName):SrcFile.Open(FileName,0); @@ -34,36 +32,36 @@ bool ReadTextFile( else SrcFile.SetHandleType(FILE_HANDLESTD); - uint DataSize=0,ReadSize; + size_t DataSize=0,ReadSize; const int ReadBlock=4096; - Array Data(ReadBlock); + std::vector Data(ReadBlock); while ((ReadSize=SrcFile.Read(&Data[DataSize],ReadBlock))!=0) { DataSize+=ReadSize; - Data.Add(ReadSize); // Always have ReadBlock available for next data. + Data.resize(DataSize+ReadBlock); // Always have ReadBlock available for next data. } // Set to really read size, so we can zero terminate it correctly. - Data.Alloc(DataSize); + Data.resize(DataSize); int LittleEndian=DataSize>=2 && Data[0]==255 && Data[1]==254 ? 1:0; int BigEndian=DataSize>=2 && Data[0]==254 && Data[1]==255 ? 1:0; bool Utf8=DataSize>=3 && Data[0]==0xef && Data[1]==0xbb && Data[2]==0xbf; if (SrcCharset==RCH_DEFAULT) - SrcCharset=DetectTextEncoding(&Data[0],DataSize); + SrcCharset=DetectTextEncoding(Data.data(),DataSize); - Array DataW; + std::vector DataW(ReadBlock); if (SrcCharset==RCH_DEFAULT || SrcCharset==RCH_OEM || SrcCharset==RCH_ANSI) { - Data.Push(0); // Zero terminate. + Data.push_back(0); // Zero terminate. #if defined(_WIN_ALL) if (SrcCharset==RCH_OEM) - OemToCharA((char *)&Data[0],(char *)&Data[0]); + OemToCharA((char *)Data.data(),(char *)Data.data()); #endif - DataW.Alloc(Data.Size()); - CharToWide((char *)&Data[0],&DataW[0],DataW.Size()); + DataW.resize(Data.size()); + CharToWide((char *)Data.data(),DataW.data(),DataW.size()); } if (SrcCharset==RCH_UNICODE) @@ -75,8 +73,8 @@ bool ReadTextFile( LittleEndian=1; } - DataW.Alloc(Data.Size()/2+1); - size_t End=Data.Size() & ~1; // We need even bytes number for UTF-16. + DataW.resize(Data.size()/2+1); + size_t End=Data.size() & ~1; // We need even bytes number for UTF-16. for (size_t I=Start;IAddString(ExpName); + Expanded=true; } #endif if (!Expanded && *CurStr!=0) diff --git a/multiarc/src/formats/rar/unrar/filestr.hpp b/multiarc/src/formats/rar/unrar/filestr.hpp index febd0a292..3c2813049 100644 --- a/multiarc/src/formats/rar/unrar/filestr.hpp +++ b/multiarc/src/formats/rar/unrar/filestr.hpp @@ -2,7 +2,7 @@ #define _RAR_FILESTR_ bool ReadTextFile( - const wchar *Name, + const std::wstring &Name, StringList *List, bool Config, bool AbortOnError=false, diff --git a/multiarc/src/formats/rar/unrar/find.cpp b/multiarc/src/formats/rar/unrar/find.cpp index 3de019216..fb210b2e3 100644 --- a/multiarc/src/formats/rar/unrar/find.cpp +++ b/multiarc/src/formats/rar/unrar/find.cpp @@ -2,7 +2,6 @@ FindFile::FindFile() { - *FindMask=0; FirstCall=true; #ifdef _WIN_ALL hFind=INVALID_HANDLE_VALUE; @@ -24,9 +23,9 @@ FindFile::~FindFile() } -void FindFile::SetMask(const wchar *Mask) +void FindFile::SetMask(const std::wstring &Mask) { - wcsncpyz(FindMask,Mask,ASIZE(FindMask)); + FindMask=Mask; FirstCall=true; } @@ -34,7 +33,7 @@ void FindFile::SetMask(const wchar *Mask) bool FindFile::Next(FindData *fd,bool GetSymLink) { fd->Error=false; - if (*FindMask==0) + if (FindMask.empty()) return false; #ifdef _WIN_ALL if (FirstCall) @@ -48,14 +47,14 @@ bool FindFile::Next(FindData *fd,bool GetSymLink) #else if (FirstCall) { - wchar DirName[NM]; - wcsncpyz(DirName,FindMask,ASIZE(DirName)); + std::wstring DirName; + DirName=FindMask; RemoveNameFromPath(DirName); - if (*DirName==0) - wcsncpyz(DirName,L".",ASIZE(DirName)); - char DirNameA[NM]; - WideToChar(DirName,DirNameA,ASIZE(DirNameA)); - if ((dirp=opendir(DirNameA))==NULL) + if (DirName.empty()) + DirName=L"."; + std::string DirNameA; + WideToChar(DirName,DirNameA); + if ((dirp=opendir(DirNameA.c_str()))==NULL) { fd->Error=(errno!=ENOENT); return false; @@ -63,32 +62,31 @@ bool FindFile::Next(FindData *fd,bool GetSymLink) } while (1) { - wchar Name[NM]; + std::wstring Name; struct dirent *ent=readdir(dirp); if (ent==NULL) return false; if (strcmp(ent->d_name,".")==0 || strcmp(ent->d_name,"..")==0) continue; - if (!CharToWide(ent->d_name,Name,ASIZE(Name))) - uiMsg(UIERROR_INVALIDNAME,UINULL,Name); + if (!CharToWide(std::string(ent->d_name),Name)) + uiMsg(UIERROR_INVALIDNAME,L"",Name); if (CmpName(FindMask,Name,MATCH_NAMES)) { - wchar FullName[NM]; - wcsncpyz(FullName,FindMask,ASIZE(FullName)); - *PointToName(FullName)=0; - if (wcslen(FullName)+wcslen(Name)>=ASIZE(FullName)-1) + std::wstring FullName=FindMask; + FullName.erase(GetNamePos(FullName)); + if (FullName.size()+Name.size()>=MAXPATHSIZE) { uiMsg(UIERROR_PATHTOOLONG,FullName,L"",Name); return false; } - wcsncatz(FullName,Name,ASIZE(FullName)); + FullName+=Name; if (!FastFind(FullName,fd,GetSymLink)) { ErrHandler.OpenErrorMsg(FullName); continue; } - wcsncpyz(fd->Name,FullName,ASIZE(fd->Name)); + fd->Name=FullName; break; } } @@ -98,14 +96,14 @@ bool FindFile::Next(FindData *fd,bool GetSymLink) fd->IsLink=IsLink(fd->FileAttr); FirstCall=false; - wchar *NameOnly=PointToName(fd->Name); - if (wcscmp(NameOnly,L".")==0 || wcscmp(NameOnly,L"..")==0) + std::wstring NameOnly=PointToName(fd->Name); + if (NameOnly==L"." || NameOnly==L"..") return Next(fd); return true; } -bool FindFile::FastFind(const wchar *FindMask,FindData *fd,bool GetSymLink) +bool FindFile::FastFind(const std::wstring &FindMask,FindData *fd,bool GetSymLink) { fd->Error=false; #ifndef _UNIX @@ -118,16 +116,16 @@ bool FindFile::FastFind(const wchar *FindMask,FindData *fd,bool GetSymLink) return false; FindClose(hFind); #elif defined(_UNIX) - char FindMaskA[NM]; - WideToChar(FindMask,FindMaskA,ASIZE(FindMaskA)); + std::string FindMaskA; + WideToChar(FindMask,FindMaskA); struct stat st; if (GetSymLink) { #ifdef SAVE_LINKS - if (lstat(FindMaskA,&st)!=0) + if (lstat(FindMaskA.c_str(),&st)!=0) #else - if (stat(FindMaskA,&st)!=0) + if (stat(FindMaskA.c_str(),&st)!=0) #endif { fd->Error=(errno!=ENOENT); @@ -135,7 +133,7 @@ bool FindFile::FastFind(const wchar *FindMask,FindData *fd,bool GetSymLink) } } else - if (stat(FindMaskA,&st)!=0) + if (stat(FindMaskA.c_str(),&st)!=0) { fd->Error=(errno!=ENOENT); return false; @@ -145,7 +143,7 @@ bool FindFile::FastFind(const wchar *FindMask,FindData *fd,bool GetSymLink) File::StatToRarTime(st,&fd->mtime,&fd->ctime,&fd->atime); - wcsncpyz(fd->Name,FindMask,ASIZE(fd->Name)); + fd->Name=FindMask; #endif fd->Flags=0; fd->IsDir=IsDir(fd->FileAttr); @@ -156,17 +154,17 @@ bool FindFile::FastFind(const wchar *FindMask,FindData *fd,bool GetSymLink) #ifdef _WIN_ALL -HANDLE FindFile::Win32Find(HANDLE hFind,const wchar *Mask,FindData *fd) +HANDLE FindFile::Win32Find(HANDLE hFind,const std::wstring &Mask,FindData *fd) { WIN32_FIND_DATA FindData; if (hFind==INVALID_HANDLE_VALUE) { - hFind=FindFirstFile(Mask,&FindData); + hFind=FindFirstFile(Mask.c_str(),&FindData); if (hFind==INVALID_HANDLE_VALUE) { - wchar LongMask[NM]; - if (GetWinLongPath(Mask,LongMask,ASIZE(LongMask))) - hFind=FindFirstFile(LongMask,&FindData); + std::wstring LongMask; + if (GetWinLongPath(Mask,LongMask)) + hFind=FindFirstFile(LongMask.c_str(),&FindData); } if (hFind==INVALID_HANDLE_VALUE) { @@ -190,8 +188,8 @@ HANDLE FindFile::Win32Find(HANDLE hFind,const wchar *Mask,FindData *fd) if (hFind!=INVALID_HANDLE_VALUE) { - wcsncpyz(fd->Name,Mask,ASIZE(fd->Name)); - SetName(fd->Name,FindData.cFileName,ASIZE(fd->Name)); + fd->Name=Mask; + SetName(fd->Name,FindData.cFileName); fd->Size=FindData.nFileSize; fd->FileAttr=FindData.dwFileAttributes; fd->ftCreationTime=FindData.ftCreationTime; diff --git a/multiarc/src/formats/rar/unrar/find.hpp b/multiarc/src/formats/rar/unrar/find.hpp index 250637f8a..6812def7e 100644 --- a/multiarc/src/formats/rar/unrar/find.hpp +++ b/multiarc/src/formats/rar/unrar/find.hpp @@ -7,7 +7,7 @@ enum FINDDATA_FLAGS { struct FindData { - wchar Name[NM]; + std::wstring Name; uint64 Size; uint FileAttr; bool IsDir; @@ -28,10 +28,10 @@ class FindFile { private: #ifdef _WIN_ALL - static HANDLE Win32Find(HANDLE hFind,const wchar *Mask,FindData *fd); + static HANDLE Win32Find(HANDLE hFind,const std::wstring &Mask,FindData *fd); #endif - wchar FindMask[NM]; + std::wstring FindMask; bool FirstCall; #ifdef _WIN_ALL HANDLE hFind; @@ -41,9 +41,9 @@ class FindFile public: FindFile(); ~FindFile(); - void SetMask(const wchar *Mask); + void SetMask(const std::wstring &Mask); bool Next(FindData *fd,bool GetSymLink=false); - static bool FastFind(const wchar *FindMask,FindData *fd,bool GetSymLink=false); + static bool FastFind(const std::wstring &FindMask,FindData *fd,bool GetSymLink=false); }; #endif diff --git a/multiarc/src/formats/rar/unrar/getbits.cpp b/multiarc/src/formats/rar/unrar/getbits.cpp index 8805f2787..a5c8c3af4 100644 --- a/multiarc/src/formats/rar/unrar/getbits.cpp +++ b/multiarc/src/formats/rar/unrar/getbits.cpp @@ -5,11 +5,11 @@ BitInput::BitInput(bool AllocBuffer) ExternalBuffer=false; if (AllocBuffer) { - // getbits*() attempt to read data from InAddr, ... InAddr+3 positions. - // So let's allocate 3 additional bytes for situation, when we need to + // getbits*() attempt to read data from InAddr, ... InAddr+8 positions. + // So let's allocate 8 additional bytes for situation, when we need to // read only 1 byte from the last position of buffer and avoid a crash - // from access to next 3 bytes, which contents we do not need. - size_t BufSize=MAX_SIZE+3; + // from access to next 8 bytes, which contents we do not need. + size_t BufSize=MAX_SIZE+8; InBuf=new byte[BufSize]; // Ensure that we get predictable results when accessing bytes in area @@ -17,7 +17,7 @@ BitInput::BitInput(bool AllocBuffer) memset(InBuf,0,BufSize); } else - InBuf=NULL; + InBuf=nullptr; } @@ -30,14 +30,14 @@ BitInput::~BitInput() void BitInput::faddbits(uint Bits) { - // Function wrapped version of inline addbits to save code size. + // Function wrapped version of inline addbits to reduce the code size. addbits(Bits); } uint BitInput::fgetbits() { - // Function wrapped version of inline getbits to save code size. + // Function wrapped version of inline getbits to reduce the code size. return getbits(); } diff --git a/multiarc/src/formats/rar/unrar/getbits.hpp b/multiarc/src/formats/rar/unrar/getbits.hpp index 00acbea94..65fb25a14 100644 --- a/multiarc/src/formats/rar/unrar/getbits.hpp +++ b/multiarc/src/formats/rar/unrar/getbits.hpp @@ -34,8 +34,7 @@ class BitInput uint getbits() { #if defined(LITTLE_ENDIAN) && defined(ALLOW_MISALIGNED) - uint32 BitField=*(uint32*)(InBuf+InAddr); - BitField=ByteSwap32(BitField); + uint32 BitField=RawGetBE4(InBuf+InAddr); BitField >>= (16-InBit); #else uint BitField=(uint)InBuf[InAddr] << 16; @@ -51,19 +50,21 @@ class BitInput // Bit at (InAddr,InBit) has the highest position in returning data. uint getbits32() { -#if defined(LITTLE_ENDIAN) && defined(ALLOW_MISALIGNED) - uint32 BitField=*(uint32*)(InBuf+InAddr); - BitField=ByteSwap32(BitField); -#else - uint BitField=(uint)InBuf[InAddr] << 24; - BitField|=(uint)InBuf[InAddr+1] << 16; - BitField|=(uint)InBuf[InAddr+2] << 8; - BitField|=(uint)InBuf[InAddr+3]; -#endif + uint BitField=RawGetBE4(InBuf+InAddr); BitField <<= InBit; BitField|=(uint)InBuf[InAddr+4] >> (8-InBit); return BitField & 0xffffffff; } + + // Return 64 bits from current position in the buffer. + // Bit at (InAddr,InBit) has the highest position in returning data. + uint64 getbits64() + { + uint64 BitField=RawGetBE8(InBuf+InAddr); + BitField <<= InBit; + BitField|=(uint)InBuf[InAddr+8] >> (8-InBit); + return BitField; + } void faddbits(uint Bits); uint fgetbits(); diff --git a/multiarc/src/formats/rar/unrar/hardlinks.cpp b/multiarc/src/formats/rar/unrar/hardlinks.cpp index 171b5fa08..5080da056 100644 --- a/multiarc/src/formats/rar/unrar/hardlinks.cpp +++ b/multiarc/src/formats/rar/unrar/hardlinks.cpp @@ -1,4 +1,4 @@ -bool ExtractHardlink(CommandData *Cmd,wchar *NameNew,wchar *NameExisting,size_t NameExistingSize) +bool ExtractHardlink(CommandData *Cmd,const std::wstring &NameNew,const std::wstring &NameExisting) { if (!FileExist(NameExisting)) { @@ -10,7 +10,7 @@ bool ExtractHardlink(CommandData *Cmd,wchar *NameNew,wchar *NameExisting,size_t CreatePath(NameNew,true,Cmd->DisableNames); #ifdef _WIN_ALL - bool Success=CreateHardLink(NameNew,NameExisting,NULL)!=0; + bool Success=CreateHardLink(NameNew.c_str(),NameExisting.c_str(),NULL)!=0; if (!Success) { uiMsg(UIERROR_HLINKCREATE,NameNew); @@ -19,10 +19,10 @@ bool ExtractHardlink(CommandData *Cmd,wchar *NameNew,wchar *NameExisting,size_t } return Success; #elif defined(_UNIX) - char NameExistingA[NM],NameNewA[NM]; - WideToChar(NameExisting,NameExistingA,ASIZE(NameExistingA)); - WideToChar(NameNew,NameNewA,ASIZE(NameNewA)); - bool Success=link(NameExistingA,NameNewA)==0; + std::string NameExistingA,NameNewA; + WideToChar(NameExisting,NameExistingA); + WideToChar(NameNew,NameNewA); + bool Success=link(NameExistingA.c_str(),NameNewA.c_str())==0; if (!Success) { uiMsg(UIERROR_HLINKCREATE,NameNew); diff --git a/multiarc/src/formats/rar/unrar/hash.cpp b/multiarc/src/formats/rar/unrar/hash.cpp index 106cc6083..cde3db913 100644 --- a/multiarc/src/formats/rar/unrar/hash.cpp +++ b/multiarc/src/formats/rar/unrar/hash.cpp @@ -76,7 +76,7 @@ void DataHash::Init(HASH_TYPE Type,uint MaxThreads) if (Type==HASH_BLAKE2) blake2sp_init(blake2ctx); #ifdef RAR_SMP - DataHash::MaxThreads=Min(MaxThreads,MaxHashThreads); + DataHash::MaxThreads=Min(MaxThreads,HASH_POOL_THREADS); #endif } @@ -88,13 +88,19 @@ void DataHash::Update(const void *Data,size_t DataSize) CurCRC32=Checksum14((ushort)CurCRC32,Data,DataSize); #endif if (HashType==HASH_CRC32) + { +#ifdef RAR_SMP + UpdateCRC32MT(Data,DataSize); +#else CurCRC32=CRC32(CurCRC32,Data,DataSize); +#endif + } if (HashType==HASH_BLAKE2) { #ifdef RAR_SMP - if (MaxThreads>1 && ThPool==NULL) - ThPool=new ThreadPool(BLAKE2_THREADS_NUMBER); + if (MaxThreads>1 && ThPool==nullptr) + ThPool=new ThreadPool(HASH_POOL_THREADS); blake2ctx->ThPool=ThPool; blake2ctx->MaxThreads=MaxThreads; #endif @@ -103,6 +109,146 @@ void DataHash::Update(const void *Data,size_t DataSize) } +#ifdef RAR_SMP +THREAD_PROC(BuildCRC32Thread) +{ + DataHash::CRC32ThreadData *td=(DataHash::CRC32ThreadData *)Data; + + // Use 0 initial value to simplify combining the result with existing CRC32. + // It doesn't affect the first initial 0xffffffff in the data beginning. + // If we used 0xffffffff here, we would need to shift 0xffffffff left to + // block width and XOR it with block CRC32 to reset its initial value to 0. + td->DataCRC=CRC32(0,td->Data,td->DataSize); +} + + +// CRC is linear and distributive over addition, so CRC(a+b)=CRC(a)+CRC(b). +// Since addition in finite field is XOR, we have CRC(a^b)=CRC(a)^CRC(b). +// So CRC(aaabbb) = CRC(aaa000) ^ CRC(000bbb) = CRC(aaa000) ^ CRC(bbb), +// because CRC ignores leading zeroes. Thus to split CRC calculations +// to "aaa" and "bbb" blocks and then to threads we need to be able to +// find CRC(aaa000) knowing "aaa" quickly. We use Galois finite field to +// calculate the power of 2 to get "1000" and multiply it by "aaa". +void DataHash::UpdateCRC32MT(const void *Data,size_t DataSize) +{ + const size_t MinBlock=0x4000; + if (DataSize<2*MinBlock || MaxThreads<2) + { + CurCRC32=CRC32(CurCRC32,Data,DataSize); + return; + } + + if (ThPool==nullptr) + ThPool=new ThreadPool(HASH_POOL_THREADS); + + size_t Threads=MaxThreads; + size_t BlockSize=DataSize/Threads; + if (BlockSizeAddTask(BuildCRC32Thread,(void*)&td[I]); +#else + BuildCRC32Thread((void*)&td[I]); +#endif + } + +#ifdef USE_THREADS + ThPool->WaitDone(); +#endif // USE_THREADS + + uint StdShift=gfExpCRC(uint(8*td[0].DataSize)); + for (size_t I=0;I>=1) + Reversed|=(N & 1)<<(31-I); + return Reversed; +} + + +// Galois field multiplication modulo POLY. +uint DataHash::gfMulCRC(uint A, uint B) +{ + // For reversed 0xEDB88320 polynomial we bit reverse CRC32 before passing + // to this function, so we must use the normal polynomial here. + // We set the highest polynomial bit 33 for proper multiplication + // in case uint is larger than 32-bit. + const uint POLY=uint(0x104c11db7); + + uint R = 0 ; // Multiplication result. + while (A != 0 && B != 0) // If any of multipliers becomes 0, quit early. + { + // For non-zero lowest B bit, add A to result. + R ^= (B & 1)!=0 ? A : 0; + + // Make A twice larger before the next iteration. + // Subtract POLY to keep it modulo POLY if high bit is set. + A = (A << 1) ^ ((A & 0x80000000)!=0 ? POLY : 0); + + B >>= 1; // Move next B bit to lowest position. + } + return R; +} + + +// Calculate 2 power N with square-and-multiply algorithm. +uint DataHash::gfExpCRC(uint N) +{ + uint S = 2; // Starts from base value and contains the current square. + uint R = 1; // Exponentiation result. + while (N > 1) + { + if ((N & 1)!=0) // If N is odd. + R = gfMulCRC(R, S); + S = gfMulCRC(S, S); // Next square. + N >>= 1; + } + // We could change the loop condition to N > 0 and return R at expense + // of one additional gfMulCRC(S, S). + return gfMulCRC(R, S); +} + + void DataHash::Result(HashValue *Result) { Result->Type=HashType; diff --git a/multiarc/src/formats/rar/unrar/hash.hpp b/multiarc/src/formats/rar/unrar/hash.hpp index 6315680e7..0189113e9 100644 --- a/multiarc/src/formats/rar/unrar/hash.hpp +++ b/multiarc/src/formats/rar/unrar/hash.hpp @@ -32,7 +32,24 @@ class DataHash; class DataHash { + public: + struct CRC32ThreadData + { + void *Data; + size_t DataSize; + uint DataCRC; + }; private: + void UpdateCRC32MT(const void *Data,size_t DataSize); + uint BitReverse32(uint N); + uint gfMulCRC(uint A, uint B); + uint gfExpCRC(uint N); + + // Speed gain seems to vanish above 8 CRC32 threads. + static const uint CRC32_POOL_THREADS=8; + // Thread pool must allow at least BLAKE2_THREADS_NUMBER threads. + static const uint HASH_POOL_THREADS=Max(BLAKE2_THREADS_NUMBER,CRC32_POOL_THREADS); + HASH_TYPE HashType; uint CurCRC32; blake2sp_state *blake2ctx; @@ -41,8 +58,6 @@ class DataHash ThreadPool *ThPool; uint MaxThreads; - // Upper limit for maximum threads to prevent wasting threads in pool. - static const uint MaxHashThreads=8; #endif public: DataHash(); diff --git a/multiarc/src/formats/rar/unrar/headers.cpp b/multiarc/src/formats/rar/unrar/headers.cpp index b441376c5..b2d00d3b9 100644 --- a/multiarc/src/formats/rar/unrar/headers.cpp +++ b/multiarc/src/formats/rar/unrar/headers.cpp @@ -2,7 +2,7 @@ void FileHeader::Reset(size_t SubDataSize) { - SubData.Alloc(SubDataSize); + SubData.resize(SubDataSize); BaseBlock::Reset(); FileHash.Init(HASH_NONE); mtime.Reset(); @@ -37,6 +37,7 @@ void FileHeader::Reset(size_t SubDataSize) } +/* FileHeader& FileHeader::operator = (FileHeader &hd) { SubData.Reset(); @@ -45,6 +46,7 @@ FileHeader& FileHeader::operator = (FileHeader &hd) SubData=hd.SubData; return *this; } +*/ void MainHeader::Reset() diff --git a/multiarc/src/formats/rar/unrar/headers.hpp b/multiarc/src/formats/rar/unrar/headers.hpp index 0941220f5..8697e3887 100644 --- a/multiarc/src/formats/rar/unrar/headers.hpp +++ b/multiarc/src/formats/rar/unrar/headers.hpp @@ -11,13 +11,14 @@ #define SIZEOF_SUBBLOCKHEAD 14 #define SIZEOF_COMMHEAD 13 #define SIZEOF_PROTECTHEAD 26 -#define SIZEOF_UOHEAD 18 #define SIZEOF_STREAMHEAD 26 #define VER_PACK 29U #define VER_PACK5 50U // It is stored as 0, but we subtract 50 when saving an archive. +#define VER_PACK7 70U // It is stored as 1, but we subtract 70 when saving an archive. #define VER_UNPACK 29U #define VER_UNPACK5 50U // It is stored as 0, but we add 50 when reading an archive. +#define VER_UNPACK7 70U // It is stored as 1, but we add 50 when reading an archive. #define VER_UNKNOWN 9999U // Just some large value. #define MHD_VOLUME 0x0001U @@ -83,7 +84,7 @@ enum HEADER_TYPE { }; -// RAR 2.9 and earlier. +// RAR 2.9 and earlier service haeders, mostly outdated and not supported. enum { EA_HEAD=0x100,UO_HEAD=0x101,MAC_HEAD=0x102,BEEA_HEAD=0x103, NTACL_HEAD=0x104,STREAM_HEAD=0x105 }; @@ -148,6 +149,14 @@ struct BaseBlock { SkipIfUnknown=false; } + + // We use it to assign this block data to inherited blocks. + // Such function seems to be cleaner than '(BaseBlock&)' cast or adding + // 'using BaseBlock::operator=;' to every inherited header. + void SetBaseBlock(BaseBlock &Src) + { + *this=Src; + } }; @@ -185,9 +194,9 @@ struct FileHeader:BlockHeader uint FileAttr; uint SubFlags; }; - wchar FileName[NM]; + std::wstring FileName; - Array SubData; + std::vector SubData; RarTime mtime; RarTime ctime; @@ -227,7 +236,7 @@ struct FileHeader:BlockHeader bool Dir; bool CommentInHeader; // RAR 2.0 file comment. bool Version; // name.ext;ver file name containing the version number. - size_t WinSize; + uint64 WinSize; bool Inherited; // New file inherits a subblock when updating a host file (for subblocks only). // 'true' if file sizes use 8 bytes instead of 4. Not used in RAR 5.0. @@ -240,7 +249,7 @@ struct FileHeader:BlockHeader HOST_SYSTEM_TYPE HSType; FILE_SYSTEM_REDIRECT RedirType; - wchar RedirName[NM]; + std::wstring RedirName; bool DirTarget; bool UnixOwnerSet,UnixOwnerNumeric,UnixGroupNumeric; @@ -257,10 +266,10 @@ struct FileHeader:BlockHeader bool CmpName(const wchar *Name) { - return(wcscmp(FileName,Name)==0); + return FileName==Name; } - FileHeader& operator = (FileHeader &hd); +// FileHeader& operator = (FileHeader &hd); }; @@ -325,16 +334,6 @@ struct ProtectHeader:BlockHeader }; -struct UnixOwnersHeader:SubBlockHeader -{ - ushort OwnerNameSize; - ushort GroupNameSize; -/* dummy */ - char OwnerName[256]; - char GroupName[256]; -}; - - struct EAHeader:SubBlockHeader { uint UnpSize; @@ -351,7 +350,7 @@ struct StreamHeader:SubBlockHeader byte Method; uint StreamCRC; ushort StreamNameSize; - char StreamName[260]; + std::string StreamName; }; diff --git a/multiarc/src/formats/rar/unrar/headers5.hpp b/multiarc/src/formats/rar/unrar/headers5.hpp index 50f5955d1..361e554df 100644 --- a/multiarc/src/formats/rar/unrar/headers5.hpp +++ b/multiarc/src/formats/rar/unrar/headers5.hpp @@ -42,20 +42,27 @@ // RAR 5.0 file compression flags. -#define FCI_ALGO_BIT0 0x0001 // Version of compression algorithm. -#define FCI_ALGO_BIT1 0x0002 // 0 .. 63. -#define FCI_ALGO_BIT2 0x0004 -#define FCI_ALGO_BIT3 0x0008 -#define FCI_ALGO_BIT4 0x0010 -#define FCI_ALGO_BIT5 0x0020 -#define FCI_SOLID 0x0040 // Solid flag. -#define FCI_METHOD_BIT0 0x0080 // Compression method. -#define FCI_METHOD_BIT1 0x0100 // 0 .. 5 (6 and 7 are not used). -#define FCI_METHOD_BIT2 0x0200 -#define FCI_DICT_BIT0 0x0400 // Dictionary size. -#define FCI_DICT_BIT1 0x0800 // 128 KB .. 4 GB. -#define FCI_DICT_BIT2 0x1000 -#define FCI_DICT_BIT3 0x2000 +#define FCI_ALGO_BIT0 0x00000001 // Version of compression algorithm. +#define FCI_ALGO_BIT1 0x00000002 // 0 .. 63. +#define FCI_ALGO_BIT2 0x00000004 +#define FCI_ALGO_BIT3 0x00000008 +#define FCI_ALGO_BIT4 0x00000010 +#define FCI_ALGO_BIT5 0x00000020 +#define FCI_SOLID 0x00000040 // Solid flag. +#define FCI_METHOD_BIT0 0x00000080 // Compression method. +#define FCI_METHOD_BIT1 0x00000100 // 0 .. 5 (6 and 7 are not used). +#define FCI_METHOD_BIT2 0x00000200 +#define FCI_DICT_BIT0 0x00000400 // Dictionary size. +#define FCI_DICT_BIT1 0x00000800 // 128 KB .. 1 TB. +#define FCI_DICT_BIT2 0x00001000 +#define FCI_DICT_BIT3 0x00002000 +#define FCI_DICT_BIT4 0x00004000 +#define FCI_DICT_FRACT0 0x00008000 // Dictionary fraction in 1/32 of size. +#define FCI_DICT_FRACT1 0x00010000 +#define FCI_DICT_FRACT2 0x00020000 +#define FCI_DICT_FRACT3 0x00040000 +#define FCI_DICT_FRACT4 0x00080000 +#define FCI_RAR5_COMPAT 0x00100000 // RAR7 compression flags and RAR5 compression algorithm. // Main header extra field values. #define MHEXTRA_LOCATOR 0x01 // Position of quick list and other blocks. diff --git a/multiarc/src/formats/rar/unrar/list.cpp b/multiarc/src/formats/rar/unrar/list.cpp index e4444e13a..3eae4de65 100644 --- a/multiarc/src/formats/rar/unrar/list.cpp +++ b/multiarc/src/formats/rar/unrar/list.cpp @@ -1,7 +1,6 @@ #include "rar.hpp" static void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bool Technical,bool Bare,bool DisableNames); -static void ListSymLink(Archive &Arc); static void ListFileAttr(uint A,HOST_SYSTEM_TYPE HostType,wchar *AttrStr,size_t AttrSize); static void ListOldSubHeader(Archive &Arc); static void ListNewSubHeader(CommandData *Cmd,Archive &Arc); @@ -15,8 +14,8 @@ void ListArchive(CommandData *Cmd) bool Bare=(Cmd->Command[1]=='B'); bool Verbose=(Cmd->Command[0]=='V'); - wchar ArcName[NM]; - while (Cmd->GetArcName(ArcName,ASIZE(ArcName))) + std::wstring ArcName; + while (Cmd->GetArcName(ArcName)) { if (Cmd->ManualPassword) Cmd->Password.Clean(); // Clean user entered password before processing next archive. @@ -35,33 +34,31 @@ void ListArchive(CommandData *Cmd) if (!Bare) { Arc.ViewComment(); - mprintf(L"\n%s: %s",St(MListArchive),Arc.FileName); + mprintf(L"\n%s: %s",St(MListArchive),Arc.FileName.c_str()); mprintf(L"\n%s: ",St(MListDetails)); - uint SetCount=0; const wchar *Fmt=Arc.Format==RARFMT14 ? L"RAR 1.4":(Arc.Format==RARFMT15 ? L"RAR 4":L"RAR 5"); - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", Fmt); + mprintf(L"%s", Fmt); if (Arc.Solid) - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", St(MListSolid)); + mprintf(L", %s", St(MListSolid)); if (Arc.SFXSize>0) - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", St(MListSFX)); + mprintf(L", %s", St(MListSFX)); if (Arc.Volume) if (Arc.Format==RARFMT50) { // RAR 5.0 archives store the volume number in main header, // so it is already available now. - if (SetCount++ > 0) - mprintf(L", "); + mprintf(L", "); mprintf(St(MVolumeNumber),Arc.VolNumber+1); } else - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", St(MListVolume)); + mprintf(L", %s", St(MListVolume)); if (Arc.Protected) - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", St(MListRR)); + mprintf(L", %s", St(MListRR)); if (Arc.Locked) - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", St(MListLock)); + mprintf(L", %s", St(MListLock)); if (Arc.Encrypted) - mprintf(L"%s%s", SetCount++ > 0 ? L", ":L"", St(MListEncHead)); + mprintf(L", %s", St(MListEncHead)); if (!Arc.MainHead.OrigName.empty()) mprintf(L"\n%s: %s",St(MOrigName),Arc.MainHead.OrigName.c_str()); @@ -100,7 +97,7 @@ void ListArchive(CommandData *Cmd) switch(HeaderType) { case HEAD_FILE: - FileMatched=Cmd->IsProcessFile(Arc.FileHead,NULL,MATCH_WILDSUBPATH,0,NULL,0)!=0; + FileMatched=Cmd->IsProcessFile(Arc.FileHead,NULL,MATCH_WILDSUBPATH,0,NULL)!=0; if (FileMatched) { ListFileHeader(Arc,Arc.FileHead,TitleShown,Verbose,Technical,Bare,Cmd->DisableNames); @@ -155,7 +152,7 @@ void ListArchive(CommandData *Cmd) ArcCount++; #ifndef NOVOLUME - if (Cmd->VolSize!=0 && (Arc.FileHead.SplitAfter || + if (Cmd->VolSize==VOLSIZE_AUTO && (Arc.FileHead.SplitAfter || Arc.GetHeaderType()==HEAD_ENDARC && Arc.EndArcHead.NextVolume) && MergeArchive(Arc,NULL,false,Cmd->Command[0])) Arc.Seek(0,SEEK_SET); @@ -166,7 +163,7 @@ void ListArchive(CommandData *Cmd) else { if (Cmd->ArcNames.ItemsCount()<2 && !Bare) - mprintf(St(MNotRAR),Arc.FileName); + mprintf(St(MNotRAR),Arc.FileName.c_str()); break; } } @@ -219,7 +216,7 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo if (DisableNames) return; - wchar *Name=hd.FileName; + const wchar *Name=hd.FileName.c_str(); RARFORMAT Format=Arc.Format; if (Bare) @@ -266,9 +263,8 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo if (!FileBlock && Arc.SubHead.CmpName(SUBHEAD_TYPE_STREAM)) { mprintf(L"\n%12ls: %ls",St(MListType),St(MListStream)); - wchar StreamName[NM]; - GetStreamNameNTFS(Arc,StreamName,ASIZE(StreamName)); - mprintf(L"\n%12ls: %ls",St(MListTarget),StreamName); + std::wstring StreamName=GetStreamNameNTFS(Arc); + mprintf(L"\n%12ls: %ls",St(MListTarget),StreamName.c_str()); } else { @@ -292,25 +288,27 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo if (hd.RedirType!=FSREDIR_NONE) if (Format==RARFMT15) { - char LinkTargetA[NM]; + std::string LinkTargetA; if (Arc.FileHead.Encrypted) { // Link data are encrypted. We would need to ask for password // and initialize decryption routine to display the link target. - strncpyz(LinkTargetA,"*<-?->",ASIZE(LinkTargetA)); + LinkTargetA="*<-?->"; } else { - int DataSize=(int)Min(hd.PackSize,ASIZE(LinkTargetA)-1); - Arc.Read(LinkTargetA,DataSize); - LinkTargetA[DataSize > 0 ? DataSize : 0] = 0; + size_t DataSize=(size_t)Min(hd.PackSize,MAXPATHSIZE); + std::vector Buf(DataSize+1); + Arc.Read(Buf.data(),DataSize); + Buf[DataSize] = 0; + LinkTargetA=Buf.data(); } - wchar LinkTarget[NM]; - CharToWide(LinkTargetA,LinkTarget,ASIZE(LinkTarget)); - mprintf(L"\n%12ls: %ls",St(MListTarget),LinkTarget); + std::wstring LinkTarget; + CharToWide(LinkTargetA,LinkTarget); + mprintf(L"\n%12ls: %ls",St(MListTarget),LinkTarget.c_str()); } else - mprintf(L"\n%12ls: %ls",St(MListTarget),hd.RedirName); + mprintf(L"\n%12ls: %ls",St(MListTarget),hd.RedirName.c_str()); } if (!hd.Dir) { @@ -341,11 +339,11 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo hd.FileHash.CRC32); if (hd.FileHash.Type==HASH_BLAKE2) { - wchar BlakeStr[BLAKE2_DIGEST_SIZE*2+1]; - BinToHex(hd.FileHash.Digest,BLAKE2_DIGEST_SIZE,NULL,BlakeStr,ASIZE(BlakeStr)); + std::wstring BlakeStr; + BinToHex(hd.FileHash.Digest,BLAKE2_DIGEST_SIZE,BlakeStr); mprintf(L"\n%12ls: %ls", hd.UseHashKey ? L"BLAKE2 MAC":hd.SplitAfter ? L"Pack-BLAKE2":L"BLAKE2", - BlakeStr); + BlakeStr.c_str()); } const wchar *HostOS=L""; @@ -362,11 +360,22 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo if (*HostOS!=0) mprintf(L"\n%12ls: %ls",St(MListHostOS),HostOS); - mprintf(L"\n%12ls: RAR %ls(v%d) -m%d -md=%d%s",St(MListCompInfo), + std::wstring WinSize; + if (!hd.Dir) + if (hd.WinSize%1073741824==0) + WinSize=L" -md=" + std::to_wstring(hd.WinSize/1073741824) + L"g"; + else + if (hd.WinSize%1048576==0) + WinSize=L" -md=" + std::to_wstring(hd.WinSize/1048576) + L"m"; + else + if (hd.WinSize>=1024) + WinSize=L" -md=" + std::to_wstring(hd.WinSize/1024) + L"k"; + else + WinSize=L" -md=?"; + + mprintf(L"\n%12ls: RAR %ls(v%d) -m%d%s",St(MListCompInfo), Format==RARFMT15 ? L"1.5":L"5.0", - hd.UnpVer==VER_UNKNOWN ? 0 : hd.UnpVer,hd.Method, - hd.WinSize>=0x100000 ? hd.WinSize/0x100000:hd.WinSize/0x400, - hd.WinSize>=0x100000 ? L"M":L"K"); + hd.UnpVer==VER_UNKNOWN ? 0 : hd.UnpVer,hd.Method,WinSize.c_str()); if (hd.Solid || hd.Encrypted) { @@ -379,7 +388,7 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo if (hd.Version) { - uint Version=ParseVersionFileName(Name,false); + uint Version=ParseVersionFileName(hd.FileName,false); if (Version!=0) mprintf(L"\n%12ls: %u",St(MListFileVer),Version); } @@ -388,13 +397,13 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo { mprintf(L"\n%12ls: ",L"Unix owner"); if (*hd.UnixOwnerName!=0) - mprintf(L"%ls",GetWide(hd.UnixOwnerName)); + mprintf(L"%ls",GetWide(hd.UnixOwnerName).c_str()); else if (hd.UnixOwnerNumeric) mprintf(L"#%d",hd.UnixOwnerID); mprintf(L":"); if (*hd.UnixGroupName!=0) - mprintf(L"%ls",GetWide(hd.UnixGroupName)); + mprintf(L"%ls",GetWide(hd.UnixGroupName).c_str()); else if (hd.UnixGroupNumeric) mprintf(L"#%d",hd.UnixGroupID); @@ -422,31 +431,11 @@ void ListFileHeader(Archive &Arc,FileHeader &hd,bool &TitleShown,bool Verbose,bo mprintf(L"%02x%02x..%02x ",S[0],S[1],S[31]); } else - mprintf(L"???????? "); + mprintf(hd.Dir ? L" ":L"???????? "); // Missing checksum is ok for folder, not for file. } mprintf(L"%ls",Name); } -/* -void ListSymLink(Archive &Arc) -{ - if (Arc.FileHead.HSType==HSYS_UNIX && (Arc.FileHead.FileAttr & 0xF000)==0xA000) - if (Arc.FileHead.Encrypted) - { - // Link data are encrypted. We would need to ask for password - // and initialize decryption routine to display the link target. - mprintf(L"\n%22ls %ls",L"-->",L"*<-?->"); - } - else - { - char FileName[NM]; - uint DataSize=(uint)Min(Arc.FileHead.PackSize,sizeof(FileName)-1); - Arc.Read(FileName,DataSize); - FileName[DataSize]=0; - mprintf(L"\n%22ls %ls",L"-->",GetWide(FileName)); - } -} -*/ void ListFileAttr(uint A,HOST_SYSTEM_TYPE HostType,wchar *AttrStr,size_t AttrSize) { diff --git a/multiarc/src/formats/rar/unrar/loclang.hpp b/multiarc/src/formats/rar/unrar/loclang.hpp index 8c7c08814..df94a8eb2 100644 --- a/multiarc/src/formats/rar/unrar/loclang.hpp +++ b/multiarc/src/formats/rar/unrar/loclang.hpp @@ -94,9 +94,8 @@ #define MCHelpSwKB L"\n kb Keep broken extracted files" #define MCHelpSwLog L"\n log[f][=name] Write names to log file" #define MCHelpSwMn L"\n m<0..5> Set compression level (0-store...3-default...5-maximal)" -#define MCHelpSwMA L"\n ma[4|5] Specify a version of archiving format" #define MCHelpSwMC L"\n mc Set advanced compression parameters" -#define MCHelpSwMD L"\n md[k,m,g] Dictionary size in KB, MB or GB" +#define MCHelpSwMD L"\n md[x][kmg] Dictionary size in KB, MB or GB" #define MCHelpSwME L"\n me[par] Set encryption parameters" #define MCHelpSwMS L"\n ms[ext;ext] Specify file types to store" #define MCHelpSwMT L"\n mt Set the number of threads" @@ -107,7 +106,7 @@ #define MCHelpSwOC L"\n oc Set NTFS Compressed attribute" #define MCHelpSwOH L"\n oh Save hard links as the link instead of the file" #define MCHelpSwOI L"\n oi[0-4][:min] Save identical files as references" -#define MCHelpSwOL L"\n ol[a] Process symbolic links as the link [absolute paths]" +#define MCHelpSwOL L"\n ol[a,-] Process symbolic links as the link [absolute paths, skip]" #define MCHelpSwONI L"\n oni Allow potentially incompatible names" #define MCHelpSwOP L"\n op Set the output path for extracted files" #define MCHelpSwOR L"\n or Rename files automatically" @@ -126,8 +125,8 @@ #define MCHelpSwSC L"\n sc[obj] Specify the character set" #define MCHelpSwSFX L"\n sfx[name] Create SFX archive" #define MCHelpSwSI L"\n si[name] Read data from standard input (stdin)" -#define MCHelpSwSL L"\n sl Process files with size less than specified" -#define MCHelpSwSM L"\n sm Process files with size more than specified" +#define MCHelpSwSL L"\n sl[u] Process files with size less than specified" +#define MCHelpSwSM L"\n sm[u] Process files with size more than specified" #define MCHelpSwT L"\n t Test files after archiving" #define MCHelpSwTK L"\n tk Keep original archive time" #define MCHelpSwTL L"\n tl Set archive time to latest file" @@ -139,10 +138,9 @@ #define MCHelpSwU L"\n u Update files" #define MCHelpSwV L"\n v Create volumes with size autodetection or list all volumes" #define MCHelpSwVUnr L"\n v List all volumes" -#define MCHelpSwVn L"\n v[k,b] Create volumes with size=*1000 [*1024, *1]" +#define MCHelpSwVn L"\n v[u] Create volumes with size in [bBkKmMgGtT] units" #define MCHelpSwVD L"\n vd Erase disk contents before creating volume" #define MCHelpSwVER L"\n ver[n] File version control" -#define MCHelpSwVN L"\n vn Use the old style volume naming scheme" #define MCHelpSwVP L"\n vp Pause before each volume" #define MCHelpSwW L"\n w Assign work directory" #define MCHelpSwX L"\n x Exclude specified file" @@ -165,7 +163,7 @@ #define MErrRename L"\nCannot rename %s to %s" #define MAbsNextVol L"\nCannot find volume %s" #define MBreak L"\nUser break\n" -#define MAskCreatVol L"\nCreate next volume ?" +#define MAskCreatVol L"\nCreate next volume?" #define MAskNextDisk L"\nDisk full. Insert next" #define MCreatVol L"\n\nCreating %sarchive %s\n" #define MAskNextVol L"\nInsert disk with %s" @@ -212,7 +210,7 @@ #define MCRCFailed L"\n%-20s - checksum error" #define MExtrTest L"\n\nTesting archive %s\n" #define MExtracting L"\n\nExtracting from %s\n" -#define MUseCurPsw L"\n%s - use current password ?" +#define MUseCurPsw L"\n%s - use current password?" #define MCreatDir L"\nCreating %-56s" #define MExtrSkipFile L"\nSkipping %-56s" #define MExtrTestFile L"\nTesting %-56s" @@ -225,18 +223,17 @@ #define MExtrAllOk L"\nAll OK" #define MExtrTotalErr L"\nTotal errors: %ld" #define MAskReplace L"\n\nWould you like to replace the existing file %s\n%6s bytes, modified on %s\nwith a new one\n%6s bytes, modified on %s\n" -#define MAskOverwrite L"\nOverwrite %s ?" +#define MAskOverwrite L"\nOverwrite %s?" #define MAskNewName L"\nEnter new name: " #define MHeaderBroken L"\nCorrupt header is found" #define MMainHeaderBroken L"\nMain archive header is corrupt" #define MLogFileHead L"\n%s - the file header is corrupt" #define MLogProtectHead L"The data recovery header is corrupt" +#define MArcComment L"\nArchive comment" #define MReadStdinCmt L"\nReading comment from stdin\n" #define MReadCommFrom L"\nReading comment from %s" #define MDelComment L"\nDeleting comment from %s" #define MAddComment L"\nAdding comment to %s" -#define MFCommAdd L"\nAdding file comments" -#define MAskFComm L"\n\nReading comment for %s : %s from stdin\n" #define MLogCommBrk L"\nThe archive comment is corrupt" #define MCommAskCont L"\nPress 'Enter' to continue or 'Q' to quit:" #define MWriteCommTo L"\nWrite comment to %s" @@ -284,11 +281,9 @@ #define MListHostOS L"Host OS" #define MListFileVer L"File version" #define MListService L"Service" -#define MListUOHead L"\n Unix Owner/Group data: %-14s %-14s" #define MListNTACLHead L"\n NTFS security data" #define MListStrmHead L"\n NTFS stream: %s" #define MListUnkHead L"\n Unknown subheader type: 0x%04x" -#define MFileComment L"\nComment: " #define MYes L"Yes" #define MNo L"No" #define MListNoFiles L" 0 files\n" @@ -296,10 +291,10 @@ #define MRprBuild L"\nBuilding %s" #define MRprOldFormat L"\nCannot repair archive with old format" #define MRprFind L"\nFound %s" -#define MRprAskIsSol L"\nThe archive header is corrupt. Mark archive as solid ?" +#define MRprAskIsSol L"\nThe archive header is corrupt. Mark archive as solid?" #define MRprNoFiles L"\nNo files found" #define MLogUnexpEOF L"\nUnexpected end of archive" -#define MRepAskReconst L"\nReconstruct archive structure ?" +#define MRepAskReconst L"\nReconstruct archive structure?" #define MRRSearch L"\nSearching for recovery record" #define MAnalyzeFileData L"\nAnalyzing file data" #define MRecRNotFound L"\nData recovery record not found" @@ -316,7 +311,7 @@ #define MSetOwnersError L"\nWARNING: Cannot set %s owner and group\n" #define MErrLnkRead L"\nWARNING: Cannot read symbolic link %s" #define MSymLinkExists L"\nWARNING: Symbolic link %s already exists" -#define MAskRetryCreate L"\nCannot create %s. Retry ?" +#define MAskRetryCreate L"\nCannot create %s. Retry?" #define MDataBadCRC L"\n%-20s : packed data checksum error in volume %s" #define MFileRO L"\n%s is read-only" #define MACLGetError L"\nWARNING: Cannot get %s security data\n" @@ -330,6 +325,7 @@ #define MCorrectingName L"\nWARNING: Attempting to correct the invalid file or directory name" #define MUnpCannotMerge L"\nWARNING: You need to start extraction from a previous volume to unpack %s" #define MUnknownOption L"\nERROR: Unknown option: %s" +#define MSwSyntaxError L"\nERROR: '-' is expected in the beginning of: %s" #define MSubHeadCorrupt L"\nERROR: Corrupt data header found, ignored" #define MSubHeadUnknown L"\nWARNING: Unknown data header format, ignored" #define MSubHeadDataCRC L"\nERROR: Corrupt %s data block" @@ -386,6 +382,8 @@ #define MNeedAdmin L"\nYou may need to run RAR as administrator" #define MDictOutMem L"\nNot enough memory for %d MB compression dictionary, changed to %d MB." #define MUseSmalllerDict L"\nPlease use a smaller compression dictionary." +#define MExtrDictOutMem L"\nNot enough memory to unpack the archive with %u GB compression dictionary." +#define MSuggest64bit L"\n64-bit RAR version is necessary." #define MOpenErrAtime L"\nYou may need to remove -tsp switch to open this file." #define MErrReadInfo L"\nChoose 'Ignore' to continue with the already read file part only, 'Ignore all' to do it for all read errors, 'Retry' to repeat read and 'Quit' to abort." #define MErrReadTrunc L"\n%s is archived incompletely because of read error.\n" @@ -399,3 +397,8 @@ #define MOrigName L"Original name" #define MOriginalTime L"Original time" #define MFileRenamed L"\n%s is renamed to %s" +#define MDictNotAllowed L"\n%u GB dictionary exceeds %u GB limit and needs more than %u GB memory to unpack." +#define MDictExtrAnyway L"\nUse -md%ug or -mdx%ug switches to extract anyway." +#define MDictComprLimit L"\n%u GB dictionary exceeds %u GB limit and not allowed when compressing data." +#define MNeedSFX64 L"\n64-bit self-extracting module is necessary for %u GB compression dictionary." +#define MSkipUnsafeLink L"\nSkipping the potentially unsafe %s -> %s link. For archives from a trustworthy source use -ola to extract it anyway." diff --git a/multiarc/src/formats/rar/unrar/log.cpp b/multiarc/src/formats/rar/unrar/log.cpp index 8bbe8ee0b..57f7648ef 100644 --- a/multiarc/src/formats/rar/unrar/log.cpp +++ b/multiarc/src/formats/rar/unrar/log.cpp @@ -1,13 +1,14 @@ #include "rar.hpp" -static wchar LogName[NM]; -static RAR_CHARSET LogCharset=RCH_DEFAULT; -void InitLogOptions(const wchar *LogFileName,RAR_CHARSET CSet) +void InitLogOptions(const std::wstring &LogFileName,RAR_CHARSET CSet) +{ +} + + +void CloseLogOptions() { - wcsncpyz(LogName,LogFileName,ASIZE(LogName)); - LogCharset=CSet; } @@ -19,17 +20,15 @@ void Log(const wchar *ArcName,const wchar *fmt,...) uiAlarm(UIALARM_ERROR); - // This buffer is for format string only, not for entire output, - // so it can be short enough. - wchar fmtw[1024]; - PrintfPrepareFmt(fmt,fmtw,ASIZE(fmtw)); - - safebuf wchar Msg[2*NM+1024]; va_list arglist; va_start(arglist,fmt); - vswprintf(Msg,ASIZE(Msg),fmtw,arglist); + + std::wstring s=vwstrprintf(fmt,arglist); + + ReplaceEsc(s); + va_end(arglist); - eprintf(L"%ls",Msg); + eprintf(L"%ls",s.c_str()); ErrHandler.SetSystemErrorCode(Code); } #endif diff --git a/multiarc/src/formats/rar/unrar/log.hpp b/multiarc/src/formats/rar/unrar/log.hpp index 008ef11a0..22eaa8e6e 100644 --- a/multiarc/src/formats/rar/unrar/log.hpp +++ b/multiarc/src/formats/rar/unrar/log.hpp @@ -1,7 +1,8 @@ #ifndef _RAR_LOG_ #define _RAR_LOG_ -void InitLogOptions(const wchar *LogFileName,RAR_CHARSET CSet); +void InitLogOptions(const std::wstring &LogFileName,RAR_CHARSET CSet); +void CloseLogOptions(); #ifdef SILENT inline void Log(const wchar *ArcName,const wchar *fmt,...) {} diff --git a/multiarc/src/formats/rar/unrar/makefile b/multiarc/src/formats/rar/unrar/makefile index 55af49bef..ce54a0247 100644 --- a/multiarc/src/formats/rar/unrar/makefile +++ b/multiarc/src/formats/rar/unrar/makefile @@ -3,7 +3,7 @@ # Linux using GCC CXX=c++ -CXXFLAGS=-O2 -Wno-logical-op-parentheses -Wno-switch -Wno-dangling-else +CXXFLAGS=-march=native -O2 -std=c++11 -Wno-logical-op-parentheses -Wno-switch -Wno-dangling-else LIBFLAGS=-fPIC DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DRAR_SMP STRIP=strip @@ -11,107 +11,6 @@ AR=ar LDFLAGS=-pthread DESTDIR=/usr -# Linux using LCC -#CXX=lcc -#CXXFLAGS=-O2 -#DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -#STRIP=strip -#AR=ar -#DESTDIR=/usr - -# CYGWIN using GCC -#CXX=c++ -#CXXFLAGS=-O2 -#LIBFLAGS= -#DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DRAR_SMP -#STRIP=strip -#AR=ar -#LDFLAGS=-pthread -#DESTDIR=/usr - -# HP UX using aCC -#CXX=aCC -#CXXFLAGS=-AA +O2 +Onolimit -#DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -#STRIP=strip -#AR=ar -#DESTDIR=/usr - -# IRIX using GCC -#CXX=g++ -#CXXFLAGS=-O2 -#DEFINES=-DBIG_ENDIAN -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_BSD_COMPAT -D_XOPEN_SOURCE -D_XOPEN_SOURCE_EXTENDED=1 -#STRIP=strip -#AR=ar -#DESTDIR=/usr - -# IRIX using MIPSPro (experimental) -#CXX=CC -#CXXFLAGS=-O2 -mips3 -woff 1234,1156,3284 -LANG:std -#DEFINES=-DBIG_ENDIAN -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_BSD_COMPAT -Dint64=int64_t -#STRIP=strip -#AR=ar -#DESTDIR=/usr - -# AIX using xlC (IBM VisualAge C++ 5.0) -#CXX=xlC -#CXXFLAGS=-O -qinline -qro -qroconst -qmaxmem=16384 -qcpluscmt -#DEFINES=-D_LARGE_FILES -D_LARGE_FILE_API -#LIBS=-lbsd -#STRIP=strip -#AR=ar -#DESTDIR=/usr - -# Solaris using CC -#CXX=CC -#CXXFLAGS=-fast -erroff=wvarhidemem -#DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -#STRIP=strip -#AR=ar -#DESTDIR=/usr - -# Solaris using GCC (optimized for UltraSPARC 1 CPU) -#CXX=g++ -#CXXFLAGS=-O3 -mcpu=v9 -mtune=ultrasparc -m32 -#DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -#STRIP=/usr/ccs/bin/strip -#AR=/usr/ccs/bin/ar -#DESTDIR=/usr - -# Tru64 5.1B using GCC3 -#CXX=g++ -#CXXFLAGS=-O2 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_XOPEN_SOURCE=500 -#STRIP=strip -#AR=ar -#LDFLAGS=-rpath /usr/local/gcc/lib -#DESTDIR=/usr - -# Tru64 5.1B using DEC C++ -#CXX=cxx -#CXXFLAGS=-O4 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -Dint64=long -#STRIP=strip -#AR=ar -#LDFLAGS= -#DESTDIR=/usr - -# QNX 6.x using GCC -#CXX=g++ -#CXXFLAGS=-O2 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -fexceptions -#STRIP=strip -#AR=ar -#LDFLAGS=-fexceptions -#DESTDIR=/usr - -# Cross-compile -# Linux using arm-linux-g++ -#CXX=arm-linux-g++ -#CXXFLAGS=-O2 -#DEFINES=-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -#STRIP=arm-linux-strip -#AR=arm-linux-ar -#LDFLAGS=-static -#DESTDIR=/usr - ########################## COMPILE=$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(DEFINES) diff --git a/multiarc/src/formats/rar/unrar/match.cpp b/multiarc/src/formats/rar/unrar/match.cpp index ec88fa61b..0a78d8dd7 100644 --- a/multiarc/src/formats/rar/unrar/match.cpp +++ b/multiarc/src/formats/rar/unrar/match.cpp @@ -3,6 +3,7 @@ static bool match(const wchar *pattern,const wchar *string,bool ForceCase); static int mwcsicompc(const wchar *Str1,const wchar *Str2,bool ForceCase); static int mwcsnicompc(const wchar *Str1,const wchar *Str2,size_t N,bool ForceCase); +static bool IsWildcard(const wchar *Str,size_t CheckSize); inline uint touppercw(uint ch,bool ForceCase) { @@ -16,12 +17,15 @@ inline uint touppercw(uint ch,bool ForceCase) } -bool CmpName(const wchar *Wildcard,const wchar *Name,int CmpMode) +bool CmpName(const wchar *Wildcard,const wchar *Name,uint CmpMode) { bool ForceCase=(CmpMode&MATCH_FORCECASESENSITIVE)!=0; CmpMode&=MATCH_MODEMASK; + wchar *Name1=PointToName(Wildcard); + wchar *Name2=PointToName(Name); + if (CmpMode!=MATCH_NAMES) { size_t WildLength=wcslen(Wildcard); @@ -32,47 +36,48 @@ bool CmpName(const wchar *Wildcard,const wchar *Name,int CmpMode) // "path1" mask must match "path1\path2\filename.ext" and "path1" names. wchar NextCh=Name[WildLength]; if (NextCh==L'\\' || NextCh==L'/' || NextCh==0) - return(true); + return true; } // Nothing more to compare for MATCH_SUBPATHONLY. if (CmpMode==MATCH_SUBPATHONLY) - return(false); - - wchar Path1[NM],Path2[NM]; - GetFilePath(Wildcard,Path1,ASIZE(Path1)); - GetFilePath(Name,Path2,ASIZE(Path2)); - - if ((CmpMode==MATCH_EXACT || CmpMode==MATCH_EXACTPATH) && - mwcsicompc(Path1,Path2,ForceCase)!=0) - return(false); + return false; + + // 2023.08.29: We tried std::wstring Path1 and Path2 here, but performance + // impact for O(n^2) complexity loop in CmdExtract::AnalyzeArchive() + // was rather noticeable, 1.7s instead of 0.9s when extracting ~300 files + // with @listfile from archive with ~7000 files. + // This function can be invoked from other O(n^2) loops. So for now + // we prefer to avoid wstring and use pointers and path sizes here. + // Another option could be using std::wstring_view. + + size_t Path1Size=Name1-Wildcard; + size_t Path2Size=Name2-Name; + + if ((CmpMode==MATCH_EXACT || CmpMode==MATCH_EXACTPATH) && + (Path1Size!=Path2Size || + mwcsnicompc(Wildcard,Name,Path1Size,ForceCase)!=0)) + return false; if (CmpMode==MATCH_ALLWILD) return match(Wildcard,Name,ForceCase); if (CmpMode==MATCH_SUBPATH || CmpMode==MATCH_WILDSUBPATH) - if (IsWildcard(Path1)) - return(match(Wildcard,Name,ForceCase)); + if (IsWildcard(Wildcard,Path1Size)) + return match(Wildcard,Name,ForceCase); else if (CmpMode==MATCH_SUBPATH || IsWildcard(Wildcard)) { - if (*Path1 && mwcsnicompc(Path1,Path2,wcslen(Path1),ForceCase)!=0) - return(false); + if (Path1Size>0 && mwcsnicompc(Wildcard,Name,Path1Size,ForceCase)!=0) + return false; } else - if (mwcsicompc(Path1,Path2,ForceCase)!=0) - return(false); + if (Path1Size!=Path2Size || mwcsnicompc(Wildcard,Name,Path1Size,ForceCase)!=0) + return false; } - wchar *Name1=PointToName(Wildcard); - wchar *Name2=PointToName(Name); - - // Always return false for RAR temporary files to exclude them - // from archiving operations. -// if (mwcsnicompc(L"__rar_",Name2,6,false)==0) -// return(false); if (CmpMode==MATCH_EXACT) - return(mwcsicompc(Name1,Name2,ForceCase)==0); + return mwcsicompc(Name1,Name2,ForceCase)==0; - return(match(Name1,Name2,ForceCase)); + return match(Name1,Name2,ForceCase); } @@ -85,18 +90,18 @@ bool match(const wchar *pattern,const wchar *string,bool ForceCase) switch (patternc) { case 0: - return(stringc==0); + return stringc==0; case '?': if (stringc == 0) - return(false); + return false; break; case '*': if (*pattern==0) - return(true); + return true; if (*pattern=='.') { if (pattern[1]=='*' && pattern[2]==0) - return(true); + return true; const wchar *dot=wcschr(string,'.'); if (pattern[1]==0) return (dot==NULL || dot[1]==0); @@ -104,22 +109,22 @@ bool match(const wchar *pattern,const wchar *string,bool ForceCase) { string=dot; if (wcspbrk(pattern,L"*?")==NULL && wcschr(string+1,'.')==NULL) - return(mwcsicompc(pattern+1,string+1,ForceCase)==0); + return mwcsicompc(pattern+1,string+1,ForceCase)==0; } } while (*string) if (match(pattern,string++,ForceCase)) - return(true); - return(false); + return true; + return false; default: if (patternc != stringc) { // Allow "name." mask match "name" and "name.\" match "name\". if (patternc=='.' && (stringc==0 || stringc=='\\' || stringc=='.')) - return(match(pattern,string,ForceCase)); + return match(pattern,string,ForceCase); else - return(false); + return false; } break; } @@ -145,3 +150,18 @@ int mwcsnicompc(const wchar *Str1,const wchar *Str2,size_t N,bool ForceCase) return wcsnicomp(Str1,Str2,N); #endif } + + +bool IsWildcard(const wchar *Str,size_t CheckSize) +{ + size_t CheckPos=0; +#ifdef _WIN_ALL + // Not treat the special NTFS \\?\d: path prefix as a wildcard. + if (Str[0]=='\\' && Str[1]=='\\' && Str[2]=='?' && Str[3]=='\\') + CheckPos+=4; +#endif + for (size_t I=CheckPos;I= 9*sf)+(cf >= 12*sf)+(cf >= 15*sf); - pc->U.SummFreq += cf; + pc->U.SummFreq += (ushort)cf; } p=pc->U.Stats+ns1; p->Successor=Successor; p->Symbol = fs.Symbol; - p->Freq = cf; - pc->NumStats=++ns1; + p->Freq = (byte)cf; + pc->NumStats=(ushort)++ns1; } MaxContext=MinContext=fs.Successor; return; @@ -542,7 +542,7 @@ inline bool RARPPM_CONTEXT::decodeSymbol2(ModelPPM *Model) Model->CharMask[(*pps)->Symbol]=Model->EscCount; pps++; } while ( --i ); - psee2c->Summ += Model->Coder.SubRange.scale; + psee2c->Summ += (ushort)Model->Coder.SubRange.scale; Model->NumMasked = NumStats; } return true; diff --git a/multiarc/src/formats/rar/unrar/model.hpp b/multiarc/src/formats/rar/unrar/model.hpp index 52abc89b3..2c9fa404f 100644 --- a/multiarc/src/formats/rar/unrar/model.hpp +++ b/multiarc/src/formats/rar/unrar/model.hpp @@ -25,7 +25,7 @@ struct RARPPM_SEE2_CONTEXT : RARPPM_DEF } uint getMean() { - uint RetVal=GET_SHORT16(Summ) >> Shift; + short RetVal=GET_SHORT16(Summ) >> Shift; Summ -= RetVal; return RetVal+(RetVal == 0); } diff --git a/multiarc/src/formats/rar/unrar/options.cpp b/multiarc/src/formats/rar/unrar/options.cpp index 22ae27ce2..ab8cff5eb 100644 --- a/multiarc/src/formats/rar/unrar/options.cpp +++ b/multiarc/src/formats/rar/unrar/options.cpp @@ -10,6 +10,7 @@ void RAROptions::Init() { memset(this,0,sizeof(RAROptions)); WinSize=0x2000000; + WinSizeLimit=0x100000000; Overwrite=OVERWRITE_DEFAULT; Method=3; MsgStream=MSG_STDOUT; diff --git a/multiarc/src/formats/rar/unrar/options.hpp b/multiarc/src/formats/rar/unrar/options.hpp index e249eb599..1061bd021 100644 --- a/multiarc/src/formats/rar/unrar/options.hpp +++ b/multiarc/src/formats/rar/unrar/options.hpp @@ -33,7 +33,7 @@ enum RECURSE_MODE RECURSE_NONE=0, // no recurse switches RECURSE_DISABLE, // switch -r- RECURSE_ALWAYS, // switch -r - RECURSE_WILDCARDS, // switch -r0 + RECURSE_WILDCARDS // switch -r0 }; enum OVERWRITE_MODE @@ -57,7 +57,13 @@ enum QOPEN_MODE { QOPEN_NONE, QOPEN_AUTO, QOPEN_ALWAYS }; enum RAR_CHARSET { RCH_DEFAULT=0,RCH_ANSI,RCH_OEM,RCH_UNICODE,RCH_UTF8 }; #define MAX_FILTER_TYPES 16 -enum FilterState {FILTER_DEFAULT=0,FILTER_AUTO,FILTER_FORCE,FILTER_DISABLE}; + +enum FilterState { + FILTER_DEFAULT=0, // No -mc switch. + FILTER_AUTO, // -mc switch is present. + FILTER_FORCE, // -mc+ switch is present. + FILTER_DISABLE // -mc- switch is present. +}; enum SAVECOPY_MODE { @@ -98,6 +104,7 @@ class RAROptions RAROptions(); void Init(); + uint ExclFileAttr; uint InclFileAttr; @@ -107,30 +114,24 @@ class RAROptions bool InclDir; bool InclAttrSet; - size_t WinSize; - wchar TempPath[NM]; - wchar SFXModule[NM]; + uint64 WinSize; + uint64 WinSizeLimit; // Switch -mdx. #ifdef USE_QOPEN QOPEN_MODE QOpenMode; #endif bool ConfigDisabled; // Switch -cfg-. - wchar ExtrPath[NM]; - wchar CommentFile[NM]; RAR_CHARSET CommentCharset; RAR_CHARSET FilelistCharset; RAR_CHARSET ErrlogCharset; RAR_CHARSET RedirectCharset; - wchar ArcPath[NM]; // For -ap. - wchar ExclArcPath[NM]; // For -ep4 switch. bool EncryptHeaders; bool SkipEncrypted; bool ManualPassword; // Password entered manually during operation, might need to clean for next archive. - wchar LogName[NM]; MESSAGE_TYPE MsgStream; SOUND_NOTIFY_MODE Sound; OVERWRITE_MODE Overwrite; @@ -164,6 +165,7 @@ class RAROptions bool SaveSymLinks; bool SaveHardLinks; bool AbsoluteLinks; + bool SkipSymLinks; int Priority; int SleepTime; bool KeepBroken; @@ -195,7 +197,6 @@ class RAROptions bool Test; bool VolumePause; FilterMode FilterModes[MAX_FILTER_TYPES]; - wchar EmailTo[NM]; uint VersionControl; APPENDARCNAME_MODE AppendArcNameToPath; POWER_MODE Shutdown; @@ -204,11 +205,6 @@ class RAROptions EXTTIME_MODE xatime; bool PreserveAtime; - // Read data from stdin and store in archive under a name specified here - // when archiving. Read an archive from stdin if any non-empty string - // is specified here when extracting. - wchar UseStdin[NM]; - uint Threads; // We use it to init hash even if RAR_SMP is not defined. @@ -216,7 +212,6 @@ class RAROptions #ifdef RARDLL - wchar DllDestName[NM]; int DllOpMode; int DllError; LPARAM UserData; @@ -224,5 +219,6 @@ class RAROptions CHANGEVOLPROC ChangeVolProc; PROCESSDATAPROC ProcessDataProc; #endif + }; #endif diff --git a/multiarc/src/formats/rar/unrar/os.hpp b/multiarc/src/formats/rar/unrar/os.hpp index 055993be7..e2d65a368 100644 --- a/multiarc/src/formats/rar/unrar/os.hpp +++ b/multiarc/src/formats/rar/unrar/os.hpp @@ -4,10 +4,6 @@ #define FALSE 0 #define TRUE 1 -#ifdef __EMX__ - #define INCL_BASE -#endif - #if defined(RARDLL) && !defined(SILENT) #define SILENT #endif @@ -15,14 +11,13 @@ #include #include #include +#include +#include // For automatic pointers. -#if defined(_WIN_ALL) || defined(_EMX) +#ifdef _WIN_ALL #define LITTLE_ENDIAN -#define NM 2048 - -#ifdef _WIN_ALL // We got a report that just "#define STRICT" is incompatible with @@ -41,15 +36,8 @@ #define _UNICODE // Set _T() macro to convert from narrow to wide strings. #endif -#if 0 -// 2021.09.05: Allow newer Vista+ APIs like IFileOpenDialog for WinRAR, -// but still keep SFX modules XP compatible. -#define WINVER _WIN32_WINNT_VISTA -#define _WIN32_WINNT _WIN32_WINNT_VISTA -#else #define WINVER _WIN32_WINNT_WINXP #define _WIN32_WINNT _WIN32_WINNT_WINXP -#endif #if !defined(ZIPSFX) #define RAR_SMP @@ -70,28 +58,23 @@ #include #include +// For WMI requests. +#include +#include +#pragma comment(lib, "wbemuuid.lib") -#endif // _WIN_ALL #include #include #include +#include +#include -#if !defined(_EMX) && !defined(_MSC_VER) - #include +// Use SSE only for x86/x64, not ARM Windows. +#if defined(_M_IX86) || defined(_M_X64) + #define USE_SSE + #define SSE_ALIGNMENT 16 #endif -#ifdef _MSC_VER - #include - #include - - // Use SSE only for x86/x64, not ARM Windows. - #if defined(_M_IX86) || defined(_M_X64) - #define USE_SSE - #define SSE_ALIGNMENT 16 - #endif -#else - #include -#endif // _MSC_VER #include #include @@ -113,7 +96,7 @@ #define SPATHDIVIDER L"\\" -#define CPATHDIVIDER '\\' +#define CPATHDIVIDER L'\\' #define MASKALL L"*" #define READBINARY "rb" @@ -123,25 +106,13 @@ #define WRITEBINARY "wb" #define APPENDTEXT "at" -#if defined(_WIN_ALL) - #ifdef _MSC_VER - #define _stdfunction __cdecl - #define _forceinline __forceinline - #else - #define _stdfunction _USERENTRY - #define _forceinline inline - #endif -#else - #define _stdfunction - #define _forceinline inline -#endif +#define _stdfunction __cdecl +#define _forceinline __forceinline -#endif // defined(_WIN_ALL) || defined(_EMX) +#endif // _WIN_ALL #ifdef _UNIX -#define NM 2048 - #include #include #include @@ -149,7 +120,7 @@ #if defined(__QNXNTO__) #include #endif -#if defined(RAR_SMP) && defined(__APPLE__) +#ifdef _APPLE #include #endif #ifndef SFX_MODULE @@ -172,6 +143,28 @@ #include #include +#ifdef __GNUC__ + #if defined(__i386__) || defined(__x86_64__) + #include + + #define USE_SSE + #define SSE_ALIGNMENT 16 + #endif +#endif + +#if defined(__aarch64__) && (defined(__ARM_FEATURE_CRYPTO) || defined(__ARM_FEATURE_CRC32)) +#include +#ifndef _APPLE +#include +#include +#endif +#ifdef __ARM_FEATURE_CRYPTO +#define USE_NEON_AES +#endif +#ifdef __ARM_FEATURE_CRC32 +#define USE_NEON_CRC32 +#endif +#endif #ifdef S_IFLNK #define SAVE_LINKS @@ -189,7 +182,7 @@ #define SPATHDIVIDER L"/" -#define CPATHDIVIDER '/' +#define CPATHDIVIDER L'/' #define MASKALL L"*" #define READBINARY "r" @@ -219,6 +212,10 @@ #endif #endif +#ifdef __VMS +# define LITTLE_ENDIAN +#endif + // Unlike Apple x64, utimensat shall be available in all Apple M1 systems. #if _POSIX_C_SOURCE >= 200809L || defined(__APPLE__) && defined(__arm64__) #define UNIX_TIME_NS // Nanosecond time precision in Unix. @@ -226,19 +223,12 @@ #endif // _UNIX -#if 0 - #define MSGID_INT - typedef int MSGID; -#else typedef const wchar* MSGID; -#endif #ifndef SSE_ALIGNMENT // No SSE use and no special data alignment is required. #define SSE_ALIGNMENT 1 #endif -#define safebuf static - // Solaris defines _LITTLE_ENDIAN or _BIG_ENDIAN. #if defined(_LITTLE_ENDIAN) && !defined(LITTLE_ENDIAN) #define LITTLE_ENDIAN @@ -269,8 +259,8 @@ #endif #endif -#if !defined(BIG_ENDIAN) && defined(_WIN_ALL) || defined(__i386__) || defined(__x86_64__) -// Allow not aligned integer access, increases speed in some operations. +#if !defined(BIG_ENDIAN) && defined(_WIN_ALL) || defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) +// Allow unaligned integer access, increases speed in some operations. #define ALLOW_MISALIGNED #endif diff --git a/multiarc/src/formats/rar/unrar/pathfn.cpp b/multiarc/src/formats/rar/unrar/pathfn.cpp index 50b6b29da..7c0f91963 100644 --- a/multiarc/src/formats/rar/unrar/pathfn.cpp +++ b/multiarc/src/formats/rar/unrar/pathfn.cpp @@ -9,6 +9,21 @@ wchar* PointToName(const wchar *Path) } +std::wstring PointToName(const std::wstring &Path) +{ + return std::wstring(Path.substr(GetNamePos(Path))); +} + + +size_t GetNamePos(const std::wstring &Path) +{ + for (int I=(int)Path.size()-1;I>=0;I--) + if (IsPathDiv(Path[I])) + return I+1; + return IsDriveLetter(Path) ? 2 : 0; +} + + wchar* PointToLastChar(const wchar *Path) { size_t Length=wcslen(Path); @@ -16,94 +31,95 @@ wchar* PointToLastChar(const wchar *Path) } -wchar* ConvertPath(const wchar *SrcPath,wchar *DestPath,size_t DestSize) +wchar GetLastChar(const std::wstring &Path) +{ + return Path.empty() ? 0:Path.back(); +} + + +size_t ConvertPath(const std::wstring *SrcPath,std::wstring *DestPath) { - const wchar *DestPtr=SrcPath; + const std::wstring &S=*SrcPath; // To avoid *SrcPath[] everywhere. + size_t DestPos=0; - // Prevent \..\ in any part of path string. - for (const wchar *s=DestPtr;*s!=0;s++) - if (IsPathDiv(s[0]) && s[1]=='.' && s[2]=='.' && IsPathDiv(s[3])) - DestPtr=s+4; + // Prevent \..\ in any part of path string and \.. at the end of string + for (size_t I=0;I:\ and any sequence of . and \ in the beginning of path string. - while (*DestPtr!=0) + while (DestPos='A' && Letter<='Z' && IsDriveDiv(Path[1]); } -int GetPathDisk(const wchar *Path) +int GetPathDisk(const std::wstring &Path) { if (IsDriveLetter(Path)) - return etoupperw(*Path)-'A'; + return etoupperw(Path[0])-'A'; else return -1; } -void AddEndSlash(wchar *Path,size_t MaxLength) +void AddEndSlash(std::wstring &Path) { - size_t Length=wcslen(Path); - if (Length>0 && Path[Length-1]!=CPATHDIVIDER && Length+12) + AddEndSlash(OutName); + OutName+=Name; + Pathname=OutName; } -// Returns file path including the trailing path separator symbol. -void GetFilePath(const wchar *FullName,wchar *Path,size_t MaxLength) +// Returns the file path including the trailing path separator symbol. +// It is allowed for both parameters to point to the same string. +void GetPathWithSep(const std::wstring &FullName,std::wstring &Path) { - if (MaxLength==0) - return; - size_t PathLength=Min(MaxLength-1,size_t(PointToName(FullName)-FullName)); - wcsncpy(Path,FullName,PathLength); - Path[PathLength]=0; + if (std::addressof(FullName)!=std::addressof(Path)) + Path=FullName; + Path.erase(GetNamePos(FullName)); } -// Removes name and returns file path without the trailing -// path separator symbol. -void RemoveNameFromPath(wchar *Path) +// Removes name and returns file path without the trailing path separator. +// But for names like d:\name return d:\ with trailing path separator. +void RemoveNameFromPath(std::wstring &Path) { - wchar *Name=PointToName(Path); - if (Name>=Path+2 && (!IsDriveDiv(Path[1]) || Name>=Path+4)) - Name--; - *Name=0; + auto NamePos=GetNamePos(Path); + if (NamePos>=2 && (!IsDriveDiv(Path[1]) || NamePos>=4)) + NamePos--; + Path.erase(NamePos); } #if defined(_WIN_ALL) && !defined(SFX_MODULE) -bool GetAppDataPath(wchar *Path,size_t MaxSize,bool Create) +bool GetAppDataPath(std::wstring &Path,bool Create) { LPMALLOC g_pMalloc; SHGetMalloc(&g_pMalloc); LPITEMIDLIST ppidl; - *Path=0; + Path.clear(); bool Success=false; if (SHGetSpecialFolderLocation(NULL,CSIDL_APPDATA,&ppidl)==NOERROR && - SHGetPathFromIDList(ppidl,Path) && *Path!=0) + SHGetPathStrFromIDList(ppidl,Path) && !Path.empty()) { - AddEndSlash(Path,MaxSize); - wcsncatz(Path,L"WinRAR",MaxSize); + AddEndSlash(Path); + Path+=L"WinRAR"; Success=FileExist(Path); if (!Success && Create) - Success=MakeDir(Path,false,0)==MKDIR_SUCCESS; + Success=CreateDir(Path); } g_pMalloc->Free(ppidl); return Success; @@ -243,24 +278,41 @@ bool GetAppDataPath(wchar *Path,size_t MaxSize,bool Create) #endif +#if defined(_WIN_ALL) +bool SHGetPathStrFromIDList(PCIDLIST_ABSOLUTE pidl,std::wstring &Path) +{ + std::vector Buf(MAX_PATH); + bool Success=SHGetPathFromIDList(pidl,Buf.data())!=FALSE; + Path=Buf.data(); + return Success; +} +#endif + + #if defined(_WIN_ALL) && !defined(SFX_MODULE) -void GetRarDataPath(wchar *Path,size_t MaxSize,bool Create) +void GetRarDataPath(std::wstring &Path,bool Create) { - *Path=0; + Path.clear(); HKEY hKey; if (RegOpenKeyEx(HKEY_CURRENT_USER,L"Software\\WinRAR\\Paths",0, KEY_QUERY_VALUE,&hKey)==ERROR_SUCCESS) { - DWORD DataSize=(DWORD)MaxSize,Type; - RegQueryValueEx(hKey,L"AppData",0,&Type,(BYTE *)Path,&DataSize); - RegCloseKey(hKey); + DWORD DataSize; + LSTATUS Code=RegQueryValueEx(hKey,L"AppData",NULL,NULL,NULL,&DataSize); + if (Code==ERROR_SUCCESS) + { + std::vector PathBuf(DataSize/sizeof(wchar)); + RegQueryValueEx(hKey,L"AppData",0,NULL,(BYTE *)PathBuf.data(),&DataSize); + Path=PathBuf.data(); + RegCloseKey(hKey); + } } - if (*Path==0 || !FileExist(Path)) - if (!GetAppDataPath(Path,MaxSize,Create)) + if (Path.empty() || !FileExist(Path)) + if (!GetAppDataPath(Path,Create)) { - GetModuleFileName(NULL,Path,(DWORD)MaxSize); + Path=GetModuleFileStr(); RemoveNameFromPath(Path); } } @@ -268,7 +320,7 @@ void GetRarDataPath(wchar *Path,size_t MaxSize,bool Create) #ifndef SFX_MODULE -bool EnumConfigPaths(uint Number,wchar *Path,size_t MaxSize,bool Create) +bool EnumConfigPaths(uint Number,std::wstring &Path,bool Create) { #ifdef _UNIX static const wchar *ConfPath[]={ @@ -278,24 +330,24 @@ bool EnumConfigPaths(uint Number,wchar *Path,size_t MaxSize,bool Create) { char *EnvStr=getenv("HOME"); if (EnvStr!=NULL) - CharToWide(EnvStr,Path,MaxSize); + CharToWide(EnvStr,Path); else - wcsncpyz(Path,ConfPath[0],MaxSize); + Path=ConfPath[0]; return true; } Number--; if (Number>=ASIZE(ConfPath)) return false; - wcsncpyz(Path,ConfPath[Number], MaxSize); + Path=ConfPath[Number]; return true; #elif defined(_WIN_ALL) if (Number>1) return false; if (Number==0) - GetRarDataPath(Path,MaxSize,Create); + GetRarDataPath(Path,Create); else { - GetModuleFileName(NULL,Path,(DWORD)MaxSize); + Path=GetModuleFileStr(); RemoveNameFromPath(Path); } return true; @@ -307,13 +359,15 @@ bool EnumConfigPaths(uint Number,wchar *Path,size_t MaxSize,bool Create) #ifndef SFX_MODULE -void GetConfigName(const wchar *Name,wchar *FullName,size_t MaxSize,bool CheckExist,bool Create) +void GetConfigName(const std::wstring &Name,std::wstring &FullName,bool CheckExist,bool Create) { - *FullName=0; - for (uint I=0;EnumConfigPaths(I,FullName,MaxSize,Create);I++) + FullName.clear(); + for (uint I=0;;I++) { - AddEndSlash(FullName,MaxSize); - wcsncatz(FullName,Name,MaxSize); + std::wstring ConfPath; + if (!EnumConfigPaths(I,ConfPath,Create)) + break; + MakeName(ConfPath,Name,FullName); if (!CheckExist || WildFileExist(FullName)) break; } @@ -321,112 +375,107 @@ void GetConfigName(const wchar *Name,wchar *FullName,size_t MaxSize,bool CheckEx #endif -// Returns a pointer to rightmost digit of volume number or to beginning +// Returns the position to rightmost digit of volume number or beginning // of file name if numeric part is missing. -wchar* GetVolNumPart(const wchar *ArcName) +size_t GetVolNumPos(const std::wstring &ArcName) { // We do not want to increment any characters in path component. - ArcName=PointToName(ArcName); + size_t NamePos=GetNamePos(ArcName); - if (*ArcName==0) - return (wchar *)ArcName; + if (NamePos==ArcName.size()) + return NamePos; // Pointing to last name character. - const wchar *ChPtr=ArcName+wcslen(ArcName)-1; + size_t Pos=ArcName.size()-1; // Skipping the archive extension. - while (!IsDigit(*ChPtr) && ChPtr>ArcName) - ChPtr--; + while (!IsDigit(ArcName[Pos]) && Pos>NamePos) + Pos--; // Skipping the numeric part of name. - const wchar *NumPtr=ChPtr; - while (IsDigit(*NumPtr) && NumPtr>ArcName) - NumPtr--; + size_t NumPos=Pos; + while (IsDigit(ArcName[NumPos]) && NumPos>NamePos) + NumPos--; // Searching for first numeric part in names like name.part##of##.rar. // Stop search on the first dot. - while (NumPtr>ArcName && *NumPtr!='.') + while (NumPos>NamePos && ArcName[NumPos]!='.') { - if (IsDigit(*NumPtr)) + if (IsDigit(ArcName[NumPos])) { // Validate the first numeric part only if it has a dot somewhere // before it. - const wchar *Dot=wcschr(ArcName,'.'); - if (Dot!=NULL && Dot|\"")==NULL; + return !Name.empty() && Name.find_first_of(L"?*<>|\"")==std::wstring::npos; } - - -void MakeNameUsable(wchar *Name,bool Extended) +void MakeNameUsable(std::wstring &Name,bool Extended) { - for (wchar *s=Name;*s!=0;s++) + for (size_t I=0;I|\"":L"?*",*s)!=NULL || Extended && (uint)*s<32) - *s='_'; + if (wcschr(Extended ? L"?*<>|\"":L"?*",Name[I])!=NULL || + Extended && (uint)Name[I]<32) + Name[I]='_'; #ifdef _UNIX // We were asked to apply Windows-like conversion in Linux in case // files are unpacked to Windows share. This code is invoked only @@ -478,26 +526,24 @@ void MakeNameUsable(wchar *Name,bool Extended) { // Windows shares in Unix do not allow the drive letter, // so unlike Windows version, we check all characters here. - if (*s==':') - *s='_'; + if (Name[I]==':') + Name[I]='_'; // No spaces or dots before the path separator are allowed on Windows - // shares. But they are allowed and automtically removed at the end of - // file or folder name, so it is useless to replace them here. + // shares. But they are allowed and automatically removed at the end of + // file or folder name, so we need to replace them only before + // the path separator, but not at the end of file name. // Since such files or folders are created successfully, a supposed - // conversion here would never be invoked. - if ((*s==' ' || *s=='.') && IsPathDiv(s[1])) - *s='_'; + // conversion at the end of file name would never be invoked here. + // While converting dots, we preserve "." and ".." path components, + // such as when specifying ".." in the destination path. + if (IsPathDiv(Name[I+1]) && (Name[I]==' ' || Name[I]=='.' && I>0 && + !IsPathDiv(Name[I-1]) && (Name[I-1]!='.' || I>1 && !IsPathDiv(Name[I-2])))) + Name[I]='_'; } #else - if (s-Name>1 && *s==':') - *s='_'; -#if 0 // We already can create such files. - // Remove ' ' and '.' before path separator, but allow .\ and ..\. - if (IsPathDiv(s[1]) && (*s==' ' || *s=='.' && s>Name && - !IsPathDiv(s[-1]) && (s[-1]!='.' || s>Name+1 && !IsPathDiv(s[-2])))) - *s='_'; -#endif + if (I>1 && Name[I]==':') + Name[I]='_'; #endif } } @@ -512,20 +558,38 @@ void UnixSlashToDos(const char *SrcName,char *DestName,size_t MaxLength) } -void DosSlashToUnix(const char *SrcName,char *DestName,size_t MaxLength) +void UnixSlashToDos(const wchar *SrcName,wchar *DestName,size_t MaxLength) { size_t Copied=0; for (;Copied0) - *Dest=0; + Dest.clear(); return; } #ifdef _WIN_ALL { - wchar FullName[NM],*NamePtr; - DWORD Code=GetFullPathName(Src,ASIZE(FullName),FullName,&NamePtr); - if (Code==0 || Code>ASIZE(FullName)) + DWORD Code=GetFullPathName(Src.c_str(),0,NULL,NULL); // Get the buffer size. + if (Code!=0) { - wchar LongName[NM]; - if (GetWinLongPath(Src,LongName,ASIZE(LongName))) - Code=GetFullPathName(LongName,ASIZE(FullName),FullName,&NamePtr); + std::vector FullName(Code); + Code=GetFullPathName(Src.c_str(),(DWORD)FullName.size(),FullName.data(),NULL); + + if (Code>0 && Code<=FullName.size()) + { + Dest=FullName.data(); + return; + } } - if (Code!=0 && Code FullName(Code); + Code=GetFullPathName(LongName.c_str(),(DWORD)FullName.size(),FullName.data(),NULL); + + if (Code>0 && Code<=FullName.size()) + { + Dest=FullName.data(); + return; + } + } + } + if (Src!=Dest) + Dest=Src; // Copy source to destination in case of failure. } #elif defined(_UNIX) if (IsFullPath(Src)) - *Dest=0; + Dest.clear(); else { - char CurDirA[NM]; - if (getcwd(CurDirA,ASIZE(CurDirA))==NULL) - *CurDirA=0; - CharToWide(CurDirA,Dest,MaxSize); - AddEndSlash(Dest,MaxSize); + std::vector CurDirA(MAXPATHSIZE); + if (getcwd(CurDirA.data(),CurDirA.size())==NULL) + CurDirA[0]=0; + CharToWide(CurDirA.data(),Dest); + AddEndSlash(Dest); } - wcsncatz(Dest,Src,MaxSize); + Dest+=Src; #else - wcsncpyz(Dest,Src,MaxSize); + Dest=Src; #endif } -bool IsFullPath(const wchar *Path) +bool IsFullPath(const std::wstring &Path) { -/* - wchar PathOnly[NM]; - GetFilePath(Path,PathOnly,ASIZE(PathOnly)); - if (IsWildcard(PathOnly)) - return true; -*/ -#if defined(_WIN_ALL) || defined(_EMX) - return Path[0]=='\\' && Path[1]=='\\' || IsDriveLetter(Path) && IsPathDiv(Path[2]); +#ifdef _WIN_ALL + return Path.size()>=2 && Path[0]=='\\' && Path[1]=='\\' || + Path.size()>=3 && IsDriveLetter(Path) && IsPathDiv(Path[2]); #else - return IsPathDiv(Path[0]); + return Path.size()>=1 && IsPathDiv(Path[0]); #endif } -bool IsFullRootPath(const wchar *Path) +bool IsFullRootPath(const std::wstring &Path) { return IsFullPath(Path) || IsPathDiv(Path[0]); } -void GetPathRoot(const wchar *Path,wchar *Root,size_t MaxSize) +// Both source and destination can point to the same string. +void GetPathRoot(const std::wstring &Path,std::wstring &Root) { - *Root=0; if (IsDriveLetter(Path)) - swprintf(Root,MaxSize,L"%c:\\",*Path); + Root=Path.substr(0,2) + L"\\"; else if (Path[0]=='\\' && Path[1]=='\\') { - const wchar *Slash=wcschr(Path+2,'\\'); - if (Slash!=NULL) + size_t Slash=Path.find('\\',2); + if (Slash!=std::wstring::npos) { size_t Length; - if ((Slash=wcschr(Slash+1,'\\'))!=NULL) - Length=Slash-Path+1; + if ((Slash=Path.find('\\',Slash+1))!=std::wstring::npos) + Length=Slash+1; else - Length=wcslen(Path); - if (Length>=MaxSize) - Length=0; - wcsncpy(Root,Path,Length); - Root[Length]=0; + Length=Path.size(); + Root=Path.substr(0,Length); } } + else + Root.clear(); } -int ParseVersionFileName(wchar *Name,bool Truncate) +int ParseVersionFileName(std::wstring &Name,bool Truncate) { int Version=0; - wchar *VerText=wcsrchr(Name,';'); - if (VerText!=NULL) + auto VerPos=Name.rfind(';'); + if (VerPos!=std::wstring::npos && VerPos+1FirstName;ChPtr--) - if (IsDigit(*ChPtr)) + for (size_t Pos=GetVolNumPos(Name);Pos>0;Pos--) + if (IsDigit(Name[Pos])) { - *ChPtr=N; // Set the rightmost digit to '1' and others to '0'. + Name[Pos]=N; // Set the rightmost digit to '1' and others to '0'. N='0'; } else - if (N=='0') + if (N=='0') // If we already set the rightmost '1' before. { - VolNumStart=ChPtr+1; // Store the position of leftmost digit in volume number. + VolNumStart=Pos+1; // Store the position of leftmost digit in volume number. break; } } else { // Old volume numbering scheme. Just set the extension to ".rar". - SetExt(FirstName,L"rar",MaxSize); - VolNumStart=GetExt(FirstName); + SetExt(Name,L"rar"); + VolNumStart=GetExtPos(Name); } - if (!FileExist(FirstName)) + if (!FileExist(Name)) { // If the first volume, which name we just generated, does not exist, // check if volume with same name and any other extension is available. // It can help in case of *.exe or *.sfx first volume. - wchar Mask[NM]; - wcsncpyz(Mask,FirstName,ASIZE(Mask)); - SetExt(Mask,L"*",ASIZE(Mask)); + std::wstring Mask=Name; + SetExt(Mask,L"*"); FindFile Find; Find.SetMask(Mask); FindData FD; @@ -689,32 +782,33 @@ wchar* VolNameToFirstName(const wchar *VolName,wchar *FirstName,size_t MaxSize,b Archive Arc; if (Arc.Open(FD.Name,0) && Arc.IsArchive(true) && Arc.FirstVolume) { - wcsncpyz(FirstName,FD.Name,MaxSize); + Name=FD.Name; break; } } } + FirstName=Name; return VolNumStart; } #endif #ifndef SFX_MODULE -static void GenArcName(wchar *ArcName,size_t MaxSize,const wchar *GenerateMask,uint ArcNumber,bool &ArcNumPresent) +static void GenArcName(std::wstring &ArcName,const std::wstring &GenerateMask,uint ArcNumber,bool &ArcNumPresent) { + size_t Pos=0; bool Prefix=false; - if (*GenerateMask=='+') + if (GenerateMask[0]=='+') { Prefix=true; // Add the time string before the archive name. - GenerateMask++; // Skip '+' in the beginning of time mask. + Pos++; // Skip '+' in the beginning of time mask. } - wchar Mask[MAX_GENERATE_MASK]; - wcsncpyz(Mask,*GenerateMask!=0 ? GenerateMask:L"yyyymmddhhmmss",ASIZE(Mask)); + std::wstring Mask=!GenerateMask.empty() ? GenerateMask.substr(Pos):L"yyyymmddhhmmss"; bool QuoteMode=false; uint MAsMinutes=0; // By default we treat 'M' as months. - for (uint I=0;Mask[I]!=0;I++) + for (uint I=0;I=4) CurWeek++; - char Field[10][6]; + const size_t FieldSize=11; + char Field[10][FieldSize]; - sprintf(Field[0],"%04u",rlt.Year); - sprintf(Field[1],"%02u",rlt.Month); - sprintf(Field[2],"%02u",rlt.Day); - sprintf(Field[3],"%02u",rlt.Hour); - sprintf(Field[4],"%02u",rlt.Minute); - sprintf(Field[5],"%02u",rlt.Second); - sprintf(Field[6],"%02u",(uint)CurWeek); - sprintf(Field[7],"%u",(uint)WeekDay+1); - sprintf(Field[8],"%03u",rlt.yDay+1); - sprintf(Field[9],"%05u",ArcNumber); + snprintf(Field[0],FieldSize,"%04u",rlt.Year); + snprintf(Field[1],FieldSize,"%02u",rlt.Month); + snprintf(Field[2],FieldSize,"%02u",rlt.Day); + snprintf(Field[3],FieldSize,"%02u",rlt.Hour); + snprintf(Field[4],FieldSize,"%02u",rlt.Minute); + snprintf(Field[5],FieldSize,"%02u",rlt.Second); + snprintf(Field[6],FieldSize,"%02u",(uint)CurWeek); + snprintf(Field[7],FieldSize,"%u",(uint)WeekDay+1); + snprintf(Field[8],FieldSize,"%03u",rlt.yDay+1); + snprintf(Field[9],FieldSize,"%05u",ArcNumber); const wchar *MaskChars=L"YMDHISWAEN"; // How many times every modifier character was encountered in the mask. - int CField[sizeof(Field)/sizeof(Field[0])]; + int CField[sizeof(Field)/sizeof(Field[0])]{}; - memset(CField,0,sizeof(CField)); QuoteMode=false; - for (uint I=0;Mask[I]!=0;I++) + for (uint I=0;I0) - DestW[DestSize-1]=0; - - return DestW; -} - - #ifdef _WIN_ALL // We should return 'true' even if resulting path is shorter than MAX_PATH, // because we can also use this function to open files with non-standard // characters, even if their path length is normal. -bool GetWinLongPath(const wchar *Src,wchar *Dest,size_t MaxSize) +bool GetWinLongPath(const std::wstring &Src,std::wstring &Dest) { - if (*Src==0) + if (Src.empty()) return false; - const wchar *Prefix=L"\\\\?\\"; - const size_t PrefixLength=4; - bool FullPath=IsDriveLetter(Src) && IsPathDiv(Src[2]); - size_t SrcLength=wcslen(Src); + const std::wstring Prefix=L"\\\\?\\"; + + bool FullPath=Src.size()>=3 && IsDriveLetter(Src) && IsPathDiv(Src[2]); if (IsFullPath(Src)) // Paths in d:\path\name format. { if (IsDriveLetter(Src)) { - if (MaxSize<=PrefixLength+SrcLength) - return false; - wcsncpyz(Dest,Prefix,MaxSize); - wcsncatz(Dest,Src,MaxSize); // "\\?\D:\very long path". + Dest=Prefix+Src; // "\\?\D:\very long path". return true; } else - if (Src[0]=='\\' && Src[1]=='\\') + if (Src.size()>2 && Src[0]=='\\' && Src[1]=='\\') { - if (MaxSize<=PrefixLength+SrcLength+2) - return false; - wcsncpyz(Dest,Prefix,MaxSize); - wcsncatz(Dest,L"UNC",MaxSize); - wcsncatz(Dest,Src+1,MaxSize); // "\\?\UNC\server\share". + Dest=Prefix+L"UNC"+Src.substr(1); // "\\?\UNC\server\share". return true; } - // We may be here only if we modify IsFullPath in the future. + // We can be here only if modify IsFullPath() in the future. return false; } else { - wchar CurDir[NM]; - DWORD DirCode=GetCurrentDirectory(ASIZE(CurDir)-1,CurDir); - if (DirCode==0 || DirCode>ASIZE(CurDir)-1) + std::wstring CurDir; + if (!GetCurDir(CurDir)) return false; if (IsPathDiv(Src[0])) // Paths in \path\name format. { - if (MaxSize<=PrefixLength+SrcLength+2) - return false; - wcsncpyz(Dest,Prefix,MaxSize); - CurDir[2]=0; - wcsncatz(Dest,CurDir,MaxSize); // Copy drive letter 'd:'. - wcsncatz(Dest,Src,MaxSize); + Dest=Prefix+CurDir[0]+L':'+Src; // Copy drive letter 'd:'. return true; } else // Paths in path\name format. { - AddEndSlash(CurDir,ASIZE(CurDir)); - if (MaxSize<=PrefixLength+wcslen(CurDir)+SrcLength) - return false; - wcsncpyz(Dest,Prefix,MaxSize); - wcsncatz(Dest,CurDir,MaxSize); + Dest=Prefix+CurDir; + AddEndSlash(Dest); + size_t Pos=0; if (Src[0]=='.' && IsPathDiv(Src[1])) // Remove leading .\ in pathname. - Src+=2; + Pos=2; - wcsncatz(Dest,Src,MaxSize); + Dest+=Src.substr(Pos); return true; } } @@ -1008,46 +1061,55 @@ bool GetWinLongPath(const wchar *Src,wchar *Dest,size_t MaxSize) // Convert Unix, OS X and Android decomposed chracters to Windows precomposed. -void ConvertToPrecomposed(wchar *Name,size_t NameSize) +void ConvertToPrecomposed(std::wstring &Name) { - wchar FileName[NM]; - if (WinNT()>=WNT_VISTA && // MAP_PRECOMPOSED is not supported in XP. - FoldString(MAP_PRECOMPOSED,Name,-1,FileName,ASIZE(FileName))!=0) - { - FileName[ASIZE(FileName)-1]=0; - wcsncpyz(Name,FileName,NameSize); - } + if (WinNT() FileName(Size); + if (FoldString(MAP_PRECOMPOSED,Name.c_str(),-1,FileName.data(),(int)FileName.size())!=0) + Name=FileName.data(); } -void MakeNameCompatible(wchar *Name,size_t MaxSize) +void MakeNameCompatible(std::wstring &Name) { // Remove trailing spaces and dots in file name and in dir names in path. - int Src=0,Dest=0; - while (true) - { - if (IsPathDiv(Name[Src]) || Name[Src]==0) - for (int I=Dest-1;I>0 && (Name[I]==' ' || Name[I]=='.');I--) + for (int I=0;I<(int)Name.size();I++) + if (I+1==Name.size() || IsPathDiv(Name[I+1])) + while (I>=0 && (Name[I]=='.' || Name[I]==' ')) { - // Permit path1/./path2 and ../path1 paths. - if (Name[I]=='.' && (IsPathDiv(Name[I-1]) || Name[I-1]=='.' && I==1)) + if (I==0 && Name[I]==' ') + { + // Windows 10 Explorer can't rename or delete " " files and folders. + Name[I]='_'; // Convert " /path" to "_/path". break; - Dest--; + } + if (Name[I]=='.') + { + // 2024.05.01: Permit ./path1, path1/./path2, ../path1, + // path1/../path2 and exotic Win32 d:.\path1, d:..\path1 paths + // requested by user. Leading dots are possible here if specified + // by user in the destination path. + if (I==0 || IsPathDiv(Name[I-1]) || I==2 && IsDriveLetter(Name)) + break; + if (I>=1 && Name[I-1]=='.' && (I==1 || IsPathDiv(Name[I-2]) || + I==3 && IsDriveLetter(Name))) + break; + } + Name.erase(I,1); + I--; } - Name[Dest]=Name[Src]; - if (Name[Src]==0) - break; - Src++; - Dest++; - } // Rename reserved device names, such as aux.txt to _aux.txt. // We check them in path components too, where they are also prohibited. - for (uint I=0;Name[I]!=0;I++) + for (size_t I=0;I0 && IsPathDiv(Name[I-1])) { static const wchar *Devices[]={L"CON",L"PRN",L"AUX",L"NUL",L"COM#",L"LPT#"}; - wchar *s=Name+I; + const wchar *s=&Name[I]; bool MatchFound=false; for (uint J=0;JI+1) // I+1, because we do not move the trailing 0. - memmove(s+1,s,(MaxSize-I-1)*sizeof(*s)); - *s='_'; + std::wstring OrigName=Name; + Name.insert(I,1,'_'); #ifndef SFX_MODULE uiMsg(UIMSG_CORRECTINGNAME,nullptr); uiMsg(UIERROR_RENAMING,nullptr,OrigName,Name); @@ -1084,3 +1143,54 @@ void MakeNameCompatible(wchar *Name,size_t MaxSize) #endif + + +#ifdef _WIN_ALL +std::wstring GetModuleFileStr() +{ + HMODULE hModule=nullptr; + + std::vector Path(256); + while (Path.size()<=MAXPATHSIZE) + { + if (GetModuleFileName(hModule,Path.data(),(DWORD)Path.size()) Buf(BufSize); + DWORD Code=GetCurrentDirectory((DWORD)Buf.size(),Buf.data()); + Dir=Buf.data(); + return Code!=0; +} +#endif + + diff --git a/multiarc/src/formats/rar/unrar/pathfn.hpp b/multiarc/src/formats/rar/unrar/pathfn.hpp index 62cae0ad2..57dab5a54 100644 --- a/multiarc/src/formats/rar/unrar/pathfn.hpp +++ b/multiarc/src/formats/rar/unrar/pathfn.hpp @@ -2,39 +2,52 @@ #define _RAR_PATHFN_ wchar* PointToName(const wchar *Path); +std::wstring PointToName(const std::wstring &Path); +size_t GetNamePos(const std::wstring &Path); wchar* PointToLastChar(const wchar *Path); -wchar* ConvertPath(const wchar *SrcPath,wchar *DestPath,size_t DestSize); -void SetName(wchar *FullName,const wchar *Name,size_t MaxSize); -void SetExt(wchar *Name,const wchar *NewExt,size_t MaxSize); -void SetSFXExt(wchar *SFXName,size_t MaxSize); +wchar GetLastChar(const std::wstring &Path); +size_t ConvertPath(const std::wstring *SrcPath,std::wstring *DestPath); +void SetName(std::wstring &FullName,const std::wstring &Name); +void SetExt(std::wstring &Name,std::wstring NewExt); +void RemoveExt(std::wstring &Name); +void SetSFXExt(std::wstring &SFXName); wchar *GetExt(const wchar *Name); -bool CmpExt(const wchar *Name,const wchar *Ext); -bool IsWildcard(const wchar *Str); +std::wstring GetExt(const std::wstring &Name); +std::wstring::size_type GetExtPos(const std::wstring &Name); +bool CmpExt(const std::wstring &Name,const std::wstring &Ext); +bool IsWildcard(const std::wstring &Str); bool IsPathDiv(int Ch); bool IsDriveDiv(int Ch); -bool IsDriveLetter(const wchar *Path); -int GetPathDisk(const wchar *Path); -void AddEndSlash(wchar *Path,size_t MaxLength); -void MakeName(const wchar *Path,const wchar *Name,wchar *Pathname,size_t MaxSize); -void GetFilePath(const wchar *FullName,wchar *Path,size_t MaxLength); -void RemoveNameFromPath(wchar *Path); +bool IsDriveLetter(const std::wstring &Path); +int GetPathDisk(const std::wstring &Path); +void AddEndSlash(std::wstring &Path); +void MakeName(const std::wstring &Path,const std::wstring &Name,std::wstring &Pathname); +void GetPathWithSep(const std::wstring &FullName,std::wstring &Path); +void RemoveNameFromPath(std::wstring &Path); #if defined(_WIN_ALL) && !defined(SFX_MODULE) -bool GetAppDataPath(wchar *Path,size_t MaxSize,bool Create); -void GetRarDataPath(wchar *Path,size_t MaxSize,bool Create); +bool GetAppDataPath(std::wstring &Path,bool Create); +void GetRarDataPath(std::wstring &Path,bool Create); +#endif +#ifdef _WIN_ALL +bool SHGetPathStrFromIDList(PCIDLIST_ABSOLUTE pidl,std::wstring &Path); #endif #ifndef SFX_MODULE -bool EnumConfigPaths(uint Number,wchar *Path,size_t MaxSize,bool Create); -void GetConfigName(const wchar *Name,wchar *FullName,size_t MaxSize,bool CheckExist,bool Create); +bool EnumConfigPaths(uint Number,std::wstring &Path,bool Create); +void GetConfigName(const std::wstring &Name,std::wstring &FullName,bool CheckExist,bool Create); #endif -wchar* GetVolNumPart(const wchar *ArcName); -void NextVolumeName(wchar *ArcName,uint MaxLength,bool OldNumbering); -bool IsNameUsable(const wchar *Name); -void MakeNameUsable(wchar *Name,bool Extended); +size_t GetVolNumPos(const std::wstring &ArcName); +void NextVolumeName(std::wstring &ArcName,bool OldNumbering); +bool IsNameUsable(const std::wstring &Name); +void MakeNameUsable(std::wstring &Name,bool Extended); void UnixSlashToDos(const char *SrcName,char *DestName,size_t MaxLength); -void DosSlashToUnix(const char *SrcName,char *DestName,size_t MaxLength); void UnixSlashToDos(const wchar *SrcName,wchar *DestName,size_t MaxLength); +void UnixSlashToDos(const std::string &SrcName,std::string &DestName); +void UnixSlashToDos(const std::wstring &SrcName,std::wstring &DestName); +void DosSlashToUnix(const char *SrcName,char *DestName,size_t MaxLength); void DosSlashToUnix(const wchar *SrcName,wchar *DestName,size_t MaxLength); +void DosSlashToUnix(const std::string &SrcName,std::string &DestName); +void DosSlashToUnix(const std::wstring &SrcName,std::wstring &DestName); inline void SlashToNative(const char *SrcName,char *DestName,size_t MaxLength) { @@ -45,6 +58,15 @@ inline void SlashToNative(const char *SrcName,char *DestName,size_t MaxLength) #endif } +inline void SlashToNative(const std::string &SrcName,std::string &DestName) +{ +#ifdef _WIN_ALL + UnixSlashToDos(SrcName,DestName); +#else + DosSlashToUnix(SrcName,DestName); +#endif +} + inline void SlashToNative(const wchar *SrcName,wchar *DestName,size_t MaxLength) { #ifdef _WIN_ALL @@ -54,22 +76,44 @@ inline void SlashToNative(const wchar *SrcName,wchar *DestName,size_t MaxLength) #endif } -void ConvertNameToFull(const wchar *Src,wchar *Dest,size_t MaxSize); -bool IsFullPath(const wchar *Path); -bool IsFullRootPath(const wchar *Path); -void GetPathRoot(const wchar *Path,wchar *Root,size_t MaxSize); -int ParseVersionFileName(wchar *Name,bool Truncate); -wchar* VolNameToFirstName(const wchar *VolName,wchar *FirstName,size_t MaxSize,bool NewNumbering); -wchar* GetWideName(const char *Name,const wchar *NameW,wchar *DestW,size_t DestSize); +inline void SlashToNative(const std::wstring &SrcName,std::wstring &DestName) +{ +#ifdef _WIN_ALL + UnixSlashToDos(SrcName,DestName); +#else + DosSlashToUnix(SrcName,DestName); +#endif +} + +void ConvertNameToFull(const std::wstring &Src,std::wstring &Dest); +bool IsFullPath(const std::wstring &Path); +bool IsFullRootPath(const std::wstring &Path); +void GetPathRoot(const std::wstring &Path,std::wstring &Root); +int ParseVersionFileName(std::wstring &Name,bool Truncate); +size_t VolNameToFirstName(const std::wstring &VolName,std::wstring &FirstName,bool NewNumbering); #ifndef SFX_MODULE -void GenerateArchiveName(wchar *ArcName,size_t MaxSize,const wchar *GenerateMask,bool Archiving); +void GenerateArchiveName(std::wstring &ArcName,const std::wstring &GenerateMask,bool Archiving); +#endif + +#ifdef _WIN_ALL +bool GetWinLongPath(const std::wstring &Src,std::wstring &Dest); +void ConvertToPrecomposed(std::wstring &Name); +void MakeNameCompatible(std::wstring &Name); +#endif + + +#ifdef _WIN_ALL +std::wstring GetModuleFileStr(); +std::wstring GetProgramFile(const std::wstring &Name); +#endif + +#if defined(_WIN_ALL) +bool SetCurDir(const std::wstring &Dir); #endif #ifdef _WIN_ALL -bool GetWinLongPath(const wchar *Src,wchar *Dest,size_t MaxSize); -void ConvertToPrecomposed(wchar *Name,size_t NameSize); -void MakeNameCompatible(wchar *Name,size_t MaxSize); +bool GetCurDir(std::wstring &Dir); #endif diff --git a/multiarc/src/formats/rar/unrar/qopen.cpp b/multiarc/src/formats/rar/unrar/qopen.cpp index d906d06bd..e4ae8a9d1 100644 --- a/multiarc/src/formats/rar/unrar/qopen.cpp +++ b/multiarc/src/formats/rar/unrar/qopen.cpp @@ -114,7 +114,7 @@ void QuickOpen::Load(uint64 BlockPos) RawDataPos=0; ReadBufSize=0; ReadBufPos=0; - LastReadHeader.Reset(); + LastReadHeader.clear(); LastReadHeaderPos=0; ReadBuffer(); @@ -126,7 +126,7 @@ bool QuickOpen::Read(void *Data,size_t Size,size_t &Result) if (!Loaded) return false; // Find next suitable cached block. - while (LastReadHeaderPos+LastReadHeader.Size()<=SeekPos) + while (LastReadHeaderPos+LastReadHeader.size()<=SeekPos) if (!ReadNext()) break; if (!Loaded) @@ -138,9 +138,9 @@ bool QuickOpen::Read(void *Data,size_t Size,size_t &Result) return false; } - if (SeekPos>=LastReadHeaderPos && SeekPos+Size<=LastReadHeaderPos+LastReadHeader.Size()) + if (SeekPos>=LastReadHeaderPos && SeekPos+Size<=LastReadHeaderPos+LastReadHeader.size()) { - memcpy(Data,LastReadHeader+size_t(SeekPos-LastReadHeaderPos),Size); + memcpy(Data,&LastReadHeader[size_t(SeekPos-LastReadHeaderPos)],Size); Result=Size; SeekPos+=Size; UnsyncSeekPos=true; @@ -292,8 +292,8 @@ bool QuickOpen::ReadNext() size_t HeaderSize=(size_t)Raw.GetV(); if (HeaderSize>MAX_HEADER_SIZE_RAR5) return false; - LastReadHeader.Alloc(HeaderSize); - Raw.GetB(&LastReadHeader[0],HeaderSize); + LastReadHeader.resize(HeaderSize); + Raw.GetB(LastReadHeader.data(),HeaderSize); // Calculate the absolute position as offset from quick open service header. LastReadHeaderPos=QOHeaderPos-Offset; return true; diff --git a/multiarc/src/formats/rar/unrar/qopen.hpp b/multiarc/src/formats/rar/unrar/qopen.hpp index d745cea80..7adca4b5e 100644 --- a/multiarc/src/formats/rar/unrar/qopen.hpp +++ b/multiarc/src/formats/rar/unrar/qopen.hpp @@ -43,7 +43,7 @@ class QuickOpen uint64 RawDataPos; // Current read position in QO data. size_t ReadBufSize; // Size of Buf data currently read from QO. size_t ReadBufPos; // Current read position in Buf data. - Array LastReadHeader; + std::vector LastReadHeader; uint64 LastReadHeaderPos; uint64 SeekPos; bool UnsyncSeekPos; // QOpen SeekPos does not match an actual file pointer. diff --git a/multiarc/src/formats/rar/unrar/rar.cpp b/multiarc/src/formats/rar/unrar/rar.cpp index 0b42b913a..5238b7686 100644 --- a/multiarc/src/formats/rar/unrar/rar.cpp +++ b/multiarc/src/formats/rar/unrar/rar.cpp @@ -15,11 +15,11 @@ int rar_main(int argc, char *argv[]) ErrHandler.SetSignalHandlers(true); #ifdef SFX_MODULE - wchar ModuleName[NM]; + std::wstring ModuleName; #ifdef _WIN_ALL - GetModuleFileName(NULL,ModuleName,ASIZE(ModuleName)); + ModuleName=GetModuleFileStr(); #else - CharToWide(argv[0],ModuleName,ASIZE(ModuleName)); + CharToWide(argv[0],ModuleName); #endif #endif @@ -38,9 +38,10 @@ int rar_main(int argc, char *argv[]) try { - CommandData *Cmd=new CommandData; + // Use std::unique_ptr to free Cmd in case of exception. + std::unique_ptr Cmd(new CommandData); #ifdef SFX_MODULE - wcsncpyz(Cmd->Command,L"X",ASIZE(Cmd->Command)); + Cmd->Command=L"X"; char *Switch=argc>1 ? argv[1]:NULL; if (Switch!=NULL && Cmd->IsSwitch(Switch[0])) { @@ -71,7 +72,7 @@ int rar_main(int argc, char *argv[]) #if defined(_WIN_ALL) && !defined(SFX_MODULE) ShutdownOnClose=Cmd->Shutdown; - if (ShutdownOnClose) + if (ShutdownOnClose!=POWERMODE_KEEP) ShutdownCheckAnother(true); #endif @@ -81,7 +82,6 @@ int rar_main(int argc, char *argv[]) Cmd->OutTitle(); Cmd->ProcessCommand(); - delete Cmd; } catch (RAR_EXIT ErrCode) { @@ -103,7 +103,6 @@ int rar_main(int argc, char *argv[]) Shutdown(ShutdownOnClose); #endif ErrHandler.MainExit=true; + CloseLogOptions(); return ErrHandler.GetErrorCode(); } - - diff --git a/multiarc/src/formats/rar/unrar/rar.hpp b/multiarc/src/formats/rar/unrar/rar.hpp index 67edb6735..38cd74446 100644 --- a/multiarc/src/formats/rar/unrar/rar.hpp +++ b/multiarc/src/formats/rar/unrar/rar.hpp @@ -16,7 +16,6 @@ #include "unicode.hpp" #include "errhnd.hpp" #include "secpassword.hpp" -#include "array.hpp" #include "strlist.hpp" #include "timefn.hpp" #include "sha1.hpp" @@ -85,9 +84,6 @@ #include "global.hpp" -#if 0 -#include "benchmark.hpp" -#endif diff --git a/multiarc/src/formats/rar/unrar/rardefs.hpp b/multiarc/src/formats/rar/unrar/rardefs.hpp index 6858d39c1..433010f22 100644 --- a/multiarc/src/formats/rar/unrar/rardefs.hpp +++ b/multiarc/src/formats/rar/unrar/rardefs.hpp @@ -17,13 +17,28 @@ #define MAXPASSWORD 512 #define MAXPASSWORD_RAR 128 +// Set some arbitrary sensible limit to maximum path length to prevent +// the excessive memory allocation for dynamically allocated strings. +#define MAXPATHSIZE 0x10000 + #define MAXSFXSIZE 0x200000 #define MAXCMTSIZE 0x40000 +#ifdef _WIN_32 +#define DefSFXName L"default32.sfx" +#else #define DefSFXName L"default.sfx" +#endif #define DefSortListName L"rarfiles.lst" +// Maximum dictionary allowed by compression. Can be less than +// maximum dictionary supported by decompression. +#define PACK_MAX_DICT 0x1000000000ULL // 64 GB. + +// Maximum dictionary allowed by decompression. +#define UNPACK_MAX_DICT 0x1000000000ULL // 64 GB. + #ifndef SFX_MODULE #define USE_QOPEN diff --git a/multiarc/src/formats/rar/unrar/raros.hpp b/multiarc/src/formats/rar/unrar/raros.hpp index 4f4f2ae79..b701637ab 100644 --- a/multiarc/src/formats/rar/unrar/raros.hpp +++ b/multiarc/src/formats/rar/unrar/raros.hpp @@ -1,15 +1,6 @@ #ifndef _RAR_RAROS_ #define _RAR_RAROS_ -#ifdef __EMX__ - #define _EMX -#endif - -#ifdef __DJGPP__ - #define _DJGPP - #define _EMX -#endif - #if defined(__WIN32__) || defined(_WIN32) #define _WIN_ALL // Defined for all Windows platforms, 32 and 64 bit, mobile and desktop. #ifdef _M_X64 @@ -29,7 +20,7 @@ #define _APPLE #endif -#if !defined(_EMX) && !defined(_WIN_ALL) && !defined(_BEOS) && !defined(_APPLE) +#if !defined(_WIN_ALL) && !defined(_UNIX) #define _UNIX #endif diff --git a/multiarc/src/formats/rar/unrar/rartypes.hpp b/multiarc/src/formats/rar/unrar/rartypes.hpp index 3d3111bc3..a612c345d 100644 --- a/multiarc/src/formats/rar/unrar/rartypes.hpp +++ b/multiarc/src/formats/rar/unrar/rartypes.hpp @@ -5,7 +5,7 @@ typedef uint8_t byte; // Unsigned 8 bits. typedef uint16_t ushort; // Preferably 16 bits, but can be more. -typedef unsigned int uint; // 32 bits or more. +typedef unsigned int uint; // Preferably 32 bits, likely can be more. typedef uint32_t uint32; // 32 bits exactly. typedef int32_t int32; // Signed 32 bits exactly. typedef uint64_t uint64; // 64 bits exactly. diff --git a/multiarc/src/formats/rar/unrar/rarvm.cpp b/multiarc/src/formats/rar/unrar/rarvm.cpp index 8d8675a39..c44e02101 100644 --- a/multiarc/src/formats/rar/unrar/rarvm.cpp +++ b/multiarc/src/formats/rar/unrar/rarvm.cpp @@ -246,7 +246,7 @@ bool RarVM::ExecuteStandardFilter(VM_StandardFilters FilterType) } else Predicted=PrevByte; - DestData[I]=PrevByte=(byte)(Predicted-*(SrcData++)); + PrevByte=DestData[I]=(byte)(Predicted-*(SrcData++)); } } for (uint I=PosR,Border=DataSize-2;I>16); D[3]=(byte)(Field>>24); #else - *(uint32 *)Data=Field; + *(uint32 *)Data=(uint32)Field; #endif } @@ -87,18 +87,32 @@ inline uint32 RawGetBE4(const byte *m) #elif defined(USE_MEM_BYTESWAP) && (defined(__clang__) || defined(__GNUC__)) return __builtin_bswap32(*(uint32 *)m); #else - return uint32(m[0]<<24) | uint32(m[1]<<16) | uint32(m[2]<<8) | m[3]; + return uint32(m[0])<<24 | uint32(m[1])<<16 | uint32(m[2])<<8 | m[3]; +#endif +} + + +// Load 8 big endian bytes from memory and return uint64. +inline uint64 RawGetBE8(const byte *m) +{ +#if defined(USE_MEM_BYTESWAP) && defined(_MSC_VER) + return _byteswap_uint64(*(uint64 *)m); +#elif defined(USE_MEM_BYTESWAP) && (defined(__clang__) || defined(__GNUC__)) + return __builtin_bswap64(*(uint64 *)m); +#else + return uint64(m[0])<<56 | uint64(m[1])<<48 | uint64(m[2])<<40 | uint64(m[3])<<32 | + uint64(m[4])<<24 | uint64(m[5])<<16 | uint64(m[6])<<8 | m[7]; #endif } // Save integer to memory as big endian. -inline void RawPutBE4(uint32 i,byte *mem) +inline void RawPutBE4(uint i,byte *mem) { #if defined(USE_MEM_BYTESWAP) && defined(_MSC_VER) - *(uint32*)mem = _byteswap_ulong(i); + *(uint32*)mem = _byteswap_ulong((uint32)i); #elif defined(USE_MEM_BYTESWAP) && (defined(__clang__) || defined(__GNUC__)) - *(uint32*)mem = __builtin_bswap32(i); + *(uint32*)mem = __builtin_bswap32((uint32)i); #else mem[0]=byte(i>>24); mem[1]=byte(i>>16); @@ -108,6 +122,26 @@ inline void RawPutBE4(uint32 i,byte *mem) } +// Save integer to memory as big endian. +inline void RawPutBE8(uint64 i,byte *mem) +{ +#if defined(USE_MEM_BYTESWAP) && defined(_MSC_VER) + *(uint64*)mem = _byteswap_uint64(i); +#elif defined(USE_MEM_BYTESWAP) && (defined(__clang__) || defined(__GNUC__)) + *(uint64*)mem = __builtin_bswap64(i); +#else + mem[0]=byte(i>>56); + mem[1]=byte(i>>48); + mem[2]=byte(i>>40); + mem[3]=byte(i>>32); + mem[4]=byte(i>>24); + mem[5]=byte(i>>16); + mem[6]=byte(i>>8); + mem[7]=byte(i); +#endif +} + + inline uint32 ByteSwap32(uint32 i) { #ifdef _MSC_VER @@ -119,4 +153,29 @@ inline uint32 ByteSwap32(uint32 i) #endif } + + + +inline bool IsPow2(uint64 n) // Check if 'n' is power of 2. +{ + return (n & (n-1))==0; +} + + +inline uint64 GetGreaterOrEqualPow2(uint64 n) +{ + uint64 p=1; + while (pRead(&Data[FullSize],AlignedReadSize); Crypt->DecryptBlock(&Data[FullSize],AlignedReadSize); DataSize+=ReadSize==0 ? 0:Size; @@ -55,7 +55,7 @@ size_t RawRead::Read(size_t Size) #endif if (Size!=0) { - Data.Add(Size); + Data.resize(Data.size()+Size); ReadSize=SrcFile->Read(&Data[DataSize],Size); DataSize+=ReadSize; } @@ -67,7 +67,7 @@ void RawRead::Read(byte *SrcData,size_t Size) { if (Size!=0) { - Data.Add(Size); + Data.resize(Data.size()+Size); memcpy(&Data[DataSize],SrcData,Size); DataSize+=Size; } @@ -96,8 +96,7 @@ uint RawRead::Get4() { if (ReadPos+3 Data; + std::vector Data; File *SrcFile; size_t DataSize; size_t ReadPos; @@ -27,7 +27,7 @@ class RawRead uint GetCRC50(); byte* GetDataPtr() {return &Data[0];} size_t Size() {return DataSize;} - size_t PaddedSize() {return Data.Size()-DataSize;} + size_t PaddedSize() {return Data.size()-DataSize;} size_t DataLeft() {return DataSize-ReadPos;} size_t GetPos() {return ReadPos;} void SetPos(size_t Pos) {ReadPos=Pos;} diff --git a/multiarc/src/formats/rar/unrar/rdwrfn.cpp b/multiarc/src/formats/rar/unrar/rdwrfn.cpp index fa7137601..a49a8bced 100644 --- a/multiarc/src/formats/rar/unrar/rdwrfn.cpp +++ b/multiarc/src/formats/rar/unrar/rdwrfn.cpp @@ -246,28 +246,28 @@ void ComprDataIO::GetUnpackedData(byte **Data,size_t *Size) } -void ComprDataIO::SetEncryption(bool Encrypt,CRYPT_METHOD Method, +// Return true if encryption or decryption mode is set correctly. +bool ComprDataIO::SetEncryption(bool Encrypt,CRYPT_METHOD Method, SecPassword *Password,const byte *Salt,const byte *InitV, uint Lg2Cnt,byte *HashKey,byte *PswCheck) { -#ifndef RAR_NOCRYPT +#ifdef RAR_NOCRYPT + return false; +#else if (Encrypt) + { Encryption=Crypt->SetCryptKeys(true,Method,Password,Salt,InitV,Lg2Cnt,HashKey,PswCheck); + return Encryption; + } else + { Decryption=Decrypt->SetCryptKeys(false,Method,Password,Salt,InitV,Lg2Cnt,HashKey,PswCheck); + return Decryption; + } #endif } -#if !defined(SFX_MODULE) && !defined(RAR_NOCRYPT) -void ComprDataIO::SetAV15Encryption() -{ - Decryption=true; - Decrypt->SetAV15Encryption(); -} -#endif - - #if !defined(SFX_MODULE) && !defined(RAR_NOCRYPT) void ComprDataIO::SetCmt13Encryption() { @@ -299,11 +299,15 @@ void ComprDataIO::AdjustTotalArcSize(Archive *Arc) // packed size to beginning of these blocks. Earlier we already calculated // the total size based on entire archive sizes. We also set LastArcSize // to start of first trailing block, to add it later to ProcessedArcSize. - int64 ArcLength=Arc->IsSeekable() ? Arc->FileLength() : 0; - if (Arc->MainHead.QOpenOffset!=0) // QO is always preceding RR record. + uint64 ArcLength=Arc->IsSeekable() ? Arc->FileLength() : 0; + // QO is always preceding RR record. + // Also we check QO and RR to be less than archive length to prevent + // negative "ArcLength-LastArcSize" and possible signed integer overflow + // when calculating TotalArcSize. + if (Arc->MainHead.QOpenOffset>0 && Arc->MainHead.QOpenOffsetMainHead.QOpenOffset; else - if (Arc->MainHead.RROffset!=0) + if (Arc->MainHead.RROffset>0 && Arc->MainHead.RROffsetMainHead.RROffset; else { diff --git a/multiarc/src/formats/rar/unrar/rdwrfn.hpp b/multiarc/src/formats/rar/unrar/rdwrfn.hpp index 3060a0ff3..58685ca36 100644 --- a/multiarc/src/formats/rar/unrar/rdwrfn.hpp +++ b/multiarc/src/formats/rar/unrar/rdwrfn.hpp @@ -6,10 +6,6 @@ class CmdAdd; class Unpack; class ArcFileSearch; -#if 0 -// We use external i/o calls for Benchmark command. -#define COMPRDATAIO_EXTIO -#endif class ComprDataIO { @@ -70,9 +66,8 @@ class ComprDataIO void SetFiles(File *SrcFile,File *DestFile); void SetCommand(CmdAdd *Cmd) {Command=Cmd;} void SetSubHeader(FileHeader *hd,int64 *Pos) {SubHead=hd;SubHeadPos=Pos;} - void SetEncryption(bool Encrypt,CRYPT_METHOD Method,SecPassword *Password, + bool SetEncryption(bool Encrypt,CRYPT_METHOD Method,SecPassword *Password, const byte *Salt,const byte *InitV,uint Lg2Cnt,byte *HashKey,byte *PswCheck); - void SetAV15Encryption(); void SetCmt13Encryption(); void SetUnpackToMemory(byte *Addr,uint Size); void SetCurrentCommand(wchar Cmd) {CurrentCommand=Cmd;} diff --git a/multiarc/src/formats/rar/unrar/recvol.cpp b/multiarc/src/formats/rar/unrar/recvol.cpp index b17820711..426d9ceeb 100644 --- a/multiarc/src/formats/rar/unrar/recvol.cpp +++ b/multiarc/src/formats/rar/unrar/recvol.cpp @@ -1,11 +1,12 @@ #include "rar.hpp" + #include "recvol3.cpp" #include "recvol5.cpp" -bool RecVolumesRestore(CommandData *Cmd,const wchar *Name,bool Silent) +bool RecVolumesRestore(CommandData *Cmd,const std::wstring &Name,bool Silent) { Archive Arc(Cmd); if (!Arc.Open(Name)) @@ -42,24 +43,20 @@ bool RecVolumesRestore(CommandData *Cmd,const wchar *Name,bool Silent) } -void RecVolumesTest(CommandData *Cmd,Archive *Arc,const wchar *Name) +void RecVolumesTest(CommandData *Cmd,Archive *Arc,const std::wstring &Name) { - wchar RevName[NM]; - *RevName=0; - if (Arc!=NULL) + std::wstring RevName; + if (Arc==NULL) + RevName=Name; + else { // We received .rar or .exe volume as a parameter, trying to find // the matching .rev file number 1. bool NewNumbering=Arc->NewNumbering; - wchar ArcName[NM]; - wcsncpyz(ArcName,Name,ASIZE(ArcName)); - - wchar *VolNumStart=VolNameToFirstName(ArcName,ArcName,ASIZE(ArcName),NewNumbering); - wchar RecVolMask[NM]; - wcsncpyz(RecVolMask,ArcName,ASIZE(RecVolMask)); - size_t BaseNamePartLength=VolNumStart-ArcName; - wcsncpyz(RecVolMask+BaseNamePartLength,L"*.rev",ASIZE(RecVolMask)-BaseNamePartLength); + std::wstring RecVolMask; + size_t VolNumStart=VolNameToFirstName(Name,RecVolMask,NewNumbering); + RecVolMask.replace(VolNumStart, std::wstring::npos, L"*.rev"); FindFile Find; Find.SetMask(RecVolMask); @@ -67,31 +64,30 @@ void RecVolumesTest(CommandData *Cmd,Archive *Arc,const wchar *Name) while (Find.Next(&RecData)) { - wchar *Num=GetVolNumPart(RecData.Name); - if (*Num!='1') // Name must have "0...01" numeric part. + size_t NumPos=GetVolNumPos(RecData.Name); + if (RecData.Name[NumPos]!='1') // Name must have "0...01" numeric part. continue; bool FirstVol=true; - while (--Num>=RecData.Name && IsDigit(*Num)) - if (*Num!='0') + while (NumPos>0 && IsDigit(RecData.Name[--NumPos])) + if (RecData.Name[NumPos]!='0') { FirstVol=false; break; } if (FirstVol) { - wcsncpyz(RevName,RecData.Name,ASIZE(RevName)); - Name=RevName; + RevName=RecData.Name; break; } } - if (*RevName==0) // First .rev file not found. + if (RevName.empty()) // First .rev file not found. return; } File RevFile; - if (!RevFile.Open(Name)) + if (!RevFile.Open(RevName)) { - ErrHandler.OpenErrorMsg(Name); // It also sets RARX_OPEN. + ErrHandler.OpenErrorMsg(RevName); // It also sets RARX_OPEN. return; } mprintf(L"\n"); @@ -101,11 +97,11 @@ void RecVolumesTest(CommandData *Cmd,Archive *Arc,const wchar *Name) if (Rev5) { RecVolumes5 RecVol(Cmd,true); - RecVol.Test(Cmd,Name); + RecVol.Test(Cmd,RevName); } else { RecVolumes3 RecVol(Cmd,true); - RecVol.Test(Cmd,Name); + RecVol.Test(Cmd,RevName); } } diff --git a/multiarc/src/formats/rar/unrar/recvol.hpp b/multiarc/src/formats/rar/unrar/recvol.hpp index 4a6d663ff..159bd1adc 100644 --- a/multiarc/src/formats/rar/unrar/recvol.hpp +++ b/multiarc/src/formats/rar/unrar/recvol.hpp @@ -8,7 +8,7 @@ class RecVolumes3 { private: File *SrcFile[256]; - Array Buf; + std::vector Buf; #ifdef RAR_SMP ThreadPool *RSThreadPool; @@ -16,16 +16,16 @@ class RecVolumes3 public: RecVolumes3(CommandData *Cmd,bool TestOnly); ~RecVolumes3(); - void Make(CommandData *Cmd,wchar *ArcName); - bool Restore(CommandData *Cmd,const wchar *Name,bool Silent); - void Test(CommandData *Cmd,const wchar *Name); + void Make(CommandData *Cmd,std::wstring ArcName); + bool Restore(CommandData *Cmd,const std::wstring &Name,bool Silent); + void Test(CommandData *Cmd,const std::wstring &Name); }; struct RecVolItem { File *f; - wchar Name[NM]; + std::wstring Name; uint CRC; uint64 FileSize; bool New; // Newly created RAR volume. @@ -52,7 +52,7 @@ class RecVolumes5 void ProcessRS(CommandData *Cmd,uint MaxRead,bool Encode); uint ReadHeader(File *RecFile,bool FirstRev); - Array RecItems; + std::vector RecItems; byte *RealReadBuffer; // Real pointer returned by 'new'. byte *ReadBuffer; // Pointer aligned for SSE instructions. @@ -78,11 +78,11 @@ class RecVolumes5 public: RecVolumes5(CommandData *Cmd,bool TestOnly); ~RecVolumes5(); - bool Restore(CommandData *Cmd,const wchar *Name,bool Silent); - void Test(CommandData *Cmd,const wchar *Name); + bool Restore(CommandData *Cmd,const std::wstring &Name,bool Silent); + void Test(CommandData *Cmd,const std::wstring &Name); }; -bool RecVolumesRestore(CommandData *Cmd,const wchar *Name,bool Silent); -void RecVolumesTest(CommandData *Cmd,Archive *Arc,const wchar *Name); +bool RecVolumesRestore(CommandData *Cmd,const std::wstring &Name,bool Silent); +void RecVolumesTest(CommandData *Cmd,Archive *Arc,const std::wstring &Name); #endif diff --git a/multiarc/src/formats/rar/unrar/recvol3.cpp b/multiarc/src/formats/rar/unrar/recvol3.cpp index ecf6dd35e..4199deed1 100644 --- a/multiarc/src/formats/rar/unrar/recvol3.cpp +++ b/multiarc/src/formats/rar/unrar/recvol3.cpp @@ -23,12 +23,6 @@ class RSEncode // Encode or decode data area, one object per one thread. #ifdef RAR_SMP -THREAD_PROC(RSEncodeThread) -{ - RSEncode *rs=(RSEncode *)Data; - rs->EncodeBuf(); -} - THREAD_PROC(RSDecodeThread) { RSEncode *rs=(RSEncode *)Data; @@ -47,8 +41,7 @@ RecVolumes3::RecVolumes3(CommandData *Cmd,bool TestOnly) } else { - Buf.Alloc(TotalBufferSize); - memset(SrcFile,0,sizeof(SrcFile)); + Buf.resize(TotalBufferSize); #ifdef RAR_SMP RSThreadPool=new ThreadPool(Cmd->Threads); #endif @@ -68,30 +61,16 @@ RecVolumes3::~RecVolumes3() -void RSEncode::EncodeBuf() -{ - for (int BufPos=BufStart;BufPosName;Ext--) - if (!IsDigit(*Ext)) - if (*Ext=='_' && IsDigit(*(Ext-1))) + for (ExtPos--;ExtPos>0;ExtPos--) + if (!IsDigit(Name[ExtPos])) + if (Name[ExtPos]=='_' && IsDigit(Name[ExtPos-1])) DigitGroup++; else break; @@ -99,19 +78,19 @@ static bool IsNewStyleRev(const wchar *Name) } -bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) +bool RecVolumes3::Restore(CommandData *Cmd,const std::wstring &Name,bool Silent) { - wchar ArcName[NM]; - wcsncpyz(ArcName,Name,ASIZE(ArcName)); - wchar *Ext=GetExt(ArcName); + std::wstring ArcName=Name; bool NewStyle=false; // New style .rev volumes are supported since RAR 3.10. - bool RevName=Ext!=NULL && wcsicomp(Ext,L".rev")==0; + bool RevName=CmpExt(ArcName,L"rev"); if (RevName) { NewStyle=IsNewStyleRev(ArcName); - while (Ext>ArcName+1 && (IsDigit(*(Ext-1)) || *(Ext-1)=='_')) - Ext--; - wcsncpyz(Ext,L"*.*",ASIZE(ArcName)-(Ext-ArcName)); + + size_t ExtPos=GetExtPos(ArcName); + while (ExtPos>1 && (IsDigit(ArcName[ExtPos-1]) || ArcName[ExtPos-1]=='_')) + ExtPos--; + ArcName.replace(ExtPos,std::wstring::npos,L"*.*"); FindFile Find; Find.SetMask(ArcName); @@ -121,7 +100,7 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) Archive Arc(Cmd); if (Arc.WOpen(fd.Name) && Arc.IsArchive(true)) { - wcsncpyz(ArcName,fd.Name,ASIZE(ArcName)); + ArcName=fd.Name; break; } } @@ -138,11 +117,10 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) bool NewNumbering=Arc.NewNumbering; Arc.Close(); - wchar *VolNumStart=VolNameToFirstName(ArcName,ArcName,ASIZE(ArcName),NewNumbering); - wchar RecVolMask[NM]; - wcsncpyz(RecVolMask,ArcName,ASIZE(RecVolMask)); - size_t BaseNamePartLength=VolNumStart-ArcName; - wcsncpyz(RecVolMask+BaseNamePartLength,L"*.rev",ASIZE(RecVolMask)-BaseNamePartLength); + size_t VolNumStart=VolNameToFirstName(ArcName,ArcName,NewNumbering); + std::wstring RecVolMask=ArcName; + RecVolMask.replace(VolNumStart,std::wstring::npos,L"*.rev"); + size_t BaseNamePartLength=VolNumStart; int64 RecFileSize=0; @@ -155,25 +133,25 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) Find.SetMask(RecVolMask); FindData RecData; int FileNumber=0,RecVolNumber=0,FoundRecVolumes=0,MissingVolumes=0; - wchar PrevName[NM]; + std::wstring PrevName; while (Find.Next(&RecData)) { - wchar *CurName=RecData.Name; + std::wstring CurName=RecData.Name; int P[3]; if (!RevName && !NewStyle) { NewStyle=true; - wchar *Dot=GetExt(CurName); - if (Dot!=NULL) + size_t DotPos=GetExtPos(CurName); + if (DotPos!=std::wstring::npos) { - int LineCount=0; - Dot--; - while (Dot>CurName && *Dot!='.') + uint LineCount=0; + DotPos--; + while (DotPos>0 && CurName[DotPos]!='.') { - if (*Dot=='_') + if (CurName[DotPos]=='_') LineCount++; - Dot--; + DotPos--; } if (LineCount==2) NewStyle=false; @@ -209,24 +187,24 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) } else { - wchar *Dot=GetExt(CurName); - if (Dot==NULL) + size_t DotPos=GetExtPos(CurName); + if (DotPos==std::wstring::npos) continue; bool WrongParam=false; for (size_t I=0;I=CurName+BaseNamePartLength); - P[I]=atoiw(Dot+1); + DotPos--; + } while (IsDigit(CurName[DotPos]) && DotPos>=BaseNamePartLength); + P[I]=atoiw(&CurName[DotPos+1]); if (P[I]==0 || P[I]>255) WrongParam=true; } if (WrongParam) continue; } - if (P[1]+P[2]>255) + if (P[0]<=0 || P[1]<=0 || P[2]<=0 || P[1]+P[2]>255 || P[0]+P[2]-1>255) continue; if (RecVolNumber!=0 && RecVolNumber!=P[1] || FileNumber!=0 && FileNumber!=P[2]) { @@ -235,10 +213,17 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) } RecVolNumber=P[1]; FileNumber=P[2]; - wcsncpyz(PrevName,CurName,ASIZE(PrevName)); + PrevName=CurName; File *NewFile=new File; NewFile->TOpen(CurName); - SrcFile[FileNumber+P[0]-1]=NewFile; + + // This check is redundant taking into account P[I]>255 and P[0]+P[2]-1>255 + // checks above. Still we keep it here for better clarity and security. + int SrcPos=FileNumber+P[0]-1; + if (SrcPos<0 || SrcPos>=ASIZE(SrcFile)) + continue; + SrcFile[SrcPos]=NewFile; + FoundRecVolumes++; if (RecFileSize==0) @@ -249,11 +234,9 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) if (FoundRecVolumes==0) return false; - bool WriteFlags[256]; - memset(WriteFlags,0,sizeof(WriteFlags)); + bool WriteFlags[256]{}; - wchar LastVolName[NM]; - *LastVolName=0; + std::wstring LastVolName; for (int CurArcNum=0;CurArcNumClose(); - wchar NewName[NM]; - wcsncpyz(NewName,ArcName,ASIZE(NewName)); - wcsncatz(NewName,L".bad",ASIZE(NewName)); + std::wstring NewName=ArcName+L".bad"; uiMsg(UIMSG_BADARCHIVE,ArcName); uiMsg(UIMSG_RENAMING,ArcName,NewName); @@ -322,13 +303,13 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) MissingVolumes++; if (CurArcNum==FileNumber-1) - wcsncpyz(LastVolName,ArcName,ASIZE(LastVolName)); + LastVolName=ArcName; uiMsg(UIMSG_MISSINGVOL,ArcName); uiMsg(UIEVENT_NEWARCHIVE,ArcName); } SrcFile[CurArcNum]=(File*)NewFile; - NextVolumeName(ArcName,ASIZE(ArcName),!NewNumbering); + NextVolumeName(ArcName,!NewNumbering); } uiMsg(UIMSG_RECVOLMISSING,MissingVolumes); @@ -453,7 +434,7 @@ bool RecVolumes3::Restore(CommandData *Cmd,const wchar *Name,bool Silent) CurFile->Close(); SrcFile[I]=NULL; } - if (*LastVolName!=0) + if (!LastVolName.empty()) { // Truncate the last volume to its real size. Archive Arc(Cmd); @@ -497,7 +478,7 @@ void RSEncode::DecodeBuf() } -void RecVolumes3::Test(CommandData *Cmd,const wchar *Name) +void RecVolumes3::Test(CommandData *Cmd,const std::wstring &Name) { if (!IsNewStyleRev(Name)) // RAR 3.0 name#_#_#.rev do not include CRC32. { @@ -505,8 +486,7 @@ void RecVolumes3::Test(CommandData *Cmd,const wchar *Name) return; } - wchar VolName[NM]; - wcsncpyz(VolName,Name,ASIZE(VolName)); + std::wstring VolName=Name; while (FileExist(VolName)) { @@ -518,7 +498,7 @@ void RecVolumes3::Test(CommandData *Cmd,const wchar *Name) } if (!uiStartFileExtract(VolName,false,true,false)) return; - mprintf(St(MExtrTestFile),VolName); + mprintf(St(MExtrTestFile),VolName.c_str()); mprintf(L" "); CurFile.Seek(0,SEEK_END); int64 Length=CurFile.Tell(); @@ -539,6 +519,6 @@ void RecVolumes3::Test(CommandData *Cmd,const wchar *Name) ErrHandler.SetErrorCode(RARX_CRC); } - NextVolumeName(VolName,ASIZE(VolName),false); + NextVolumeName(VolName,false); } } diff --git a/multiarc/src/formats/rar/unrar/recvol5.cpp b/multiarc/src/formats/rar/unrar/recvol5.cpp index 2d9c94711..e094b1711 100644 --- a/multiarc/src/formats/rar/unrar/recvol5.cpp +++ b/multiarc/src/formats/rar/unrar/recvol5.cpp @@ -48,8 +48,8 @@ RecVolumes5::~RecVolumes5() { delete[] RealBuf; delete[] RealReadBuffer; - for (uint I=0;IArcName && IsDigit(*(Num-1))) - Num--; - if (Num<=PointToName(ArcName)) + size_t NumPos=GetVolNumPos(ArcName); + while (NumPos>0 && IsDigit(ArcName[NumPos-1])) + NumPos--; + if (NumPos<=GetNamePos(ArcName)) return false; // Numeric part is missing or entire volume name is numeric, not possible for RAR or REV volume. - wcsncpyz(Num,L"*.*",ASIZE(ArcName)-(Num-ArcName)); + ArcName.replace(NumPos,std::wstring::npos,L"*.*"); - wchar FirstVolName[NM]; - *FirstVolName=0; - - wchar LongestRevName[NM]; - *LongestRevName=0; + std::wstring FirstVolName; + std::wstring LongestRevName; int64 RecFileSize=0; @@ -184,8 +180,8 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) ItemPos=RecNum; FoundRecVolumes++; - if (wcslen(fd.Name)>wcslen(LongestRevName)) - wcsncpyz(LongestRevName,fd.Name,ASIZE(LongestRevName)); + if (fd.Name.size()>LongestRevName.size()) + LongestRevName=fd.Name; } } else @@ -204,35 +200,35 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) // RAR volume found. Get its number, store the handle in appropriate // array slot, clean slots in between if we had to grow the array. - wchar *Num=GetVolNumPart(fd.Name); + size_t NumPos=GetVolNumPos(fd.Name); uint VolNum=0; - for (uint K=1;Num>=fd.Name && IsDigit(*Num);K*=10,Num--) - VolNum+=(*Num-'0')*K; + for (uint K=1;(int)NumPos>=0 && IsDigit(fd.Name[NumPos]);K*=10,NumPos--) + VolNum+=(fd.Name[NumPos]-'0')*K; if (VolNum==0 || VolNum>MaxVolumes) continue; - size_t CurSize=RecItems.Size(); + size_t CurSize=RecItems.size(); if (VolNum>CurSize) { - RecItems.Alloc(VolNum); - for (size_t I=CurSize;If=Vol; Item->New=false; - wcsncpyz(Item->Name,fd.Name,ASIZE(Item->Name)); + Item->Name=fd.Name; } } @@ -244,10 +240,10 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) // If we did not find even a single .rar volume, create .rar volume name // based on the longest .rev file name. Use longest .rev, so we have // enough space for volume number. - if (*FirstVolName==0) + if (FirstVolName.empty()) { - SetExt(LongestRevName,L"rar",ASIZE(LongestRevName)); - VolNameToFirstName(LongestRevName,FirstVolName,ASIZE(FirstVolName),true); + SetExt(LongestRevName,L"rar"); + VolNameToFirstName(LongestRevName,FirstVolName,true); } uiMsg(UIMSG_RECVOLCALCCHECKSUM); @@ -309,9 +305,8 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) { Item->f->Close(); - wchar NewName[NM]; - wcsncpyz(NewName,Item->Name,ASIZE(NewName)); - wcsncatz(NewName,L".bad",ASIZE(NewName)); + std::wstring NewName; + NewName=Item->Name+L".bad"; uiMsg(UIMSG_BADARCHIVE,Item->Name); uiMsg(UIMSG_RENAMING,Item->Name,NewName); @@ -322,12 +317,12 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) if ((Item->New=(Item->f==NULL))==true) { - wcsncpyz(Item->Name,FirstVolName,ASIZE(Item->Name)); + Item->Name=FirstVolName; uiMsg(UIMSG_CREATING,Item->Name); uiMsg(UIEVENT_NEWARCHIVE,Item->Name); File *NewVol=new File; bool UserReject; - if (!FileCreate(Cmd,NewVol,Item->Name,ASIZE(Item->Name),&UserReject)) + if (!FileCreate(Cmd,NewVol,Item->Name,&UserReject)) { if (!UserReject) ErrHandler.CreateErrorMsg(Item->Name); @@ -336,7 +331,7 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) NewVol->Prealloc(Item->FileSize); Item->f=NewVol; } - NextVolumeName(FirstVolName,ASIZE(FirstVolName),false); + NextVolumeName(FirstVolName,false); } @@ -389,7 +384,7 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) J++; VolNum=J++; // Use next valid REV volume data instead of RAR. } - RecVolItem *Item=RecItems+VolNum; + RecVolItem *Item=&RecItems[VolNum]; byte *B=&ReadBuf[0]; int ReadSize=0; @@ -411,7 +406,7 @@ bool RecVolumes5::Restore(CommandData *Cmd,const wchar *Name,bool Silent) for (uint I=0,J=0;IFileSize); Item->f->Write(Buf+(J++)*RecBufferSize,WriteSize); Item->FileSize-=WriteSize; @@ -477,10 +472,10 @@ uint RecVolumes5::ReadHeader(File *RecFile,bool FirstRev) { // If we have read the first valid REV file, init data structures // using information from REV header. - size_t CurSize=RecItems.Size(); - RecItems.Alloc(TotalCount); - for (size_t I=CurSize;I=1) // Check the maximum supported cpuid function. @@ -102,8 +111,24 @@ void Rijndael::Init(bool Encrypt,const byte *key,uint keyLen,const byte * initVe } else AES_NI=false; -#elif defined(USE_NEON) - AES_Neon=(getauxval(AT_HWCAP) & HWCAP_AES)!=0; +#elif defined(__GNUC__) + AES_NI=__builtin_cpu_supports("aes"); +#endif + +#elif defined(USE_NEON_AES) + #ifdef _APPLE + // getauxval isn't available in OS X + uint Value=0; + size_t Size=sizeof(Value); + int RetCode=sysctlbyname("hw.optional.arm.FEAT_AES",&Value,&Size,NULL,0); + + // We treat sysctlbyname failure with -1 return code as AES presence, + // because "hw.optional.arm.FEAT_AES" was missing in OS X 11, but AES + // still was supported by Neon. + AES_Neon=RetCode!=0 || Value!=0; + #else + AES_Neon=(getauxval(AT_HWCAP) & HWCAP_AES)!=0; + #endif #endif // Other developers asked us to initialize it to suppress "may be used @@ -156,7 +181,7 @@ void Rijndael::blockEncrypt(const byte *input,size_t inputLen,byte *outBuffer) blockEncryptSSE(input,numBlocks,outBuffer); return; } -#elif defined(USE_NEON) +#elif defined(USE_NEON_AES) if (AES_Neon) { blockEncryptNeon(input,numBlocks,outBuffer); @@ -249,7 +274,7 @@ void Rijndael::blockEncryptSSE(const byte *input,size_t numBlocks,byte *outBuffe #endif -#ifdef USE_NEON +#ifdef USE_NEON_AES void Rijndael::blockEncryptNeon(const byte *input,size_t numBlocks,byte *outBuffer) { byte *prevBlock = m_initVector; @@ -295,7 +320,7 @@ void Rijndael::blockDecrypt(const byte *input, size_t inputLen, byte *outBuffer) blockDecryptSSE(input,numBlocks,outBuffer); return; } -#elif defined(USE_NEON) +#elif defined(USE_NEON_AES) if (AES_Neon) { blockDecryptNeon(input,numBlocks,outBuffer); @@ -392,7 +417,7 @@ void Rijndael::blockDecryptSSE(const byte *input, size_t numBlocks, byte *outBuf #endif -#ifdef USE_NEON +#ifdef USE_NEON_AES void Rijndael::blockDecryptNeon(const byte *input, size_t numBlocks, byte *outBuffer) { byte iv[16]; @@ -585,16 +610,16 @@ void TestRijndael() for (uint L=0;L<3;L++) { byte Out[16]; - wchar Str[sizeof(Out)*2+1]; + std::wstring Str; uint KeyLength=128+L*64; rij.Init(true,Key[L],KeyLength,IV); for (uint I=0;I>8; - ((byte *)&T1L)[I]=gfMul(I<<4,M); - ((byte *)&T1H)[I]=gfMul(I<<4,M)>>8; - ((byte *)&T2L)[I]=gfMul(I<<8,M); - ((byte *)&T2H)[I]=gfMul(I<<8,M)>>8; - ((byte *)&T3L)[I]=gfMul(I<<12,M); - ((byte *)&T3H)[I]=gfMul(I<<12,M)>>8; + ((byte *)&T0L)[I]=byte(gfMul(I,M)); + ((byte *)&T0H)[I]=byte(gfMul(I,M)>>8); + ((byte *)&T1L)[I]=byte(gfMul(I<<4,M)); + ((byte *)&T1H)[I]=byte(gfMul(I<<4,M)>>8); + ((byte *)&T2L)[I]=byte(gfMul(I<<8,M)); + ((byte *)&T2H)[I]=byte(gfMul(I<<8,M)>>8); + ((byte *)&T3L)[I]=byte(gfMul(I<<12,M)); + ((byte *)&T3H)[I]=byte(gfMul(I<<12,M)>>8); } size_t Pos=0; diff --git a/multiarc/src/formats/rar/unrar/rs16.hpp b/multiarc/src/formats/rar/unrar/rs16.hpp index b67a7ca86..3833313c5 100644 --- a/multiarc/src/formats/rar/unrar/rs16.hpp +++ b/multiarc/src/formats/rar/unrar/rs16.hpp @@ -17,6 +17,9 @@ class RSCoder16 void InvertDecoderMatrix(); #ifdef USE_SSE +#if defined(USE_SSE) && defined(__GNUC__) + __attribute__((target("ssse3"))) +#endif bool SSE_UpdateECC(uint DataNum, uint ECCNum, const byte *Data, byte *ECC, size_t BlockSize); #endif diff --git a/multiarc/src/formats/rar/unrar/scantree.cpp b/multiarc/src/formats/rar/unrar/scantree.cpp index dbaf1e427..310559724 100644 --- a/multiarc/src/formats/rar/unrar/scantree.cpp +++ b/multiarc/src/formats/rar/unrar/scantree.cpp @@ -10,12 +10,11 @@ ScanTree::ScanTree(StringList *FileMasks,RECURSE_MODE Recurse,bool GetLinks,SCAN ScanEntireDisk=false; FolderWildcards=false; + FindStack.push_back(NULL); // We need a single NULL pointer for initial Depth==0. + SetAllMaskDepth=0; - *CurMask=0; - memset(FindStack,0,sizeof(FindStack)); Depth=0; Errors=0; - *ErrArcName=0; Cmd=NULL; ErrDirList=NULL; ErrDirSpecPathLength=NULL; @@ -42,7 +41,7 @@ SCAN_CODE ScanTree::GetNext(FindData *FD) SCAN_CODE FindCode; while (1) { - if (*CurMask==0 && !GetNextMask()) + if (CurMask.empty() && !GetNextMask()) return SCAN_DONE; #ifndef SILENT @@ -79,7 +78,7 @@ bool ScanTree::ExpandFolderMask() { bool WildcardFound=false; uint SlashPos=0; - for (int I=0;CurMask[I]!=0;I++) + for (uint I=0;I0 && ExpandedFolderList.GetString(CurMask,ASIZE(CurMask))) + if (ExpandedFolderList.ItemsCount()>0 && ExpandedFolderList.GetString(CurMask)) return true; FolderWildcards=false; FilterList.Reset(); - if (!FileMasks->GetString(CurMask,ASIZE(CurMask))) + if (!FileMasks->GetString(CurMask)) return false; // Check if folder wildcards present. @@ -144,10 +141,10 @@ bool ScanTree::GetFilteredMask() uint SlashPos=0; uint StartPos=0; #ifdef _WIN_ALL // Not treat the special NTFS \\?\d: path prefix as a wildcard. - if (CurMask[0]=='\\' && CurMask[1]=='\\' && CurMask[2]=='?' && CurMask[3]=='\\') + if (CurMask.rfind(L"\\\\?\\",0)==0) StartPos=4; #endif - for (uint I=StartPos;CurMask[I]!=0;I++) + for (uint I=StartPos;I2 && CurMask[0]=='\\' && CurMask[1]=='\\') { - const wchar *Slash=wcschr(CurMask+2,'\\'); - if (Slash!=NULL) + auto Slash=CurMask.find('\\',2); + if (Slash!=std::wstring::npos) { - Slash=wcschr(Slash+1,'\\'); - ScanEntireDisk=Slash!=NULL && *(Slash+1)==0; + Slash=CurMask.find('\\',Slash+1); + // If backslash is found and it is the last string character. + ScanEntireDisk=Slash!=std::wstring::npos && Slash+1==CurMask.size(); } } else ScanEntireDisk=IsDriveLetter(CurMask) && IsPathDiv(CurMask[2]) && CurMask[3]==0; - wchar *Name=PointToName(CurMask); - if (*Name==0) - wcsncatz(CurMask,MASKALL,ASIZE(CurMask)); - if (Name[0]=='.' && (Name[1]==0 || Name[1]=='.' && Name[2]==0)) + auto NamePos=GetNamePos(CurMask); + std::wstring Name=CurMask.substr(NamePos); + if (Name.empty()) + CurMask+=MASKALL; + if (Name==L"." || Name==L"..") { - AddEndSlash(CurMask,ASIZE(CurMask)); - wcsncatz(CurMask,MASKALL,ASIZE(CurMask)); + AddEndSlash(CurMask); + CurMask+=MASKALL; } - SpecPathLength=Name-CurMask; + SpecPathLength=NamePos; Depth=0; - wcsncpyz(OrigCurMask,CurMask,ASIZE(OrigCurMask)); + OrigCurMask=CurMask; return true; } @@ -250,7 +249,7 @@ bool ScanTree::GetNextMask() SCAN_CODE ScanTree::FindProc(FindData *FD) { - if (*CurMask==0) + if (CurMask.empty()) return SCAN_NEXT; bool FastFindFile=false; @@ -283,10 +282,9 @@ SCAN_CODE ScanTree::FindProc(FindData *FD) // Create the new FindFile object for wildcard based search. FindStack[Depth]=new FindFile; - wchar SearchMask[NM]; - wcsncpyz(SearchMask,CurMask,ASIZE(SearchMask)); + std::wstring SearchMask=CurMask; if (SearchAll) - SetName(SearchMask,MASKALL,ASIZE(SearchMask)); + SetName(SearchMask,MASKALL); FindStack[Depth]->SetMask(SearchMask); } else @@ -325,7 +323,7 @@ SCAN_CODE ScanTree::FindProc(FindData *FD) // It is not necessary for directories, because even in "fast find" // mode, directory recursing will quit by (Depth < 0) condition, // which returns SCAN_DONE to calling function. - *CurMask=0; + CurMask.clear(); return RetCode; } @@ -344,9 +342,6 @@ SCAN_CODE ScanTree::FindProc(FindData *FD) if (Error) ScanError(Error); - wchar DirName[NM]; - *DirName=0; - // Going to at least one directory level higher. delete FindStack[Depth]; FindStack[Depth--]=NULL; @@ -362,29 +357,32 @@ SCAN_CODE ScanTree::FindProc(FindData *FD) return SCAN_DONE; } - wchar *Slash=wcsrchr(CurMask,CPATHDIVIDER); - if (Slash!=NULL) + auto Slash=CurMask.rfind(CPATHDIVIDER); + if (Slash!=std::wstring::npos) { - wchar Mask[NM]; - wcsncpyz(Mask,Slash,ASIZE(Mask)); + std::wstring Mask; + Mask=CurMask.substr(Slash); // Name mask with leading slash like \*.* if (DepthIsDir) + { + FD->Flags|=FDDF_SECONDDIR; + return Error ? SCAN_ERROR:SCAN_SUCCESS; } - } - if (GetDirs==SCAN_GETDIRSTWICE && - FindFile::FastFind(DirName,FD,GetLinks) && FD->IsDir) - { - FD->Flags|=FDDF_SECONDDIR; - return Error ? SCAN_ERROR:SCAN_SUCCESS; } return Error ? SCAN_ERROR:SCAN_NEXT; } @@ -414,22 +412,22 @@ SCAN_CODE ScanTree::FindProc(FindData *FD) return FastFindFile ? SCAN_DONE:SCAN_NEXT; } - wchar Mask[NM]; - - wcsncpyz(Mask,FastFindFile ? MASKALL:PointToName(CurMask),ASIZE(Mask)); - wcsncpyz(CurMask,FD->Name,ASIZE(CurMask)); + std::wstring Mask=FastFindFile ? MASKALL:PointToName(CurMask); + CurMask=FD->Name; - if (wcslen(CurMask)+wcslen(Mask)+1>=NM || Depth>=MAXSCANDEPTH-1) + if (CurMask.size()+Mask.size()+1>=MAXPATHSIZE || Depth>=MAXSCANDEPTH-1) { uiMsg(UIERROR_PATHTOOLONG,CurMask,SPATHDIVIDER,Mask); return SCAN_ERROR; } - AddEndSlash(CurMask,ASIZE(CurMask)); - wcsncatz(CurMask,Mask,ASIZE(CurMask)); + AddEndSlash(CurMask); + CurMask+=Mask; Depth++; + FindStack.resize(Depth+1); + // We need to use OrigCurMask for depths less than SetAllMaskDepth // and "*" for depths equal or larger than SetAllMaskDepth. // It is important when "fast finding" directories at Depth > 0. @@ -470,19 +468,18 @@ void ScanTree::ScanError(bool &Error) // We cannot just check FD->FileAttr here, it can be undefined // if we process "folder\*" mask or if we process "folder" mask, // but "folder" is inaccessible. - wchar *Slash=PointToName(CurMask); - if (Slash>CurMask) + auto Slash=GetNamePos(CurMask); + if (Slash>1) { - *(Slash-1)=0; - DWORD Attr=GetFileAttributes(CurMask); - *(Slash-1)=CPATHDIVIDER; + std::wstring Parent=CurMask.substr(0,Slash-1); + DWORD Attr=GetFileAttr(Parent); if (Attr!=0xffffffff && (Attr & FILE_ATTRIBUTE_REPARSE_POINT)!=0) Error=false; } // Do not display an error if we cannot scan contents of // "System Volume Information" folder. Normally it is not accessible. - if (wcsstr(CurMask,L"System Volume Information\\")!=NULL) + if (CurMask.find(L"System Volume Information\\")!=std::wstring::npos) Error=false; } #endif @@ -495,10 +492,10 @@ void ScanTree::ScanError(bool &Error) if (ErrDirList!=NULL) ErrDirList->AddString(CurMask); if (ErrDirSpecPathLength!=NULL) - ErrDirSpecPathLength->Push((uint)SpecPathLength); - wchar FullName[NM]; + ErrDirSpecPathLength->push_back((uint)SpecPathLength); + std::wstring FullName; // This conversion works for wildcard masks too. - ConvertNameToFull(CurMask,FullName,ASIZE(FullName)); + ConvertNameToFull(CurMask,FullName); uiMsg(UIERROR_DIRSCAN,FullName); ErrHandler.SysErrMsg(); } diff --git a/multiarc/src/formats/rar/unrar/scantree.hpp b/multiarc/src/formats/rar/unrar/scantree.hpp index 7ebe69ad1..06bf545c6 100644 --- a/multiarc/src/formats/rar/unrar/scantree.hpp +++ b/multiarc/src/formats/rar/unrar/scantree.hpp @@ -11,20 +11,21 @@ enum SCAN_DIRS enum SCAN_CODE { SCAN_SUCCESS,SCAN_DONE,SCAN_ERROR,SCAN_NEXT }; -#define MAXSCANDEPTH (NM/2) - class CommandData; class ScanTree { private: + static constexpr size_t MAXSCANDEPTH = MAXPATHSIZE/2; + bool ExpandFolderMask(); bool GetFilteredMask(); bool GetNextMask(); SCAN_CODE FindProc(FindData *FD); void ScanError(bool &Error); - FindFile *FindStack[MAXSCANDEPTH]; +// FindFile *FindStack[MAXSCANDEPTH]; + std::vector FindStack; int Depth; int SetAllMaskDepth; @@ -33,13 +34,13 @@ class ScanTree RECURSE_MODE Recurse; bool GetLinks; SCAN_DIRS GetDirs; - int Errors; + uint Errors; // Set when processing paths like c:\ (root directory without wildcards). bool ScanEntireDisk; - wchar CurMask[NM]; - wchar OrigCurMask[NM]; + std::wstring CurMask; + std::wstring OrigCurMask; // Store all folder masks generated from folder wildcard mask in non-recursive mode. StringList ExpandedFolderList; @@ -49,7 +50,7 @@ class ScanTree // Save the list of unreadable dirs here. StringList *ErrDirList; - Array *ErrDirSpecPathLength; + std::vector *ErrDirSpecPathLength; // Set if processing a folder wildcard mask. bool FolderWildcards; @@ -57,7 +58,7 @@ class ScanTree bool SearchAllInRoot; size_t SpecPathLength; - wchar ErrArcName[NM]; + std::wstring ErrArcName; CommandData *Cmd; public: @@ -65,10 +66,10 @@ class ScanTree ~ScanTree(); SCAN_CODE GetNext(FindData *FindData); size_t GetSpecPathLength() {return SpecPathLength;} - int GetErrors() {return Errors;}; - void SetErrArcName(const wchar *Name) {wcsncpyz(ErrArcName,Name,ASIZE(ErrArcName));} + uint GetErrors() {return Errors;}; + void SetErrArcName(const std::wstring &Name) {ErrArcName=Name;} void SetCommandData(CommandData *Cmd) {ScanTree::Cmd=Cmd;} - void SetErrDirList(StringList *List,Array *Lengths) + void SetErrDirList(StringList *List,std::vector *Lengths) { ErrDirList=List; ErrDirSpecPathLength=Lengths; diff --git a/multiarc/src/formats/rar/unrar/secpassword.cpp b/multiarc/src/formats/rar/unrar/secpassword.cpp index 42ed47d5e..8d8f29860 100644 --- a/multiarc/src/formats/rar/unrar/secpassword.cpp +++ b/multiarc/src/formats/rar/unrar/secpassword.cpp @@ -69,8 +69,8 @@ SecPassword::~SecPassword() void SecPassword::Clean() { PasswordSet=false; - if (Password.size()>0) - cleandata(&Password[0],Password.size()); + if (!Password.empty()) + cleandata(Password.data(),Password.size()*sizeof(Password[0])); } @@ -79,7 +79,7 @@ void SecPassword::Clean() // So we use our own function for this purpose. void cleandata(void *data,size_t size) { - if (data==NULL || size==0) + if (data==nullptr || size==0) return; #if defined(_WIN_ALL) && defined(_MSC_VER) SecureZeroMemory(data,size); @@ -120,6 +120,14 @@ void SecPassword::Get(wchar *Psw,size_t MaxSize) } +void SecPassword::Get(std::wstring &Psw) +{ + wchar PswBuf[MAXPASSWORD]; + Get(PswBuf,ASIZE(PswBuf)); + Psw=PswBuf; +} + + void SecPassword::Set(const wchar *Psw) @@ -141,7 +149,7 @@ size_t SecPassword::Length() wchar Plain[MAXPASSWORD]; Get(Plain,ASIZE(Plain)); size_t Length=wcslen(Plain); - cleandata(Plain,ASIZE(Plain)); + cleandata(Plain,sizeof(Plain)); return Length; } @@ -156,8 +164,8 @@ bool SecPassword::operator == (SecPassword &psw) Get(Plain1,ASIZE(Plain1)); psw.Get(Plain2,ASIZE(Plain2)); bool Result=wcscmp(Plain1,Plain2)==0; - cleandata(Plain1,ASIZE(Plain1)); - cleandata(Plain2,ASIZE(Plain2)); + cleandata(Plain1,sizeof(Plain1)); + cleandata(Plain2,sizeof(Plain2)); return Result; } diff --git a/multiarc/src/formats/rar/unrar/secpassword.hpp b/multiarc/src/formats/rar/unrar/secpassword.hpp index 5284bce12..44e073c97 100644 --- a/multiarc/src/formats/rar/unrar/secpassword.hpp +++ b/multiarc/src/formats/rar/unrar/secpassword.hpp @@ -15,6 +15,7 @@ class SecPassword ~SecPassword(); void Clean(); void Get(wchar *Psw,size_t MaxSize); + void Get(std::wstring &Psw); void Set(const wchar *Psw); bool IsSet() {return PasswordSet;} size_t Length(); @@ -23,6 +24,7 @@ class SecPassword void cleandata(void *data,size_t size); +inline void cleandata(std::wstring &s) {cleandata(&s[0],s.size()*sizeof(s[0]));} void SecHideData(void *Data,size_t DataSize,bool Encode,bool CrossProcess); #endif diff --git a/multiarc/src/formats/rar/unrar/sha256.cpp b/multiarc/src/formats/rar/unrar/sha256.cpp index f90d2c095..4c93aba78 100644 --- a/multiarc/src/formats/rar/unrar/sha256.cpp +++ b/multiarc/src/formats/rar/unrar/sha256.cpp @@ -63,7 +63,7 @@ static void sha256_transform(sha256_context *ctx) for (uint I = 0; I < 64; I++) { - uint T1 = v[7] + Sg1(v[4]) + Ch(v[4], v[5], v[6]) + K[I] + W[I]; + uint32 T1 = v[7] + Sg1(v[4]) + Ch(v[4], v[5], v[6]) + K[I] + W[I]; // It is possible to eliminate variable copying if we unroll loop // and rename variables every time. But my test did not show any speed @@ -74,7 +74,7 @@ static void sha256_transform(sha256_context *ctx) v[4] = v[3] + T1; // It works a little faster when moved here from beginning of loop. - uint T2 = Sg0(v[0]) + Maj(v[0], v[1], v[2]); + uint32 T2 = Sg0(v[0]) + Maj(v[0], v[1], v[2]); v[3] = v[2]; v[2] = v[1]; @@ -146,3 +146,13 @@ void sha256_done(sha256_context *ctx, byte *Digest) sha256_init(ctx); } + + +void sha256_get(const void *Data, size_t Size, byte *Digest) +{ + sha256_context ctx; + sha256_init(&ctx); + sha256_process(&ctx, Data, Size); + sha256_done(&ctx, Digest); +} + diff --git a/multiarc/src/formats/rar/unrar/sha256.hpp b/multiarc/src/formats/rar/unrar/sha256.hpp index b6837e760..186297c07 100644 --- a/multiarc/src/formats/rar/unrar/sha256.hpp +++ b/multiarc/src/formats/rar/unrar/sha256.hpp @@ -13,5 +13,6 @@ typedef struct void sha256_init(sha256_context *ctx); void sha256_process(sha256_context *ctx, const void *Data, size_t Size); void sha256_done(sha256_context *ctx, byte *Digest); +void sha256_get(const void *Data, size_t Size, byte *Digest); #endif diff --git a/multiarc/src/formats/rar/unrar/strfn.cpp b/multiarc/src/formats/rar/unrar/strfn.cpp index 7617f7a59..3fdaca59b 100644 --- a/multiarc/src/formats/rar/unrar/strfn.cpp +++ b/multiarc/src/formats/rar/unrar/strfn.cpp @@ -2,66 +2,67 @@ const char *NullToEmpty(const char *Str) { - return Str==NULL ? "":Str; + return Str==nullptr ? "":Str; } const wchar *NullToEmpty(const wchar *Str) { - return Str==NULL ? L"":Str; + return Str==nullptr ? L"":Str; } -void IntToExt(const char *Src,char *Dest,size_t DestSize) +void IntToExt(const std::string &Src,std::string &Dest) { #ifdef _WIN_ALL - // OemToCharBuff does not stop at 0, so let's check source length. - size_t SrcLength=strlen(Src)+1; - if (DestSize>SrcLength) - DestSize=SrcLength; - OemToCharBuffA(Src,Dest,(DWORD)DestSize); - Dest[DestSize-1]=0; + if (std::addressof(Src)!=std::addressof(Dest)) + Dest=Src; + // OemToCharA use seems to be discouraged. So we use OemToCharBuffA, + // which doesn't stop at 0 and converts the entire passed length. + OemToCharBuffA(&Dest[0],&Dest[0],(DWORD)Dest.size()); + + std::string::size_type Pos=Dest.find('\0'); // Avoid zeroes inside of Dest. + if (Pos!=std::string::npos) + Dest.erase(Pos); + #else - if (Dest!=Src) - strncpyz(Dest,Src,DestSize); + if (std::addressof(Src)!=std::addressof(Dest)) + Dest=Src; #endif } // Convert archived names and comments to Unicode. // Allows user to select a code page in GUI. -void ArcCharToWide(const char *Src,wchar *Dest,size_t DestSize,ACTW_ENCODING Encoding) +void ArcCharToWide(const char *Src,std::wstring &Dest,ACTW_ENCODING Encoding) { #if defined(_WIN_ALL) // Console Windows RAR. if (Encoding==ACTW_UTF8) - UtfToWide(Src,Dest,DestSize); + UtfToWide(Src,Dest); else { - Array NameA; + std::string NameA; if (Encoding==ACTW_OEM) { - NameA.Alloc(DestSize+1); - IntToExt(Src,&NameA[0],NameA.Size()); - Src=&NameA[0]; + IntToExt(Src,NameA); + Src=NameA.data(); } - CharToWide(Src,Dest,DestSize); + CharToWide(Src,Dest); } #else // RAR for Unix. if (Encoding==ACTW_UTF8) - UtfToWide(Src,Dest,DestSize); + UtfToWide(Src,Dest); else - CharToWide(Src,Dest,DestSize); + CharToWide(Src,Dest); #endif - // Ensure that we return a zero terminate string for security reason. - // While [Jni]CharToWide might already do it, be protected in case of future - // changes in these functions. - if (DestSize>0) - Dest[DestSize-1]=0; + TruncateAtZero(Dest); // Ensure there are no zeroes inside of string. } + + int stricomp(const char *s1,const char *s2) { #ifdef _WIN_ALL @@ -113,6 +114,19 @@ wchar* RemoveEOL(wchar *Str) } +void RemoveEOL(std::wstring &Str) +{ + while (!Str.empty()) + { + wchar c=Str.back(); + if (c=='\r' || c=='\n' || c==' ' || c=='\t') + Str.pop_back(); + else + break; + } +} + + wchar* RemoveLF(wchar *Str) { for (int I=(int)wcslen(Str)-1;I>=0 && (Str[I]=='\r' || Str[I]=='\n');I--) @@ -121,6 +135,13 @@ wchar* RemoveLF(wchar *Str) } +void RemoveLF(std::wstring &Str) +{ + for (int I=(int)Str.size()-1;I>=0 && (Str[I]=='\r' || Str[I]=='\n');I--) + Str.erase(I); +} + + #if defined(SFX_MODULE) // char version of etoupperw. Used in console SFX module only. // Fast toupper for English only input and output. Additionally to speed, @@ -176,30 +197,18 @@ bool IsAlpha(int ch) -void BinToHex(const byte *Bin,size_t BinSize,char *HexA,wchar *HexW,size_t HexSize) +void BinToHex(const byte *Bin,size_t BinSize,std::wstring &Hex) { - uint A=0,W=0; // ASCII and Unicode hex output positions. + Hex.clear(); for (uint I=0;I> 4; uint Low=Bin[I] & 0xf; - uint HighHex=High>9 ? 'a'+High-10:'0'+High; - uint LowHex=Low>9 ? 'a'+Low-10:'0'+Low; - if (HexA!=NULL && A9 ? 'a'+High-10 : '0'+High; + uint LowHex=Low>9 ? 'a'+Low-10 : '0'+Low; + Hex+=HighHex; + Hex+=LowHex; } - if (HexA!=NULL && HexSize>0) - HexA[A]=0; - if (HexW!=NULL && HexSize>0) - HexW[W]=0; } @@ -217,22 +226,25 @@ uint GetDigits(uint Number) #endif -bool LowAscii(const char *Str) +bool LowAscii(const std::string &Str) { - for (size_t I=0;Str[I]!=0;I++) - if (/*(byte)Str[I]<32 || */(byte)Str[I]>127) + for (char Ch : Str) + { + // We convert char to byte in case char is signed. + if (/*(uint)Ch<32 || */(byte)Ch>127) return false; + } return true; } -bool LowAscii(const wchar *Str) +bool LowAscii(const std::wstring &Str) { - for (size_t I=0;Str[I]!=0;I++) + for (wchar Ch : Str) { // We convert wchar_t to uint just in case if some compiler // uses signed wchar_t. - if (/*(uint)Str[I]<32 || */(uint)Str[I]>127) + if (/*(uint)Ch<32 || */(uint)Ch>127) return false; } return true; @@ -249,6 +261,12 @@ int wcsicompc(const wchar *s1,const wchar *s2) // For path comparison. } +int wcsicompc(const std::wstring &s1,const std::wstring &s2) +{ + return wcsicompc(s1.c_str(),s2.c_str()); +} + + int wcsnicompc(const wchar *s1,const wchar *s2,size_t n) { #if defined(_UNIX) @@ -259,6 +277,12 @@ int wcsnicompc(const wchar *s1,const wchar *s2,size_t n) } +int wcsnicompc(const std::wstring &s1,const std::wstring &s2,size_t n) +{ + return wcsnicompc(s1.c_str(),s2.c_str(),n); +} + + // Safe copy: copies maxlen-1 max and for maxlen>0 returns zero terminated dest. void strncpyz(char *dest, const char *src, size_t maxlen) { @@ -383,88 +407,164 @@ void fmtitoa(int64 n,wchar *Str,size_t MaxSize) } -const wchar* GetWide(const char *Src) +std::wstring GetWide(const char *Src) { - const size_t MaxLength=NM; - static wchar StrTable[4][MaxLength]; - static uint StrNum=0; - if (++StrNum >= ASIZE(StrTable)) - StrNum=0; - wchar *Str=StrTable[StrNum]; - CharToWide(Src,Str,MaxLength); - Str[MaxLength-1]=0; + std::wstring Str; + CharToWide(Src,Str); return Str; } // Parse string containing parameters separated with spaces. -// Support quote marks. Param can be NULL to return the pointer to next -// parameter, which can be used to estimate the buffer size for Param. -const wchar* GetCmdParam(const wchar *CmdLine,wchar *Param,size_t MaxSize) +// Support quote marks. Accepts and updates the current position in the string. +// Returns false if there is nothing to parse. +bool GetCmdParam(const std::wstring &CmdLine,std::wstring::size_type &Pos,std::wstring &Param) { - while (IsSpace(*CmdLine)) - CmdLine++; - if (*CmdLine==0) - return NULL; + Param.clear(); + + while (IsSpace(CmdLine[Pos])) + Pos++; + if (Pos==CmdLine.size()) + return false; - size_t ParamSize=0; bool Quote=false; - while (*CmdLine!=0 && (Quote || !IsSpace(*CmdLine))) + while (Pos=0 || Msg.size()>MaxAllocSize) + break; + Msg.resize(Msg.size()*4); + } + std::wstring::size_type ZeroPos=Msg.find(L'\0'); + if (ZeroPos!=std::wstring::npos) + Msg.resize(ZeroPos); // Remove excessive zeroes at the end. + + return Msg; +} +#endif + + +#ifdef _WIN_ALL +bool ExpandEnvironmentStr(std::wstring &Str) +{ + DWORD ExpCode=ExpandEnvironmentStrings(Str.c_str(),nullptr,0); + if (ExpCode==0) + return false; + std::vector Buf(ExpCode); + ExpCode=ExpandEnvironmentStrings(Str.c_str(),Buf.data(),(DWORD)Buf.size()); + if (ExpCode==0 || ExpCode>Buf.size()) + return false; + Str=Buf.data(); + return true; } #endif + + +void TruncateAtZero(std::wstring &Str) +{ + std::wstring::size_type Pos=Str.find(L'\0'); + if (Pos!=std::wstring::npos) + Str.erase(Pos); +} + + +void ReplaceEsc(std::wstring &Str) +{ + std::wstring::size_type Pos=0; + while (true) + { + Pos=Str.find(L'\033',Pos); + if (Pos==std::wstring::npos) + break; + Str[Pos]=L'\''; + Str.insert(Pos+1,L"\\033'"); + Pos+=6; + } +} diff --git a/multiarc/src/formats/rar/unrar/strfn.hpp b/multiarc/src/formats/rar/unrar/strfn.hpp index 2a21feae0..fd4793c57 100644 --- a/multiarc/src/formats/rar/unrar/strfn.hpp +++ b/multiarc/src/formats/rar/unrar/strfn.hpp @@ -3,16 +3,18 @@ const char* NullToEmpty(const char *Str); const wchar* NullToEmpty(const wchar *Str); -void IntToExt(const char *Src,char *Dest,size_t DestSize); +void IntToExt(const std::string &Src,std::string &Dest); enum ACTW_ENCODING { ACTW_DEFAULT, ACTW_OEM, ACTW_UTF8}; -void ArcCharToWide(const char *Src,wchar *Dest,size_t DestSize,ACTW_ENCODING Encoding); +void ArcCharToWide(const char *Src,std::wstring &Dest,ACTW_ENCODING Encoding); int stricomp(const char *s1,const char *s2); int strnicomp(const char *s1,const char *s2,size_t n); wchar* RemoveEOL(wchar *Str); +void RemoveEOL(std::wstring &Str); wchar* RemoveLF(wchar *Str); +void RemoveLF(std::wstring &Str); void strncpyz(char *dest, const char *src, size_t maxlen); void wcsncpyz(wchar *dest, const wchar *src, size_t maxlen); @@ -28,25 +30,36 @@ bool IsDigit(int ch); bool IsSpace(int ch); bool IsAlpha(int ch); -void BinToHex(const byte *Bin,size_t BinSize,char *Hex,wchar *HexW,size_t HexSize); +void BinToHex(const byte *Bin,size_t BinSize,std::wstring &Hex); #ifndef SFX_MODULE uint GetDigits(uint Number); #endif -bool LowAscii(const char *Str); -bool LowAscii(const wchar *Str); +bool LowAscii(const std::string &Str); +bool LowAscii(const std::wstring &Str); int wcsicompc(const wchar *s1,const wchar *s2); +int wcsicompc(const std::wstring &s1,const std::wstring &s2); int wcsnicompc(const wchar *s1,const wchar *s2,size_t n); +int wcsnicompc(const std::wstring &s1,const std::wstring &s2,size_t n); void itoa(int64 n,char *Str,size_t MaxSize); void itoa(int64 n,wchar *Str,size_t MaxSize); void fmtitoa(int64 n,wchar *Str,size_t MaxSize); -const wchar* GetWide(const char *Src); -const wchar* GetCmdParam(const wchar *CmdLine,wchar *Param,size_t MaxSize); +std::wstring GetWide(const char *Src); +bool GetCmdParam(const std::wstring &CmdLine,std::wstring::size_type &Pos,std::wstring &Param); #ifndef RARDLL -void PrintfPrepareFmt(const wchar *Org,wchar *Cvt,size_t MaxSize); +void PrintfPrepareFmt(const wchar *Org,std::wstring &Cvt); +std::wstring wstrprintf(const wchar *fmt,...); +std::wstring vwstrprintf(const wchar *fmt,va_list arglist); #endif +#ifdef _WIN_ALL +bool ExpandEnvironmentStr(std::wstring &Str); +#endif + +void TruncateAtZero(std::wstring &Str); +void ReplaceEsc(std::wstring &Str); + #endif diff --git a/multiarc/src/formats/rar/unrar/strlist.cpp b/multiarc/src/formats/rar/unrar/strlist.cpp index 50d69c715..e76654fcd 100644 --- a/multiarc/src/formats/rar/unrar/strlist.cpp +++ b/multiarc/src/formats/rar/unrar/strlist.cpp @@ -9,18 +9,20 @@ StringList::StringList() void StringList::Reset() { Rewind(); - StringData.Reset(); + StringData.clear(); StringsCount=0; SavePosNumber=0; } +/* void StringList::AddStringA(const char *Str) { - Array StrW(strlen(Str)); - CharToWide(Str,&StrW[0],StrW.Size()); - AddString(&StrW[0]); + std::wstring StrW; + CharToWide(Str,StrW); + AddString(StrW); } +*/ void StringList::AddString(const wchar *Str) @@ -28,22 +30,30 @@ void StringList::AddString(const wchar *Str) if (Str==NULL) Str=L""; - size_t PrevSize=StringData.Size(); - StringData.Add(wcslen(Str)+1); + size_t PrevSize=StringData.size(); + StringData.resize(PrevSize+wcslen(Str)+1); wcscpy(&StringData[PrevSize],Str); StringsCount++; } +void StringList::AddString(const std::wstring &Str) +{ + AddString(Str.c_str()); +} + + +/* bool StringList::GetStringA(char *Str,size_t MaxLength) { - Array StrW(MaxLength); - if (!GetString(&StrW[0],StrW.Size())) + std::wstring StrW; + if (!GetString(StrW)) return false; - WideToChar(&StrW[0],Str,MaxLength); + WideToChar(StrW.c_str(),Str,MaxLength); return true; } +*/ bool StringList::GetString(wchar *Str,size_t MaxLength) @@ -56,6 +66,16 @@ bool StringList::GetString(wchar *Str,size_t MaxLength) } +bool StringList::GetString(std::wstring &Str) +{ + wchar *StrPtr; + if (!GetString(&StrPtr)) + return false; + Str=StrPtr; + return true; +} + + #ifndef SFX_MODULE bool StringList::GetString(wchar *Str,size_t MaxLength,int StringNum) { @@ -71,6 +91,22 @@ bool StringList::GetString(wchar *Str,size_t MaxLength,int StringNum) RestorePosition(); return RetCode; } + + +bool StringList::GetString(std::wstring &Str,int StringNum) +{ + SavePosition(); + Rewind(); + bool RetCode=true; + while (StringNum-- >=0) + if (!GetString(Str)) + { + RetCode=false; + break; + } + RestorePosition(); + return RetCode; +} #endif @@ -84,7 +120,7 @@ wchar* StringList::GetString() bool StringList::GetString(wchar **Str) { - if (CurPos>=StringData.Size()) // No more strings left unprocessed. + if (CurPos>=StringData.size()) // No more strings left unprocessed. { if (Str!=NULL) *Str=NULL; @@ -107,7 +143,7 @@ void StringList::Rewind() #ifndef SFX_MODULE -bool StringList::Search(const wchar *Str,bool CaseSensitive) +bool StringList::Search(const std::wstring &Str,bool CaseSensitive) { SavePosition(); Rewind(); @@ -115,8 +151,8 @@ bool StringList::Search(const wchar *Str,bool CaseSensitive) wchar *CurStr; while (GetString(&CurStr)) { - if (Str!=NULL && CurStr!=NULL) - if ((CaseSensitive ? wcscmp(Str,CurStr):wcsicomp(Str,CurStr))!=0) + if (CurStr!=NULL) + if (CaseSensitive && Str!=CurStr || !CaseSensitive && wcsicomp(Str,CurStr)!=0) continue; Found=true; break; diff --git a/multiarc/src/formats/rar/unrar/strlist.hpp b/multiarc/src/formats/rar/unrar/strlist.hpp index 16a2cbb03..d89ba5c95 100644 --- a/multiarc/src/formats/rar/unrar/strlist.hpp +++ b/multiarc/src/formats/rar/unrar/strlist.hpp @@ -4,7 +4,7 @@ class StringList { private: - Array StringData; + std::vector StringData; size_t CurPos; size_t StringsCount; @@ -13,17 +13,20 @@ class StringList public: StringList(); void Reset(); - void AddStringA(const char *Str); +// void AddStringA(const char *Str); void AddString(const wchar *Str); - bool GetStringA(char *Str,size_t MaxLength); + void AddString(const std::wstring &Str); +// bool GetStringA(char *Str,size_t MaxLength); bool GetString(wchar *Str,size_t MaxLength); + bool GetString(std::wstring &Str); bool GetString(wchar *Str,size_t MaxLength,int StringNum); + bool GetString(std::wstring &Str,int StringNum); wchar* GetString(); bool GetString(wchar **Str); void Rewind(); size_t ItemsCount() {return StringsCount;}; - size_t GetCharCount() {return StringData.Size();} - bool Search(const wchar *Str,bool CaseSensitive); + size_t GetCharCount() {return StringData.size();} + bool Search(const std::wstring &Str,bool CaseSensitive); void SavePosition(); void RestorePosition(); }; diff --git a/multiarc/src/formats/rar/unrar/system.cpp b/multiarc/src/formats/rar/unrar/system.cpp index 9e5362274..cf9d1836e 100644 --- a/multiarc/src/formats/rar/unrar/system.cpp +++ b/multiarc/src/formats/rar/unrar/system.cpp @@ -150,16 +150,18 @@ bool ShutdownCheckAnother(bool Open) + #if defined(_WIN_ALL) // Load library from Windows System32 folder. Use this function to prevent // loading a malicious code from current folder or same folder as exe. HMODULE WINAPI LoadSysLibrary(const wchar *Name) { - wchar SysDir[NM]; - if (GetSystemDirectory(SysDir,ASIZE(SysDir))==0) - return NULL; - MakeName(SysDir,Name,SysDir,ASIZE(SysDir)); - return LoadLibrary(SysDir); + std::vector SysDir(MAX_PATH); + if (GetSystemDirectory(SysDir.data(),(UINT)SysDir.size())==0) + return nullptr; + std::wstring FullName; + MakeName(SysDir.data(),Name,FullName); + return LoadLibrary(FullName.c_str()); } @@ -186,6 +188,7 @@ SSE_VERSION _SSE_Version=GetSSEVersion(); SSE_VERSION GetSSEVersion() { +#ifdef _MSC_VER int CPUInfo[4]; __cpuid(CPUInfo, 0); @@ -210,6 +213,18 @@ SSE_VERSION GetSSEVersion() if ((CPUInfo[3] & 0x2000000)!=0) return SSE_SSE; } +#elif defined(__GNUC__) + if (__builtin_cpu_supports("avx2")) + return SSE_AVX2; + if (__builtin_cpu_supports("sse4.1")) + return SSE_SSE41; + if (__builtin_cpu_supports("ssse3")) + return SSE_SSSE3; + if (__builtin_cpu_supports("sse2")) + return SSE_SSE2; + if (__builtin_cpu_supports("sse")) + return SSE_SSE; +#endif return SSE_NONE; } #endif diff --git a/multiarc/src/formats/rar/unrar/system.hpp b/multiarc/src/formats/rar/unrar/system.hpp index a56d6b7fc..de4c14a6f 100644 --- a/multiarc/src/formats/rar/unrar/system.hpp +++ b/multiarc/src/formats/rar/unrar/system.hpp @@ -21,7 +21,7 @@ void InitSystemOptions(int SleepTime); void SetPriority(int Priority); clock_t MonoClock(); void Wait(); -bool EmailFile(const wchar *FileName,const wchar *MailToW); +bool EmailFile(const std::wstring &FileName,std::wstring MailToW); void Shutdown(POWER_MODE Mode); bool ShutdownCheckAnother(bool Open); @@ -30,7 +30,6 @@ HMODULE WINAPI LoadSysLibrary(const wchar *Name); bool IsUserAdmin(); #endif - #ifdef USE_SSE enum SSE_VERSION {SSE_NONE,SSE_SSE,SSE_SSE2,SSE_SSSE3,SSE_SSE41,SSE_AVX2}; SSE_VERSION GetSSEVersion(); diff --git a/multiarc/src/formats/rar/unrar/timefn.cpp b/multiarc/src/formats/rar/unrar/timefn.cpp index 0abf49de9..aa90715da 100644 --- a/multiarc/src/formats/rar/unrar/timefn.cpp +++ b/multiarc/src/formats/rar/unrar/timefn.cpp @@ -71,12 +71,12 @@ void RarTime::SetLocal(RarLocalTime *lt) { #ifdef _WIN_ALL SYSTEMTIME st; - st.wYear=lt->Year; - st.wMonth=lt->Month; - st.wDay=lt->Day; - st.wHour=lt->Hour; - st.wMinute=lt->Minute; - st.wSecond=lt->Second; + st.wYear=(WORD)lt->Year; + st.wMonth=(WORD)lt->Month; + st.wDay=(WORD)lt->Day; + st.wHour=(WORD)lt->Hour; + st.wMinute=(WORD)lt->Minute; + st.wSecond=(WORD)lt->Second; st.wMilliseconds=0; st.wDayOfWeek=0; FILETIME lft; @@ -183,8 +183,7 @@ void RarTime::SetUnix(time_t ut) // Get the high precision Unix time in nanoseconds since 01-01-1970. uint64 RarTime::GetUnixNS() { - // 11644473600000000000 - number of ns between 01-01-1601 and 01-01-1970. - uint64 ushift=INT32TO64(0xA1997B0B,0x4C6A0000); + const uint64 ushift=11644473600000000000ULL; // ns between 01-01-1601 and 01-01-1970. return itime*(1000000000/TICKS_PER_SECOND)-ushift; } @@ -192,8 +191,7 @@ uint64 RarTime::GetUnixNS() // Set the high precision Unix time in nanoseconds since 01-01-1970. void RarTime::SetUnixNS(uint64 ns) { - // 11644473600000000000 - number of ns between 01-01-1601 and 01-01-1970. - uint64 ushift=INT32TO64(0xA1997B0B,0x4C6A0000); + const uint64 ushift=11644473600000000000ULL; // ns between 01-01-1601 and 01-01-1970. itime=(ns+ushift)/(1000000000/TICKS_PER_SECOND); } @@ -327,14 +325,14 @@ void RarTime::Adjust(int64 ns) #ifndef SFX_MODULE -const wchar *GetMonthName(int Month) +const wchar *GetMonthName(uint Month) { return uiGetMonthName(Month); } #endif -bool IsLeapYear(int Year) +bool IsLeapYear(uint Year) { return (Year&3)==0 && (Year%100!=0 || Year%400==0); } diff --git a/multiarc/src/formats/rar/unrar/timefn.hpp b/multiarc/src/formats/rar/unrar/timefn.hpp index 49b61e85d..6c265ec02 100644 --- a/multiarc/src/formats/rar/unrar/timefn.hpp +++ b/multiarc/src/formats/rar/unrar/timefn.hpp @@ -70,7 +70,7 @@ class RarTime void Adjust(int64 ns); }; -const wchar *GetMonthName(int Month); -bool IsLeapYear(int Year); +const wchar *GetMonthName(uint Month); +bool IsLeapYear(uint Year); #endif diff --git a/multiarc/src/formats/rar/unrar/ui.hpp b/multiarc/src/formats/rar/unrar/ui.hpp index 5def26df1..f37001bec 100644 --- a/multiarc/src/formats/rar/unrar/ui.hpp +++ b/multiarc/src/formats/rar/unrar/ui.hpp @@ -19,8 +19,8 @@ enum UIMESSAGE_CODE { UIERROR_SUBHEADERBROKEN, UIERROR_SUBHEADERUNKNOWN, UIERROR_SUBHEADERDATABROKEN, UIERROR_RRDAMAGED, UIERROR_UNKNOWNMETHOD, UIERROR_UNKNOWNENCMETHOD, UIERROR_RENAMING, UIERROR_NEWERRAR, - UIERROR_NOTSFX, UIERROR_OLDTOSFX, - UIERROR_WRONGSFXVER, UIERROR_HEADENCMISMATCH, UIERROR_DICTOUTMEM, + UIERROR_NOTSFX, UIERROR_OLDTOSFX,UIERROR_WRONGSFXVER, + UIERROR_HEADENCMISMATCH, UIERROR_DICTOUTMEM,UIERROR_EXTRDICTOUTMEM, UIERROR_USESMALLERDICT, UIERROR_MODIFYUNKNOWN, UIERROR_MODIFYOLD, UIERROR_MODIFYLOCKED, UIERROR_MODIFYVOLUME, UIERROR_NOTVOLUME, UIERROR_NOTFIRSTVOLUME, UIERROR_RECVOLLIMIT, UIERROR_RECVOLDIFFSETS, @@ -32,14 +32,15 @@ enum UIMESSAGE_CODE { UIERROR_NOFILESTOADD, UIERROR_NOFILESTODELETE, UIERROR_NOFILESTOEXTRACT, UIERROR_MISSINGVOL, UIERROR_NEEDPREVVOL, UIERROR_UNKNOWNEXTRA, UIERROR_CORRUPTEXTRA, UIERROR_NTFSREQUIRED, UIERROR_ZIPVOLSFX, - UIERROR_FILERO, UIERROR_TOOLARGESFX, UIERROR_NOZIPSFX, UIERROR_EMAIL, - UIERROR_ACLGET, UIERROR_ACLBROKEN, UIERROR_ACLUNKNOWN, UIERROR_ACLSET, - UIERROR_STREAMBROKEN, UIERROR_STREAMUNKNOWN, UIERROR_INCOMPATSWITCH, - UIERROR_PATHTOOLONG, UIERROR_DIRSCAN, UIERROR_UOWNERGET, - UIERROR_UOWNERBROKEN, UIERROR_UOWNERGETOWNERID, UIERROR_UOWNERGETGROUPID, - UIERROR_UOWNERSET, UIERROR_ULINKREAD, UIERROR_ULINKEXIST, - UIERROR_OPENPRESERVEATIME, UIERROR_READERRTRUNCATED, UIERROR_READERRCOUNT, - UIERROR_DIRNAMEEXISTS,UIERROR_TRUNCPSW,UIERROR_ADJUSTVALUE, + UIERROR_FILERO, UIERROR_TOOLARGESFX, UIERROR_NOZIPSFX, UIERROR_NEEEDSFX64, + UIERROR_EMAIL, UIERROR_ACLGET, UIERROR_ACLBROKEN, UIERROR_ACLUNKNOWN, + UIERROR_ACLSET, UIERROR_STREAMBROKEN, UIERROR_STREAMUNKNOWN, + UIERROR_INCOMPATSWITCH, UIERROR_PATHTOOLONG, UIERROR_DIRSCAN, + UIERROR_UOWNERGET, UIERROR_UOWNERBROKEN, UIERROR_UOWNERGETOWNERID, + UIERROR_UOWNERGETGROUPID, UIERROR_UOWNERSET, UIERROR_ULINKREAD, + UIERROR_ULINKEXIST, UIERROR_OPENPRESERVEATIME, UIERROR_READERRTRUNCATED, + UIERROR_READERRCOUNT, UIERROR_DIRNAMEEXISTS,UIERROR_TRUNCPSW, + UIERROR_ADJUSTVALUE, UIERROR_SKIPUNSAFELINK, UIMSG_FIRST, UIMSG_STRING, UIMSG_BUILD, UIMSG_RRSEARCH, UIMSG_ANALYZEFILEDATA, @@ -76,19 +77,25 @@ enum UIASKREP_RESULT { UIASKREP_R_RENAME,UIASKREP_R_RENAMEAUTO,UIASKREP_R_CANCEL,UIASKREP_R_UNUSED }; -UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTime *FileTime,uint Flags); -UIASKREP_RESULT uiAskReplaceEx(CommandData *Cmd,wchar *Name,size_t MaxNameSize,int64 FileSize,RarTime *FileTime,uint Flags); +UIASKREP_RESULT uiAskReplace(std::wstring &Name,int64 FileSize,RarTime *FileTime,uint Flags); +UIASKREP_RESULT uiAskReplaceEx(CommandData *Cmd,std::wstring &Name,int64 FileSize,RarTime *FileTime,uint Flags); void uiInit(SOUND_NOTIFY_MODE Sound); -void uiStartArchiveExtract(bool Extract,const wchar *ArcName); -bool uiStartFileExtract(const wchar *FileName,bool Extract,bool Test,bool Skip); +void uiStartArchiveExtract(bool Extract,const std::wstring &ArcName); +bool uiStartFileExtract(const std::wstring &FileName,bool Extract,bool Test,bool Skip); void uiExtractProgress(int64 CurFileSize,int64 TotalFileSize,int64 CurSize,int64 TotalSize); void uiProcessProgress(const char *Command,int64 CurSize,int64 TotalSize); -enum UIPASSWORD_TYPE {UIPASSWORD_GLOBAL,UIPASSWORD_FILE,UIPASSWORD_ARCHIVE}; -bool uiGetPassword(UIPASSWORD_TYPE Type,const wchar *FileName,SecPassword *Password,CheckPassword *CheckPwd); +enum UIPASSWORD_TYPE +{ + UIPASSWORD_GLOBAL, // For -p, -hp without parameter or Ctrl+P in WinRAR. + UIPASSWORD_FILE, // Extracting an encrypted file. + UIPASSWORD_ARCHIVE, // Extracting or opening an encrypted header archive. +}; + +bool uiGetPassword(UIPASSWORD_TYPE Type,const std::wstring &FileName,SecPassword *Password,CheckPassword *CheckPwd); bool uiIsGlobalPasswordSet(); enum UIALARM_TYPE {UIALARM_ERROR, UIALARM_INFO, UIALARM_QUESTION}; @@ -96,14 +103,16 @@ void uiAlarm(UIALARM_TYPE Type); void uiEolAfterMsg(); -bool uiAskNextVolume(wchar *VolName,size_t MaxSize); +bool uiAskNextVolume(std::wstring &VolName); #if !defined(SILENT) && !defined(SFX_MODULE) -void uiAskRepeatRead(const wchar *FileName,bool &Ignore,bool &All,bool &Retry,bool &Quit); +void uiAskRepeatRead(const std::wstring &FileName,bool &Ignore,bool &All,bool &Retry,bool &Quit); #endif -bool uiAskRepeatWrite(const wchar *FileName,bool DiskFull); +bool uiAskRepeatWrite(const std::wstring &FileName,bool DiskFull); + +bool uiDictLimit(CommandData *Cmd,const std::wstring &FileName,uint64 DictSize,uint64 MaxDictSize); #ifndef SFX_MODULE -const wchar *uiGetMonthName(int Month); +const wchar *uiGetMonthName(uint Month); #endif class uiMsgStore @@ -131,6 +140,12 @@ class uiMsgStore Str[StrSize++]=s; return *this; } + uiMsgStore& operator << (const std::wstring &s) + { + if (StrSizeOverwrite==OVERWRITE_NONE) return UIASKREP_R_SKIP; #if !defined(SFX_MODULE) && !defined(SILENT) // Must be before Cmd->AllYes check or -y switch would override -or. - if (Cmd->Overwrite==OVERWRITE_AUTORENAME && GetAutoRenamedName(Name,MaxNameSize)) + if (Cmd->Overwrite==OVERWRITE_AUTORENAME && GetAutoRenamedName(Name)) return UIASKREP_R_REPLACE; #endif - // This check must be after OVERWRITE_AUTORENAME processing or -y switch - // would override -or. - if (Cmd->AllYes || Cmd->Overwrite==OVERWRITE_ALL) - { - PrepareToDelete(Name); - return UIASKREP_R_REPLACE; - } - - wchar NewName[NM]; - wcsncpyz(NewName,Name,ASIZE(NewName)); - UIASKREP_RESULT Choice=uiAskReplace(NewName,ASIZE(NewName),FileSize,FileTime,Flags); + std::wstring NewName=Name; + UIASKREP_RESULT Choice=Cmd->AllYes || Cmd->Overwrite==OVERWRITE_ALL ? + UIASKREP_R_REPLACE : uiAskReplace(NewName,FileSize,FileTime,Flags); if (Choice==UIASKREP_R_REPLACE || Choice==UIASKREP_R_REPLACEALL) + { PrepareToDelete(Name); + // Overwrite the link itself instead of its target. + // For normal files we prefer to inherit file attributes, permissions + // and hard links. + FindData FD; + if (FindFile::FastFind(Name,&FD,true) && FD.IsLink) + DelFile(Name); + } + if (Choice==UIASKREP_R_REPLACEALL) { Cmd->Overwrite=OVERWRITE_ALL; @@ -46,16 +48,16 @@ UIASKREP_RESULT uiAskReplaceEx(CommandData *Cmd,wchar *Name,size_t MaxNameSize,i } if (Choice==UIASKREP_R_RENAME) { - if (PointToName(NewName)==NewName) - SetName(Name,NewName,MaxNameSize); + if (GetNamePos(NewName)==0) + SetName(Name,NewName); else - wcsncpyz(Name,NewName,MaxNameSize); + Name=NewName; if (FileExist(Name)) - return uiAskReplaceEx(Cmd,Name,MaxNameSize,FileSize,FileTime,Flags); + return uiAskReplaceEx(Cmd,Name,FileSize,FileTime,Flags); return UIASKREP_R_REPLACE; } #if !defined(SFX_MODULE) && !defined(SILENT) - if (Choice==UIASKREP_R_RENAMEAUTO && GetAutoRenamedName(Name,MaxNameSize)) + if (Choice==UIASKREP_R_RENAMEAUTO && GetAutoRenamedName(Name)) { Cmd->Overwrite=OVERWRITE_AUTORENAME; return UIASKREP_R_REPLACE; @@ -63,3 +65,23 @@ UIASKREP_RESULT uiAskReplaceEx(CommandData *Cmd,wchar *Name,size_t MaxNameSize,i #endif return Choice; } + + +bool GetAutoRenamedName(std::wstring &Name) +{ + std::wstring Ext=GetExt(Name); + for (uint FileVer=1;FileVer<1000000;FileVer++) + { + std::wstring NewName=Name; + RemoveExt(NewName); + wchar Ver[10]; + itoa(FileVer,Ver,ASIZE(Ver)); + NewName = NewName + L"(" + Ver + L")" + Ext; + if (!FileExist(NewName)) + { + Name=NewName; + return true; + } + } + return false; +} diff --git a/multiarc/src/formats/rar/unrar/uiconsole.cpp b/multiarc/src/formats/rar/unrar/uiconsole.cpp index b1ac51322..cc1c72d6d 100644 --- a/multiarc/src/formats/rar/unrar/uiconsole.cpp +++ b/multiarc/src/formats/rar/unrar/uiconsole.cpp @@ -1,12 +1,11 @@ static bool AnyMessageDisplayed=false; // For console -idn switch. // Purely user interface function. Gets and returns user input. -UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTime *FileTime,uint Flags) +UIASKREP_RESULT uiAskReplace(std::wstring &Name,int64 FileSize,RarTime *FileTime,uint Flags) { wchar SizeText1[20],DateStr1[50],SizeText2[20],DateStr2[50]; - FindData ExistingFD; - memset(&ExistingFD,0,sizeof(ExistingFD)); // In case find fails. + FindData ExistingFD={}; // Init in case find fails. FindFile::FastFind(Name,&ExistingFD); itoa(ExistingFD.Size,SizeText1,ASIZE(SizeText1)); ExistingFD.mtime.GetText(DateStr1,ASIZE(DateStr1),false); @@ -14,16 +13,16 @@ UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTi if (FileSize==INT64NDF || FileTime==NULL) { eprintf(L"\n"); - eprintf(St(MAskOverwrite),Name); + eprintf(St(MAskOverwrite),Name.c_str()); } else { itoa(FileSize,SizeText2,ASIZE(SizeText2)); FileTime->GetText(DateStr2,ASIZE(DateStr2),false); if ((Flags & UIASKREP_F_EXCHSRCDEST)==0) - eprintf(St(MAskReplace),Name,SizeText1,DateStr1,SizeText2,DateStr2); + eprintf(St(MAskReplace),Name.c_str(),SizeText1,DateStr1,SizeText2,DateStr2); else - eprintf(St(MAskReplace),Name,SizeText2,DateStr2,SizeText1,DateStr1); + eprintf(St(MAskReplace),Name.c_str(),SizeText2,DateStr2,SizeText1,DateStr1); } bool AllowRename=(Flags & UIASKREP_F_NORENAME)==0; @@ -46,7 +45,7 @@ UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTi if (AllowRename && Choice==5) { mprintf(St(MAskNewName)); - if (getwstr(Name,MaxNameSize)) + if (getwstr(Name)) return UIASKREP_R_RENAME; else return UIASKREP_R_SKIP; // Process fwgets failure as if user answered 'No'. @@ -57,13 +56,13 @@ UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTi -void uiStartArchiveExtract(bool Extract,const wchar *ArcName) +void uiStartArchiveExtract(bool Extract,const std::wstring &ArcName) { - mprintf(St(Extract ? MExtracting : MExtrTest), ArcName); + mprintf(St(Extract ? MExtracting : MExtrTest), ArcName.c_str()); } -bool uiStartFileExtract(const wchar *FileName,bool Extract,bool Test,bool Skip) +bool uiStartFileExtract(const std::wstring &FileName,bool Extract,bool Test,bool Skip) { return true; } @@ -183,6 +182,7 @@ void uiMsgStore::Msg() Log(NULL,St(MNeedAdmin)); break; case UIERROR_ARCBROKEN: + mprintf(L"\n"); // So it is not merged with preceding UIERROR_HEADERBROKEN. Log(Str[0],St(MErrBrokenArc)); break; case UIERROR_HEADERBROKEN: @@ -236,6 +236,12 @@ void uiMsgStore::Msg() case UIERROR_RECVOLCANNOTFIX: mprintf(St(MRecVolCannotFix)); break; + case UIERROR_EXTRDICTOUTMEM: + Log(Str[0],St(MExtrDictOutMem),Num[0]); +#ifdef _WIN_32 + Log(Str[0],St(MSuggest64bit)); +#endif + break; case UIERROR_UNEXPEOF: Log(Str[0],St(MLogUnexpEOF)); break; @@ -343,6 +349,9 @@ void uiMsgStore::Msg() case UIERROR_ADJUSTVALUE: Log(NULL,St(MAdjustValue),Str[0],Str[1]); break; + case UIERROR_SKIPUNSAFELINK: + Log(NULL,St(MSkipUnsafeLink),Str[0],Str[1]); + break; #ifndef SFX_MODULE case UIMSG_STRING: @@ -388,7 +397,6 @@ void uiMsgStore::Msg() break; - case UIEVENT_RRTESTINGSTART: mprintf(L"%s ",St(MTestingRR)); break; @@ -396,7 +404,7 @@ void uiMsgStore::Msg() } -bool uiGetPassword(UIPASSWORD_TYPE Type,const wchar *FileName, +bool uiGetPassword(UIPASSWORD_TYPE Type,const std::wstring &FileName, SecPassword *Password,CheckPassword *CheckPwd) { // Unlike GUI we cannot provide Cancel button here, so we use the empty @@ -432,14 +440,14 @@ void uiAlarm(UIALARM_TYPE Type) -bool uiAskNextVolume(wchar *VolName,size_t MaxSize) +bool uiAskNextVolume(std::wstring &VolName) { - eprintf(St(MAskNextVol),VolName); + eprintf(St(MAskNextVol),VolName.c_str()); return Ask(St(MContinueQuit))!=2; } -void uiAskRepeatRead(const wchar *FileName,bool &Ignore,bool &All,bool &Retry,bool &Quit) +void uiAskRepeatRead(const std::wstring &FileName,bool &Ignore,bool &All,bool &Retry,bool &Quit) { eprintf(St(MErrReadInfo)); int Code=Ask(St(MIgnoreAllRetryQuit)); @@ -451,16 +459,37 @@ void uiAskRepeatRead(const wchar *FileName,bool &Ignore,bool &All,bool &Retry,bo } -bool uiAskRepeatWrite(const wchar *FileName,bool DiskFull) +bool uiAskRepeatWrite(const std::wstring &FileName,bool DiskFull) { mprintf(L"\n"); - Log(NULL,St(DiskFull ? MNotEnoughDisk:MErrWrite),FileName); + Log(NULL,St(DiskFull ? MNotEnoughDisk:MErrWrite),FileName.c_str()); return Ask(St(MRetryAbort))==1; } +bool uiDictLimit(CommandData *Cmd,const std::wstring &FileName,uint64 DictSize,uint64 MaxDictSize) +{ + mprintf(L"\n%s",FileName.c_str()); + const uint64 GB=1024*1024*1024; // We display sizes in GB here. + + // Include the reminder for switches like -md6400m. + DictSize=DictSize/GB+(DictSize%GB!=0 ? 1:0); + + // Exclude reminder for in case it is less than archive dictionary size, + // but still more than nearest GB value. Otherwise we could have message + // like "7 GB dictionary exceeds 7 GB limit", where first 7 might be actually + // 6272 MB and second is 6400 MB. + MaxDictSize/=GB; + + mprintf(St(MDictNotAllowed),(uint)DictSize,(uint)MaxDictSize,(uint)DictSize); + mprintf(St(MDictExtrAnyway),(uint)DictSize,(uint)DictSize); + mprintf(L"\n"); + return false; // Stop extracting. +} + + #ifndef SFX_MODULE -const wchar *uiGetMonthName(int Month) +const wchar *uiGetMonthName(uint Month) { static MSGID MonthID[12]={ MMonthJan,MMonthFeb,MMonthMar,MMonthApr,MMonthMay,MMonthJun, diff --git a/multiarc/src/formats/rar/unrar/uisilent.cpp b/multiarc/src/formats/rar/unrar/uisilent.cpp index 815588574..806e46818 100644 --- a/multiarc/src/formats/rar/unrar/uisilent.cpp +++ b/multiarc/src/formats/rar/unrar/uisilent.cpp @@ -1,5 +1,5 @@ // Purely user interface function. Gets and returns user input. -UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTime *FileTime,uint Flags) +UIASKREP_RESULT uiAskReplace(std::wstring &Name,int64 FileSize,RarTime *FileTime,uint Flags) { return UIASKREP_R_REPLACE; } @@ -7,12 +7,12 @@ UIASKREP_RESULT uiAskReplace(wchar *Name,size_t MaxNameSize,int64 FileSize,RarTi -void uiStartArchiveExtract(bool Extract,const wchar *ArcName) +void uiStartArchiveExtract(bool Extract,const std::wstring &ArcName) { } -bool uiStartFileExtract(const wchar *FileName,bool Extract,bool Test,bool Skip) +bool uiStartFileExtract(const std::wstring &FileName,bool Extract,bool Test,bool Skip) { return true; } @@ -33,7 +33,7 @@ void uiMsgStore::Msg() } -bool uiGetPassword(UIPASSWORD_TYPE Type,const wchar *FileName, +bool uiGetPassword(UIPASSWORD_TYPE Type,const std::wstring &FileName, SecPassword *Password,CheckPassword *CheckPwd) { return false; @@ -62,8 +62,19 @@ void uiGiveTick() } +bool uiDictLimit(CommandData *Cmd,const std::wstring &FileName,uint64 DictSize,uint64 MaxDictSize) +{ +#ifdef RARDLL + if (Cmd->Callback!=nullptr && + Cmd->Callback(UCM_LARGEDICT,Cmd->UserData,(LPARAM)(DictSize/1024),(LPARAM)(MaxDictSize/1024))==1) + return true; // Continue extracting if unrar.dll callback permits it. +#endif + return false; // Stop extracting. +} + + #ifndef SFX_MODULE -const wchar *uiGetMonthName(int Month) +const wchar *uiGetMonthName(uint Month) { return L""; } diff --git a/multiarc/src/formats/rar/unrar/ulinks.cpp b/multiarc/src/formats/rar/unrar/ulinks.cpp index eb2658489..f76f2abda 100644 --- a/multiarc/src/formats/rar/unrar/ulinks.cpp +++ b/multiarc/src/formats/rar/unrar/ulinks.cpp @@ -1,6 +1,6 @@ -static bool UnixSymlink(CommandData *Cmd,const char *Target,const wchar *LinkName,RarTime *ftm,RarTime *fta) +static bool UnixSymlink(CommandData *Cmd,const std::string &Target,const wchar *LinkName,RarTime *ftm,RarTime *fta) { CreatePath(LinkName,true,Cmd->DisableNames); @@ -9,15 +9,15 @@ static bool UnixSymlink(CommandData *Cmd,const char *Target,const wchar *LinkNam // called earlier inside of uiAskReplaceEx. DelFile(LinkName); - char LinkNameA[NM]; - WideToChar(LinkName,LinkNameA,ASIZE(LinkNameA)); - if (symlink(Target,LinkNameA)==-1) // Error. + std::string LinkNameA; + WideToChar(LinkName,LinkNameA); + if (symlink(Target.c_str(),LinkNameA.c_str())==-1) // Error. { if (errno==EEXIST) uiMsg(UIERROR_ULINKEXIST,LinkName); else { - uiMsg(UIERROR_SLINKCREATE,UINULL,LinkName); + uiMsg(UIERROR_SLINKCREATE,L"",LinkName); ErrHandler.SetErrorCode(RARX_WARNING); } return false; @@ -29,14 +29,14 @@ static bool UnixSymlink(CommandData *Cmd,const char *Target,const wchar *LinkNam times[0].tv_nsec=fta->IsSet() ? long(fta->GetUnixNS()%1000000000) : UTIME_NOW; times[1].tv_sec=ftm->GetUnix(); times[1].tv_nsec=ftm->IsSet() ? long(ftm->GetUnixNS()%1000000000) : UTIME_NOW; - utimensat(AT_FDCWD,LinkNameA,times,AT_SYMLINK_NOFOLLOW); + utimensat(AT_FDCWD,LinkNameA.c_str(),times,AT_SYMLINK_NOFOLLOW); #else struct timeval tv[2]; tv[0].tv_sec=fta->GetUnix(); tv[0].tv_usec=suseconds_t(fta->GetUnixNS()%1000000000/1000); tv[1].tv_sec=ftm->GetUnix(); tv[1].tv_usec=suseconds_t(ftm->GetUnixNS()%1000000000/1000); - lutimes(LinkNameA,tv); + lutimes(LinkNameA.c_str(),tv); #endif #endif @@ -55,9 +55,9 @@ static bool IsFullPath(const char *PathA) // Unix ASCII version. // it didn't affect the number of path related characters we analyze // in IsRelativeSymlinkSafe later. // This check is likely to be excessive, but let's keep it anyway. -static bool SafeCharToWide(const char *Src,wchar *Dest,size_t DestSize) +static bool SafeCharToWide(const std::string &Src,std::wstring &Dest) { - if (!CharToWide(Src,Dest,DestSize) || *Dest==0) + if (!CharToWide(Src,Dest) || Dest.empty()) return false; uint SrcChars=0,DestChars=0; for (uint I=0;Src[I]!=0;I++) @@ -73,18 +73,18 @@ static bool SafeCharToWide(const char *Src,wchar *Dest,size_t DestSize) static bool ExtractUnixLink30(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc, const wchar *LinkName,bool &UpLink) { - char Target[NM]; if (IsLink(Arc.FileHead.FileAttr)) { size_t DataSize=(size_t)Arc.FileHead.PackSize; - if (DataSize>ASIZE(Target)-1) + if (DataSize>MAXPATHSIZE) return false; - if ((size_t)DataIO.UnpRead((byte *)Target,DataSize)!=DataSize) + std::vector TargetBuf(DataSize+1); + if ((size_t)DataIO.UnpRead((byte*)TargetBuf.data(),DataSize)!=DataSize) return false; - Target[DataSize]=0; + std::string Target(TargetBuf.data(),TargetBuf.size()); DataIO.UnpHash.Init(Arc.FileHead.FileHash.Type,1); - DataIO.UnpHash.Update(Target,strlen(Target)); + DataIO.UnpHash.Update(Target.data(),strlen(Target.data())); DataIO.UnpHash.Result(&Arc.FileHead.FileHash); // Return true in case of bad checksum, so link will be processed further @@ -92,16 +92,21 @@ static bool ExtractUnixLink30(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc, if (!DataIO.UnpHash.Cmp(&Arc.FileHead.FileHash,Arc.FileHead.UseHashKey ? Arc.FileHead.HashKey:NULL)) return true; - wchar TargetW[NM]; - if (!SafeCharToWide(Target,TargetW,ASIZE(TargetW))) + std::wstring TargetW; + if (!SafeCharToWide(Target.data(),TargetW)) return false; + TruncateAtZero(TargetW); // Use Arc.FileHead.FileName instead of LinkName, since LinkName // can include the destination path as a prefix, which can // confuse IsRelativeSymlinkSafe algorithm. if (!Cmd->AbsoluteLinks && (IsFullPath(TargetW) || - !IsRelativeSymlinkSafe(Cmd,Arc.FileHead.FileName,LinkName,TargetW))) + !IsRelativeSymlinkSafe(Cmd,Arc.FileHead.FileName.c_str(),LinkName,TargetW.c_str()))) + { + uiMsg(UIERROR_SKIPUNSAFELINK,Arc.FileHead.FileName,TargetW); + ErrHandler.SetErrorCode(RARX_WARNING); return false; - UpLink=strstr(Target,"..")!=NULL; + } + UpLink=Target.find("..")!=std::string::npos; return UnixSymlink(Cmd,Target,LinkName,&Arc.FileHead.mtime,&Arc.FileHead.atime); } return false; @@ -110,27 +115,36 @@ static bool ExtractUnixLink30(CommandData *Cmd,ComprDataIO &DataIO,Archive &Arc, static bool ExtractUnixLink50(CommandData *Cmd,const wchar *Name,FileHeader *hd) { - char Target[NM]; - WideToChar(hd->RedirName,Target,ASIZE(Target)); + std::string Target; + WideToChar(hd->RedirName,Target); if (hd->RedirType==FSREDIR_WINSYMLINK || hd->RedirType==FSREDIR_JUNCTION) { // Cannot create Windows absolute path symlinks in Unix. Only relative path // Windows symlinks can be created here. RAR 5.0 used \??\ prefix // for Windows absolute symlinks, since RAR 5.1 /??/ is used. // We escape ? as \? to avoid "trigraph" warning - if (strncmp(Target,"\\??\\",4)==0 || strncmp(Target,"/\?\?/",4)==0) + if (Target.rfind("\\??\\",0)!=std::string::npos || + Target.rfind("/\?\?/",0)!=std::string::npos) + { + uiMsg(UIERROR_SLINKCREATE,nullptr,L"\"" + hd->FileName + L"\" -> \"" + hd->RedirName + L"\""); + ErrHandler.SetErrorCode(RARX_WARNING); return false; - DosSlashToUnix(Target,Target,ASIZE(Target)); + } + DosSlashToUnix(Target,Target); } - wchar TargetW[NM]; - if (!SafeCharToWide(Target,TargetW,ASIZE(TargetW))) + std::wstring TargetW; + if (!SafeCharToWide(Target,TargetW)) return false; // Use hd->FileName instead of LinkName, since LinkName can include // the destination path as a prefix, which can confuse // IsRelativeSymlinkSafe algorithm. if (!Cmd->AbsoluteLinks && (IsFullPath(TargetW) || - !IsRelativeSymlinkSafe(Cmd,hd->FileName,Name,TargetW))) + !IsRelativeSymlinkSafe(Cmd,hd->FileName.c_str(),Name,TargetW.c_str()))) + { + uiMsg(UIERROR_SKIPUNSAFELINK,hd->FileName,TargetW); + ErrHandler.SetErrorCode(RARX_WARNING); return false; + } return UnixSymlink(Cmd,Target,Name,&hd->mtime,&hd->atime); } diff --git a/multiarc/src/formats/rar/unrar/unicode.cpp b/multiarc/src/formats/rar/unrar/unicode.cpp index 73b09bb22..812e35ba8 100644 --- a/multiarc/src/formats/rar/unrar/unicode.cpp +++ b/multiarc/src/formats/rar/unrar/unicode.cpp @@ -128,6 +128,26 @@ bool CharToWide(const char *Src,wchar *Dest,size_t DestSize) } +bool WideToChar(const std::wstring &Src,std::string &Dest) +{ + // We need more than 1 char per wchar_t for DBCS and up to 4 for UTF-8. + std::vector DestA(4*Src.size()+1); // "+1" for terminating zero. + bool Result=WideToChar(Src.c_str(),DestA.data(),DestA.size()); + Dest=DestA.data(); + return Result; +} + + +bool CharToWide(const std::string &Src,std::wstring &Dest) +{ + // 2 wchar_t per char in case char is converted to UTF-16 surrogate pair. + std::vector DestW(2*Src.size()+1); // "+1" for terminating zero. + bool Result=CharToWide(Src.c_str(),DestW.data(),DestW.size()); + Dest=DestW.data(); + return Result; +} + + #if defined(_UNIX) && defined(MBFUNCTIONS) // Convert and restore mapped inconvertible Unicode characters. // We use it for extended ASCII names in Unix. @@ -244,6 +264,19 @@ byte* WideToRaw(const wchar *Src,size_t SrcSize,byte *Dest,size_t DestSize) } +// Store UTF-16 raw byte stream. +void WideToRaw(const std::wstring &Src,std::vector &Dest) +{ + for (wchar C : Src) + { + Dest.push_back((byte)C); + Dest.push_back((byte)(C>>8)); + } + Dest.push_back(0); // 2 bytes of trailing UTF-16 zero. + Dest.push_back(0); +} + + wchar* RawToWide(const byte *Src,wchar *Dest,size_t DestSize) { for (size_t I=0;I &Src) +{ + std::wstring Dest; + for (size_t I=0;I+1>6)); + Dest.push_back(0x80|(c&0x3f)); + } + else + { + if (c>=0xd800 && c<=0xdbff && I=0xdc00 && Src[I]<=0xdfff) // Surrogate pair. + { + c=((c-0xd800)<<10)+(Src[I]-0xdc00)+0x10000; + I++; + } + if (c<0x10000) + { + Dest.push_back(0xe0|(c>>12)); + Dest.push_back(0x80|((c>>6)&0x3f)); + Dest.push_back(0x80|(c&0x3f)); + } + else + if (c < 0x200000) + { + Dest.push_back(0xf0|(c>>18)); + Dest.push_back(0x80|((c>>12)&0x3f)); + Dest.push_back(0x80|((c>>6)&0x3f)); + Dest.push_back(0x80|(c&0x3f)); + } + } + } +} + + + size_t WideToUtfSize(const wchar *Src) { size_t Size=0; @@ -397,6 +484,146 @@ bool UtfToWide(const char *Src,wchar *Dest,size_t DestSize) } +bool UtfToWide(const char *Src,std::wstring &Dest) +{ + bool Success=true; + Dest.clear(); + while (*Src!=0) + { + uint c=byte(*(Src++)),d; + if (c<0x80) + d=c; + else + if ((c>>5)==6) + { + if ((*Src&0xc0)!=0x80) + { + Success=false; + break; + } + d=((c&0x1f)<<6)|(*Src&0x3f); + Src++; + } + else + if ((c>>4)==14) + { + if ((Src[0]&0xc0)!=0x80 || (Src[1]&0xc0)!=0x80) + { + Success=false; + break; + } + d=((c&0xf)<<12)|((Src[0]&0x3f)<<6)|(Src[1]&0x3f); + Src+=2; + } + else + if ((c>>3)==30) + { + if ((Src[0]&0xc0)!=0x80 || (Src[1]&0xc0)!=0x80 || (Src[2]&0xc0)!=0x80) + { + Success=false; + break; + } + d=((c&7)<<18)|((Src[0]&0x3f)<<12)|((Src[1]&0x3f)<<6)|(Src[2]&0x3f); + Src+=3; + } + else + { + Success=false; + break; + } + if (d>0xffff) + { + if (d>0x10ffff) // UTF-8 must end at 0x10ffff according to RFC 3629. + { + Success=false; + continue; + } + if (sizeof(wchar_t)==2) // Use the surrogate pair. + { + Dest.push_back( ((d-0x10000)>>10)+0xd800 ); + Dest.push_back( (d&0x3ff)+0xdc00 ); + } + else + Dest.push_back( d ); + } + else + Dest.push_back( d ); + } + return Success; +} + + +/* +bool UtfToWide(const std::vector &Src,std::wstring &Dest) +{ + bool Success=true; + Dest.clear(); + for (size_t I=0;I>5)==6) + { + if (Src.size()-I<1 || (Src[I]&0xc0)!=0x80) + { + Success=false; + break; + } + d=((c&0x1f)<<6)|(Src[I]&0x3f); + I++; + } + else + if ((c>>4)==14) + { + if (Src.size()-I<2 || (Src[I]&0xc0)!=0x80 || (Src[I+1]&0xc0)!=0x80) + { + Success=false; + break; + } + d=((c&0xf)<<12)|((Src[I]&0x3f)<<6)|(Src[I+1]&0x3f); + I+=2; + } + else + if ((c>>3)==30) + { + if (Src.size()-I<3 || (Src[I]&0xc0)!=0x80 || (Src[I+1]&0xc0)!=0x80 || (Src[I+2]&0xc0)!=0x80) + { + Success=false; + break; + } + d=((c&7)<<18)|((Src[I]&0x3f)<<12)|((Src[I+1]&0x3f)<<6)|(Src[I+2]&0x3f); + I+=3; + } + else + { + Success=false; + break; + } + if (d>0xffff) + { + if (d>0x10ffff) // UTF-8 must end at 0x10ffff according to RFC 3629. + { + Success=false; + continue; + } + if (sizeof(Dest[0])==2) // Use the surrogate pair. + { + Dest.push_back( ((d-0x10000)>>10)+0xd800 ); + Dest.push_back( (d&0x3ff)+0xdc00 ); + } + else + Dest.push_back( d ); + } + else + Dest.push_back( d ); + } + return Success; +} +*/ + + // For zero terminated strings. bool IsTextUtf8(const byte *Src) { @@ -450,8 +677,10 @@ int wcsnicomp(const wchar *s1,const wchar *s2,size_t n) // If we specify 'n' exceeding the actual string length, CompareString goes // beyond the trailing zero and compares garbage. So we need to limit 'n' // to real string length. - size_t l1=Min(wcslen(s1)+1,n); - size_t l2=Min(wcslen(s2)+1,n); + size_t sl1=wcslen(s1); // Pre-compute to not call wcslen() in Min() twice. + size_t l1=Min(sl1+1,n); + size_t sl2=wcslen(s2); // Pre-compute to not call wcslen() in Min() twice. + size_t l2=Min(sl2+1,n); return CompareStringW(LOCALE_USER_DEFAULT,NORM_IGNORECASE|SORT_STRINGSORT,s1,(int)l1,s2,(int)l2)-2; #else if (n==0) @@ -483,7 +712,15 @@ const wchar_t* wcscasestr(const wchar_t *str, const wchar_t *search) if (tolowerw(str[i+j])!=tolowerw(search[j])) break; } - return NULL; + return nullptr; +} + + +// Case insensitive std::wstring substring search. +std::wstring::size_type wcscasestr(const std::wstring &str, const std::wstring &search) +{ + const wchar *Found=wcscasestr(str.c_str(),search.c_str()); + return Found==nullptr ? std::wstring::npos : Found-str.c_str(); } @@ -500,10 +737,14 @@ wchar* wcslower(wchar *s) #endif return s; } -#endif -#ifndef SFX_MODULE +void wcslower(std::wstring &s) +{ + wcslower(&s[0]); +} + + wchar* wcsupper(wchar *s) { #ifdef _WIN_ALL @@ -516,6 +757,12 @@ wchar* wcsupper(wchar *s) #endif return s; } + + +void wcsupper(std::wstring &s) +{ + wcsupper(&s[0]); +} #endif @@ -548,27 +795,28 @@ int tolowerw(int ch) } -int atoiw(const wchar *s) +int atoiw(const std::wstring &s) { return (int)atoilw(s); } -int64 atoilw(const wchar *s) +int64 atoilw(const std::wstring &s) { bool sign=false; - if (*s=='-') // We do use signed integers here, for example, in GUI SFX. + size_t Pos=0; + if (s[Pos]=='-') // We do use signed integers here, for example, in GUI SFX. { - s++; + Pos++; sign=true; } // Use unsigned type here, since long string can overflow the variable // and signed integer overflow is undefined behavior in C++. uint64 n=0; - while (*s>='0' && *s<='9') + while (s[Pos]>='0' && s[Pos]<='9') { - n=n*10+(*s-'0'); - s++; + n=n*10+(s[Pos]-'0'); + Pos++; } // Check int64(n)>=0 to avoid the signed overflow with undefined behavior // when negating 0x8000000000000000. diff --git a/multiarc/src/formats/rar/unrar/unicode.hpp b/multiarc/src/formats/rar/unrar/unicode.hpp index 9bfd9c5dc..2d867b3ae 100644 --- a/multiarc/src/formats/rar/unrar/unicode.hpp +++ b/multiarc/src/formats/rar/unrar/unicode.hpp @@ -7,25 +7,37 @@ bool WideToChar(const wchar *Src,char *Dest,size_t DestSize); bool CharToWide(const char *Src,wchar *Dest,size_t DestSize); +bool WideToChar(const std::wstring &Src,std::string &Dest); +bool CharToWide(const std::string &Src,std::wstring &Dest); byte* WideToRaw(const wchar *Src,size_t SrcSize,byte *Dest,size_t DestSize); +void WideToRaw(const std::wstring &Src,std::vector &Dest); wchar* RawToWide(const byte *Src,wchar *Dest,size_t DestSize); +std::wstring RawToWide(const std::vector &Src); void WideToUtf(const wchar *Src,char *Dest,size_t DestSize); +void WideToUtf(const std::wstring &Src,std::string &Dest); size_t WideToUtfSize(const wchar *Src); bool UtfToWide(const char *Src,wchar *Dest,size_t DestSize); +bool UtfToWide(const char *Src,std::wstring &Dest); +//bool UtfToWide(const std::vector &Src,std::wstring &Dest); bool IsTextUtf8(const byte *Src); bool IsTextUtf8(const byte *Src,size_t SrcSize); int wcsicomp(const wchar *s1,const wchar *s2); +inline int wcsicomp(const std::wstring &s1,const std::wstring &s2) {return wcsicomp(s1.c_str(),s2.c_str());} int wcsnicomp(const wchar *s1,const wchar *s2,size_t n); +inline int wcsnicomp(const std::wstring &s1,const std::wstring &s2,size_t n) {return wcsnicomp(s1.c_str(),s2.c_str(),n);} const wchar_t* wcscasestr(const wchar_t *str, const wchar_t *search); +std::wstring::size_type wcscasestr(const std::wstring &str, const std::wstring &search); #ifndef SFX_MODULE wchar* wcslower(wchar *s); +void wcslower(std::wstring &s); wchar* wcsupper(wchar *s); +void wcsupper(std::wstring &s); #endif int toupperw(int ch); int tolowerw(int ch); -int atoiw(const wchar *s); -int64 atoilw(const wchar *s); +int atoiw(const std::wstring &s); +int64 atoilw(const std::wstring &s); #ifdef DBCS_SUPPORTED class SupportDBCS diff --git a/multiarc/src/formats/rar/unrar/unpack.cpp b/multiarc/src/formats/rar/unrar/unpack.cpp index 9236e748b..99c6f15db 100644 --- a/multiarc/src/formats/rar/unrar/unpack.cpp +++ b/multiarc/src/formats/rar/unrar/unpack.cpp @@ -10,8 +10,8 @@ #ifndef SFX_MODULE #include "unpack15.cpp" #include "unpack20.cpp" -#endif #include "unpack30.cpp" +#endif #include "unpack50.cpp" #include "unpack50frag.cpp" @@ -24,18 +24,20 @@ Unpack::Unpack(ComprDataIO *DataIO) Suspended=false; UnpAllBuf=false; UnpSomeRead=false; + ExtraDist=false; #ifdef RAR_SMP MaxUserThreads=1; UnpThreadPool=NULL; ReadBufMT=NULL; UnpThreadData=NULL; #endif + AllocWinSize=0; MaxWinSize=0; MaxWinMask=0; // Perform initialization, which should be done only once for all files. - // It prevents crash if first DoUnpack call is later made with wrong - // (true) 'Solid' value. + // It prevents crash if first unpacked file has the wrong "true" Solid flag, + // so first DoUnpack call is made with the wrong "true" Solid value later. UnpInitData(false); #ifndef SFX_MODULE // RAR 1.5 decompression initialization @@ -47,10 +49,11 @@ Unpack::Unpack(ComprDataIO *DataIO) Unpack::~Unpack() { +#ifndef SFX_MODULE InitFilters30(false); +#endif - if (Window!=NULL) - free(Window); + free(Window); #ifdef RAR_SMP delete UnpThreadPool; delete[] ReadBufMT; @@ -70,13 +73,11 @@ void Unpack::SetThreads(uint Threads) #endif -void Unpack::Init(size_t WinSize,bool Solid) +// We get 64-bit WinSize, so we still can check and quit for dictionaries +// exceeding a threshold in 32-bit builds. Then we convert WinSize to size_t +// MaxWinSize. +void Unpack::Init(uint64 WinSize,bool Solid) { - // If 32-bit RAR unpacks an archive with 4 GB dictionary, the window size - // will be 0 because of size_t overflow. Let's issue the memory error. - if (WinSize==0) - ErrHandler.MemoryError(); - // Minimum window size must be at least twice more than maximum possible // size of filter block, which is 0x10000 in RAR now. If window size is // smaller, we can have a block with never cleared flt->NextWindow flag @@ -86,39 +87,53 @@ void Unpack::Init(size_t WinSize,bool Solid) if (WinSize>16)>0x10000) // Window size must not exceed 4 GB. + if (WinSize>Min(0x10000000000ULL,UNPACK_MAX_DICT)) // Window size must not exceed 1 TB. + throw std::bad_alloc(); + + // 32-bit build can't unpack dictionaries exceeding 32-bit even in theory. + // Also we've not verified if WrapUp and WrapDown work properly in 32-bit + // version and >2GB dictionary and if 32-bit version can handle >2GB + // distances. Since such version is unlikely to allocate >2GB anyway, + // we prohibit >2GB dictionaries for 32-bit build here. + if (WinSize>0x80000000 && sizeof(size_t)<=4) + throw std::bad_alloc(); + + // Solid block shall use the same window size for all files. + // But if Window isn't initialized when Solid is set, it means that + // first file in solid block doesn't have the solid flag. We initialize + // the window anyway for such malformed archive. + // Non-solid files shall use their specific window sizes, + // so current window size and unpack routine behavior doesn't depend on + // previously unpacked files and their extraction order. + if (!Solid || Window==nullptr) + { + MaxWinSize=(size_t)WinSize; + MaxWinMask=MaxWinSize-1; + } + + // Use the already allocated window when processing non-solid files + // with reducing dictionary sizes. + if (WinSize<=AllocWinSize) return; // Archiving code guarantees that window size does not grow in the same // solid stream. So if we are here, we are either creating a new window // or increasing the size of non-solid window. So we could safely reject - // current window data without copying them to a new window, though being - // extra cautious, we still handle the solid window grow case below. - bool Grow=Solid && (Window!=NULL || Fragmented); - - // We do not handle growth for existing fragmented window. - if (Grow && Fragmented) + // current window data without copying them to a new window. + if (Solid && (Window!=NULL || Fragmented && WinSize>FragWindow.GetWinSize())) throw std::bad_alloc(); - byte *NewWindow=Fragmented ? NULL : (byte *)malloc(WinSize); + free(Window); + + Window=Fragmented ? NULL : (byte *)malloc((size_t)WinSize); - if (NewWindow==NULL) - if (Grow || WinSize<0x1000000) - { - // We do not support growth for new fragmented window. - // Also exclude RAR4 and small dictionaries. - throw std::bad_alloc(); - } + if (Window==NULL) + if (WinSize<0x1000000 || sizeof(size_t)>4) + throw std::bad_alloc(); // Exclude RAR4, small dictionaries and 64-bit. else { - if (Window!=NULL) // If allocated by preceding files. - { - free(Window); - Window=NULL; - } - FragWindow.Init(WinSize); + if (WinSize>FragWindow.GetWinSize()) + FragWindow.Init((size_t)WinSize); Fragmented=true; } @@ -126,23 +141,12 @@ void Unpack::Init(size_t WinSize,bool Solid) { // Clean the window to generate the same output when unpacking corrupt // RAR files, which may access unused areas of sliding dictionary. - memset(NewWindow,0,WinSize); - - // If Window is not NULL, it means that window size has grown. - // In solid streams we need to copy data to a new window in such case. - // RAR archiving code does not allow it in solid streams now, - // but let's implement it anyway just in case we'll change it sometimes. - if (Grow) - for (size_t I=1;I<=MaxWinSize;I++) - NewWindow[(UnpPtr-I)&(WinSize-1)]=Window[(UnpPtr-I)&(MaxWinSize-1)]; - - if (Window!=NULL) - free(Window); - Window=NewWindow; - } + // 2023.10.31: We've added FirstWinDone based unused area access check + // in Unpack::CopyString(), so this memset might be unnecessary now. +// memset(Window,0,(size_t)WinSize); - MaxWinSize=WinSize; - MaxWinMask=MaxWinSize-1; + AllocWinSize=WinSize; + } } @@ -154,21 +158,23 @@ void Unpack::DoUnpack(uint Method,bool Solid) switch(Method) { #ifndef SFX_MODULE - case 15: // rar 1.5 compression + case 15: // RAR 1.5 compression. if (!Fragmented) Unpack15(Solid); break; - case 20: // rar 2.x compression - case 26: // files larger than 2GB + case 20: // RAR 2.x compression. + case 26: // Files larger than 2GB. if (!Fragmented) Unpack20(Solid); break; -#endif - case 29: // rar 3.x compression + case 29: // RAR 3.x compression. if (!Fragmented) Unpack29(Solid); break; - case 50: // RAR 5.0 compression algorithm. +#endif + case VER_PACK5: // 50. RAR 5.0 and 7.0 compression algorithms. + case VER_PACK7: // 70. + ExtraDist=(Method==VER_PACK7); #ifdef RAR_SMP if (MaxUserThreads>1) { @@ -194,13 +200,19 @@ void Unpack::UnpInitData(bool Solid) { if (!Solid) { - memset(OldDist,0,sizeof(OldDist)); + OldDist[0]=OldDist[1]=OldDist[2]=OldDist[3]=(size_t)-1; + OldDistPtr=0; - LastDist=LastLength=0; + + LastDist=(uint)-1; // Initialize it to -1 like LastDist. + LastLength=0; + // memset(Window,0,MaxWinSize); memset(&BlockTables,0,sizeof(BlockTables)); UnpPtr=WrPtr=0; - WriteBorder=Min(MaxWinSize,UNPACK_MAX_WRITE)&MaxWinMask; + PrevPtr=0; + FirstWinDone=false; + WriteBorder=Min(MaxWinSize,UNPACK_MAX_WRITE); } // Filters never share several solid files, so we can safely reset them // even in solid archive. @@ -215,8 +227,8 @@ void Unpack::UnpInitData(bool Solid) BlockHeader.BlockSize=-1; // '-1' means not defined yet. #ifndef SFX_MODULE UnpInitData20(Solid); -#endif UnpInitData30(Solid); +#endif UnpInitData50(Solid); } diff --git a/multiarc/src/formats/rar/unrar/unpack.hpp b/multiarc/src/formats/rar/unrar/unpack.hpp index 737b31c2c..5ed904a8c 100644 --- a/multiarc/src/formats/rar/unrar/unpack.hpp +++ b/multiarc/src/formats/rar/unrar/unpack.hpp @@ -2,7 +2,7 @@ #define _RAR_UNPACK_ // Maximum allowed number of compressed bits processed in quick mode. -#define MAX_QUICK_DECODE_BITS 10 +#define MAX_QUICK_DECODE_BITS 9 // Maximum number of filters per entire data block. Must be at least // twice more than MAX_PACK_FILTERS to store filters from two data blocks. @@ -102,7 +102,7 @@ struct UnpackDecodedItem ushort Length; union { - uint Distance; + size_t Distance; byte Literal[8]; // Store up to 8 chars here to speed up extraction. }; }; @@ -143,13 +143,13 @@ struct UnpackThreadData struct UnpackFilter { + // Groop 'byte' and 'bool' together to reduce the actual struct size. byte Type; - uint BlockStart; - uint BlockLength; byte Channels; -// uint Width; -// byte PosR; bool NextWindow; + + size_t BlockStart; + uint BlockLength; }; @@ -188,14 +188,16 @@ class FragmentedWindow void Reset(); byte *Mem[MAX_MEM_BLOCKS]; size_t MemSize[MAX_MEM_BLOCKS]; + size_t LastAllocated; public: FragmentedWindow(); ~FragmentedWindow(); void Init(size_t WinSize); byte& operator [](size_t Item); - void CopyString(uint Length,uint Distance,size_t &UnpPtr,size_t MaxWinMask); + void CopyString(uint Length,size_t Distance,size_t &UnpPtr,bool FirstWinDone,size_t MaxWinSize); void CopyData(byte *Dest,size_t WinPos,size_t Size); size_t GetBlockSize(size_t StartPos,size_t RequiredSize); + size_t GetWinSize() {return LastAllocated;} }; @@ -216,10 +218,9 @@ class Unpack:PackDef bool ReadTables(BitInput &Inp,UnpackBlockHeader &Header,UnpackBlockTables &Tables); void MakeDecodeTables(byte *LengthTable,DecodeTable *Dec,uint Size); _forceinline uint DecodeNumber(BitInput &Inp,DecodeTable *Dec); - void CopyString(); - inline void InsertOldDist(unsigned int Distance); + inline void InsertOldDist(size_t Distance); void UnpInitData(bool Solid); - _forceinline void CopyString(uint Length,uint Distance); + _forceinline void CopyString(uint Length,size_t Distance); uint ReadFilterData(BitInput &Inp); bool ReadFilter(BitInput &Inp,UnpackFilter &Filter); bool AddFilter(UnpackFilter &Filter); @@ -240,20 +241,25 @@ class Unpack:PackDef byte *ReadBufMT; #endif - Array FilterSrcMemory; - Array FilterDstMemory; + std::vector FilterSrcMemory; + std::vector FilterDstMemory; // Filters code, one entry per filter. - Array Filters; + std::vector Filters; - uint OldDist[4],OldDistPtr; + size_t OldDist[4],OldDistPtr; uint LastLength; // LastDist is necessary only for RAR2 and older with circular OldDist // array. In RAR3 last distance is always stored in OldDist[0]. uint LastDist; - size_t UnpPtr,WrPtr; + size_t UnpPtr; // Current position in window. + + size_t PrevPtr; // UnpPtr value for previous loop iteration. + bool FirstWinDone; // At least one dictionary was processed. + + size_t WrPtr; // Last written unpacked data position. // Top border of read packed data. int ReadTop; @@ -266,7 +272,7 @@ class Unpack:PackDef UnpackBlockHeader BlockHeader; UnpackBlockTables BlockTables; - size_t WriteBorder; + size_t WriteBorder; // Perform write when reaching this border. byte *Window; @@ -289,7 +295,7 @@ class Unpack:PackDef void LongLZ(); void HuffDecode(); void GetFlagsBuf(); - void UnpInitData15(int Solid); + void UnpInitData15(bool Solid); void InitHuff(); void CorrHuff(ushort *CharSet,byte *NumToPlace); void CopyString15(uint Distance,uint Length); @@ -359,15 +365,15 @@ class Unpack:PackDef BitInput VMCodeInp; // Filters code, one entry per filter. - Array Filters30; + std::vector Filters30; // Filters stack, several entrances of same filter are possible. - Array PrgStack; + std::vector PrgStack; // Lengths of preceding data blocks, one length of one last block // for every filter. Used to reduce the size required to write // the data block length if lengths are repeating. - Array OldFilterLengths; + std::vector OldFilterLengths; int LastFilter; /***************************** Unpack v 3.0 *********************************/ @@ -375,9 +381,9 @@ class Unpack:PackDef public: Unpack(ComprDataIO *DataIO); ~Unpack(); - void Init(size_t WinSize,bool Solid); + void Init(uint64 WinSize,bool Solid); void DoUnpack(uint Method,bool Solid); - bool IsFileExtracted() {return(FileExtracted);} + bool IsFileExtracted() {return FileExtracted;} void SetDestSize(int64 DestSize) {DestUnpSize=DestSize;FileExtracted=false;} void SetSuspended(bool Suspended) {Unpack::Suspended=Suspended;} @@ -386,10 +392,13 @@ class Unpack:PackDef void UnpackDecode(UnpackThreadData &D); #endif + uint64 AllocWinSize; size_t MaxWinSize; size_t MaxWinMask; - uint GetChar() + bool ExtraDist; // Allow distances up to 1 TB. + + byte GetChar() { if (Inp.InAddr>BitInput::MAX_SIZE-30) { @@ -399,6 +408,30 @@ class Unpack:PackDef } return Inp.InBuf[Inp.InAddr++]; } + + + // If window position crosses the window beginning, wrap it to window end. + // Replaces &MaxWinMask for non-power 2 window sizes. + // We can't use %WinSize everywhere not only for performance reason, + // but also because C++ % is reminder instead of modulo. + // We need additional checks in the code if WinPos distance from 0 + // can exceed MaxWinSize. Alternatively we could add such check here. + inline size_t WrapDown(size_t WinPos) + { + return WinPos >= MaxWinSize ? WinPos + MaxWinSize : WinPos; + } + + // If window position crosses the window end, wrap it to window beginning. + // Replaces &MaxWinMask for non-power 2 window sizes. + // Unlike WrapDown, we can use %WinSize here if there was no size_t + // overflow when calculating WinPos. + // We need additional checks in the code if WinPos distance from MaxWinSize + // can be MaxWinSize or more. Alternatively we could add such check here + // or use %WinSize. + inline size_t WrapUp(size_t WinPos) + { + return WinPos >= MaxWinSize ? WinPos - MaxWinSize : WinPos; + } }; #endif diff --git a/multiarc/src/formats/rar/unrar/unpack15.cpp b/multiarc/src/formats/rar/unrar/unpack15.cpp index 1e7cf76c2..1019c354a 100644 --- a/multiarc/src/formats/rar/unrar/unpack15.cpp +++ b/multiarc/src/formats/rar/unrar/unpack15.cpp @@ -1,40 +1,40 @@ #define STARTL1 2 -static unsigned int DecL1[]={0x8000,0xa000,0xc000,0xd000,0xe000,0xea00, +static uint DecL1[]={0x8000,0xa000,0xc000,0xd000,0xe000,0xea00, 0xee00,0xf000,0xf200,0xf200,0xffff}; -static unsigned int PosL1[]={0,0,0,2,3,5,7,11,16,20,24,32,32}; +static uint PosL1[]={0,0,0,2,3,5,7,11,16,20,24,32,32}; #define STARTL2 3 -static unsigned int DecL2[]={0xa000,0xc000,0xd000,0xe000,0xea00,0xee00, +static uint DecL2[]={0xa000,0xc000,0xd000,0xe000,0xea00,0xee00, 0xf000,0xf200,0xf240,0xffff}; -static unsigned int PosL2[]={0,0,0,0,5,7,9,13,18,22,26,34,36}; +static uint PosL2[]={0,0,0,0,5,7,9,13,18,22,26,34,36}; #define STARTHF0 4 -static unsigned int DecHf0[]={0x8000,0xc000,0xe000,0xf200,0xf200,0xf200, +static uint DecHf0[]={0x8000,0xc000,0xe000,0xf200,0xf200,0xf200, 0xf200,0xf200,0xffff}; -static unsigned int PosHf0[]={0,0,0,0,0,8,16,24,33,33,33,33,33}; +static uint PosHf0[]={0,0,0,0,0,8,16,24,33,33,33,33,33}; #define STARTHF1 5 -static unsigned int DecHf1[]={0x2000,0xc000,0xe000,0xf000,0xf200,0xf200, +static uint DecHf1[]={0x2000,0xc000,0xe000,0xf000,0xf200,0xf200, 0xf7e0,0xffff}; -static unsigned int PosHf1[]={0,0,0,0,0,0,4,44,60,76,80,80,127}; +static uint PosHf1[]={0,0,0,0,0,0,4,44,60,76,80,80,127}; #define STARTHF2 5 -static unsigned int DecHf2[]={0x1000,0x2400,0x8000,0xc000,0xfa00,0xffff, +static uint DecHf2[]={0x1000,0x2400,0x8000,0xc000,0xfa00,0xffff, 0xffff,0xffff}; -static unsigned int PosHf2[]={0,0,0,0,0,0,2,7,53,117,233,0,0}; +static uint PosHf2[]={0,0,0,0,0,0,2,7,53,117,233,0,0}; #define STARTHF3 6 -static unsigned int DecHf3[]={0x800,0x2400,0xee00,0xfe80,0xffff,0xffff, +static uint DecHf3[]={0x800,0x2400,0xee00,0xfe80,0xffff,0xffff, 0xffff}; -static unsigned int PosHf3[]={0,0,0,0,0,0,0,2,16,218,251,0,0}; +static uint PosHf3[]={0,0,0,0,0,0,0,2,16,218,251,0,0}; #define STARTHF4 8 -static unsigned int DecHf4[]={0xff00,0xffff,0xffff,0xffff,0xffff,0xffff}; -static unsigned int PosHf4[]={0,0,0,0,0,0,0,0,0,255,0,0,0}; +static uint DecHf4[]={0xff00,0xffff,0xffff,0xffff,0xffff,0xffff}; +static uint PosHf4[]={0,0,0,0,0,0,0,0,0,255,0,0,0}; void Unpack::Unpack15(bool Solid) @@ -60,6 +60,9 @@ void Unpack::Unpack15(bool Solid) { UnpPtr&=MaxWinMask; + FirstWinDone|=(PrevPtr>UnpPtr); + PrevPtr=UnpPtr; + if (Inp.InAddr>ReadTop-30 && !UnpReadBuf()) break; if (((WrPtr-UnpPtr) & MaxWinMask)<270 && WrPtr!=UnpPtr) @@ -116,27 +119,27 @@ void Unpack::Unpack15(bool Solid) void Unpack::ShortLZ() { - static unsigned int ShortLen1[]={1,3,4,4,5,6,7,8,8,4,4,5,6,6,4,0}; - static unsigned int ShortXor1[]={0,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xfe, + static uint ShortLen1[]={1,3,4,4,5,6,7,8,8,4,4,5,6,6,4,0}; + static uint ShortXor1[]={0,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xfe, 0xff,0xc0,0x80,0x90,0x98,0x9c,0xb0}; - static unsigned int ShortLen2[]={2,3,3,3,4,4,5,6,6,4,4,5,6,6,4,0}; - static unsigned int ShortXor2[]={0,0x40,0x60,0xa0,0xd0,0xe0,0xf0,0xf8, + static uint ShortLen2[]={2,3,3,3,4,4,5,6,6,4,4,5,6,6,4,0}; + static uint ShortXor2[]={0,0x40,0x60,0xa0,0xd0,0xe0,0xf0,0xf8, 0xfc,0xc0,0x80,0x90,0x98,0x9c,0xb0}; - unsigned int Length,SaveLength; - unsigned int LastDistance; - unsigned int Distance; + uint Length,SaveLength; + uint LastDistance; + uint Distance; int DistancePlace; NumHuf=0; - unsigned int BitField=Inp.fgetbits(); + uint BitField=Inp.fgetbits(); if (LCount==2) { Inp.faddbits(1); if (BitField >= 0x8000) { - CopyString15((unsigned int)LastDist,LastLength); + CopyString15(LastDist,LastLength); return; } BitField <<= 1; @@ -168,7 +171,7 @@ void Unpack::ShortLZ() if (Length == 9) { LCount++; - CopyString15((unsigned int)LastDist,LastLength); + CopyString15(LastDist,LastLength); return; } if (Length == 14) @@ -185,7 +188,7 @@ void Unpack::ShortLZ() LCount=0; SaveLength=Length; - Distance=OldDist[(OldDistPtr-(Length-9)) & 3]; + Distance=(uint)OldDist[(OldDistPtr-(Length-9)) & 3]; Length=DecodeNum(Inp.fgetbits(),STARTL1,DecL1,PosL1)+2; if (Length==0x101 && SaveLength==10) { @@ -214,8 +217,8 @@ void Unpack::ShortLZ() if (--DistancePlace != -1) { LastDistance=ChSetA[DistancePlace]; - ChSetA[DistancePlace+1]=LastDistance; - ChSetA[DistancePlace]=Distance; + ChSetA[DistancePlace+1]=(ushort)LastDistance; + ChSetA[DistancePlace]=(ushort)Distance; } Length+=2; OldDist[OldDistPtr++] = ++Distance; @@ -228,10 +231,10 @@ void Unpack::ShortLZ() void Unpack::LongLZ() { - unsigned int Length; - unsigned int Distance; - unsigned int DistancePlace,NewDistancePlace; - unsigned int OldAvr2,OldAvr3; + uint Length; + uint Distance; + uint DistancePlace,NewDistancePlace; + uint OldAvr2,OldAvr3; NumHuf=0; Nlzb+=16; @@ -242,7 +245,7 @@ void Unpack::LongLZ() } OldAvr2=AvrLn2; - unsigned int BitField=Inp.fgetbits(); + uint BitField=Inp.fgetbits(); if (AvrLn2 >= 122) Length=DecodeNum(BitField,STARTL2,DecL2,PosL2); else @@ -286,7 +289,7 @@ void Unpack::LongLZ() } ChSetB[DistancePlace & 0xff]=ChSetB[NewDistancePlace]; - ChSetB[NewDistancePlace]=Distance; + ChSetB[NewDistancePlace]=(ushort)Distance; Distance=((Distance & 0xff00) | (Inp.fgetbits() >> 8)) >> 1; Inp.faddbits(7); @@ -320,12 +323,12 @@ void Unpack::LongLZ() void Unpack::HuffDecode() { - unsigned int CurByte,NewBytePlace; - unsigned int Length; - unsigned int Distance; + uint CurByte,NewBytePlace; + uint Length; + uint Distance; int BytePlace; - unsigned int BitField=Inp.fgetbits(); + uint BitField=Inp.fgetbits(); if (AvrPlc > 0x75ff) BytePlace=DecodeNum(BitField,STARTHF4,DecHf4,PosHf4); @@ -392,14 +395,14 @@ void Unpack::HuffDecode() } ChSet[BytePlace]=ChSet[NewBytePlace]; - ChSet[NewBytePlace]=CurByte; + ChSet[NewBytePlace]=(ushort)CurByte; } void Unpack::GetFlagsBuf() { - unsigned int Flags,NewFlagsPlace; - unsigned int FlagsPlace=DecodeNum(Inp.fgetbits(),STARTHF2,DecHf2,PosHf2); + uint Flags,NewFlagsPlace; + uint FlagsPlace=DecodeNum(Inp.fgetbits(),STARTHF2,DecHf2,PosHf2); // Our Huffman table stores 257 items and needs all them in other parts // of code such as when StMode is on, so the first item is control item. @@ -420,11 +423,11 @@ void Unpack::GetFlagsBuf() } ChSetC[FlagsPlace]=ChSetC[NewFlagsPlace]; - ChSetC[NewFlagsPlace]=Flags; + ChSetC[NewFlagsPlace]=(ushort)Flags; } -void Unpack::UnpInitData15(int Solid) +void Unpack::UnpInitData15(bool Solid) { if (!Solid) { @@ -443,7 +446,7 @@ void Unpack::UnpInitData15(int Solid) void Unpack::InitHuff() { - for (unsigned int I=0;I<256;I++) + for (ushort I=0;I<256;I++) { ChSet[I]=ChSetB[I]=I<<8; ChSetA[I]=I; @@ -471,11 +474,19 @@ void Unpack::CorrHuff(ushort *CharSet,byte *NumToPlace) void Unpack::CopyString15(uint Distance,uint Length) { DestUnpSize-=Length; - while (Length--) - { - Window[UnpPtr]=Window[(UnpPtr-Distance) & MaxWinMask]; - UnpPtr=(UnpPtr+1) & MaxWinMask; - } + // 2024.04.18: Distance can be 0 in corrupt RAR 1.5 archives. + if (!FirstWinDone && Distance>UnpPtr || Distance>MaxWinSize || Distance==0) + while (Length-- > 0) + { + Window[UnpPtr]=0; + UnpPtr=(UnpPtr+1) & MaxWinMask; + } + else + while (Length-- > 0) + { + Window[UnpPtr]=Window[(UnpPtr-Distance) & MaxWinMask]; + UnpPtr=(UnpPtr+1) & MaxWinMask; + } } diff --git a/multiarc/src/formats/rar/unrar/unpack20.cpp b/multiarc/src/formats/rar/unrar/unpack20.cpp index 93c8ba05a..a0e179d6a 100644 --- a/multiarc/src/formats/rar/unrar/unpack20.cpp +++ b/multiarc/src/formats/rar/unrar/unpack20.cpp @@ -2,7 +2,8 @@ void Unpack::CopyString20(uint Length,uint Distance) { - LastDist=OldDist[OldDistPtr++]=Distance; + LastDist=Distance; + OldDist[OldDistPtr++]=Distance; OldDistPtr = OldDistPtr & 3; // Needed if RAR 1.5 file is called after RAR 2.0. LastLength=Length; DestUnpSize-=Length; @@ -36,6 +37,9 @@ void Unpack::Unpack20(bool Solid) { UnpPtr&=MaxWinMask; + FirstWinDone|=(PrevPtr>UnpPtr); + PrevPtr=UnpPtr; + if (Inp.InAddr>ReadTop-30) if (!UnpReadBuf()) break; @@ -109,7 +113,7 @@ void Unpack::Unpack20(bool Solid) } if (Number<261) { - uint Distance=OldDist[(OldDistPtr-(Number-256)) & 3]; + uint Distance=(uint)OldDist[(OldDistPtr-(Number-256)) & 3]; uint LengthNumber=DecodeNumber(Inp,&BlockTables.RD); uint Length=LDecode[LengthNumber]+2; if ((Bits=LBits[LengthNumber])>0) diff --git a/multiarc/src/formats/rar/unrar/unpack30.cpp b/multiarc/src/formats/rar/unrar/unpack30.cpp index 7c2adfab2..1fb0615a6 100644 --- a/multiarc/src/formats/rar/unrar/unpack30.cpp +++ b/multiarc/src/formats/rar/unrar/unpack30.cpp @@ -17,8 +17,8 @@ void Unpack::Unpack29(bool Solid) { static unsigned char LDecode[]={0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224}; static unsigned char LBits[]= {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5}; - static int DDecode[DC]; - static byte DBits[DC]; + static int DDecode[DC30]; + static byte DBits[DC30]; static int DBitLengthCounts[]= {4,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,14,0,12}; static unsigned char SDDecode[]={0,4,8,16,32,64,128,192}; static unsigned char SDBits[]= {2,2,3, 4, 5, 6, 6, 6}; @@ -50,6 +50,9 @@ void Unpack::Unpack29(bool Solid) { UnpPtr&=MaxWinMask; + FirstWinDone|=(PrevPtr>UnpPtr); + PrevPtr=UnpPtr; + if (Inp.InAddr>ReadBorder) { if (!UnpReadBuf30()) @@ -219,7 +222,7 @@ void Unpack::Unpack29(bool Solid) if (Number<263) { uint DistNum=Number-259; - uint Distance=OldDist[DistNum]; + uint Distance=(uint)OldDist[DistNum]; for (uint I=DistNum;I>0;I--) OldDist[I]=OldDist[I-1]; OldDist[0]=Distance; @@ -306,7 +309,7 @@ bool Unpack::ReadVMCode() } if (Length==0) return false; - Array VMCode(Length); + std::vector VMCode(Length); for (uint I=0;I>8; Inp.addbits(8); } - return AddVMCode(FirstByte,&VMCode[0],Length); + return AddVMCode(FirstByte,VMCode.data(),Length); } @@ -346,7 +349,7 @@ bool Unpack::ReadVMCodePPM() } if (Length==0) return false; - Array VMCode(Length); + std::vector VMCode(Length); for (uint I=0;IFilters30.Size() || FiltPos>OldFilterLengths.Size()) + if (FiltPos>Filters30.size() || FiltPos>OldFilterLengths.size()) return false; LastFilter=FiltPos; - bool NewFilter=(FiltPos==Filters30.Size()); + bool NewFilter=(FiltPos==Filters30.size()); UnpackFilter30 *StackFilter=new UnpackFilter30; // New filter for PrgStack. @@ -393,15 +396,15 @@ bool Unpack::AddVMCode(uint FirstByte,byte *Code,uint CodeSize) return false; } - Filters30.Add(1); - Filters30[Filters30.Size()-1]=Filter=new UnpackFilter30; - StackFilter->ParentFilter=(uint)(Filters30.Size()-1); + StackFilter->ParentFilter=(uint)Filters30.size(); + Filter=new UnpackFilter30; + Filters30.push_back(Filter); // Reserve one item to store the data block length of our new filter // entry. We'll set it to real block length below, after reading it. // But we need to initialize it now, because when processing corrupt // data, we can access this item even before we set it to real value. - OldFilterLengths.Push(0); + OldFilterLengths.push_back(0); } else // Filter was used in the past. { @@ -410,7 +413,7 @@ bool Unpack::AddVMCode(uint FirstByte,byte *Code,uint CodeSize) } uint EmptyCount=0; - for (uint I=0;IMAX3_UNPACK_FILTERS) + if (PrgStack.size()>MAX3_UNPACK_FILTERS) { delete StackFilter; return false; } - PrgStack.Add(1); + PrgStack.resize(PrgStack.size()+1); EmptyCount=1; } - size_t StackPos=PrgStack.Size()-EmptyCount; + size_t StackPos=PrgStack.size()-EmptyCount; PrgStack[StackPos]=StackFilter; uint BlockStart=RarVM::ReadData(VMCodeInp); @@ -448,7 +451,7 @@ bool Unpack::AddVMCode(uint FirstByte,byte *Code,uint CodeSize) // for same filter. It is possible for corrupt data to access a new // and not filled yet item of OldFilterLengths array here. This is why // we set new OldFilterLengths items to zero above. - StackFilter->BlockLength=FiltPosBlockLength=FiltPosNextWindow=WrPtr!=UnpPtr && ((WrPtr-UnpPtr)&MaxWinMask)<=BlockStart; @@ -472,7 +475,7 @@ bool Unpack::AddVMCode(uint FirstByte,byte *Code,uint CodeSize) uint VMCodeSize=RarVM::ReadData(VMCodeInp); if (VMCodeSize>=0x10000 || VMCodeSize==0 || VMCodeInp.InAddr+VMCodeSize>CodeSize) return false; - Array VMCode(VMCodeSize); + std::vector VMCode(VMCodeSize); for (uint I=0;I>8; VMCodeInp.faddbits(8); } - VM.Prepare(&VMCode[0],VMCodeSize,&Filter->Prg); + VM.Prepare(VMCode.data(),VMCodeSize,&Filter->Prg); } StackFilter->Prg.Type=Filter->Prg.Type; @@ -520,7 +523,7 @@ void Unpack::UnpWriteBuf30() { uint WrittenBorder=(uint)WrPtr; uint WriteSize=(uint)((UnpPtr-WrittenBorder)&MaxWinMask); - for (size_t I=0;IFilteredDataSize; delete PrgStack[I]; - PrgStack[I]=NULL; - while (I+1FilteredDataSize; I++; delete PrgStack[I]; - PrgStack[I]=NULL; + PrgStack[I]=nullptr; } UnpIO->UnpWrite(FilteredData,FilteredDataSize); UnpSomeRead=true; @@ -601,10 +604,10 @@ void Unpack::UnpWriteBuf30() { // Current filter intersects the window write border, so we adjust // the window border to process this filter next time, not now. - for (size_t J=I;JNextWindow) + if (flt!=nullptr && flt->NextWindow) flt->NextWindow=false; } WrPtr=WrittenBorder; @@ -752,14 +755,14 @@ void Unpack::InitFilters30(bool Solid) { if (!Solid) { - OldFilterLengths.SoftReset(); + OldFilterLengths.clear(); LastFilter=0; - for (size_t I=0;IUnpPtr); + PrevPtr=UnpPtr; if (Inp.InAddr>=ReadBorder) { @@ -42,7 +47,8 @@ void Unpack::Unpack5(bool Solid) break; } - if (((WriteBorder-UnpPtr) & MaxWinMask)<=MAX_INC_LZ_MATCH && WriteBorder!=UnpPtr) + // WriteBorder==UnpPtr means that we have MaxWinSize data ahead. + if (WrapDown(WriteBorder-UnpPtr)<=MAX_INC_LZ_MATCH && WriteBorder!=UnpPtr) { UnpWriteBuf(); if (WrittenFileSize>DestUnpSize) @@ -67,7 +73,8 @@ void Unpack::Unpack5(bool Solid) { uint Length=SlotToLength(Inp,MainSlot-262); - uint DBits,Distance=1,DistSlot=DecodeNumber(Inp,&BlockTables.DD); + size_t Distance=1; + uint DBits,DistSlot=DecodeNumber(Inp,&BlockTables.DD); if (DistSlot<4) { DBits=0; @@ -76,7 +83,7 @@ void Unpack::Unpack5(bool Solid) else { DBits=DistSlot/2 - 1; - Distance+=(2 | (DistSlot & 1)) << DBits; + Distance+=size_t(2 | (DistSlot & 1)) << DBits; } if (DBits>0) @@ -85,11 +92,23 @@ void Unpack::Unpack5(bool Solid) { if (DBits>4) { - Distance+=((Inp.getbits32()>>(36-DBits))<<4); + // It is also possible to always use getbits64() here. + if (DBits>36) + Distance+=( ( size_t(Inp.getbits64() ) >> (68-DBits) ) << 4 ); + else + Distance+=( ( size_t(Inp.getbits32() ) >> (36-DBits) ) << 4 ); Inp.addbits(DBits-4); } uint LowDist=DecodeNumber(Inp,&BlockTables.LDD); Distance+=LowDist; + + // Distance can be 0 for multiples of 4 GB as result of size_t + // overflow in 32-bit build. Its lower 32-bit can also erroneously + // fit into dictionary after truncating upper 32-bits. Replace such + // invalid distances with -1, so CopyString sets 0 data for them. + // DBits>=30 also as DistSlot>=62 indicate distances >=0x80000001. + if (sizeof(Distance)==4 && DBits>=30) + Distance=(size_t)-1; } else { @@ -112,7 +131,7 @@ void Unpack::Unpack5(bool Solid) InsertOldDist(Distance); LastLength=Length; if (Fragmented) - FragWindow.CopyString(Length,Distance,UnpPtr,MaxWinMask); + FragWindow.CopyString(Length,Distance,UnpPtr,FirstWinDone,MaxWinSize); else CopyString(Length,Distance); continue; @@ -128,7 +147,7 @@ void Unpack::Unpack5(bool Solid) { if (LastLength!=0) if (Fragmented) - FragWindow.CopyString(LastLength,OldDist[0],UnpPtr,MaxWinMask); + FragWindow.CopyString(LastLength,OldDist[0],UnpPtr,FirstWinDone,MaxWinSize); else CopyString(LastLength,OldDist[0]); continue; @@ -136,7 +155,7 @@ void Unpack::Unpack5(bool Solid) if (MainSlot<262) { uint DistNum=MainSlot-258; - uint Distance=OldDist[DistNum]; + size_t Distance=OldDist[DistNum]; for (uint I=DistNum;I>0;I--) OldDist[I]=OldDist[I-1]; OldDist[0]=Distance; @@ -145,7 +164,7 @@ void Unpack::Unpack5(bool Solid) uint Length=SlotToLength(Inp,LengthSlot); LastLength=Length; if (Fragmented) - FragWindow.CopyString(Length,Distance,UnpPtr,MaxWinMask); + FragWindow.CopyString(Length,Distance,UnpPtr,FirstWinDone,MaxWinSize); else CopyString(Length,Distance); continue; @@ -196,20 +215,25 @@ bool Unpack::ReadFilter(BitInput &Inp,UnpackFilter &Filter) bool Unpack::AddFilter(UnpackFilter &Filter) { - if (Filters.Size()>=MAX_UNPACK_FILTERS) + if (Filters.size()>=MAX_UNPACK_FILTERS) { UnpWriteBuf(); // Write data, apply and flush filters. - if (Filters.Size()>=MAX_UNPACK_FILTERS) + if (Filters.size()>=MAX_UNPACK_FILTERS) InitFilters(); // Still too many filters, prevent excessive memory use. } // If distance to filter start is that large that due to circular dictionary // mode now it points to old not written yet data, then we set 'NextWindow' // flag and process this filter only after processing that older data. - Filter.NextWindow=WrPtr!=UnpPtr && ((WrPtr-UnpPtr)&MaxWinMask)<=Filter.BlockStart; - - Filter.BlockStart=uint((Filter.BlockStart+UnpPtr)&MaxWinMask); - Filters.Push(Filter); + Filter.NextWindow=WrPtr!=UnpPtr && WrapDown(WrPtr-UnpPtr)<=Filter.BlockStart; + + // In malformed archive Filter.BlockStart can be many times larger + // than window size, so here we must use the reminder instead of + // subtracting the single window size as WrapUp can do. So the result + // is always within the window. Since we add and not subtract here, + // reminder always provides the valid result in valid archives. + Filter.BlockStart=(Filter.BlockStart+UnpPtr)%MaxWinSize; + Filters.push_back(Filter); return true; } @@ -255,10 +279,10 @@ bool Unpack::UnpReadBuf() void Unpack::UnpWriteBuf() { size_t WrittenBorder=WrPtr; - size_t FullWriteSize=(UnpPtr-WrittenBorder)&MaxWinMask; + size_t FullWriteSize=WrapDown(UnpPtr-WrittenBorder); size_t WriteSizeLeft=FullWriteSize; bool NotAllFiltersProcessed=false; - for (size_t I=0;IBlockStart-WrPtr)&MaxWinMask)<=FullWriteSize) + // to next block and no further wrap arounds is possible. + if (WrapDown(flt->BlockStart-WrPtr)<=FullWriteSize) flt->NextWindow=false; continue; } - uint BlockStart=flt->BlockStart; + size_t BlockStart=flt->BlockStart; uint BlockLength=flt->BlockLength; - if (((BlockStart-WrittenBorder)&MaxWinMask)0) // We set it to 0 also for invalid filters. { - uint BlockEnd=(BlockStart+BlockLength)&MaxWinMask; + size_t BlockEnd=WrapUp(BlockStart+BlockLength); - FilterSrcMemory.Alloc(BlockLength); - byte *Mem=&FilterSrcMemory[0]; + FilterSrcMemory.resize(BlockLength); + byte *Mem=FilterSrcMemory.data(); if (BlockStartType!=FILTER_NONE) @@ -365,7 +389,7 @@ void Unpack::UnpWriteBuf() // Remove processed filters from queue. size_t EmptyCount=0; - for (size_t I=0;I0) Filters[I-EmptyCount]=Filters[I]; @@ -373,7 +397,7 @@ void Unpack::UnpWriteBuf() EmptyCount++; } if (EmptyCount>0) - Filters.Alloc(Filters.Size()-EmptyCount); + Filters.resize(Filters.size()-EmptyCount); if (!NotAllFiltersProcessed) // Only if all filters are processed. { @@ -385,12 +409,12 @@ void Unpack::UnpWriteBuf() // We prefer to write data in blocks not exceeding UNPACK_MAX_WRITE // instead of potentially huge MaxWinSize blocks. It also allows us // to keep the size of Filters array reasonable. - WriteBorder=(UnpPtr+Min(MaxWinSize,UNPACK_MAX_WRITE))&MaxWinMask; + WriteBorder=WrapUp(UnpPtr+Min(MaxWinSize,UNPACK_MAX_WRITE)); // Choose the nearest among WriteBorder and WrPtr actual written border. // If border is equal to UnpPtr, it means that we have MaxWinSize data ahead. if (WriteBorder==UnpPtr || - WrPtr!=UnpPtr && ((WrPtr-UnpPtr)&MaxWinMask)<((WriteBorder-UnpPtr)&MaxWinMask)) + WrPtr!=UnpPtr && WrapDown(WrPtr-UnpPtr)Channels,SrcPos=0; - FilterDstMemory.Alloc(DataSize); - byte *DstData=&FilterDstMemory[0]; + FilterDstMemory.resize(DataSize); + byte *DstData=FilterDstMemory.data(); // Bytes from same channels are grouped to continual data blocks, // so we need to place them back to their interleaving positions. @@ -492,13 +516,13 @@ void Unpack::UnpWriteArea(size_t StartPtr,size_t EndPtr) if (Fragmented) { - size_t SizeToWrite=(EndPtr-StartPtr) & MaxWinMask; + size_t SizeToWrite=WrapDown(EndPtr-StartPtr); while (SizeToWrite>0) { size_t BlockSize=FragWindow.GetBlockSize(StartPtr,SizeToWrite); UnpWriteData(&FragWindow[StartPtr],BlockSize); SizeToWrite-=BlockSize; - StartPtr=(StartPtr+BlockSize) & MaxWinMask; + StartPtr=WrapUp(StartPtr+BlockSize); } } else @@ -541,7 +565,7 @@ bool Unpack::ReadBlockHeader(BitInput &Inp,UnpackBlockHeader &Header) return false; Inp.faddbits((8-Inp.InBit)&7); - byte BlockFlags=Inp.fgetbits()>>8; + byte BlockFlags=byte(Inp.fgetbits()>>8); Inp.faddbits(8); uint ByteCount=((BlockFlags>>3)&3)+1; // Block size byte count. @@ -564,14 +588,24 @@ bool Unpack::ReadBlockHeader(BitInput &Inp,UnpackBlockHeader &Header) Header.BlockSize=BlockSize; byte CheckSum=byte(0x5a^BlockFlags^BlockSize^(BlockSize>>8)^(BlockSize>>16)); + + // 2024.01.04: In theory the valid block can have Header.BlockSize == 0 + // and Header.TablePresent == false in case the only block purpose is to + // store Header.LastBlockInFile flag if it didn't fit into previous block. + // So we do not reject Header.BlockSize == 0. Though currently RAR doesn't + // seem to issue such zero length blocks. if (CheckSum!=SavedCheckSum) return false; Header.BlockStart=Inp.InAddr; + + // We called Inp.faddbits(8) above, thus Header.BlockStart can't be 0 here. + // So there is no overflow even if Header.BlockSize is 0. ReadBorder=Min(ReadBorder,Header.BlockStart+Header.BlockSize-1); Header.LastBlockInFile=(BlockFlags & 0x40)!=0; Header.TablePresent=(BlockFlags & 0x80)!=0; + return true; } @@ -610,8 +644,8 @@ bool Unpack::ReadTables(BitInput &Inp,UnpackBlockHeader &Header,UnpackBlockTable MakeDecodeTables(BitLength,&Tables.BD,BC); - byte Table[HUFF_TABLE_SIZE]; - const uint TableSize=HUFF_TABLE_SIZE; + byte Table[HUFF_TABLE_SIZEX]; + const uint TableSize=ExtraDist ? HUFF_TABLE_SIZEX:HUFF_TABLE_SIZEB; for (uint I=0;IReadTop-5) @@ -674,14 +708,15 @@ bool Unpack::ReadTables(BitInput &Inp,UnpackBlockHeader &Header,UnpackBlockTable if (!Inp.ExternalBuffer && Inp.InAddr>ReadTop) return false; MakeDecodeTables(&Table[0],&Tables.LD,NC); - MakeDecodeTables(&Table[NC],&Tables.DD,DC); - MakeDecodeTables(&Table[NC+DC],&Tables.LDD,LDC); - MakeDecodeTables(&Table[NC+DC+LDC],&Tables.RD,RC); + uint DCodes=ExtraDist ? DCX : DCB; + MakeDecodeTables(&Table[NC],&Tables.DD,DCodes); + MakeDecodeTables(&Table[NC+DCodes],&Tables.LDD,LDC); + MakeDecodeTables(&Table[NC+DCodes+LDC],&Tables.RD,RC); return true; } void Unpack::InitFilters() { - Filters.SoftReset(); + Filters.clear(); } diff --git a/multiarc/src/formats/rar/unrar/unpack50frag.cpp b/multiarc/src/formats/rar/unrar/unpack50frag.cpp index 3c008ff24..9208405e0 100644 --- a/multiarc/src/formats/rar/unrar/unpack50frag.cpp +++ b/multiarc/src/formats/rar/unrar/unpack50frag.cpp @@ -2,6 +2,7 @@ FragmentedWindow::FragmentedWindow() { memset(Mem,0,sizeof(Mem)); memset(MemSize,0,sizeof(MemSize)); + LastAllocated=0; } @@ -13,6 +14,7 @@ FragmentedWindow::~FragmentedWindow() void FragmentedWindow::Reset() { + LastAllocated=0; for (uint I=0;IUnpPtr) + { + SrcPtr+=MaxWinSize; + + if (Distance>MaxWinSize || !FirstWinDone) + { + while (Length-- > 0) + { + (*this)[UnpPtr]=0; + if (++UnpPtr>=MaxWinSize) + UnpPtr-=MaxWinSize; + } + return; + } + } + while (Length-- > 0) { - (*this)[UnpPtr]=(*this)[SrcPtr++ & MaxWinMask]; - // We need to have masked UnpPtr after quit from loop, so it must not - // be replaced with '(*this)[UnpPtr++ & MaxWinMask]' - UnpPtr=(UnpPtr+1) & MaxWinMask; + (*this)[UnpPtr]=(*this)[SrcPtr]; + if (++SrcPtr>=MaxWinSize) + SrcPtr-=MaxWinSize; + if (++UnpPtr>=MaxWinSize) + UnpPtr-=MaxWinSize; } } diff --git a/multiarc/src/formats/rar/unrar/unpack50mt.cpp b/multiarc/src/formats/rar/unrar/unpack50mt.cpp index 82c9c4a8c..3d3fde59c 100644 --- a/multiarc/src/formats/rar/unrar/unpack50mt.cpp +++ b/multiarc/src/formats/rar/unrar/unpack50mt.cpp @@ -1,3 +1,5 @@ +// 2023.09.09: 0x400000 and 2 are optimal for i9-12900K. +// Further increasing the buffer size reduced the extraction speed. #define UNP_READ_SIZE_MT 0x400000 #define UNP_BLOCKS_PER_THREAD 2 @@ -277,7 +279,7 @@ void Unpack::Unpack5MT(bool Solid) } } } - UnpPtr&=MaxWinMask; // ProcessDecoded and maybe others can leave UnpPtr > MaxWinMask here. + UnpPtr=WrapUp(UnpPtr); // ProcessDecoded and maybe others can leave UnpPtr >= MaxWinSize here. UnpWriteBuf(); BlockHeader=UnpThreadData[LastBlockNum].BlockHeader; @@ -362,7 +364,8 @@ void Unpack::UnpackDecode(UnpackThreadData &D) { uint Length=SlotToLength(D.Inp,MainSlot-262); - uint DBits,Distance=1,DistSlot=DecodeNumber(D.Inp,&D.BlockTables.DD); + size_t Distance=1; + uint DBits,DistSlot=DecodeNumber(D.Inp,&D.BlockTables.DD); if (DistSlot<4) { DBits=0; @@ -371,7 +374,7 @@ void Unpack::UnpackDecode(UnpackThreadData &D) else { DBits=DistSlot/2 - 1; - Distance+=(2 | (DistSlot & 1)) << DBits; + Distance+=size_t(2 | (DistSlot & 1)) << DBits; } if (DBits>0) @@ -380,11 +383,23 @@ void Unpack::UnpackDecode(UnpackThreadData &D) { if (DBits>4) { - Distance+=((D.Inp.getbits32()>>(36-DBits))<<4); + // It is also possible to always use getbits64() here. + if (DBits>36) + Distance+=( ( size_t(D.Inp.getbits64() ) >> (68-DBits) ) << 4 ); + else + Distance+=( ( size_t(D.Inp.getbits32() ) >> (36-DBits) ) << 4 ); D.Inp.addbits(DBits-4); } uint LowDist=DecodeNumber(D.Inp,&D.BlockTables.LDD); Distance+=LowDist; + + // Distance can be 0 for multiples of 4 GB as result of size_t + // overflow in 32-bit build. Its lower 32-bit can also erroneously + // fit into dictionary after truncating upper 32-bits. Replace such + // invalid distances with -1, so CopyString sets 0 data for them. + // DBits>=30 also as DistSlot>=62 indicate distances >=0x80000001. + if (sizeof(Distance)==4 && DBits>=30) + Distance=(size_t)-1; } else { @@ -450,8 +465,12 @@ bool Unpack::ProcessDecoded(UnpackThreadData &D) UnpackDecodedItem *Item=D.Decoded,*Border=D.Decoded+D.DecodedSize; while (ItemUnpPtr); + PrevPtr=UnpPtr; + + if (WrapDown(WriteBorder-UnpPtr)<=MAX_INC_LZ_MATCH && WriteBorder!=UnpPtr) { UnpWriteBuf(); if (WrittenFileSize>DestUnpSize) @@ -469,7 +488,7 @@ bool Unpack::ProcessDecoded(UnpackThreadData &D) else #endif for (uint I=0;I<=Item->Length;I++) - Window[UnpPtr++ & MaxWinMask]=Item->Literal[I]; + Window[WrapUp(UnpPtr++)]=Item->Literal[I]; } else if (Item->Type==UNPDT_MATCH) @@ -481,8 +500,8 @@ bool Unpack::ProcessDecoded(UnpackThreadData &D) else if (Item->Type==UNPDT_REP) { - uint Distance=OldDist[Item->Distance]; - for (uint I=Item->Distance;I>0;I--) + size_t Distance=OldDist[Item->Distance]; + for (size_t I=Item->Distance;I>0;I--) OldDist[I]=OldDist[I-1]; OldDist[0]=Distance; LastLength=Item->Length; @@ -505,7 +524,7 @@ bool Unpack::ProcessDecoded(UnpackThreadData &D) Item++; Filter.Channels=(byte)Item->Length; - Filter.BlockLength=Item->Distance; + Filter.BlockLength=(uint)Item->Distance; AddFilter(Filter); } @@ -543,7 +562,11 @@ bool Unpack::UnpackLargeBlock(UnpackThreadData &D) while (true) { - UnpPtr&=MaxWinMask; + UnpPtr=WrapUp(UnpPtr); + + FirstWinDone|=(PrevPtr>UnpPtr); + PrevPtr=UnpPtr; + if (D.Inp.InAddr>=ReadBorder) { if (D.Inp.InAddr>BlockBorder || D.Inp.InAddr==BlockBorder && @@ -559,7 +582,7 @@ bool Unpack::UnpackLargeBlock(UnpackThreadData &D) break; } } - if (((WriteBorder-UnpPtr) & MaxWinMask)<=MAX_INC_LZ_MATCH && WriteBorder!=UnpPtr) + if (WrapDown(WriteBorder-UnpPtr)<=MAX_INC_LZ_MATCH && WriteBorder!=UnpPtr) { UnpWriteBuf(); if (WrittenFileSize>DestUnpSize) @@ -576,7 +599,8 @@ bool Unpack::UnpackLargeBlock(UnpackThreadData &D) { uint Length=SlotToLength(D.Inp,MainSlot-262); - uint DBits,Distance=1,DistSlot=DecodeNumber(D.Inp,&D.BlockTables.DD); + size_t Distance=1; + uint DBits,DistSlot=DecodeNumber(D.Inp,&D.BlockTables.DD); if (DistSlot<4) { DBits=0; @@ -585,7 +609,7 @@ bool Unpack::UnpackLargeBlock(UnpackThreadData &D) else { DBits=DistSlot/2 - 1; - Distance+=(2 | (DistSlot & 1)) << DBits; + Distance+=size_t(2 | (DistSlot & 1)) << DBits; } if (DBits>0) @@ -594,11 +618,23 @@ bool Unpack::UnpackLargeBlock(UnpackThreadData &D) { if (DBits>4) { - Distance+=((D.Inp.getbits32()>>(36-DBits))<<4); + // It is also possible to always use getbits64() here. + if (DBits>36) + Distance+=( ( size_t(D.Inp.getbits64() ) >> (68-DBits) ) << 4 ); + else + Distance+=( ( size_t(D.Inp.getbits32() ) >> (36-DBits) ) << 4 ); D.Inp.addbits(DBits-4); } uint LowDist=DecodeNumber(D.Inp,&D.BlockTables.LDD); Distance+=LowDist; + + // Distance can be 0 for multiples of 4 GB as result of size_t + // overflow in 32-bit build. Its lower 32-bit can also erroneously + // fit into dictionary after truncating upper 32-bits. Replace such + // invalid distances with -1, so CopyString sets 0 data for them. + // DBits>=30 also as DistSlot>=62 indicate distances >=0x80000001. + if (sizeof(Distance)==4 && DBits>=30) + Distance=(size_t)-1; } else { @@ -639,7 +675,7 @@ bool Unpack::UnpackLargeBlock(UnpackThreadData &D) if (MainSlot<262) { uint DistNum=MainSlot-258; - uint Distance=OldDist[DistNum]; + size_t Distance=OldDist[DistNum]; for (uint I=DistNum;I>0;I--) OldDist[I]=OldDist[I-1]; OldDist[0]=Distance; diff --git a/multiarc/src/formats/rar/unrar/unpackinline.cpp b/multiarc/src/formats/rar/unrar/unpackinline.cpp index 04c3d1f7d..68aec9170 100644 --- a/multiarc/src/formats/rar/unrar/unpackinline.cpp +++ b/multiarc/src/formats/rar/unrar/unpackinline.cpp @@ -1,4 +1,4 @@ -_forceinline void Unpack::InsertOldDist(uint Distance) +_forceinline void Unpack::InsertOldDist(size_t Distance) { OldDist[3]=OldDist[2]; OldDist[2]=OldDist[1]; @@ -6,23 +6,58 @@ _forceinline void Unpack::InsertOldDist(uint Distance) OldDist[0]=Distance; } -#ifdef _MSC_VER -#define FAST_MEMCPY +#if defined(LITTLE_ENDIAN) && defined(ALLOW_MISALIGNED) +#define UNPACK_COPY8 // We can copy 8 bytes at any position as uint64. #endif -_forceinline void Unpack::CopyString(uint Length,uint Distance) +_forceinline void Unpack::CopyString(uint Length,size_t Distance) { size_t SrcPtr=UnpPtr-Distance; + + // Perform the correction here instead of "else", so matches crossing + // the window beginning can also be processed by first "if" part. + if (Distance>UnpPtr) // Unlike SrcPtr>=MaxWinSize, it catches invalid distances like 0xfffffff0 in 32-bit build. + { + // Same as WrapDown(SrcPtr), needed because of UnpPtr-Distance above. + // We need the same condition below, so we expanded WrapDown() here. + SrcPtr+=MaxWinSize; + + // About Distance>MaxWinSize check. + // SrcPtr can be >=MaxWinSize if distance exceeds MaxWinSize + // in a malformed archive. Our WrapDown replacement above might not + // correct it, so to prevent out of bound Window read we check it here. + // Unlike SrcPtr>=MaxWinSize check, it allows MaxWinSize>0x80000000 + // in 32-bit build, which could cause overflow in SrcPtr. + // About !FirstWinDone check. + // If first window hasn't filled yet and it points outside of window, + // set data to 0 instead of copying preceding file data, so result doesn't + // depend on previously extracted files in non-solid archive. + if (Distance>MaxWinSize || !FirstWinDone) + { + // Fill area of specified length with 0 instead of returning. + // So if only the distance is broken and rest of packed data is valid, + // it preserves offsets and allows to continue extraction. + // If we set SrcPtr to random offset instead, let's say, 0, + // we still will be copying preceding file data if UnpPtr is also 0. + while (Length-- > 0) + { + Window[UnpPtr]=0; + UnpPtr=WrapUp(UnpPtr+1); + } + return; + } + } + if (SrcPtr=8) @@ -40,7 +75,7 @@ _forceinline void Unpack::CopyString(uint Length,uint Distance) Dest+=8; Length-=8; } -#ifdef FAST_MEMCPY +#ifdef UNPACK_COPY8 else while (Length>=8) { @@ -49,9 +84,7 @@ _forceinline void Unpack::CopyString(uint Length,uint Distance) // But for real RAR archives Distance <= MaxWinSize - MAX_INC_LZ_MATCH // always, so overlap here is impossible. - // This memcpy expanded inline by MSVC. We could also use uint64 - // assignment, which seems to provide about the same speed. - memcpy(Dest,Src,8); + RawPut8(RawGet8(Src),Dest); Src+=8; Dest+=8; @@ -71,10 +104,10 @@ _forceinline void Unpack::CopyString(uint Length,uint Distance) else while (Length-- > 0) // Slow copying with all possible precautions. { - Window[UnpPtr]=Window[SrcPtr++ & MaxWinMask]; + Window[UnpPtr]=Window[WrapUp(SrcPtr++)]; // We need to have masked UnpPtr after quit from loop, so it must not - // be replaced with 'Window[UnpPtr++ & MaxWinMask]' - UnpPtr=(UnpPtr+1) & MaxWinMask; + // be replaced with 'Window[WrapUp(UnpPtr++)]' + UnpPtr=WrapUp(UnpPtr+1); } } diff --git a/multiarc/src/formats/rar/unrar/uowners.cpp b/multiarc/src/formats/rar/unrar/uowners.cpp index 27f82d992..cbe2f4044 100644 --- a/multiarc/src/formats/rar/unrar/uowners.cpp +++ b/multiarc/src/formats/rar/unrar/uowners.cpp @@ -1,65 +1,17 @@ -void ExtractUnixOwner20(Archive &Arc,const wchar *FileName) -{ - char NameA[NM]; - WideToChar(FileName,NameA,ASIZE(NameA)); - - if (Arc.BrokenHeader) - { - uiMsg(UIERROR_UOWNERBROKEN,Arc.FileName,FileName); - ErrHandler.SetErrorCode(RARX_CRC); - return; - } - - struct passwd *pw; - errno=0; // Required by getpwnam specification if we need to check errno. - if ((pw=getpwnam(Arc.UOHead.OwnerName))==NULL) - { - uiMsg(UIERROR_UOWNERGETOWNERID,Arc.FileName,GetWide(Arc.UOHead.OwnerName)); - ErrHandler.SysErrMsg(); - ErrHandler.SetErrorCode(RARX_WARNING); - return; - } - uid_t OwnerID=pw->pw_uid; - - struct group *gr; - errno=0; // Required by getgrnam specification if we need to check errno. - if ((gr=getgrnam(Arc.UOHead.GroupName))==NULL) - { - uiMsg(UIERROR_UOWNERGETGROUPID,Arc.FileName,GetWide(Arc.UOHead.GroupName)); - ErrHandler.SysErrMsg(); - ErrHandler.SetErrorCode(RARX_CRC); - return; - } - uint Attr=GetFileAttr(FileName); - gid_t GroupID=gr->gr_gid; -#if defined(SAVE_LINKS) && !defined(_APPLE) - if (lchown(NameA,OwnerID,GroupID)!=0) -#else - if (chown(NameA,OwnerID,GroupID)!=0) -#endif - { - uiMsg(UIERROR_UOWNERSET,Arc.FileName,FileName); - ErrHandler.SetErrorCode(RARX_CREATE); - } - SetFileAttr(FileName,Attr); -} - - void ExtractUnixOwner30(Archive &Arc,const wchar *FileName) { - char NameA[NM]; - WideToChar(FileName,NameA,ASIZE(NameA)); + // There must be 0 byte between owner and group strings. + // Otherwise strlen call below wouldn't be safe. + if (memchr(Arc.SubHead.SubData.data(),0,Arc.SubHead.SubData.size())==NULL) + return; - char *OwnerName=(char *)&Arc.SubHead.SubData[0]; + char *OwnerName=(char *)Arc.SubHead.SubData.data(); int OwnerSize=strlen(OwnerName)+1; - int GroupSize=Arc.SubHead.SubData.Size()-OwnerSize; - char GroupName[NM]; - if (GroupSize >= (int)sizeof(GroupName)) - GroupSize = sizeof(GroupName) - 1; - strncpy(GroupName,(char *)&Arc.SubHead.SubData[OwnerSize],GroupSize); - GroupName[GroupSize]=0; + int GroupSize=Arc.SubHead.SubData.size()-OwnerSize; + char *GroupName=(char *)&Arc.SubHead.SubData[OwnerSize]; + std::string GroupStr(GroupName,GroupName+GroupSize); struct passwd *pw; if ((pw=getpwnam(OwnerName))==NULL) @@ -71,7 +23,7 @@ void ExtractUnixOwner30(Archive &Arc,const wchar *FileName) uid_t OwnerID=pw->pw_uid; struct group *gr; - if ((gr=getgrnam(GroupName))==NULL) + if ((gr=getgrnam(GroupStr.c_str()))==NULL) { uiMsg(UIERROR_UOWNERGETGROUPID,Arc.FileName,GetWide(GroupName)); ErrHandler.SetErrorCode(RARX_WARNING); @@ -79,10 +31,14 @@ void ExtractUnixOwner30(Archive &Arc,const wchar *FileName) } uint Attr=GetFileAttr(FileName); gid_t GroupID=gr->gr_gid; + + std::string NameA; + WideToChar(FileName,NameA); + #if defined(SAVE_LINKS) && !defined(_APPLE) - if (lchown(NameA,OwnerID,GroupID)!=0) + if (lchown(NameA.c_str(),OwnerID,GroupID)!=0) #else - if (chown(NameA,OwnerID,GroupID)!=0) + if (chown(NameA.c_str(),OwnerID,GroupID)!=0) #endif { uiMsg(UIERROR_UOWNERSET,Arc.FileName,FileName); @@ -92,11 +48,8 @@ void ExtractUnixOwner30(Archive &Arc,const wchar *FileName) } -void SetUnixOwner(Archive &Arc,const wchar *FileName) +void SetUnixOwner(Archive &Arc,const std::wstring &FileName) { - char NameA[NM]; - WideToChar(FileName,NameA,ASIZE(NameA)); - // First, we try to resolve symbolic names. If they are missing or cannot // be resolved, we try to use numeric values if any. If numeric values // are missing too, function fails. @@ -131,10 +84,14 @@ void SetUnixOwner(Archive &Arc,const wchar *FileName) else hd.UnixGroupID=gr->gr_gid; } + + std::string NameA; + WideToChar(FileName,NameA); + #if defined(SAVE_LINKS) && !defined(_APPLE) - if (lchown(NameA,hd.UnixOwnerID,hd.UnixGroupID)!=0) + if (lchown(NameA.c_str(),hd.UnixOwnerID,hd.UnixGroupID)!=0) #else - if (chown(NameA,hd.UnixOwnerID,hd.UnixGroupID)!=0) + if (chown(NameA.c_str(),hd.UnixOwnerID,hd.UnixGroupID)!=0) #endif { uiMsg(UIERROR_UOWNERSET,Arc.FileName,FileName); diff --git a/multiarc/src/formats/rar/unrar/version.hpp b/multiarc/src/formats/rar/unrar/version.hpp index fa887fa0e..6987a9afb 100644 --- a/multiarc/src/formats/rar/unrar/version.hpp +++ b/multiarc/src/formats/rar/unrar/version.hpp @@ -1,6 +1,6 @@ -#define RARVER_MAJOR 6 -#define RARVER_MINOR 21 +#define RARVER_MAJOR 7 +#define RARVER_MINOR 1 #define RARVER_BETA 0 -#define RARVER_DAY 16 -#define RARVER_MONTH 2 -#define RARVER_YEAR 2023 +#define RARVER_DAY 12 +#define RARVER_MONTH 5 +#define RARVER_YEAR 2024 diff --git a/multiarc/src/formats/rar/unrar/volume.cpp b/multiarc/src/formats/rar/unrar/volume.cpp index 4924d8d0e..e17151559 100644 --- a/multiarc/src/formats/rar/unrar/volume.cpp +++ b/multiarc/src/formats/rar/unrar/volume.cpp @@ -1,8 +1,8 @@ #include "rar.hpp" #ifdef RARDLL -static bool DllVolChange(CommandData *Cmd,wchar *NextName,size_t NameSize); -static bool DllVolNotify(CommandData *Cmd,wchar *NextName); +bool DllVolChange(CommandData *Cmd,std::wstring &NextName); +static bool DllVolNotify(CommandData *Cmd,const std::wstring &NextName); #endif @@ -35,9 +35,8 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma Arc.Close(); - wchar NextName[NM]; - wcsncpyz(NextName,Arc.FileName,ASIZE(NextName)); - NextVolumeName(NextName,ASIZE(NextName),!Arc.NewNumbering); + std::wstring NextName=Arc.FileName; + NextVolumeName(NextName,!Arc.NewNumbering); #if !defined(SFX_MODULE) && !defined(RARDLL) bool RecoveryDone=false; @@ -56,7 +55,7 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma // the next file might not be fully decoded yet. They write chunks of data // and then close the file again until the next chunk comes in. - if (Cmd->VolumePause && !uiAskNextVolume(NextName,ASIZE(NextName))) + if (Cmd->VolumePause && !uiAskNextVolume(NextName)) FailedOpen=true; #endif @@ -76,18 +75,17 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma { // Checking for new style volumes renamed by user to old style // name format. Some users did it for unknown reason. - wchar AltNextName[NM]; - wcsncpyz(AltNextName,Arc.FileName,ASIZE(AltNextName)); - NextVolumeName(AltNextName,ASIZE(AltNextName),true); + std::wstring AltNextName=Arc.FileName; + NextVolumeName(AltNextName,true); OldSchemeTested=true; if (Arc.Open(AltNextName,OpenMode)) { - wcsncpyz(NextName,AltNextName,ASIZE(NextName)); + NextName=AltNextName; break; } } #ifdef RARDLL - if (!DllVolChange(Cmd,NextName,ASIZE(NextName))) + if (!DllVolChange(Cmd,NextName)) { FailedOpen=true; break; @@ -109,7 +107,7 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma break; } #ifndef SILENT - if (Cmd->AllYes || !uiAskNextVolume(NextName,ASIZE(NextName))) + if (Cmd->AllYes || !uiAskNextVolume(NextName)) #endif { FailedOpen=true; @@ -128,7 +126,7 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma } if (Command=='T' || Command=='X' || Command=='E') - mprintf(St(Command=='T' ? MTestVol:MExtrVol),Arc.FileName); + mprintf(St(Command=='T' ? MTestVol:MExtrVol),Arc.FileName.c_str()); Arc.CheckArc(true); @@ -158,7 +156,7 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma } if (ShowFileName && !Cmd->DisableNames) { - mprintf(St(MExtrPoints),Arc.FileHead.FileName); + mprintf(St(MExtrPoints),Arc.FileHead.FileName.c_str()); if (!Cmd->DisablePercentage) mprintf(L" "); } @@ -190,45 +188,65 @@ bool MergeArchive(Archive &Arc,ComprDataIO *DataIO,bool ShowFileName,wchar Comma #ifdef RARDLL -bool DllVolChange(CommandData *Cmd,wchar *NextName,size_t NameSize) +bool DllVolChange(CommandData *Cmd,std::wstring &NextName) { bool DllVolChanged=false,DllVolAborted=false; if (Cmd->Callback!=NULL) { - wchar OrgNextName[NM]; - wcsncpyz(OrgNextName,NextName,ASIZE(OrgNextName)); - if (Cmd->Callback(UCM_CHANGEVOLUMEW,Cmd->UserData,(LPARAM)NextName,RAR_VOL_ASK)==-1) + std::wstring OrgNextName=NextName; + + std::vector NameBuf(MAXPATHSIZE); + std::copy(NextName.data(), NextName.data() + NextName.size() + 1, NameBuf.begin()); + + if (Cmd->Callback(UCM_CHANGEVOLUMEW,Cmd->UserData,(LPARAM)NameBuf.data(),RAR_VOL_ASK)==-1) DllVolAborted=true; else - if (wcscmp(OrgNextName,NextName)!=0) + { + NextName=NameBuf.data(); + if (OrgNextName!=NextName) DllVolChanged=true; else { - char NextNameA[NM],OrgNextNameA[NM]; - WideToChar(NextName,NextNameA,ASIZE(NextNameA)); - strncpyz(OrgNextNameA,NextNameA,ASIZE(OrgNextNameA)); - if (Cmd->Callback(UCM_CHANGEVOLUME,Cmd->UserData,(LPARAM)NextNameA,RAR_VOL_ASK)==-1) + std::string NextNameA; + WideToChar(NextName,NextNameA); + std::string OrgNextNameA=NextNameA; + + std::vector NameBufA(MAXPATHSIZE); + std::copy(NextNameA.data(), NextNameA.data() + NextNameA.size() + 1, NameBufA.begin()); + + if (Cmd->Callback(UCM_CHANGEVOLUME,Cmd->UserData,(LPARAM)NameBufA.data(),RAR_VOL_ASK)==-1) DllVolAborted=true; else - if (strcmp(OrgNextNameA,NextNameA)!=0) + { + NextNameA=NameBufA.data(); + if (OrgNextNameA!=NextNameA) { // We can damage some Unicode characters by U->A->U conversion, // so set Unicode name only if we see that ANSI name is changed. - CharToWide(NextNameA,NextName,NameSize); + CharToWide(NextNameA,NextName); DllVolChanged=true; } + } } + } } if (!DllVolChanged && Cmd->ChangeVolProc!=NULL) { - char NextNameA[NM]; - WideToChar(NextName,NextNameA,ASIZE(NextNameA)); - int RetCode=Cmd->ChangeVolProc(NextNameA,RAR_VOL_ASK); + std::string NextNameA; + WideToChar(NextName,NextNameA); + + std::vector NameBufA(MAXPATHSIZE); + std::copy(NextNameA.data(), NextNameA.data() + NextNameA.size() + 1, NameBufA.begin()); + + int RetCode=Cmd->ChangeVolProc(NameBufA.data(),RAR_VOL_ASK); if (RetCode==0) DllVolAborted=true; else - CharToWide(NextNameA,NextName,NameSize); + { + NextNameA=NameBufA.data(); + CharToWide(NextNameA,NextName); + } } // We quit only on 'abort' condition, but not on 'name not changed'. @@ -246,20 +264,21 @@ bool DllVolChange(CommandData *Cmd,wchar *NextName,size_t NameSize) #ifdef RARDLL -bool DllVolNotify(CommandData *Cmd,wchar *NextName) +static bool DllVolNotify(CommandData *Cmd,const std::wstring &NextName) { - char NextNameA[NM]; - WideToChar(NextName,NextNameA,ASIZE(NextNameA)); + std::string NextNameA; + WideToChar(NextName,NextNameA); + if (Cmd->Callback!=NULL) { - if (Cmd->Callback(UCM_CHANGEVOLUMEW,Cmd->UserData,(LPARAM)NextName,RAR_VOL_NOTIFY)==-1) + if (Cmd->Callback(UCM_CHANGEVOLUMEW,Cmd->UserData,(LPARAM)NextName.data(),RAR_VOL_NOTIFY)==-1) return false; - if (Cmd->Callback(UCM_CHANGEVOLUME,Cmd->UserData,(LPARAM)NextNameA,RAR_VOL_NOTIFY)==-1) + if (Cmd->Callback(UCM_CHANGEVOLUME,Cmd->UserData,(LPARAM)NextNameA.data(),RAR_VOL_NOTIFY)==-1) return false; } if (Cmd->ChangeVolProc!=NULL) { - int RetCode=Cmd->ChangeVolProc(NextNameA,RAR_VOL_NOTIFY); + int RetCode=Cmd->ChangeVolProc((char *)NextNameA.data(),RAR_VOL_NOTIFY); if (RetCode==0) return false; } diff --git a/multiarc/src/formats/rar/unrar/win32acl.cpp b/multiarc/src/formats/rar/unrar/win32acl.cpp index d4797bde0..36cb8525e 100644 --- a/multiarc/src/formats/rar/unrar/win32acl.cpp +++ b/multiarc/src/formats/rar/unrar/win32acl.cpp @@ -5,7 +5,7 @@ static bool ReadSacl=false; #ifndef SFX_MODULE -void ExtractACL20(Archive &Arc,const wchar *FileName) +void ExtractACL20(Archive &Arc,const std::wstring &FileName) { SetACLPrivileges(); @@ -27,7 +27,7 @@ void ExtractACL20(Archive &Arc,const wchar *FileName) Unpack Unpack(&DataIO); Unpack.Init(0x10000,false); - Array UnpData(Arc.EAHead.UnpSize); + std::vector UnpData(Arc.EAHead.UnpSize); DataIO.SetUnpackToMemory(&UnpData[0],Arc.EAHead.UnpSize); DataIO.SetPackedSizeToRead(Arc.EAHead.DataSize); DataIO.EnableShowProgress(false); @@ -49,7 +49,7 @@ void ExtractACL20(Archive &Arc,const wchar *FileName) si|=SACL_SECURITY_INFORMATION; SECURITY_DESCRIPTOR *sd=(SECURITY_DESCRIPTOR *)&UnpData[0]; - int SetCode=SetFileSecurity(FileName,si,sd); + int SetCode=SetFileSecurity(FileName.c_str(),si,sd); if (!SetCode) { @@ -64,9 +64,9 @@ void ExtractACL20(Archive &Arc,const wchar *FileName) #endif -void ExtractACL(Archive &Arc,const wchar *FileName) +void ExtractACL(Archive &Arc,const std::wstring &FileName) { - Array SubData; + std::vector SubData; if (!Arc.ReadSubData(&SubData,NULL,false)) return; @@ -78,12 +78,12 @@ void ExtractACL(Archive &Arc,const wchar *FileName) si|=SACL_SECURITY_INFORMATION; SECURITY_DESCRIPTOR *sd=(SECURITY_DESCRIPTOR *)&SubData[0]; - int SetCode=SetFileSecurity(FileName,si,sd); + int SetCode=SetFileSecurity(FileName.c_str(),si,sd); if (!SetCode) { - wchar LongName[NM]; - if (GetWinLongPath(FileName,LongName,ASIZE(LongName))) - SetCode=SetFileSecurity(LongName,si,sd); + std::wstring LongName; + if (GetWinLongPath(FileName,LongName)) + SetCode=SetFileSecurity(LongName.c_str(),si,sd); } if (!SetCode) diff --git a/multiarc/src/formats/rar/unrar/win32lnk.cpp b/multiarc/src/formats/rar/unrar/win32lnk.cpp index 84ab63ff6..677c27357 100644 --- a/multiarc/src/formats/rar/unrar/win32lnk.cpp +++ b/multiarc/src/formats/rar/unrar/win32lnk.cpp @@ -40,26 +40,24 @@ bool CreateReparsePoint(CommandData *Cmd,const wchar *Name,FileHeader *hd) PrivSet=true; } - const DWORD BufSize=sizeof(REPARSE_DATA_BUFFER)+2*NM+1024; - Array Buf(BufSize); - REPARSE_DATA_BUFFER *rdb=(REPARSE_DATA_BUFFER *)&Buf[0]; - - wchar SubstName[NM]; - wcsncpyz(SubstName,hd->RedirName,ASIZE(SubstName)); - size_t SubstLength=wcslen(SubstName); - - wchar PrintName[NM],*PrintNameSrc=SubstName,*PrintNameDst=PrintName; - bool WinPrefix=wcsncmp(PrintNameSrc,L"\\??\\",4)==0; - if (WinPrefix) - PrintNameSrc+=4; - if (WinPrefix && wcsncmp(PrintNameSrc,L"UNC\\",4)==0) - { - *(PrintNameDst++)='\\'; // Insert second \ in beginning of share name. - PrintNameSrc+=3; - } - wcscpy(PrintNameDst,PrintNameSrc); + const std::wstring &SubstName=hd->RedirName; + size_t SubstLength=SubstName.size(); + + // REPARSE_DATA_BUFFER receives both SubstName and PrintName strings, + // thus "*2" below. PrintName is either shorter or same length as SubstName. + const DWORD BufSize=sizeof(REPARSE_DATA_BUFFER)+((DWORD)SubstLength+1)*2*sizeof(wchar); + + std::vector Buf(BufSize); + REPARSE_DATA_BUFFER *rdb=(REPARSE_DATA_BUFFER *)Buf.data(); - size_t PrintLength=wcslen(PrintName); + // Remove \??\ NTFS junction prefix of present. + bool WinPrefix=SubstName.rfind(L"\\??\\",0)!=std::wstring::npos; + std::wstring PrintName=WinPrefix ? SubstName.substr(4):SubstName; + + if (WinPrefix && PrintName.rfind(L"UNC\\",0)!=std::wstring::npos) + PrintName=L"\\"+PrintName.substr(3); // Convert UNC\server\share to \\server\share. + + size_t PrintLength=PrintName.size(); bool AbsPath=WinPrefix; // IsFullPath is not really needed here, AbsPath check is enough. @@ -69,7 +67,11 @@ bool CreateReparsePoint(CommandData *Cmd,const wchar *Name,FileHeader *hd) // path as a prefix, which can confuse IsRelativeSymlinkSafe algorithm. if (!Cmd->AbsoluteLinks && (AbsPath || IsFullPath(hd->RedirName) || !IsRelativeSymlinkSafe(Cmd,hd->FileName,Name,hd->RedirName))) + { + uiMsg(UIERROR_SKIPUNSAFELINK,hd->FileName,hd->RedirName); + ErrHandler.SetErrorCode(RARX_WARNING); return false; + } CreatePath(Name,true,Cmd->DisableNames); @@ -86,9 +88,9 @@ bool CreateReparsePoint(CommandData *Cmd,const wchar *Name,FileHeader *hd) // Unix symlinks do not have their own 'directory' attribute. if (hd->Dir || hd->DirTarget) { - if (!CreateDirectory(Name,NULL)) + if (!CreateDir(Name)) { - uiMsg(UIERROR_DIRCREATE,UINULL,Name); + uiMsg(UIERROR_DIRCREATE,L"",Name); ErrHandler.SetErrorCode(RARX_CREATE); return false; } @@ -118,11 +120,11 @@ bool CreateReparsePoint(CommandData *Cmd,const wchar *Name,FileHeader *hd) rdb->MountPointReparseBuffer.SubstituteNameOffset=0; rdb->MountPointReparseBuffer.SubstituteNameLength=USHORT(SubstLength*sizeof(WCHAR)); - wcscpy(rdb->MountPointReparseBuffer.PathBuffer,SubstName); + wcscpy(rdb->MountPointReparseBuffer.PathBuffer,SubstName.data()); rdb->MountPointReparseBuffer.PrintNameOffset=USHORT((SubstLength+1)*sizeof(WCHAR)); rdb->MountPointReparseBuffer.PrintNameLength=USHORT(PrintLength*sizeof(WCHAR)); - wcscpy(rdb->MountPointReparseBuffer.PathBuffer+SubstLength+1,PrintName); + wcscpy(rdb->MountPointReparseBuffer.PathBuffer+SubstLength+1,PrintName.data()); } else if (hd->RedirType==FSREDIR_WINSYMLINK || hd->RedirType==FSREDIR_UNIXSYMLINK) @@ -139,11 +141,11 @@ bool CreateReparsePoint(CommandData *Cmd,const wchar *Name,FileHeader *hd) rdb->SymbolicLinkReparseBuffer.SubstituteNameOffset=0; rdb->SymbolicLinkReparseBuffer.SubstituteNameLength=USHORT(SubstLength*sizeof(WCHAR)); - wcscpy(rdb->SymbolicLinkReparseBuffer.PathBuffer,SubstName); + wcscpy(rdb->SymbolicLinkReparseBuffer.PathBuffer,SubstName.data()); rdb->SymbolicLinkReparseBuffer.PrintNameOffset=USHORT((SubstLength+1)*sizeof(WCHAR)); rdb->SymbolicLinkReparseBuffer.PrintNameLength=USHORT(PrintLength*sizeof(WCHAR)); - wcscpy(rdb->SymbolicLinkReparseBuffer.PathBuffer+SubstLength+1,PrintName); + wcscpy(rdb->SymbolicLinkReparseBuffer.PathBuffer+SubstLength+1,PrintName.data()); rdb->SymbolicLinkReparseBuffer.Flags=AbsPath ? 0:SYMLINK_FLAG_RELATIVE; } @@ -166,7 +168,7 @@ bool CreateReparsePoint(CommandData *Cmd,const wchar *Name,FileHeader *hd) rdb->ReparseDataLength,NULL,0,&Returned,NULL)) { CloseHandle(hFile); - uiMsg(UIERROR_SLINKCREATE,UINULL,Name); + uiMsg(UIERROR_SLINKCREATE,L"",Name); DWORD LastError=GetLastError(); if ((LastError==ERROR_ACCESS_DENIED || LastError==ERROR_PRIVILEGE_NOT_HELD) && diff --git a/multiarc/src/formats/rar/unrar/win32stm.cpp b/multiarc/src/formats/rar/unrar/win32stm.cpp index 048fd86b6..0c9c72a4a 100644 --- a/multiarc/src/formats/rar/unrar/win32stm.cpp +++ b/multiarc/src/formats/rar/unrar/win32stm.cpp @@ -1,7 +1,25 @@ +#ifdef _WIN_ALL +// StreamName must include the leading ':'. +static bool IsNtfsReservedStream(const std::wstring &StreamName) +{ + const wchar *Reserved[]{ + L"::$ATTRIBUTE_LIST",L"::$BITMAP",L"::$DATA",L"::$EA",L"::$EA_INFORMATION", + L"::$FILE_NAME",L"::$INDEX_ALLOCATION",L":$I30:$INDEX_ALLOCATION", + L"::$INDEX_ROOT",L"::$LOGGED_UTILITY_STREAM",L":$EFS:$LOGGED_UTILITY_STREAM", + L":$TXF_DATA:$LOGGED_UTILITY_STREAM",L"::$OBJECT_ID",L"::$REPARSE_POINT" + }; + for (const wchar *Name : Reserved) + if (wcsicomp(StreamName,Name)==0) + return true; + return false; +} +#endif + + #if !defined(SFX_MODULE) && defined(_WIN_ALL) -void ExtractStreams20(Archive &Arc,const wchar *FileName) +void ExtractStreams20(Archive &Arc,const std::wstring &FileName) { if (Arc.BrokenHeader) { @@ -17,35 +35,39 @@ void ExtractStreams20(Archive &Arc,const wchar *FileName) return; } - wchar StreamName[NM+2]; - if (FileName[0]!=0 && FileName[1]==0) + std::wstring StreamName; + if (FileName.size()==1) { // Convert single character names like f:stream to .\f:stream to // resolve the ambiguity with drive letters. - wcsncpyz(StreamName,L".\\",ASIZE(StreamName)); - wcsncatz(StreamName,FileName,ASIZE(StreamName)); + StreamName=L".\\"+FileName; } else - wcsncpyz(StreamName,FileName,ASIZE(StreamName)); - if (wcslen(StreamName)+strlen(Arc.StreamHead.StreamName)>=ASIZE(StreamName) || - Arc.StreamHead.StreamName[0]!=':') + StreamName=FileName; + if (Arc.StreamHead.StreamName[0]!=':') { uiMsg(UIERROR_STREAMBROKEN,Arc.FileName,FileName); ErrHandler.SetErrorCode(RARX_CRC); return; } - wchar StoredName[NM]; - CharToWide(Arc.StreamHead.StreamName,StoredName,ASIZE(StoredName)); - ConvertPath(StoredName+1,StoredName+1,ASIZE(StoredName)-1); + std::wstring StoredName; + // "substr(1)" to exclude ':', so we can use ConvertPath() below. + CharToWide(Arc.StreamHead.StreamName.substr(1),StoredName); + ConvertPath(&StoredName,&StoredName); - wcsncatz(StreamName,StoredName,ASIZE(StreamName)); - FindData fd; - bool Found=FindFile::FastFind(FileName,&fd); + StoredName=L":"+StoredName; + if (IsNtfsReservedStream(StoredName)) + return; + + StreamName+=StoredName; - if ((fd.FileAttr & FILE_ATTRIBUTE_READONLY)!=0) - SetFileAttr(FileName,fd.FileAttr & ~FILE_ATTRIBUTE_READONLY); + FindData FD; + bool Found=FindFile::FastFind(FileName,&FD); + + if ((FD.FileAttr & FILE_ATTRIBUTE_READONLY)!=0) + SetFileAttr(FileName,FD.FileAttr & ~FILE_ATTRIBUTE_READONLY); File CurFile; if (CurFile.WCreate(StreamName)) @@ -71,31 +93,29 @@ void ExtractStreams20(Archive &Arc,const wchar *FileName) } File HostFile; if (Found && HostFile.Open(FileName,FMF_OPENSHARED|FMF_UPDATE)) - SetFileTime(HostFile.GetHandle(),&fd.ftCreationTime,&fd.ftLastAccessTime, - &fd.ftLastWriteTime); - if ((fd.FileAttr & FILE_ATTRIBUTE_READONLY)!=0) - SetFileAttr(FileName,fd.FileAttr); + SetFileTime(HostFile.GetHandle(),&FD.ftCreationTime,&FD.ftLastAccessTime, + &FD.ftLastWriteTime); + if ((FD.FileAttr & FILE_ATTRIBUTE_READONLY)!=0) + SetFileAttr(FileName,FD.FileAttr); } #endif #ifdef _WIN_ALL -void ExtractStreams(Archive &Arc,const wchar *FileName,bool TestMode) +void ExtractStreams(Archive &Arc,const std::wstring &FileName,bool TestMode) { - wchar FullName[NM+2]; + std::wstring FullName; if (FileName[0]!=0 && FileName[1]==0) { // Convert single character names like f:stream to .\f:stream to // resolve the ambiguity with drive letters. - wcsncpyz(FullName,L".\\",ASIZE(FullName)); - wcsncatz(FullName,FileName,ASIZE(FullName)); + FullName=L".\\"+FileName; } else - wcsncpyz(FullName,FileName,ASIZE(FullName)); + FullName=FileName; - wchar StreamName[NM]; - GetStreamNameNTFS(Arc,StreamName,ASIZE(StreamName)); - if (*StreamName!=':') + std::wstring StreamName=GetStreamNameNTFS(Arc); + if (StreamName[0]!=':') { uiMsg(UIERROR_STREAMBROKEN,Arc.FileName,FileName); ErrHandler.SetErrorCode(RARX_CRC); @@ -109,14 +129,17 @@ void ExtractStreams(Archive &Arc,const wchar *FileName,bool TestMode) return; } - wcsncatz(FullName,StreamName,ASIZE(FullName)); + FullName+=StreamName; - FindData fd; - bool HostFound=FindFile::FastFind(FileName,&fd); + if (IsNtfsReservedStream(StreamName)) + return; - if ((fd.FileAttr & FILE_ATTRIBUTE_READONLY)!=0) - SetFileAttr(FileName,fd.FileAttr & ~FILE_ATTRIBUTE_READONLY); + FindData FD; + bool HostFound=FindFile::FastFind(FileName,&FD); + + if ((FD.FileAttr & FILE_ATTRIBUTE_READONLY)!=0) + SetFileAttr(FileName,FD.FileAttr & ~FILE_ATTRIBUTE_READONLY); File CurFile; if (CurFile.WCreate(FullName)) @@ -128,32 +151,26 @@ void ExtractStreams(Archive &Arc,const wchar *FileName,bool TestMode) // Restoring original file timestamps. File HostFile; if (HostFound && HostFile.Open(FileName,FMF_OPENSHARED|FMF_UPDATE)) - SetFileTime(HostFile.GetHandle(),&fd.ftCreationTime,&fd.ftLastAccessTime, - &fd.ftLastWriteTime); + SetFileTime(HostFile.GetHandle(),&FD.ftCreationTime,&FD.ftLastAccessTime, + &FD.ftLastWriteTime); // Restoring original file attributes. Important if file was read only // or did not have "Archive" attribute - SetFileAttr(FileName,fd.FileAttr); + SetFileAttr(FileName,FD.FileAttr); } #endif -void GetStreamNameNTFS(Archive &Arc,wchar *StreamName,size_t MaxSize) +std::wstring GetStreamNameNTFS(Archive &Arc) { - byte *Data=&Arc.SubHead.SubData[0]; - size_t DataSize=Arc.SubHead.SubData.Size(); + std::wstring Dest; if (Arc.Format==RARFMT15) - { - size_t DestSize=Min(DataSize/2,MaxSize-1); - RawToWide(Data,StreamName,DestSize); - StreamName[DestSize]=0; - } + Dest=RawToWide(Arc.SubHead.SubData); else { - char UtfString[NM*4]; - size_t DestSize=Min(DataSize,ASIZE(UtfString)-1); - memcpy(UtfString,Data,DestSize); - UtfString[DestSize]=0; - UtfToWide(UtfString,StreamName,MaxSize); + std::vector Src=Arc.SubHead.SubData; + Src.push_back(0); // Needed for our UtfToWide. + UtfToWide((char *)Src.data(),Dest); } + return Dest; } From 405104b81903c6c8a5c4734175752a3be3d358d8 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 15:04:37 +0300 Subject: [PATCH 14/22] ma: updated bundled 7z sources to 2408 --- multiarc/src/formats/7z/7z.cpp | 2 +- multiarc/src/formats/7z/7zMain.cpp | 2 +- multiarc/src/formats/7z/C/7z.h | 12 +- multiarc/src/formats/7z/C/7zAlloc.c | 69 +- multiarc/src/formats/7z/C/7zAlloc.h | 6 +- multiarc/src/formats/7z/C/7zArcIn.c | 409 +-- multiarc/src/formats/7z/C/7zBuf.c | 0 multiarc/src/formats/7z/C/7zBuf.h | 6 +- multiarc/src/formats/7z/C/7zBuf2.c | 0 multiarc/src/formats/7z/C/7zCrc.c | 489 ++- multiarc/src/formats/7z/C/7zCrc.h | 15 +- multiarc/src/formats/7z/C/7zCrcOpt.c | 244 +- multiarc/src/formats/7z/C/7zDec.c | 198 +- multiarc/src/formats/7z/C/7zFile.c | 33 +- multiarc/src/formats/7z/C/7zFile.h | 9 +- multiarc/src/formats/7z/C/7zStream.c | 67 +- multiarc/src/formats/7z/C/7zTypes.h | 272 +- multiarc/src/formats/7z/C/7zVersion.h | 10 +- multiarc/src/formats/7z/C/7zVersion.rc | 0 multiarc/src/formats/7z/C/Aes.c | 162 +- multiarc/src/formats/7z/C/Aes.h | 36 +- multiarc/src/formats/7z/C/AesOpt.c | 552 ++-- multiarc/src/formats/7z/C/Alloc.c | 330 +- multiarc/src/formats/7z/C/Alloc.h | 32 +- multiarc/src/formats/7z/C/Bcj2.c | 319 +- multiarc/src/formats/7z/C/Bcj2.h | 268 +- multiarc/src/formats/7z/C/Bcj2Enc.c | 559 ++-- multiarc/src/formats/7z/C/Blake2.h | 115 +- multiarc/src/formats/7z/C/Blake2s.c | 2724 +++++++++++++++- multiarc/src/formats/7z/C/Bra.c | 765 ++++- multiarc/src/formats/7z/C/Bra.h | 121 +- multiarc/src/formats/7z/C/Bra86.c | 221 +- multiarc/src/formats/7z/C/BraIA64.c | 57 +- multiarc/src/formats/7z/C/BwtSort.c | 13 +- multiarc/src/formats/7z/C/BwtSort.h | 6 +- multiarc/src/formats/7z/C/Compiler.h | 237 +- multiarc/src/formats/7z/C/CpuArch.c | 880 +++-- multiarc/src/formats/7z/C/CpuArch.h | 350 +- multiarc/src/formats/7z/C/Delta.c | 0 multiarc/src/formats/7z/C/Delta.h | 6 +- multiarc/src/formats/7z/C/DllSecur.c | 117 +- multiarc/src/formats/7z/C/DllSecur.h | 6 +- multiarc/src/formats/7z/C/HuffEnc.c | 12 +- multiarc/src/formats/7z/C/HuffEnc.h | 6 +- multiarc/src/formats/7z/C/LzFind.c | 636 ++-- multiarc/src/formats/7z/C/LzFind.h | 56 +- multiarc/src/formats/7z/C/LzFindMt.c | 122 +- multiarc/src/formats/7z/C/LzFindMt.h | 15 +- multiarc/src/formats/7z/C/LzFindOpt.c | 14 +- multiarc/src/formats/7z/C/LzHash.h | 8 +- multiarc/src/formats/7z/C/Lzma2Dec.c | 16 +- multiarc/src/formats/7z/C/Lzma2Dec.h | 15 +- multiarc/src/formats/7z/C/Lzma2DecMt.c | 155 +- multiarc/src/formats/7z/C/Lzma2DecMt.h | 20 +- multiarc/src/formats/7z/C/Lzma2Enc.c | 174 +- multiarc/src/formats/7z/C/Lzma2Enc.h | 20 +- multiarc/src/formats/7z/C/Lzma86.h | 6 +- multiarc/src/formats/7z/C/Lzma86Dec.c | 7 +- multiarc/src/formats/7z/C/Lzma86Enc.c | 7 +- multiarc/src/formats/7z/C/LzmaDec.c | 190 +- multiarc/src/formats/7z/C/LzmaDec.h | 17 +- multiarc/src/formats/7z/C/LzmaEnc.c | 402 ++- multiarc/src/formats/7z/C/LzmaEnc.h | 23 +- multiarc/src/formats/7z/C/LzmaLib.c | 8 +- multiarc/src/formats/7z/C/LzmaLib.h | 12 +- multiarc/src/formats/7z/C/MtCoder.c | 104 +- multiarc/src/formats/7z/C/MtCoder.h | 50 +- multiarc/src/formats/7z/C/MtDec.c | 117 +- multiarc/src/formats/7z/C/MtDec.h | 34 +- multiarc/src/formats/7z/C/Ppmd.h | 12 +- multiarc/src/formats/7z/C/Ppmd7.c | 217 +- multiarc/src/formats/7z/C/Ppmd7.h | 10 +- multiarc/src/formats/7z/C/Ppmd7Dec.c | 79 +- multiarc/src/formats/7z/C/Ppmd7Enc.c | 102 +- multiarc/src/formats/7z/C/Ppmd7aDec.c | 76 +- multiarc/src/formats/7z/C/Ppmd8.c | 279 +- multiarc/src/formats/7z/C/Ppmd8.h | 10 +- multiarc/src/formats/7z/C/Ppmd8Dec.c | 78 +- multiarc/src/formats/7z/C/Ppmd8Enc.c | 101 +- multiarc/src/formats/7z/C/Precomp.h | 127 +- multiarc/src/formats/7z/C/RotateDefs.h | 28 +- multiarc/src/formats/7z/C/Sha1.c | 205 +- multiarc/src/formats/7z/C/Sha1.h | 14 +- multiarc/src/formats/7z/C/Sha1Opt.c | 279 +- multiarc/src/formats/7z/C/Sha256.c | 184 +- multiarc/src/formats/7z/C/Sha256.h | 12 +- multiarc/src/formats/7z/C/Sha256Opt.c | 252 +- multiarc/src/formats/7z/C/Sort.c | 0 multiarc/src/formats/7z/C/Sort.h | 6 +- multiarc/src/formats/7z/C/SwapBytes.c | 835 +++++ multiarc/src/formats/7z/C/SwapBytes.h | 17 + multiarc/src/formats/7z/C/Threads.c | 127 +- multiarc/src/formats/7z/C/Threads.h | 68 +- multiarc/src/formats/7z/C/Xxh64.c | 327 ++ multiarc/src/formats/7z/C/Xxh64.h | 50 + multiarc/src/formats/7z/C/Xz.c | 6 +- multiarc/src/formats/7z/C/Xz.h | 61 +- multiarc/src/formats/7z/C/XzCrc64.c | 140 +- multiarc/src/formats/7z/C/XzCrc64.h | 16 +- multiarc/src/formats/7z/C/XzCrc64Opt.c | 266 +- multiarc/src/formats/7z/C/XzDec.c | 368 ++- multiarc/src/formats/7z/C/XzEnc.c | 310 +- multiarc/src/formats/7z/C/XzEnc.h | 23 +- multiarc/src/formats/7z/C/XzIn.c | 83 +- multiarc/src/formats/7z/C/ZstdDec.c | 4067 ++++++++++++++++++++++++ multiarc/src/formats/7z/C/ZstdDec.h | 173 + 106 files changed, 16301 insertions(+), 4672 deletions(-) mode change 100755 => 100644 multiarc/src/formats/7z/C/7z.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zAlloc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zAlloc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zArcIn.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zBuf.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zBuf.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zBuf2.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zCrc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zCrc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zCrcOpt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zDec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zFile.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zFile.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zStream.c mode change 100755 => 100644 multiarc/src/formats/7z/C/7zTypes.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zVersion.h mode change 100755 => 100644 multiarc/src/formats/7z/C/7zVersion.rc mode change 100755 => 100644 multiarc/src/formats/7z/C/Aes.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Aes.h mode change 100755 => 100644 multiarc/src/formats/7z/C/AesOpt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Alloc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Alloc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Bcj2.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Bcj2.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Bcj2Enc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Blake2.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Blake2s.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Bra.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Bra.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Bra86.c mode change 100755 => 100644 multiarc/src/formats/7z/C/BraIA64.c mode change 100755 => 100644 multiarc/src/formats/7z/C/BwtSort.c mode change 100755 => 100644 multiarc/src/formats/7z/C/BwtSort.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Compiler.h mode change 100755 => 100644 multiarc/src/formats/7z/C/CpuArch.c mode change 100755 => 100644 multiarc/src/formats/7z/C/CpuArch.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Delta.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Delta.h mode change 100755 => 100644 multiarc/src/formats/7z/C/DllSecur.c mode change 100755 => 100644 multiarc/src/formats/7z/C/DllSecur.h mode change 100755 => 100644 multiarc/src/formats/7z/C/HuffEnc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/HuffEnc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/LzFind.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzFind.h mode change 100755 => 100644 multiarc/src/formats/7z/C/LzFindMt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzFindMt.h mode change 100755 => 100644 multiarc/src/formats/7z/C/LzFindOpt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzHash.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma2Dec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma2Dec.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma2DecMt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma2DecMt.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma2Enc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma2Enc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma86.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma86Dec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Lzma86Enc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzmaDec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzmaDec.h mode change 100755 => 100644 multiarc/src/formats/7z/C/LzmaEnc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzmaEnc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/LzmaLib.c mode change 100755 => 100644 multiarc/src/formats/7z/C/LzmaLib.h mode change 100755 => 100644 multiarc/src/formats/7z/C/MtCoder.c mode change 100755 => 100644 multiarc/src/formats/7z/C/MtCoder.h mode change 100755 => 100644 multiarc/src/formats/7z/C/MtDec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/MtDec.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd7.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd7.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd7Dec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd7Enc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd7aDec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd8.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd8.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd8Dec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Ppmd8Enc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Precomp.h mode change 100755 => 100644 multiarc/src/formats/7z/C/RotateDefs.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Sha1.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Sha1.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Sha1Opt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Sha256.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Sha256.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Sha256Opt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Sort.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Sort.h create mode 100644 multiarc/src/formats/7z/C/SwapBytes.c create mode 100644 multiarc/src/formats/7z/C/SwapBytes.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Threads.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Threads.h create mode 100644 multiarc/src/formats/7z/C/Xxh64.c create mode 100644 multiarc/src/formats/7z/C/Xxh64.h mode change 100755 => 100644 multiarc/src/formats/7z/C/Xz.c mode change 100755 => 100644 multiarc/src/formats/7z/C/Xz.h mode change 100755 => 100644 multiarc/src/formats/7z/C/XzCrc64.c mode change 100755 => 100644 multiarc/src/formats/7z/C/XzCrc64.h mode change 100755 => 100644 multiarc/src/formats/7z/C/XzCrc64Opt.c mode change 100755 => 100644 multiarc/src/formats/7z/C/XzDec.c mode change 100755 => 100644 multiarc/src/formats/7z/C/XzEnc.c mode change 100755 => 100644 multiarc/src/formats/7z/C/XzEnc.h mode change 100755 => 100644 multiarc/src/formats/7z/C/XzIn.c create mode 100644 multiarc/src/formats/7z/C/ZstdDec.c create mode 100644 multiarc/src/formats/7z/C/ZstdDec.h diff --git a/multiarc/src/formats/7z/7z.cpp b/multiarc/src/formats/7z/7z.cpp index 6767cbf3f..a29227f00 100644 --- a/multiarc/src/formats/7z/7z.cpp +++ b/multiarc/src/formats/7z/7z.cpp @@ -107,7 +107,7 @@ class Traverser _look_stream.bufSize = inbuf_len; _look_stream.realStream = &_file_stream.vt; - LookToRead2_Init(&_look_stream); + LookToRead2_INIT(&_look_stream); static volatile int i = (CrcGenerateTable(), 0); i; diff --git a/multiarc/src/formats/7z/7zMain.cpp b/multiarc/src/formats/7z/7zMain.cpp index 34a374029..ac1c9d73c 100644 --- a/multiarc/src/formats/7z/7zMain.cpp +++ b/multiarc/src/formats/7z/7zMain.cpp @@ -472,7 +472,7 @@ extern "C" int sevenz_main(int numargs, char *args[]) { lookStream.bufSize = kInputBufSize; lookStream.realStream = &archiveStream.vt; - LookToRead2_Init(&lookStream); + LookToRead2_INIT(&lookStream); } } diff --git a/multiarc/src/formats/7z/C/7z.h b/multiarc/src/formats/7z/C/7z.h old mode 100755 new mode 100644 index 304f75ffc..9e27c0152 --- a/multiarc/src/formats/7z/C/7z.h +++ b/multiarc/src/formats/7z/C/7z.h @@ -1,8 +1,8 @@ /* 7z.h -- 7z interface -2018-07-02 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __7Z_H -#define __7Z_H +#ifndef ZIP7_INC_7Z_H +#define ZIP7_INC_7Z_H #include "7zTypes.h" @@ -98,7 +98,7 @@ typedef struct UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, - ILookInStream *stream, UInt64 startPos, + ILookInStreamPtr stream, UInt64 startPos, Byte *outBuffer, size_t outSize, ISzAllocPtr allocMain); @@ -174,7 +174,7 @@ UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 SRes SzArEx_Extract( const CSzArEx *db, - ILookInStream *inStream, + ILookInStreamPtr inStream, UInt32 fileIndex, /* index of file */ UInt32 *blockIndex, /* index of solid block */ Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ @@ -196,7 +196,7 @@ SZ_ERROR_INPUT_EOF SZ_ERROR_FAIL */ -SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, +SRes SzArEx_Open(CSzArEx *p, ILookInStreamPtr inStream, ISzAllocPtr allocMain, ISzAllocPtr allocTemp); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/7zAlloc.c b/multiarc/src/formats/7z/C/7zAlloc.c old mode 100755 new mode 100644 index c924a529f..2f0659af6 --- a/multiarc/src/formats/7z/C/7zAlloc.c +++ b/multiarc/src/formats/7z/C/7zAlloc.c @@ -1,5 +1,5 @@ -/* 7zAlloc.c -- Allocation functions -2017-04-03 : Igor Pavlov : Public domain */ +/* 7zAlloc.c -- Allocation functions for 7z processing +2023-03-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,74 +7,83 @@ #include "7zAlloc.h" -/* #define _SZ_ALLOC_DEBUG */ -/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ +/* #define SZ_ALLOC_DEBUG */ +/* use SZ_ALLOC_DEBUG to debug alloc/free operations */ -#ifdef _SZ_ALLOC_DEBUG +#ifdef SZ_ALLOC_DEBUG +/* #ifdef _WIN32 -#include +#include "7zWindows.h" #endif +*/ #include -int g_allocCount = 0; -int g_allocCountTemp = 0; +static int g_allocCount = 0; +static int g_allocCountTemp = 0; +static void Print_Alloc(const char *s, size_t size, int *counter) +{ + const unsigned size2 = (unsigned)size; + fprintf(stderr, "\n%s count = %10d : %10u bytes; ", s, *counter, size2); + (*counter)++; +} +static void Print_Free(const char *s, int *counter) +{ + (*counter)--; + fprintf(stderr, "\n%s count = %10d", s, *counter); +} #endif void *SzAlloc(ISzAllocPtr p, size_t size) { - UNUSED_VAR(p); + UNUSED_VAR(p) if (size == 0) return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc %10u bytes; count = %10d", (unsigned)size, g_allocCount); - g_allocCount++; + #ifdef SZ_ALLOC_DEBUG + Print_Alloc("Alloc", size, &g_allocCount); #endif return malloc(size); } void SzFree(ISzAllocPtr p, void *address) { - UNUSED_VAR(p); - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - { - g_allocCount--; - fprintf(stderr, "\nFree; count = %10d", g_allocCount); - } + UNUSED_VAR(p) + #ifdef SZ_ALLOC_DEBUG + if (address) + Print_Free("Free ", &g_allocCount); #endif free(address); } void *SzAllocTemp(ISzAllocPtr p, size_t size) { - UNUSED_VAR(p); + UNUSED_VAR(p) if (size == 0) return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_temp %10u bytes; count = %10d", (unsigned)size, g_allocCountTemp); - g_allocCountTemp++; + #ifdef SZ_ALLOC_DEBUG + Print_Alloc("Alloc_temp", size, &g_allocCountTemp); + /* #ifdef _WIN32 return HeapAlloc(GetProcessHeap(), 0, size); #endif + */ #endif return malloc(size); } void SzFreeTemp(ISzAllocPtr p, void *address) { - UNUSED_VAR(p); - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - { - g_allocCountTemp--; - fprintf(stderr, "\nFree_temp; count = %10d", g_allocCountTemp); - } + UNUSED_VAR(p) + #ifdef SZ_ALLOC_DEBUG + if (address) + Print_Free("Free_temp ", &g_allocCountTemp); + /* #ifdef _WIN32 HeapFree(GetProcessHeap(), 0, address); return; #endif + */ #endif free(address); } diff --git a/multiarc/src/formats/7z/C/7zAlloc.h b/multiarc/src/formats/7z/C/7zAlloc.h old mode 100755 new mode 100644 index 44778f9b2..b2b8b0cdd --- a/multiarc/src/formats/7z/C/7zAlloc.h +++ b/multiarc/src/formats/7z/C/7zAlloc.h @@ -1,8 +1,8 @@ /* 7zAlloc.h -- Allocation functions -2017-04-03 : Igor Pavlov : Public domain */ +2023-03-04 : Igor Pavlov : Public domain */ -#ifndef __7Z_ALLOC_H -#define __7Z_ALLOC_H +#ifndef ZIP7_INC_7Z_ALLOC_H +#define ZIP7_INC_7Z_ALLOC_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/7zArcIn.c b/multiarc/src/formats/7z/C/7zArcIn.c old mode 100755 new mode 100644 index eb7bbe363..45adce42b --- a/multiarc/src/formats/7z/C/7zArcIn.c +++ b/multiarc/src/formats/7z/C/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2021-02-09 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -10,10 +10,11 @@ #include "7zCrc.h" #include "CpuArch.h" -#define MY_ALLOC(T, p, size, alloc) { \ - if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } +#define MY_ALLOC(T, p, size, alloc) \ + { if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } -#define MY_ALLOC_ZE(T, p, size, alloc) { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) } +#define MY_ALLOC_ZE(T, p, size, alloc) \ + { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) } #define MY_ALLOC_AND_CPY(to, size, from, alloc) \ { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } @@ -58,7 +59,7 @@ enum EIdEnum const Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; -#define SzBitUi32s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } +#define SzBitUi32s_INIT(p) { (p)->Defs = NULL; (p)->Vals = NULL; } static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) { @@ -69,8 +70,8 @@ static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) } else { - MY_ALLOC(Byte, p->Defs, (num + 7) >> 3, alloc); - MY_ALLOC(UInt32, p->Vals, num, alloc); + MY_ALLOC(Byte, p->Defs, (num + 7) >> 3, alloc) + MY_ALLOC(UInt32, p->Vals, num, alloc) } return SZ_OK; } @@ -81,7 +82,7 @@ static void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; } -#define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } +#define SzBitUi64s_INIT(p) { (p)->Defs = NULL; (p)->Vals = NULL; } static void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) { @@ -96,7 +97,7 @@ static void SzAr_Init(CSzAr *p) p->NumFolders = 0; p->PackPositions = NULL; - SzBitUi32s_Init(&p->FolderCRCs); + SzBitUi32s_INIT(&p->FolderCRCs) p->FoCodersOffsets = NULL; p->FoStartPackStreamIndex = NULL; @@ -142,11 +143,11 @@ void SzArEx_Init(CSzArEx *p) p->FileNameOffsets = NULL; p->FileNames = NULL; - SzBitUi32s_Init(&p->CRCs); - SzBitUi32s_Init(&p->Attribs); - // SzBitUi32s_Init(&p->Parents); - SzBitUi64s_Init(&p->MTime); - SzBitUi64s_Init(&p->CTime); + SzBitUi32s_INIT(&p->CRCs) + SzBitUi32s_INIT(&p->Attribs) + // SzBitUi32s_INIT(&p->Parents) + SzBitUi64s_INIT(&p->MTime) + SzBitUi64s_INIT(&p->CTime) } void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc) @@ -180,11 +181,20 @@ static int TestSignatureCandidate(const Byte *testBytes) return 1; } -#define SzData_Clear(p) { (p)->Data = NULL; (p)->Size = 0; } +#define SzData_CLEAR(p) { (p)->Data = NULL; (p)->Size = 0; } + +#define SZ_READ_BYTE_SD_NOCHECK(_sd_, dest) \ + (_sd_)->Size--; dest = *(_sd_)->Data++; + +#define SZ_READ_BYTE_SD(_sd_, dest) \ + if ((_sd_)->Size == 0) return SZ_ERROR_ARCHIVE; \ + SZ_READ_BYTE_SD_NOCHECK(_sd_, dest) -#define SZ_READ_BYTE_SD(_sd_, dest) if ((_sd_)->Size == 0) return SZ_ERROR_ARCHIVE; (_sd_)->Size--; dest = *(_sd_)->Data++; #define SZ_READ_BYTE(dest) SZ_READ_BYTE_SD(sd, dest) -#define SZ_READ_BYTE_2(dest) if (sd.Size == 0) return SZ_ERROR_ARCHIVE; sd.Size--; dest = *sd.Data++; + +#define SZ_READ_BYTE_2(dest) \ + if (sd.Size == 0) return SZ_ERROR_ARCHIVE; \ + sd.Size--; dest = *sd.Data++; #define SKIP_DATA(sd, size) { sd->Size -= (size_t)(size); sd->Data += (size_t)(size); } #define SKIP_DATA2(sd, size) { sd.Size -= (size_t)(size); sd.Data += (size_t)(size); } @@ -192,25 +202,25 @@ static int TestSignatureCandidate(const Byte *testBytes) #define SZ_READ_32(dest) if (sd.Size < 4) return SZ_ERROR_ARCHIVE; \ dest = GetUi32(sd.Data); SKIP_DATA2(sd, 4); -static MY_NO_INLINE SRes ReadNumber(CSzData *sd, UInt64 *value) +static Z7_NO_INLINE SRes ReadNumber(CSzData *sd, UInt64 *value) { Byte firstByte, mask; unsigned i; UInt32 v; - SZ_READ_BYTE(firstByte); + SZ_READ_BYTE(firstByte) if ((firstByte & 0x80) == 0) { *value = firstByte; return SZ_OK; } - SZ_READ_BYTE(v); + SZ_READ_BYTE(v) if ((firstByte & 0x40) == 0) { *value = (((UInt32)firstByte & 0x3F) << 8) | v; return SZ_OK; } - SZ_READ_BYTE(mask); + SZ_READ_BYTE(mask) *value = v | ((UInt32)mask << 8); mask = 0x20; for (i = 2; i < 8; i++) @@ -218,11 +228,11 @@ static MY_NO_INLINE SRes ReadNumber(CSzData *sd, UInt64 *value) Byte b; if ((firstByte & mask) == 0) { - UInt64 highPart = (unsigned)firstByte & (unsigned)(mask - 1); + const UInt64 highPart = (unsigned)firstByte & (unsigned)(mask - 1); *value |= (highPart << (8 * i)); return SZ_OK; } - SZ_READ_BYTE(b); + SZ_READ_BYTE(b) *value |= ((UInt64)b << (8 * i)); mask >>= 1; } @@ -230,7 +240,7 @@ static MY_NO_INLINE SRes ReadNumber(CSzData *sd, UInt64 *value) } -static MY_NO_INLINE SRes SzReadNumber32(CSzData *sd, UInt32 *value) +static Z7_NO_INLINE SRes SzReadNumber32(CSzData *sd, UInt32 *value) { Byte firstByte; UInt64 value64; @@ -244,7 +254,7 @@ static MY_NO_INLINE SRes SzReadNumber32(CSzData *sd, UInt32 *value) sd->Size--; return SZ_OK; } - RINOK(ReadNumber(sd, &value64)); + RINOK(ReadNumber(sd, &value64)) if (value64 >= (UInt32)0x80000000 - 1) return SZ_ERROR_UNSUPPORTED; if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 4))) @@ -258,10 +268,10 @@ static MY_NO_INLINE SRes SzReadNumber32(CSzData *sd, UInt32 *value) static SRes SkipData(CSzData *sd) { UInt64 size; - RINOK(ReadNumber(sd, &size)); + RINOK(ReadNumber(sd, &size)) if (size > sd->Size) return SZ_ERROR_ARCHIVE; - SKIP_DATA(sd, size); + SKIP_DATA(sd, size) return SZ_OK; } @@ -270,28 +280,28 @@ static SRes WaitId(CSzData *sd, UInt32 id) for (;;) { UInt64 type; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == id) return SZ_OK; if (type == k7zIdEnd) return SZ_ERROR_ARCHIVE; - RINOK(SkipData(sd)); + RINOK(SkipData(sd)) } } static SRes RememberBitVector(CSzData *sd, UInt32 numItems, const Byte **v) { - UInt32 numBytes = (numItems + 7) >> 3; + const UInt32 numBytes = (numItems + 7) >> 3; if (numBytes > sd->Size) return SZ_ERROR_ARCHIVE; *v = sd->Data; - SKIP_DATA(sd, numBytes); + SKIP_DATA(sd, numBytes) return SZ_OK; } static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) { - Byte b = 0; + unsigned b = 0; unsigned m = 0; UInt32 sum = 0; for (; numItems != 0; numItems--) @@ -302,53 +312,53 @@ static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) m = 8; } m--; - sum += ((b >> m) & 1); + sum += (UInt32)((b >> m) & 1); } return sum; } -static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) +static Z7_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) { Byte allAreDefined; Byte *v2; - UInt32 numBytes = (numItems + 7) >> 3; + const UInt32 numBytes = (numItems + 7) >> 3; *v = NULL; - SZ_READ_BYTE(allAreDefined); + SZ_READ_BYTE(allAreDefined) if (numBytes == 0) return SZ_OK; if (allAreDefined == 0) { if (numBytes > sd->Size) return SZ_ERROR_ARCHIVE; - MY_ALLOC_AND_CPY(*v, numBytes, sd->Data, alloc); - SKIP_DATA(sd, numBytes); + MY_ALLOC_AND_CPY(*v, numBytes, sd->Data, alloc) + SKIP_DATA(sd, numBytes) return SZ_OK; } - MY_ALLOC(Byte, *v, numBytes, alloc); + MY_ALLOC(Byte, *v, numBytes, alloc) v2 = *v; memset(v2, 0xFF, (size_t)numBytes); { - unsigned numBits = (unsigned)numItems & 7; + const unsigned numBits = (unsigned)numItems & 7; if (numBits != 0) v2[(size_t)numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); } return SZ_OK; } -static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) +static Z7_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { UInt32 i; CSzData sd; UInt32 *vals; const Byte *defs; - MY_ALLOC_ZE(UInt32, crcs->Vals, numItems, alloc); + MY_ALLOC_ZE(UInt32, crcs->Vals, numItems, alloc) sd = *sd2; defs = crcs->Defs; vals = crcs->Vals; for (i = 0; i < numItems; i++) if (SzBitArray_Check(defs, i)) { - SZ_READ_32(vals[i]); + SZ_READ_32(vals[i]) } else vals[i] = 0; @@ -359,7 +369,7 @@ static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *c static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { SzBitUi32s_Free(crcs, alloc); - RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc)); + RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc)) return ReadUi32s(sd, numItems, crcs, alloc); } @@ -367,36 +377,36 @@ static SRes SkipBitUi32s(CSzData *sd, UInt32 numItems) { Byte allAreDefined; UInt32 numDefined = numItems; - SZ_READ_BYTE(allAreDefined); + SZ_READ_BYTE(allAreDefined) if (!allAreDefined) { - size_t numBytes = (numItems + 7) >> 3; + const size_t numBytes = (numItems + 7) >> 3; if (numBytes > sd->Size) return SZ_ERROR_ARCHIVE; numDefined = CountDefinedBits(sd->Data, numItems); - SKIP_DATA(sd, numBytes); + SKIP_DATA(sd, numBytes) } if (numDefined > (sd->Size >> 2)) return SZ_ERROR_ARCHIVE; - SKIP_DATA(sd, (size_t)numDefined * 4); + SKIP_DATA(sd, (size_t)numDefined * 4) return SZ_OK; } static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) { - RINOK(SzReadNumber32(sd, &p->NumPackStreams)); + RINOK(SzReadNumber32(sd, &p->NumPackStreams)) - RINOK(WaitId(sd, k7zIdSize)); - MY_ALLOC(UInt64, p->PackPositions, (size_t)p->NumPackStreams + 1, alloc); + RINOK(WaitId(sd, k7zIdSize)) + MY_ALLOC(UInt64, p->PackPositions, (size_t)p->NumPackStreams + 1, alloc) { UInt64 sum = 0; UInt32 i; - UInt32 numPackStreams = p->NumPackStreams; + const UInt32 numPackStreams = p->NumPackStreams; for (i = 0; i < numPackStreams; i++) { UInt64 packSize; p->PackPositions[i] = sum; - RINOK(ReadNumber(sd, &packSize)); + RINOK(ReadNumber(sd, &packSize)) sum += packSize; if (sum < packSize) return SZ_ERROR_ARCHIVE; @@ -407,16 +417,16 @@ static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) for (;;) { UInt64 type; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == k7zIdEnd) return SZ_OK; if (type == k7zIdCRC) { /* CRC of packed streams is unused now */ - RINOK(SkipBitUi32s(sd, p->NumPackStreams)); + RINOK(SkipBitUi32s(sd, p->NumPackStreams)) continue; } - RINOK(SkipData(sd)); + RINOK(SkipData(sd)) } } @@ -442,7 +452,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) f->NumPackStreams = 0; f->UnpackStream = 0; - RINOK(SzReadNumber32(sd, &numCoders)); + RINOK(SzReadNumber32(sd, &numCoders)) if (numCoders == 0 || numCoders > SZ_NUM_CODERS_IN_FOLDER_MAX) return SZ_ERROR_UNSUPPORTED; @@ -453,7 +463,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) unsigned idSize, j; UInt64 id; - SZ_READ_BYTE(mainByte); + SZ_READ_BYTE(mainByte) if ((mainByte & 0xC0) != 0) return SZ_ERROR_UNSUPPORTED; @@ -481,12 +491,12 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) { UInt32 numStreams; - RINOK(SzReadNumber32(sd, &numStreams)); + RINOK(SzReadNumber32(sd, &numStreams)) if (numStreams > k_NumCodersStreams_in_Folder_MAX) return SZ_ERROR_UNSUPPORTED; coder->NumStreams = (Byte)numStreams; - RINOK(SzReadNumber32(sd, &numStreams)); + RINOK(SzReadNumber32(sd, &numStreams)) if (numStreams != 1) return SZ_ERROR_UNSUPPORTED; } @@ -499,7 +509,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) if ((mainByte & 0x20) != 0) { UInt32 propsSize = 0; - RINOK(SzReadNumber32(sd, &propsSize)); + RINOK(SzReadNumber32(sd, &propsSize)) if (propsSize > sd->Size) return SZ_ERROR_ARCHIVE; if (propsSize >= 0x80) @@ -549,12 +559,12 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) { CSzBond *bp = f->Bonds + i; - RINOK(SzReadNumber32(sd, &bp->InIndex)); + RINOK(SzReadNumber32(sd, &bp->InIndex)) if (bp->InIndex >= numInStreams || streamUsed[bp->InIndex]) return SZ_ERROR_ARCHIVE; streamUsed[bp->InIndex] = True; - RINOK(SzReadNumber32(sd, &bp->OutIndex)); + RINOK(SzReadNumber32(sd, &bp->OutIndex)) if (bp->OutIndex >= numCoders || coderUsed[bp->OutIndex]) return SZ_ERROR_ARCHIVE; coderUsed[bp->OutIndex] = True; @@ -584,7 +594,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) for (i = 0; i < numPackStreams; i++) { UInt32 index; - RINOK(SzReadNumber32(sd, &index)); + RINOK(SzReadNumber32(sd, &index)) if (index >= numInStreams || streamUsed[index]) return SZ_ERROR_ARCHIVE; streamUsed[index] = True; @@ -598,7 +608,7 @@ SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) } -static MY_NO_INLINE SRes SkipNumbers(CSzData *sd2, UInt32 num) +static Z7_NO_INLINE SRes SkipNumbers(CSzData *sd2, UInt32 num) { CSzData sd; sd = *sd2; @@ -606,7 +616,7 @@ static MY_NO_INLINE SRes SkipNumbers(CSzData *sd2, UInt32 num) { Byte firstByte, mask; unsigned i; - SZ_READ_BYTE_2(firstByte); + SZ_READ_BYTE_2(firstByte) if ((firstByte & 0x80) == 0) continue; if ((firstByte & 0x40) == 0) @@ -622,7 +632,7 @@ static MY_NO_INLINE SRes SkipNumbers(CSzData *sd2, UInt32 num) mask >>= 1; if (i > sd.Size) return SZ_ERROR_ARCHIVE; - SKIP_DATA2(sd, i); + SKIP_DATA2(sd, i) } *sd2 = sd; return SZ_OK; @@ -645,30 +655,30 @@ static SRes ReadUnpackInfo(CSzAr *p, const Byte *startBufPtr; Byte external; - RINOK(WaitId(sd2, k7zIdFolder)); + RINOK(WaitId(sd2, k7zIdFolder)) - RINOK(SzReadNumber32(sd2, &numFolders)); + RINOK(SzReadNumber32(sd2, &numFolders)) if (numFolders > numFoldersMax) return SZ_ERROR_UNSUPPORTED; p->NumFolders = numFolders; - SZ_READ_BYTE_SD(sd2, external); + SZ_READ_BYTE_SD(sd2, external) if (external == 0) sd = *sd2; else { UInt32 index; - RINOK(SzReadNumber32(sd2, &index)); + RINOK(SzReadNumber32(sd2, &index)) if (index >= numTempBufs) return SZ_ERROR_ARCHIVE; sd.Data = tempBufs[index].data; sd.Size = tempBufs[index].size; } - MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); - MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); - MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc) + MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc) + MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc) + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc) startBufPtr = sd.Data; @@ -681,7 +691,7 @@ static SRes ReadUnpackInfo(CSzAr *p, p->FoCodersOffsets[fo] = (size_t)(sd.Data - startBufPtr); - RINOK(SzReadNumber32(&sd, &numCoders)); + RINOK(SzReadNumber32(&sd, &numCoders)) if (numCoders == 0 || numCoders > k_Scan_NumCoders_MAX) return SZ_ERROR_UNSUPPORTED; @@ -691,7 +701,7 @@ static SRes ReadUnpackInfo(CSzAr *p, unsigned idSize; UInt32 coderInStreams; - SZ_READ_BYTE_2(mainByte); + SZ_READ_BYTE_2(mainByte) if ((mainByte & 0xC0) != 0) return SZ_ERROR_UNSUPPORTED; idSize = (mainByte & 0xF); @@ -699,15 +709,15 @@ static SRes ReadUnpackInfo(CSzAr *p, return SZ_ERROR_UNSUPPORTED; if (idSize > sd.Size) return SZ_ERROR_ARCHIVE; - SKIP_DATA2(sd, idSize); + SKIP_DATA2(sd, idSize) coderInStreams = 1; if ((mainByte & 0x10) != 0) { UInt32 coderOutStreams; - RINOK(SzReadNumber32(&sd, &coderInStreams)); - RINOK(SzReadNumber32(&sd, &coderOutStreams)); + RINOK(SzReadNumber32(&sd, &coderInStreams)) + RINOK(SzReadNumber32(&sd, &coderOutStreams)) if (coderInStreams > k_Scan_NumCodersStreams_in_Folder_MAX || coderOutStreams != 1) return SZ_ERROR_UNSUPPORTED; } @@ -717,10 +727,10 @@ static SRes ReadUnpackInfo(CSzAr *p, if ((mainByte & 0x20) != 0) { UInt32 propsSize; - RINOK(SzReadNumber32(&sd, &propsSize)); + RINOK(SzReadNumber32(&sd, &propsSize)) if (propsSize > sd.Size) return SZ_ERROR_ARCHIVE; - SKIP_DATA2(sd, propsSize); + SKIP_DATA2(sd, propsSize) } } @@ -734,7 +744,7 @@ static SRes ReadUnpackInfo(CSzAr *p, Byte coderUsed[k_Scan_NumCoders_MAX]; UInt32 i; - UInt32 numBonds = numCoders - 1; + const UInt32 numBonds = numCoders - 1; if (numInStreams < numBonds) return SZ_ERROR_ARCHIVE; @@ -750,12 +760,12 @@ static SRes ReadUnpackInfo(CSzAr *p, { UInt32 index; - RINOK(SzReadNumber32(&sd, &index)); + RINOK(SzReadNumber32(&sd, &index)) if (index >= numInStreams || streamUsed[index]) return SZ_ERROR_ARCHIVE; streamUsed[index] = True; - RINOK(SzReadNumber32(&sd, &index)); + RINOK(SzReadNumber32(&sd, &index)) if (index >= numCoders || coderUsed[index]) return SZ_ERROR_ARCHIVE; coderUsed[index] = True; @@ -767,7 +777,7 @@ static SRes ReadUnpackInfo(CSzAr *p, for (i = 0; i < numPackStreams; i++) { UInt32 index; - RINOK(SzReadNumber32(&sd, &index)); + RINOK(SzReadNumber32(&sd, &index)) if (index >= numInStreams || streamUsed[index]) return SZ_ERROR_ARCHIVE; streamUsed[index] = True; @@ -802,7 +812,7 @@ static SRes ReadUnpackInfo(CSzAr *p, const size_t dataSize = (size_t)(sd.Data - startBufPtr); p->FoStartPackStreamIndex[fo] = packStreamIndex; p->FoCodersOffsets[fo] = dataSize; - MY_ALLOC_ZE_AND_CPY(p->CodersData, dataSize, startBufPtr, alloc); + MY_ALLOC_ZE_AND_CPY(p->CodersData, dataSize, startBufPtr, alloc) } if (external != 0) @@ -812,21 +822,21 @@ static SRes ReadUnpackInfo(CSzAr *p, sd = *sd2; } - RINOK(WaitId(&sd, k7zIdCodersUnpackSize)); + RINOK(WaitId(&sd, k7zIdCodersUnpackSize)) - MY_ALLOC_ZE(UInt64, p->CoderUnpackSizes, (size_t)numCodersOutStreams, alloc); + MY_ALLOC_ZE(UInt64, p->CoderUnpackSizes, (size_t)numCodersOutStreams, alloc) { UInt32 i; for (i = 0; i < numCodersOutStreams; i++) { - RINOK(ReadNumber(&sd, p->CoderUnpackSizes + i)); + RINOK(ReadNumber(&sd, p->CoderUnpackSizes + i)) } } for (;;) { UInt64 type; - RINOK(ReadID(&sd, &type)); + RINOK(ReadID(&sd, &type)) if (type == k7zIdEnd) { *sd2 = sd; @@ -834,10 +844,10 @@ static SRes ReadUnpackInfo(CSzAr *p, } if (type == k7zIdCRC) { - RINOK(ReadBitUi32s(&sd, numFolders, &p->FolderCRCs, alloc)); + RINOK(ReadBitUi32s(&sd, numFolders, &p->FolderCRCs, alloc)) continue; } - RINOK(SkipData(&sd)); + RINOK(SkipData(&sd)) } } @@ -862,13 +872,13 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) { UInt64 type = 0; UInt32 numSubDigests = 0; - UInt32 numFolders = p->NumFolders; + const UInt32 numFolders = p->NumFolders; UInt32 numUnpackStreams = numFolders; UInt32 numUnpackSizesInData = 0; for (;;) { - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == k7zIdNumUnpackStream) { UInt32 i; @@ -878,7 +888,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) for (i = 0; i < numFolders; i++) { UInt32 numStreams; - RINOK(SzReadNumber32(sd, &numStreams)); + RINOK(SzReadNumber32(sd, &numStreams)) if (numUnpackStreams > numUnpackStreams + numStreams) return SZ_ERROR_UNSUPPORTED; numUnpackStreams += numStreams; @@ -892,7 +902,7 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) } if (type == k7zIdCRC || type == k7zIdSize || type == k7zIdEnd) break; - RINOK(SkipData(sd)); + RINOK(SkipData(sd)) } if (!ssi->sdNumSubStreams.Data) @@ -908,9 +918,9 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) if (type == k7zIdSize) { ssi->sdSizes.Data = sd->Data; - RINOK(SkipNumbers(sd, numUnpackSizesInData)); + RINOK(SkipNumbers(sd, numUnpackSizesInData)) ssi->sdSizes.Size = (size_t)(sd->Data - ssi->sdSizes.Data); - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) } for (;;) @@ -920,14 +930,14 @@ static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) if (type == k7zIdCRC) { ssi->sdCRCs.Data = sd->Data; - RINOK(SkipBitUi32s(sd, numSubDigests)); + RINOK(SkipBitUi32s(sd, numSubDigests)) ssi->sdCRCs.Size = (size_t)(sd->Data - ssi->sdCRCs.Data); } else { - RINOK(SkipData(sd)); + RINOK(SkipData(sd)) } - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) } } @@ -940,31 +950,31 @@ static SRes SzReadStreamsInfo(CSzAr *p, { UInt64 type; - SzData_Clear(&ssi->sdSizes); - SzData_Clear(&ssi->sdCRCs); - SzData_Clear(&ssi->sdNumSubStreams); + SzData_CLEAR(&ssi->sdSizes) + SzData_CLEAR(&ssi->sdCRCs) + SzData_CLEAR(&ssi->sdNumSubStreams) *dataOffset = 0; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == k7zIdPackInfo) { - RINOK(ReadNumber(sd, dataOffset)); + RINOK(ReadNumber(sd, dataOffset)) if (*dataOffset > p->RangeLimit) return SZ_ERROR_ARCHIVE; - RINOK(ReadPackInfo(p, sd, alloc)); + RINOK(ReadPackInfo(p, sd, alloc)) if (p->PackPositions[p->NumPackStreams] > p->RangeLimit - *dataOffset) return SZ_ERROR_ARCHIVE; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) } if (type == k7zIdUnpackInfo) { - RINOK(ReadUnpackInfo(p, sd, numFoldersMax, tempBufs, numTempBufs, alloc)); - RINOK(ReadID(sd, &type)); + RINOK(ReadUnpackInfo(p, sd, numFoldersMax, tempBufs, numTempBufs, alloc)) + RINOK(ReadID(sd, &type)) } if (type == k7zIdSubStreamsInfo) { - RINOK(ReadSubStreamsInfo(p, sd, ssi)); - RINOK(ReadID(sd, &type)); + RINOK(ReadSubStreamsInfo(p, sd, ssi)) + RINOK(ReadID(sd, &type)) } else { @@ -976,7 +986,7 @@ static SRes SzReadStreamsInfo(CSzAr *p, } static SRes SzReadAndDecodePackedStreams( - ILookInStream *inStream, + ILookInStreamPtr inStream, CSzData *sd, CBuf *tempBufs, UInt32 numFoldersMax, @@ -988,7 +998,7 @@ static SRes SzReadAndDecodePackedStreams( UInt32 fo; CSubStreamInfo ssi; - RINOK(SzReadStreamsInfo(p, sd, numFoldersMax, NULL, 0, &dataStartPos, &ssi, allocTemp)); + RINOK(SzReadStreamsInfo(p, sd, numFoldersMax, NULL, 0, &dataStartPos, &ssi, allocTemp)) dataStartPos += baseOffset; if (p->NumFolders == 0) @@ -1000,7 +1010,7 @@ static SRes SzReadAndDecodePackedStreams( for (fo = 0; fo < p->NumFolders; fo++) { CBuf *tempBuf = tempBufs + fo; - UInt64 unpackSize = SzAr_GetFolderUnpackSize(p, fo); + const UInt64 unpackSize = SzAr_GetFolderUnpackSize(p, fo); if ((size_t)unpackSize != unpackSize) return SZ_ERROR_MEM; if (!Buf_Create(tempBuf, (size_t)unpackSize, allocTemp)) @@ -1010,8 +1020,8 @@ static SRes SzReadAndDecodePackedStreams( for (fo = 0; fo < p->NumFolders; fo++) { const CBuf *tempBuf = tempBufs + fo; - RINOK(LookInStream_SeekTo(inStream, dataStartPos)); - RINOK(SzAr_DecodeFolder(p, fo, inStream, dataStartPos, tempBuf->data, tempBuf->size, allocTemp)); + RINOK(LookInStream_SeekTo(inStream, dataStartPos)) + RINOK(SzAr_DecodeFolder(p, fo, inStream, dataStartPos, tempBuf->data, tempBuf->size, allocTemp)) } return SZ_OK; @@ -1046,7 +1056,7 @@ static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; } -static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, +static Z7_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, CSzData *sd2, const CBuf *tempBufs, UInt32 numTempBufs, ISzAllocPtr alloc) @@ -1057,22 +1067,22 @@ static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, Byte *defs; Byte external; - RINOK(ReadBitVector(sd2, num, &p->Defs, alloc)); + RINOK(ReadBitVector(sd2, num, &p->Defs, alloc)) - SZ_READ_BYTE_SD(sd2, external); + SZ_READ_BYTE_SD(sd2, external) if (external == 0) sd = *sd2; else { UInt32 index; - RINOK(SzReadNumber32(sd2, &index)); + RINOK(SzReadNumber32(sd2, &index)) if (index >= numTempBufs) return SZ_ERROR_ARCHIVE; sd.Data = tempBufs[index].data; sd.Size = tempBufs[index].size; } - MY_ALLOC_ZE(CNtfsFileTime, p->Vals, num, alloc); + MY_ALLOC_ZE(CNtfsFileTime, p->Vals, num, alloc) vals = p->Vals; defs = p->Defs; for (i = 0; i < num; i++) @@ -1082,7 +1092,7 @@ static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, return SZ_ERROR_ARCHIVE; vals[i].Low = GetUi32(sd.Data); vals[i].High = GetUi32(sd.Data + 4); - SKIP_DATA2(sd, 8); + SKIP_DATA2(sd, 8) } else vals[i].High = vals[i].Low = 0; @@ -1100,7 +1110,7 @@ static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, static SRes SzReadHeader2( CSzArEx *p, /* allocMain */ CSzData *sd, - ILookInStream *inStream, + ILookInStreamPtr inStream, CBuf *tempBufs, UInt32 *numTempBufs, ISzAllocPtr allocMain, ISzAllocPtr allocTemp @@ -1111,26 +1121,26 @@ static SRes SzReadHeader2( { UInt64 type; - SzData_Clear(&ssi.sdSizes); - SzData_Clear(&ssi.sdCRCs); - SzData_Clear(&ssi.sdNumSubStreams); + SzData_CLEAR(&ssi.sdSizes) + SzData_CLEAR(&ssi.sdCRCs) + SzData_CLEAR(&ssi.sdNumSubStreams) ssi.NumSubDigests = 0; ssi.NumTotalSubStreams = 0; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == k7zIdArchiveProperties) { for (;;) { UInt64 type2; - RINOK(ReadID(sd, &type2)); + RINOK(ReadID(sd, &type2)) if (type2 == k7zIdEnd) break; - RINOK(SkipData(sd)); + RINOK(SkipData(sd)) } - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) } if (type == k7zIdAdditionalStreamsInfo) @@ -1148,15 +1158,15 @@ static SRes SzReadHeader2( if (res != SZ_OK) return res; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) } if (type == k7zIdMainStreamsInfo) { RINOK(SzReadStreamsInfo(&p->db, sd, (UInt32)1 << 30, tempBufs, *numTempBufs, - &p->dataPos, &ssi, allocMain)); + &p->dataPos, &ssi, allocMain)) p->dataPos += p->startPosAfterHeader; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) } if (type == k7zIdEnd) @@ -1174,23 +1184,23 @@ static SRes SzReadHeader2( const Byte *emptyStreams = NULL; const Byte *emptyFiles = NULL; - RINOK(SzReadNumber32(sd, &numFiles)); + RINOK(SzReadNumber32(sd, &numFiles)) p->NumFiles = numFiles; for (;;) { UInt64 type; UInt64 size; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == k7zIdEnd) break; - RINOK(ReadNumber(sd, &size)); + RINOK(ReadNumber(sd, &size)) if (size > sd->Size) return SZ_ERROR_ARCHIVE; if (type >= ((UInt32)1 << 8)) { - SKIP_DATA(sd, size); + SKIP_DATA(sd, size) } else switch ((unsigned)type) { @@ -1200,7 +1210,7 @@ static SRes SzReadHeader2( const Byte *namesData; Byte external; - SZ_READ_BYTE(external); + SZ_READ_BYTE(external) if (external == 0) { namesSize = (size_t)size - 1; @@ -1209,7 +1219,7 @@ static SRes SzReadHeader2( else { UInt32 index; - RINOK(SzReadNumber32(sd, &index)); + RINOK(SzReadNumber32(sd, &index)) if (index >= *numTempBufs) return SZ_ERROR_ARCHIVE; namesData = (tempBufs)[index].data; @@ -1218,25 +1228,25 @@ static SRes SzReadHeader2( if ((namesSize & 1) != 0) return SZ_ERROR_ARCHIVE; - MY_ALLOC(size_t, p->FileNameOffsets, numFiles + 1, allocMain); - MY_ALLOC_ZE_AND_CPY(p->FileNames, namesSize, namesData, allocMain); + MY_ALLOC(size_t, p->FileNameOffsets, numFiles + 1, allocMain) + MY_ALLOC_ZE_AND_CPY(p->FileNames, namesSize, namesData, allocMain) RINOK(SzReadFileNames(p->FileNames, namesSize, numFiles, p->FileNameOffsets)) if (external == 0) { - SKIP_DATA(sd, namesSize); + SKIP_DATA(sd, namesSize) } break; } case k7zIdEmptyStream: { - RINOK(RememberBitVector(sd, numFiles, &emptyStreams)); + RINOK(RememberBitVector(sd, numFiles, &emptyStreams)) numEmptyStreams = CountDefinedBits(emptyStreams, numFiles); emptyFiles = NULL; break; } case k7zIdEmptyFile: { - RINOK(RememberBitVector(sd, numEmptyStreams, &emptyFiles)); + RINOK(RememberBitVector(sd, numEmptyStreams, &emptyFiles)) break; } case k7zIdWinAttrib: @@ -1245,22 +1255,22 @@ static SRes SzReadHeader2( CSzData sdSwitch; CSzData *sdPtr; SzBitUi32s_Free(&p->Attribs, allocMain); - RINOK(ReadBitVector(sd, numFiles, &p->Attribs.Defs, allocMain)); + RINOK(ReadBitVector(sd, numFiles, &p->Attribs.Defs, allocMain)) - SZ_READ_BYTE(external); + SZ_READ_BYTE(external) if (external == 0) sdPtr = sd; else { UInt32 index; - RINOK(SzReadNumber32(sd, &index)); + RINOK(SzReadNumber32(sd, &index)) if (index >= *numTempBufs) return SZ_ERROR_ARCHIVE; sdSwitch.Data = (tempBufs)[index].data; sdSwitch.Size = (tempBufs)[index].size; sdPtr = &sdSwitch; } - RINOK(ReadUi32s(sdPtr, numFiles, &p->Attribs, allocMain)); + RINOK(ReadUi32s(sdPtr, numFiles, &p->Attribs, allocMain)) break; } /* @@ -1273,11 +1283,11 @@ static SRes SzReadHeader2( break; } */ - case k7zIdMTime: RINOK(ReadTime(&p->MTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)); break; - case k7zIdCTime: RINOK(ReadTime(&p->CTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)); break; + case k7zIdMTime: RINOK(ReadTime(&p->MTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)) break; + case k7zIdCTime: RINOK(ReadTime(&p->CTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)) break; default: { - SKIP_DATA(sd, size); + SKIP_DATA(sd, size) } } } @@ -1288,10 +1298,10 @@ static SRes SzReadHeader2( for (;;) { UInt64 type; - RINOK(ReadID(sd, &type)); + RINOK(ReadID(sd, &type)) if (type == k7zIdEnd) break; - RINOK(SkipData(sd)); + RINOK(SkipData(sd)) } { @@ -1303,40 +1313,37 @@ static SRes SzReadHeader2( UInt64 unpackPos = 0; const Byte *digestsDefs = NULL; const Byte *digestsVals = NULL; - UInt32 digestsValsIndex = 0; - UInt32 digestIndex; - Byte allDigestsDefined = 0; + UInt32 digestIndex = 0; Byte isDirMask = 0; Byte crcMask = 0; Byte mask = 0x80; - MY_ALLOC(UInt32, p->FolderToFile, p->db.NumFolders + 1, allocMain); - MY_ALLOC_ZE(UInt32, p->FileToFolder, p->NumFiles, allocMain); - MY_ALLOC(UInt64, p->UnpackPositions, p->NumFiles + 1, allocMain); - MY_ALLOC_ZE(Byte, p->IsDirs, (p->NumFiles + 7) >> 3, allocMain); + MY_ALLOC(UInt32, p->FolderToFile, p->db.NumFolders + 1, allocMain) + MY_ALLOC_ZE(UInt32, p->FileToFolder, p->NumFiles, allocMain) + MY_ALLOC(UInt64, p->UnpackPositions, p->NumFiles + 1, allocMain) + MY_ALLOC_ZE(Byte, p->IsDirs, (p->NumFiles + 7) >> 3, allocMain) - RINOK(SzBitUi32s_Alloc(&p->CRCs, p->NumFiles, allocMain)); + RINOK(SzBitUi32s_Alloc(&p->CRCs, p->NumFiles, allocMain)) if (ssi.sdCRCs.Size != 0) { - SZ_READ_BYTE_SD(&ssi.sdCRCs, allDigestsDefined); + Byte allDigestsDefined = 0; + SZ_READ_BYTE_SD_NOCHECK(&ssi.sdCRCs, allDigestsDefined) if (allDigestsDefined) digestsVals = ssi.sdCRCs.Data; else { - size_t numBytes = (ssi.NumSubDigests + 7) >> 3; + const size_t numBytes = (ssi.NumSubDigests + 7) >> 3; digestsDefs = ssi.sdCRCs.Data; digestsVals = digestsDefs + numBytes; } } - digestIndex = 0; - for (i = 0; i < numFiles; i++, mask >>= 1) { if (mask == 0) { - UInt32 byteIndex = (i - 1) >> 3; + const UInt32 byteIndex = (i - 1) >> 3; p->IsDirs[byteIndex] = isDirMask; p->CRCs.Defs[byteIndex] = crcMask; isDirMask = 0; @@ -1374,18 +1381,17 @@ static SRes SzReadHeader2( numSubStreams = 1; if (ssi.sdNumSubStreams.Data) { - RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)); + RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)) } remSubStreams = numSubStreams; if (numSubStreams != 0) break; { - UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + const UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); unpackPos += folderUnpackSize; if (unpackPos < folderUnpackSize) return SZ_ERROR_ARCHIVE; } - folderIndex++; } } @@ -1397,47 +1403,44 @@ static SRes SzReadHeader2( if (--remSubStreams == 0) { - UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); - UInt64 startFolderUnpackPos = p->UnpackPositions[p->FolderToFile[folderIndex]]; + const UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + const UInt64 startFolderUnpackPos = p->UnpackPositions[p->FolderToFile[folderIndex]]; if (folderUnpackSize < unpackPos - startFolderUnpackPos) return SZ_ERROR_ARCHIVE; unpackPos = startFolderUnpackPos + folderUnpackSize; if (unpackPos < folderUnpackSize) return SZ_ERROR_ARCHIVE; - if (numSubStreams == 1 && SzBitWithVals_Check(&p->db.FolderCRCs, i)) + if (numSubStreams == 1 && SzBitWithVals_Check(&p->db.FolderCRCs, folderIndex)) { p->CRCs.Vals[i] = p->db.FolderCRCs.Vals[folderIndex]; crcMask |= mask; } - else if (allDigestsDefined || (digestsDefs && SzBitArray_Check(digestsDefs, digestIndex))) - { - p->CRCs.Vals[i] = GetUi32(digestsVals + (size_t)digestsValsIndex * 4); - digestsValsIndex++; - crcMask |= mask; - } - folderIndex++; } else { UInt64 v; - RINOK(ReadNumber(&ssi.sdSizes, &v)); + RINOK(ReadNumber(&ssi.sdSizes, &v)) unpackPos += v; if (unpackPos < v) return SZ_ERROR_ARCHIVE; - if (allDigestsDefined || (digestsDefs && SzBitArray_Check(digestsDefs, digestIndex))) + } + if ((crcMask & mask) == 0 && digestsVals) + { + if (!digestsDefs || SzBitArray_Check(digestsDefs, digestIndex)) { - p->CRCs.Vals[i] = GetUi32(digestsVals + (size_t)digestsValsIndex * 4); - digestsValsIndex++; + p->CRCs.Vals[i] = GetUi32(digestsVals); + digestsVals += 4; crcMask |= mask; } + digestIndex++; } } if (mask != 0x80) { - UInt32 byteIndex = (i - 1) >> 3; + const UInt32 byteIndex = (i - 1) >> 3; p->IsDirs[byteIndex] = isDirMask; p->CRCs.Defs[byteIndex] = crcMask; } @@ -1454,7 +1457,7 @@ static SRes SzReadHeader2( break; if (!ssi.sdNumSubStreams.Data) return SZ_ERROR_ARCHIVE; - RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)); + RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)) if (numSubStreams != 0) return SZ_ERROR_ARCHIVE; /* @@ -1479,7 +1482,7 @@ static SRes SzReadHeader2( static SRes SzReadHeader( CSzArEx *p, CSzData *sd, - ILookInStream *inStream, + ILookInStreamPtr inStream, ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { @@ -1498,7 +1501,7 @@ static SRes SzReadHeader( for (i = 0; i < NUM_ADDITIONAL_STREAMS_MAX; i++) Buf_Free(tempBufs + i, allocTemp); - RINOK(res); + RINOK(res) if (sd->Size != 0) return SZ_ERROR_FAIL; @@ -1508,7 +1511,7 @@ static SRes SzReadHeader( static SRes SzArEx_Open2( CSzArEx *p, - ILookInStream *inStream, + ILookInStreamPtr inStream, ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { @@ -1521,9 +1524,9 @@ static SRes SzArEx_Open2( SRes res; startArcPos = 0; - RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)); + RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)) - RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)); + RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)) if (!TestSignatureCandidate(header)) return SZ_ERROR_NO_ARCHIVE; @@ -1552,14 +1555,14 @@ static SRes SzArEx_Open2( { Int64 pos = 0; - RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)) if ((UInt64)pos < (UInt64)startArcPos + nextHeaderOffset || (UInt64)pos < (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset || (UInt64)pos < (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) return SZ_ERROR_INPUT_EOF; } - RINOK(LookInStream_SeekTo(inStream, (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset)); + RINOK(LookInStream_SeekTo(inStream, (UInt64)startArcPos + k7zStartHeaderSize + nextHeaderOffset)) if (!Buf_Create(&buf, nextHeaderSizeT, allocTemp)) return SZ_ERROR_MEM; @@ -1634,10 +1637,10 @@ static SRes SzArEx_Open2( } -SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, +SRes SzArEx_Open(CSzArEx *p, ILookInStreamPtr inStream, ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { - SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); + const SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); if (res != SZ_OK) SzArEx_Free(p, allocMain); return res; @@ -1646,7 +1649,7 @@ SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, SRes SzArEx_Extract( const CSzArEx *p, - ILookInStream *inStream, + ILookInStreamPtr inStream, UInt32 fileIndex, UInt32 *blockIndex, Byte **tempBuf, @@ -1656,7 +1659,7 @@ SRes SzArEx_Extract( ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { - UInt32 folderIndex = p->FileToFolder[fileIndex]; + const UInt32 folderIndex = p->FileToFolder[fileIndex]; SRes res = SZ_OK; *offset = 0; @@ -1673,13 +1676,13 @@ SRes SzArEx_Extract( if (*tempBuf == NULL || *blockIndex != folderIndex) { - UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + const UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); /* UInt64 unpackSizeSpec = p->UnpackPositions[p->FolderToFile[(size_t)folderIndex + 1]] - p->UnpackPositions[p->FolderToFile[folderIndex]]; */ - size_t unpackSize = (size_t)unpackSizeSpec; + const size_t unpackSize = (size_t)unpackSizeSpec; if (unpackSize != unpackSizeSpec) return SZ_ERROR_MEM; @@ -1707,7 +1710,7 @@ SRes SzArEx_Extract( if (res == SZ_OK) { - UInt64 unpackPos = p->UnpackPositions[fileIndex]; + const UInt64 unpackPos = p->UnpackPositions[fileIndex]; *offset = (size_t)(unpackPos - p->UnpackPositions[p->FolderToFile[folderIndex]]); *outSizeProcessed = (size_t)(p->UnpackPositions[(size_t)fileIndex + 1] - unpackPos); if (*offset + *outSizeProcessed > *outBufferSize) @@ -1726,8 +1729,8 @@ size_t SzArEx_GetFileNameUtf16(const CSzArEx *p, size_t fileIndex, UInt16 *dest) if (!p->FileNameOffsets) return 0; - size_t offs = p->FileNameOffsets[fileIndex]; - size_t len = p->FileNameOffsets[fileIndex + 1] - offs; + const size_t offs = p->FileNameOffsets[fileIndex]; + const size_t len = p->FileNameOffsets[fileIndex + 1] - offs; if (dest != 0) { size_t i; diff --git a/multiarc/src/formats/7z/C/7zBuf.c b/multiarc/src/formats/7z/C/7zBuf.c old mode 100755 new mode 100644 diff --git a/multiarc/src/formats/7z/C/7zBuf.h b/multiarc/src/formats/7z/C/7zBuf.h old mode 100755 new mode 100644 index 81d1b5b64..c0ba8a7b6 --- a/multiarc/src/formats/7z/C/7zBuf.h +++ b/multiarc/src/formats/7z/C/7zBuf.h @@ -1,8 +1,8 @@ /* 7zBuf.h -- Byte Buffer -2017-04-03 : Igor Pavlov : Public domain */ +2023-03-04 : Igor Pavlov : Public domain */ -#ifndef __7Z_BUF_H -#define __7Z_BUF_H +#ifndef ZIP7_INC_7Z_BUF_H +#define ZIP7_INC_7Z_BUF_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/7zBuf2.c b/multiarc/src/formats/7z/C/7zBuf2.c old mode 100755 new mode 100644 diff --git a/multiarc/src/formats/7z/C/7zCrc.c b/multiarc/src/formats/7z/C/7zCrc.c old mode 100755 new mode 100644 index bbd3e5f02..6e2db9eab --- a/multiarc/src/formats/7z/C/7zCrc.c +++ b/multiarc/src/formats/7z/C/7zCrc.c @@ -1,113 +1,218 @@ -/* 7zCrc.c -- CRC32 init -2021-04-01 : Igor Pavlov : Public domain */ +/* 7zCrc.c -- CRC32 calculation and init +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "7zCrc.h" #include "CpuArch.h" -#define kCrcPoly 0xEDB88320 +// for debug: +// #define __ARM_FEATURE_CRC32 1 -#ifdef MY_CPU_LE - #define CRC_NUM_TABLES 8 -#else - #define CRC_NUM_TABLES 9 +#ifdef __ARM_FEATURE_CRC32 +// #pragma message("__ARM_FEATURE_CRC32") +#define Z7_CRC_HW_FORCE +#endif - #define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24)) +// #define Z7_CRC_DEBUG_BE +#ifdef Z7_CRC_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif - UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table); - UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table); +#ifdef Z7_CRC_HW_FORCE + #define Z7_CRC_NUM_TABLES_USE 1 +#else +#ifdef Z7_CRC_NUM_TABLES + #define Z7_CRC_NUM_TABLES_USE Z7_CRC_NUM_TABLES +#else + #define Z7_CRC_NUM_TABLES_USE 12 +#endif #endif -#ifndef MY_CPU_BE - UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); - UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); -#endif - -typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table); - -extern -CRC_FUNC g_CrcUpdateT4; -CRC_FUNC g_CrcUpdateT4; -extern -CRC_FUNC g_CrcUpdateT8; -CRC_FUNC g_CrcUpdateT8; -extern -CRC_FUNC g_CrcUpdateT0_32; -CRC_FUNC g_CrcUpdateT0_32; -extern -CRC_FUNC g_CrcUpdateT0_64; -CRC_FUNC g_CrcUpdateT0_64; -extern -CRC_FUNC g_CrcUpdate; -CRC_FUNC g_CrcUpdate; - -UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; - -UInt32 MY_FAST_CALL CrcUpdate(UInt32 v, const void *data, size_t size) -{ - return g_CrcUpdate(v, data, size, g_CrcTable); -} +#if Z7_CRC_NUM_TABLES_USE < 1 + #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif -UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size) -{ - return g_CrcUpdate(CRC_INIT_VAL, data, size, g_CrcTable) ^ CRC_INIT_VAL; -} +#if defined(MY_CPU_LE) || (Z7_CRC_NUM_TABLES_USE == 1) + #define Z7_CRC_NUM_TABLES_TOTAL Z7_CRC_NUM_TABLES_USE +#else + #define Z7_CRC_NUM_TABLES_TOTAL (Z7_CRC_NUM_TABLES_USE + 1) +#endif -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#ifndef Z7_CRC_HW_FORCE -UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); -UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) +#if Z7_CRC_NUM_TABLES_USE == 1 \ + || (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) +#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define Z7_CRC_UPDATE_T1_FUNC_NAME CrcUpdateGT1 +static UInt32 Z7_FASTCALL Z7_CRC_UPDATE_T1_FUNC_NAME(UInt32 v, const void *data, size_t size) { + const UInt32 *table = g_CrcTable; const Byte *p = (const Byte *)data; - const Byte *pEnd = p + size; - for (; p != pEnd; p++) + const Byte *lim = p + size; + for (; p != lim; p++) v = CRC_UPDATE_BYTE_2(v, *p); return v; } +#endif + +#if Z7_CRC_NUM_TABLES_USE != 1 +#ifndef MY_CPU_BE + #define FUNC_NAME_LE_2(s) CrcUpdateT ## s + #define FUNC_NAME_LE_1(s) FUNC_NAME_LE_2(s) + #define FUNC_NAME_LE FUNC_NAME_LE_1(Z7_CRC_NUM_TABLES_USE) + UInt32 Z7_FASTCALL FUNC_NAME_LE (UInt32 v, const void *data, size_t size, const UInt32 *table); +#endif +#ifndef MY_CPU_LE + #define FUNC_NAME_BE_2(s) CrcUpdateT1_BeT ## s + #define FUNC_NAME_BE_1(s) FUNC_NAME_BE_2(s) + #define FUNC_NAME_BE FUNC_NAME_BE_1(Z7_CRC_NUM_TABLES_USE) + UInt32 Z7_FASTCALL FUNC_NAME_BE (UInt32 v, const void *data, size_t size, const UInt32 *table); +#endif +#endif + +#endif // Z7_CRC_HW_FORCE /* ---------- hardware CRC ---------- */ #ifdef MY_CPU_LE -#if defined(MY_CPU_ARM_OR_ARM64) && defined(__ARM_FEATURE_CRC32) - #define USE_ARM_ACLE - #include -#endif // defined(MY_CPU_ARM64) && defined(MY_CPU_LE) +#if defined(MY_CPU_ARM_OR_ARM64) +// #pragma message("ARM*") + + #if (defined(__clang__) && (__clang_major__ >= 3)) \ + || defined(__GNUC__) && (__GNUC__ >= 6) && defined(MY_CPU_ARM64) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #if !defined(__ARM_FEATURE_CRC32) +// #pragma message("!defined(__ARM_FEATURE_CRC32)") +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define __ARM_FEATURE_CRC32 1 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRC32_WAS_SET + #if defined(__clang__) + #if defined(MY_CPU_ARM64) + #define ATTRIB_CRC __attribute__((__target__("crc"))) + #else + #define ATTRIB_CRC __attribute__((__target__("armv8-a,crc"))) + #endif + #else + #if defined(MY_CPU_ARM64) +#if !defined(Z7_GCC_VERSION) || (Z7_GCC_VERSION >= 60000) + #define ATTRIB_CRC __attribute__((__target__("+crc"))) +#endif + #else +#if !defined(Z7_GCC_VERSION) || (__GNUC__ >= 8) +#if defined(__ARM_FP) && __GNUC__ >= 8 +// for -mfloat-abi=hard: similar to + #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc+simd"))) +#else + #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc"))) +#endif +#endif + #endif + #endif + #endif + #if defined(__ARM_FEATURE_CRC32) + // #pragma message("") +/* +arm_acle.h (GGC): + before Nov 17, 2017: +#ifdef __ARM_FEATURE_CRC32 + + Nov 17, 2017: gcc10.0 (gcc 9.2.0) checked" +#if __ARM_ARCH >= 8 +#pragma GCC target ("arch=armv8-a+crc") + + Aug 22, 2019: GCC 8.4?, 9.2.1, 10.1: +#ifdef __ARM_FEATURE_CRC32 +#ifdef __ARM_FP +#pragma GCC target ("arch=armv8-a+crc+simd") +#else +#pragma GCC target ("arch=armv8-a+crc") +#endif +*/ +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 +#if defined(Z7_GCC_VERSION) && (__GNUC__ == 8) && (Z7_GCC_VERSION < 80400) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 9) && (Z7_GCC_VERSION < 90201) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 10) && (Z7_GCC_VERSION < 100100) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") +#undef __ARM_ARCH +#define __ARM_ARCH 8 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif + #define Z7_CRC_HW_USE + #include + #endif + #elif defined(_MSC_VER) + #if defined(MY_CPU_ARM64) + #if (_MSC_VER >= 1910) + #ifdef __clang__ + // #define Z7_CRC_HW_USE + // #include + #else + #define Z7_CRC_HW_USE + #include + #endif + #endif + #endif + #endif + +#else // non-ARM* -#ifdef USE_ARM_ACLE +// #define Z7_CRC_HW_USE // for debug : we can test HW-branch of code +#ifdef Z7_CRC_HW_USE +#include "7zCrcEmu.h" +#endif - #define T0_32_UNROLL_BYTES (4 * 4) - #define T0_64_UNROLL_BYTES (4 * 8) +#endif // non-ARM* + + + +#if defined(Z7_CRC_HW_USE) - #ifndef ATTRIB_CRC - #define ATTRIB_CRC - #endif // #pragma message("USE ARM HW CRC") -ATTRIB_CRC -UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table); -ATTRIB_CRC -UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table) +#ifdef MY_CPU_64BIT + #define CRC_HW_WORD_TYPE UInt64 + #define CRC_HW_WORD_FUNC __crc32d +#else + #define CRC_HW_WORD_TYPE UInt32 + #define CRC_HW_WORD_FUNC __crc32w +#endif + +#define CRC_HW_UNROLL_BYTES (sizeof(CRC_HW_WORD_TYPE) * 4) + +#ifdef ATTRIB_CRC + ATTRIB_CRC +#endif +Z7_NO_INLINE +#ifdef Z7_CRC_HW_FORCE + UInt32 Z7_FASTCALL CrcUpdate +#else + static UInt32 Z7_FASTCALL CrcUpdate_HW +#endif + (UInt32 v, const void *data, size_t size) { const Byte *p = (const Byte *)data; - UNUSED_VAR(table); - - for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_32_UNROLL_BYTES - 1)) != 0; size--) + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (CRC_HW_UNROLL_BYTES - 1)) != 0; size--) v = __crc32b(v, *p++); - - if (size >= T0_32_UNROLL_BYTES) + if (size >= CRC_HW_UNROLL_BYTES) { const Byte *lim = p + size; - size &= (T0_32_UNROLL_BYTES - 1); + size &= CRC_HW_UNROLL_BYTES - 1; lim -= size; do { - v = __crc32w(v, *(const UInt32 *)(const void *)(p)); - v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; - v = __crc32w(v, *(const UInt32 *)(const void *)(p)); - v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p)); + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p + sizeof(CRC_HW_WORD_TYPE))); + p += 2 * sizeof(CRC_HW_WORD_TYPE); + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p)); + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p + sizeof(CRC_HW_WORD_TYPE))); + p += 2 * sizeof(CRC_HW_WORD_TYPE); } while (p != lim); } @@ -118,130 +223,198 @@ UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, cons return v; } -ATTRIB_CRC -UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table); -ATTRIB_CRC -UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table) +#ifdef Z7_ARM_FEATURE_CRC32_WAS_SET +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#undef __ARM_FEATURE_CRC32 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#undef Z7_ARM_FEATURE_CRC32_WAS_SET +#endif + +#endif // defined(Z7_CRC_HW_USE) +#endif // MY_CPU_LE + + + +#ifndef Z7_CRC_HW_FORCE + +#if defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) +/* +typedef UInt32 (Z7_FASTCALL *Z7_CRC_UPDATE_WITH_TABLE_FUNC) + (UInt32 v, const void *data, size_t size, const UInt32 *table); +Z7_CRC_UPDATE_WITH_TABLE_FUNC g_CrcUpdate; +*/ +static unsigned g_Crc_Algo; +#if (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) +static unsigned g_Crc_Be; +#endif +#endif // defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) + + + +Z7_NO_INLINE +#ifdef Z7_CRC_HW_USE + static UInt32 Z7_FASTCALL CrcUpdate_Base +#else + UInt32 Z7_FASTCALL CrcUpdate +#endif + (UInt32 crc, const void *data, size_t size) { - const Byte *p = (const Byte *)data; - UNUSED_VAR(table); +#if Z7_CRC_NUM_TABLES_USE == 1 + return Z7_CRC_UPDATE_T1_FUNC_NAME(crc, data, size); +#else // Z7_CRC_NUM_TABLES_USE != 1 +#ifdef Z7_CRC_UPDATE_T1_FUNC_NAME + if (g_Crc_Algo == 1) + return Z7_CRC_UPDATE_T1_FUNC_NAME(crc, data, size); +#endif - for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_64_UNROLL_BYTES - 1)) != 0; size--) - v = __crc32b(v, *p++); +#ifdef MY_CPU_LE + return FUNC_NAME_LE(crc, data, size, g_CrcTable); +#elif defined(MY_CPU_BE) + return FUNC_NAME_BE(crc, data, size, g_CrcTable); +#else + if (g_Crc_Be) + return FUNC_NAME_BE(crc, data, size, g_CrcTable); + else + return FUNC_NAME_LE(crc, data, size, g_CrcTable); +#endif +#endif // Z7_CRC_NUM_TABLES_USE != 1 +} - if (size >= T0_64_UNROLL_BYTES) - { - const Byte *lim = p + size; - size &= (T0_64_UNROLL_BYTES - 1); - lim -= size; - do - { - v = __crc32d(v, *(const UInt64 *)(const void *)(p)); - v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; - v = __crc32d(v, *(const UInt64 *)(const void *)(p)); - v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; - } - while (p != lim); - } - - for (; size != 0; size--) - v = __crc32b(v, *p++); - return v; +#ifdef Z7_CRC_HW_USE +Z7_NO_INLINE +UInt32 Z7_FASTCALL CrcUpdate(UInt32 crc, const void *data, size_t size) +{ + if (g_Crc_Algo == 0) + return CrcUpdate_HW(crc, data, size); + return CrcUpdate_Base(crc, data, size); } +#endif -#endif // defined(USE_ARM_ACLE) +#endif // !defined(Z7_CRC_HW_FORCE) -#endif // MY_CPU_LE +UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size) +{ + return CrcUpdate(CRC_INIT_VAL, data, size) ^ CRC_INIT_VAL; +} + + +MY_ALIGN(64) +UInt32 g_CrcTable[256 * Z7_CRC_NUM_TABLES_TOTAL]; -void MY_FAST_CALL CrcGenerateTable() +void Z7_FASTCALL CrcGenerateTable(void) { UInt32 i; for (i = 0; i < 256; i++) { +#if defined(Z7_CRC_HW_FORCE) + g_CrcTable[i] = __crc32b(i, 0); +#else + #define kCrcPoly 0xEDB88320 UInt32 r = i; unsigned j; for (j = 0; j < 8; j++) r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); g_CrcTable[i] = r; +#endif } - for (i = 256; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * Z7_CRC_NUM_TABLES_USE; i++) { - UInt32 r = g_CrcTable[(size_t)i - 256]; + const UInt32 r = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); } - #if CRC_NUM_TABLES < 4 - - g_CrcUpdate = CrcUpdateT1; - - #else - - #ifdef MY_CPU_LE - - g_CrcUpdateT4 = CrcUpdateT4; - g_CrcUpdate = CrcUpdateT4; +#if !defined(Z7_CRC_HW_FORCE) && \ + (defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) || defined(MY_CPU_BE)) - #if CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT8; - - #ifdef MY_CPU_X86_OR_AMD64 - if (!CPU_Is_InOrder()) - #endif - g_CrcUpdate = CrcUpdateT8; - #endif +#if Z7_CRC_NUM_TABLES_USE <= 1 + g_Crc_Algo = 1; +#else // Z7_CRC_NUM_TABLES_USE <= 1 - #else +#if defined(MY_CPU_LE) + g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; +#else // !defined(MY_CPU_LE) { - #ifndef MY_CPU_BE +#ifndef MY_CPU_BE UInt32 k = 0x01020304; const Byte *p = (const Byte *)&k; if (p[0] == 4 && p[1] == 3) - { - g_CrcUpdateT4 = CrcUpdateT4; - g_CrcUpdate = CrcUpdateT4; - #if CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT8; - g_CrcUpdate = CrcUpdateT8; - #endif - } + g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; else if (p[0] != 1 || p[1] != 2) - g_CrcUpdate = CrcUpdateT1; + g_Crc_Algo = 1; else - #endif +#endif // MY_CPU_BE { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + for (i = 256 * Z7_CRC_NUM_TABLES_TOTAL - 1; i >= 256; i--) { - UInt32 x = g_CrcTable[(size_t)i - 256]; - g_CrcTable[i] = CRC_UINT32_SWAP(x); + const UInt32 x = g_CrcTable[(size_t)i - 256]; + g_CrcTable[i] = Z7_BSWAP32(x); } - g_CrcUpdateT4 = CrcUpdateT1_BeT4; - g_CrcUpdate = CrcUpdateT1_BeT4; - #if CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT1_BeT8; - g_CrcUpdate = CrcUpdateT1_BeT8; - #endif +#if defined(Z7_CRC_UPDATE_T1_FUNC_NAME) + g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; +#endif +#if (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) + g_Crc_Be = 1; +#endif } } - #endif - #endif +#endif // !defined(MY_CPU_LE) - #ifdef MY_CPU_LE - #ifdef USE_ARM_ACLE - if (CPU_IsSupported_CRC32()) - { - g_CrcUpdateT0_32 = CrcUpdateT0_32; - g_CrcUpdateT0_64 = CrcUpdateT0_64; - g_CrcUpdate = - #if defined(MY_CPU_ARM) - CrcUpdateT0_32; - #else - CrcUpdateT0_64; - #endif - } - #endif +#ifdef MY_CPU_LE +#ifdef Z7_CRC_HW_USE + if (CPU_IsSupported_CRC32()) + g_Crc_Algo = 0; +#endif // Z7_CRC_HW_USE +#endif // MY_CPU_LE + +#endif // Z7_CRC_NUM_TABLES_USE <= 1 +#endif // g_Crc_Algo was declared +} + +Z7_CRC_UPDATE_FUNC z7_GetFunc_CrcUpdate(unsigned algo) +{ + if (algo == 0) + return &CrcUpdate; + +#if defined(Z7_CRC_HW_USE) + if (algo == sizeof(CRC_HW_WORD_TYPE) * 8) + { +#ifdef Z7_CRC_HW_FORCE + return &CrcUpdate; +#else + if (g_Crc_Algo == 0) + return &CrcUpdate_HW; +#endif + } +#endif + +#ifndef Z7_CRC_HW_FORCE + if (algo == Z7_CRC_NUM_TABLES_USE) + return + #ifdef Z7_CRC_HW_USE + &CrcUpdate_Base; + #else + &CrcUpdate; #endif +#endif + + return NULL; } + +#undef kCrcPoly +#undef Z7_CRC_NUM_TABLES_USE +#undef Z7_CRC_NUM_TABLES_TOTAL +#undef CRC_UPDATE_BYTE_2 +#undef FUNC_NAME_LE_2 +#undef FUNC_NAME_LE_1 +#undef FUNC_NAME_LE +#undef FUNC_NAME_BE_2 +#undef FUNC_NAME_BE_1 +#undef FUNC_NAME_BE + +#undef CRC_HW_UNROLL_BYTES +#undef CRC_HW_WORD_FUNC +#undef CRC_HW_WORD_TYPE diff --git a/multiarc/src/formats/7z/C/7zCrc.h b/multiarc/src/formats/7z/C/7zCrc.h old mode 100755 new mode 100644 index 8fd579587..3e6d408b9 --- a/multiarc/src/formats/7z/C/7zCrc.h +++ b/multiarc/src/formats/7z/C/7zCrc.h @@ -1,8 +1,8 @@ /* 7zCrc.h -- CRC32 calculation -2013-01-18 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ -#ifndef __7Z_CRC_H -#define __7Z_CRC_H +#ifndef ZIP7_INC_7Z_CRC_H +#define ZIP7_INC_7Z_CRC_H #include "7zTypes.h" @@ -11,14 +11,17 @@ EXTERN_C_BEGIN extern UInt32 g_CrcTable[]; /* Call CrcGenerateTable one time before other CRC functions */ -void MY_FAST_CALL CrcGenerateTable(void); +void Z7_FASTCALL CrcGenerateTable(void); #define CRC_INIT_VAL 0xFFFFFFFF #define CRC_GET_DIGEST(crc) ((crc) ^ CRC_INIT_VAL) #define CRC_UPDATE_BYTE(crc, b) (g_CrcTable[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) -UInt32 MY_FAST_CALL CrcUpdate(UInt32 crc, const void *data, size_t size); -UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size); +UInt32 Z7_FASTCALL CrcUpdate(UInt32 crc, const void *data, size_t size); +UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size); + +typedef UInt32 (Z7_FASTCALL *Z7_CRC_UPDATE_FUNC)(UInt32 v, const void *data, size_t size); +Z7_CRC_UPDATE_FUNC z7_GetFunc_CrcUpdate(unsigned algo); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/7zCrcOpt.c b/multiarc/src/formats/7z/C/7zCrcOpt.c old mode 100755 new mode 100644 index 69fad9ca2..9408017ed --- a/multiarc/src/formats/7z/C/7zCrcOpt.c +++ b/multiarc/src/formats/7z/C/7zCrcOpt.c @@ -1,117 +1,199 @@ -/* 7zCrcOpt.c -- CRC32 calculation -2021-02-09 : Igor Pavlov : Public domain */ +/* 7zCrcOpt.c -- CRC32 calculation (optimized functions) +2023-12-07 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" +#if !defined(Z7_CRC_NUM_TABLES) || Z7_CRC_NUM_TABLES > 1 + +// for debug only : define Z7_CRC_DEBUG_BE to test big-endian code in little-endian cpu +// #define Z7_CRC_DEBUG_BE +#ifdef Z7_CRC_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif + +// the value Z7_CRC_NUM_TABLES_USE must be defined to same value as in 7zCrc.c +#ifdef Z7_CRC_NUM_TABLES +#define Z7_CRC_NUM_TABLES_USE Z7_CRC_NUM_TABLES +#else +#define Z7_CRC_NUM_TABLES_USE 12 +#endif + +#if Z7_CRC_NUM_TABLES_USE % 4 || \ + Z7_CRC_NUM_TABLES_USE < 4 * 1 || \ + Z7_CRC_NUM_TABLES_USE > 4 * 6 + #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif + + #ifndef MY_CPU_BE -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) -UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); -UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x300)[((v ) & 0xFF)] - ^ (table + 0x200)[((v >> 8) & 0xFF)] - ^ (table + 0x100)[((v >> 16) & 0xFF)] - ^ (table + 0x000)[((v >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - return v; -} +#define Q(n, d) \ + ( (table + ((n) * 4 + 3) * 0x100)[(Byte)(d)] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 0) * 0x100)[((d) >> 3 * 8)] ) + +#define R(a) *((const UInt32 *)(const void *)p + (a)) + +#define CRC_FUNC_PRE_LE2(step) \ +UInt32 Z7_FASTCALL CrcUpdateT ## step (UInt32 v, const void *data, size_t size, const UInt32 *table) -UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); -UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table) +#define CRC_FUNC_PRE_LE(step) \ + CRC_FUNC_PRE_LE2(step); \ + CRC_FUNC_PRE_LE2(step) + +CRC_FUNC_PRE_LE(Z7_CRC_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) + const Byte *lim; + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p); - for (; size >= 8; size -= 8, p += 8) + lim = p + size; + if (size >= Z7_CRC_NUM_TABLES_USE) { - UInt32 d; - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x700)[((v ) & 0xFF)] - ^ (table + 0x600)[((v >> 8) & 0xFF)] - ^ (table + 0x500)[((v >> 16) & 0xFF)] - ^ (table + 0x400)[((v >> 24))]; - d = *((const UInt32 *)(const void *)p + 1); - v ^= - (table + 0x300)[((d ) & 0xFF)] - ^ (table + 0x200)[((d >> 8) & 0xFF)] - ^ (table + 0x100)[((d >> 16) & 0xFF)] - ^ (table + 0x000)[((d >> 24))]; + lim -= Z7_CRC_NUM_TABLES_USE; + do + { + v ^= R(0); + { +#if Z7_CRC_NUM_TABLES_USE == 1 * 4 + v = Q(0, v); +#else +#define U2(r, op) \ + { d = R(r); x op Q(Z7_CRC_NUM_TABLES_USE / 4 - 1 - (r), d); } + UInt32 d, x; + U2(1, =) +#if Z7_CRC_NUM_TABLES_USE >= 3 * 4 +#define U(r) U2(r, ^=) + U(2) +#if Z7_CRC_NUM_TABLES_USE >= 4 * 4 + U(3) +#if Z7_CRC_NUM_TABLES_USE >= 5 * 4 + U(4) +#if Z7_CRC_NUM_TABLES_USE >= 6 * 4 + U(5) +#if Z7_CRC_NUM_TABLES_USE >= 7 * 4 +#error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif +#endif +#endif +#endif +#endif +#undef U +#undef U2 + v = x ^ Q(Z7_CRC_NUM_TABLES_USE / 4 - 1, v); +#endif + } + p += Z7_CRC_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC_UPDATE_BYTE_2(v, *p); return v; } +#undef CRC_UPDATE_BYTE_2 +#undef R +#undef Q +#undef CRC_FUNC_PRE_LE +#undef CRC_FUNC_PRE_LE2 + #endif + + #ifndef MY_CPU_LE -#define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24)) +#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[((crc) >> 24) ^ (b)] ^ ((crc) << 8)) -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(((crc) >> 24) ^ (b))] ^ ((crc) << 8)) +#define Q(n, d) \ + ( (table + ((n) * 4 + 0) * 0x100)[((d)) & 0xFF] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 3) * 0x100)[((d) >> 3 * 8)] ) -UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - table += 0x100; - v = CRC_UINT32_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x000)[((v ) & 0xFF)] - ^ (table + 0x100)[((v >> 8) & 0xFF)] - ^ (table + 0x200)[((v >> 16) & 0xFF)] - ^ (table + 0x300)[((v >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT32_SWAP(v); -} +#ifdef Z7_CRC_DEBUG_BE + #define R(a) GetBe32a((const UInt32 *)(const void *)p + (a)) +#else + #define R(a) *((const UInt32 *)(const void *)p + (a)) +#endif + + +#define CRC_FUNC_PRE_BE2(step) \ +UInt32 Z7_FASTCALL CrcUpdateT1_BeT ## step (UInt32 v, const void *data, size_t size, const UInt32 *table) -UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table) +#define CRC_FUNC_PRE_BE(step) \ + CRC_FUNC_PRE_BE2(step); \ + CRC_FUNC_PRE_BE2(step) + +CRC_FUNC_PRE_BE(Z7_CRC_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; + const Byte *lim; table += 0x100; - v = CRC_UINT32_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) + v = Z7_BSWAP32(v); + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 8; size -= 8, p += 8) + lim = p + size; + if (size >= Z7_CRC_NUM_TABLES_USE) { - UInt32 d; - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x400)[((v ) & 0xFF)] - ^ (table + 0x500)[((v >> 8) & 0xFF)] - ^ (table + 0x600)[((v >> 16) & 0xFF)] - ^ (table + 0x700)[((v >> 24))]; - d = *((const UInt32 *)(const void *)p + 1); - v ^= - (table + 0x000)[((d ) & 0xFF)] - ^ (table + 0x100)[((d >> 8) & 0xFF)] - ^ (table + 0x200)[((d >> 16) & 0xFF)] - ^ (table + 0x300)[((d >> 24))]; + lim -= Z7_CRC_NUM_TABLES_USE; + do + { + v ^= R(0); + { +#if Z7_CRC_NUM_TABLES_USE == 1 * 4 + v = Q(0, v); +#else +#define U2(r, op) \ + { d = R(r); x op Q(Z7_CRC_NUM_TABLES_USE / 4 - 1 - (r), d); } + UInt32 d, x; + U2(1, =) +#if Z7_CRC_NUM_TABLES_USE >= 3 * 4 +#define U(r) U2(r, ^=) + U(2) +#if Z7_CRC_NUM_TABLES_USE >= 4 * 4 + U(3) +#if Z7_CRC_NUM_TABLES_USE >= 5 * 4 + U(4) +#if Z7_CRC_NUM_TABLES_USE >= 6 * 4 + U(5) +#if Z7_CRC_NUM_TABLES_USE >= 7 * 4 +#error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif +#endif +#endif +#endif +#endif +#undef U +#undef U2 + v = x ^ Q(Z7_CRC_NUM_TABLES_USE / 4 - 1, v); +#endif + } + p += Z7_CRC_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT32_SWAP(v); + return Z7_BSWAP32(v); } +#undef CRC_UPDATE_BYTE_2_BE +#undef R +#undef Q +#undef CRC_FUNC_PRE_BE +#undef CRC_FUNC_PRE_BE2 + +#endif +#undef Z7_CRC_NUM_TABLES_USE #endif diff --git a/multiarc/src/formats/7z/C/7zDec.c b/multiarc/src/formats/7z/C/7zDec.c old mode 100755 new mode 100644 index fbfd016e1..a5a3e6ba7 --- a/multiarc/src/formats/7z/C/7zDec.c +++ b/multiarc/src/formats/7z/C/7zDec.c @@ -1,11 +1,11 @@ /* 7zDec.c -- Decoding from 7z folder -2021-02-09 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include -/* #define _7ZIP_PPMD_SUPPPORT */ +#define Z7_PPMD_SUPPORT #include "7z.h" #include "7zCrc.h" @@ -16,27 +16,50 @@ #include "Delta.h" #include "LzmaDec.h" #include "Lzma2Dec.h" -#ifdef _7ZIP_PPMD_SUPPPORT +#ifdef Z7_PPMD_SUPPORT #include "Ppmd7.h" #endif #define k_Copy 0 -#ifndef _7Z_NO_METHOD_LZMA2 +#ifndef Z7_NO_METHOD_LZMA2 #define k_LZMA2 0x21 #endif #define k_LZMA 0x30101 #define k_BCJ2 0x303011B -#ifndef _7Z_NO_METHODS_FILTERS + +#if !defined(Z7_NO_METHODS_FILTERS) +#define Z7_USE_BRANCH_FILTER +#endif + +#if !defined(Z7_NO_METHODS_FILTERS) || \ + defined(Z7_USE_NATIVE_BRANCH_FILTER) && defined(MY_CPU_ARM64) +#define Z7_USE_FILTER_ARM64 +#ifndef Z7_USE_BRANCH_FILTER +#define Z7_USE_BRANCH_FILTER +#endif +#define k_ARM64 0xa +#endif + +#if !defined(Z7_NO_METHODS_FILTERS) || \ + defined(Z7_USE_NATIVE_BRANCH_FILTER) && defined(MY_CPU_ARMT) +#define Z7_USE_FILTER_ARMT +#ifndef Z7_USE_BRANCH_FILTER +#define Z7_USE_BRANCH_FILTER +#endif +#define k_ARMT 0x3030701 +#endif + +#ifndef Z7_NO_METHODS_FILTERS #define k_Delta 3 +#define k_RISCV 0xb #define k_BCJ 0x3030103 #define k_PPC 0x3030205 #define k_IA64 0x3030401 #define k_ARM 0x3030501 -#define k_ARMT 0x3030701 #define k_SPARC 0x3030805 #endif -#ifdef _7ZIP_PPMD_SUPPPORT +#ifdef Z7_PPMD_SUPPORT #define k_PPMD 0x30401 @@ -49,12 +72,12 @@ typedef struct UInt64 processed; BoolInt extra; SRes res; - const ILookInStream *inStream; + ILookInStreamPtr inStream; } CByteInToLook; -static Byte ReadByte(const IByteIn *pp) +static Byte ReadByte(IByteInPtr pp) { - CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CByteInToLook) if (p->cur != p->end) return *p->cur++; if (p->res == SZ_OK) @@ -67,13 +90,13 @@ static Byte ReadByte(const IByteIn *pp) p->cur = p->begin; p->end = p->begin + size; if (size != 0) - return *p->cur++;; + return *p->cur++; } p->extra = True; return 0; } -static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, const ILookInStream *inStream, +static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStreamPtr inStream, Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CPpmd7 ppmd; @@ -138,14 +161,14 @@ static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, c #endif -static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, +static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStreamPtr inStream, Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzmaDec state; SRes res = SZ_OK; - LzmaDec_Construct(&state); - RINOK(LzmaDec_AllocateProbs(&state, props, propsSize, allocMain)); + LzmaDec_CONSTRUCT(&state) + RINOK(LzmaDec_AllocateProbs(&state, props, propsSize, allocMain)) state.dic = outBuffer; state.dicBufSize = outSize; LzmaDec_Init(&state); @@ -196,18 +219,18 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I } -#ifndef _7Z_NO_METHOD_LZMA2 +#ifndef Z7_NO_METHOD_LZMA2 -static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, +static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStreamPtr inStream, Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzma2Dec state; SRes res = SZ_OK; - Lzma2Dec_Construct(&state); + Lzma2Dec_CONSTRUCT(&state) if (propsSize != 1) return SZ_ERROR_DATA; - RINOK(Lzma2Dec_AllocateProbs(&state, props[0], allocMain)); + RINOK(Lzma2Dec_AllocateProbs(&state, props[0], allocMain)) state.decoder.dic = outBuffer; state.decoder.dicBufSize = outSize; Lzma2Dec_Init(&state); @@ -257,7 +280,7 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, #endif -static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer) +static SRes SzDecodeCopy(UInt64 inSize, ILookInStreamPtr inStream, Byte *outBuffer) { while (inSize > 0) { @@ -265,13 +288,13 @@ static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer size_t curSize = (1 << 18); if (curSize > inSize) curSize = (size_t)inSize; - RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)); + RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)) if (curSize == 0) return SZ_ERROR_INPUT_EOF; memcpy(outBuffer, inBuf, curSize); outBuffer += curSize; inSize -= curSize; - RINOK(ILookInStream_Skip(inStream, curSize)); + RINOK(ILookInStream_Skip(inStream, curSize)) } return SZ_OK; } @@ -282,12 +305,12 @@ static BoolInt IS_MAIN_METHOD(UInt32 m) { case k_Copy: case k_LZMA: - #ifndef _7Z_NO_METHOD_LZMA2 + #ifndef Z7_NO_METHOD_LZMA2 case k_LZMA2: - #endif - #ifdef _7ZIP_PPMD_SUPPPORT + #endif + #ifdef Z7_PPMD_SUPPORT case k_PPMD: - #endif + #endif return True; } return False; @@ -317,7 +340,7 @@ static SRes CheckSupportedFolder(const CSzFolder *f) } - #ifndef _7Z_NO_METHODS_FILTERS + #if defined(Z7_USE_BRANCH_FILTER) if (f->NumCoders == 2) { @@ -333,13 +356,21 @@ static SRes CheckSupportedFolder(const CSzFolder *f) return SZ_ERROR_UNSUPPORTED; switch ((UInt32)c->MethodID) { + #if !defined(Z7_NO_METHODS_FILTERS) case k_Delta: case k_BCJ: case k_PPC: case k_IA64: case k_SPARC: case k_ARM: + case k_RISCV: + #endif + #ifdef Z7_USE_FILTER_ARM64 + case k_ARM64: + #endif + #ifdef Z7_USE_FILTER_ARMT case k_ARMT: + #endif break; default: return SZ_ERROR_UNSUPPORTED; @@ -372,15 +403,16 @@ static SRes CheckSupportedFolder(const CSzFolder *f) return SZ_ERROR_UNSUPPORTED; } -#ifndef _7Z_NO_METHODS_FILTERS -#define CASE_BRA_CONV(isa) case k_ ## isa: isa ## _Convert(outBuffer, outSize, 0, 0); break; -#endif + + + + static SRes SzFolder_Decode2(const CSzFolder *folder, const Byte *propsData, const UInt64 *unpackSizes, const UInt64 *packPositions, - ILookInStream *inStream, UInt64 startPos, + ILookInStreamPtr inStream, UInt64 startPos, Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain, Byte *tempBuf[]) { @@ -389,7 +421,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, SizeT tempSize3 = 0; Byte *tempBuf3 = 0; - RINOK(CheckSupportedFolder(folder)); + RINOK(CheckSupportedFolder(folder)) for (ci = 0; ci < folder->NumCoders; ci++) { @@ -404,8 +436,8 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, SizeT outSizeCur = outSize; if (folder->NumCoders == 4) { - UInt32 indices[] = { 3, 2, 0 }; - UInt64 unpackSize = unpackSizes[ci]; + const UInt32 indices[] = { 3, 2, 0 }; + const UInt64 unpackSize = unpackSizes[ci]; si = indices[ci]; if (ci < 2) { @@ -431,37 +463,37 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, } offset = packPositions[si]; inSize = packPositions[(size_t)si + 1] - offset; - RINOK(LookInStream_SeekTo(inStream, startPos + offset)); + RINOK(LookInStream_SeekTo(inStream, startPos + offset)) if (coder->MethodID == k_Copy) { if (inSize != outSizeCur) /* check it */ return SZ_ERROR_DATA; - RINOK(SzDecodeCopy(inSize, inStream, outBufCur)); + RINOK(SzDecodeCopy(inSize, inStream, outBufCur)) } else if (coder->MethodID == k_LZMA) { - RINOK(SzDecodeLzma(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); + RINOK(SzDecodeLzma(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)) } - #ifndef _7Z_NO_METHOD_LZMA2 + #ifndef Z7_NO_METHOD_LZMA2 else if (coder->MethodID == k_LZMA2) { - RINOK(SzDecodeLzma2(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); + RINOK(SzDecodeLzma2(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)) } - #endif - #ifdef _7ZIP_PPMD_SUPPPORT + #endif + #ifdef Z7_PPMD_SUPPORT else if (coder->MethodID == k_PPMD) { - RINOK(SzDecodePpmd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); + RINOK(SzDecodePpmd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)) } - #endif + #endif else return SZ_ERROR_UNSUPPORTED; } else if (coder->MethodID == k_BCJ2) { - UInt64 offset = packPositions[1]; - UInt64 s3Size = packPositions[2] - offset; + const UInt64 offset = packPositions[1]; + const UInt64 s3Size = packPositions[2] - offset; if (ci != 3) return SZ_ERROR_UNSUPPORTED; @@ -473,8 +505,8 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, if (!tempBuf[2] && tempSizes[2] != 0) return SZ_ERROR_MEM; - RINOK(LookInStream_SeekTo(inStream, startPos + offset)); - RINOK(SzDecodeCopy(s3Size, inStream, tempBuf[2])); + RINOK(LookInStream_SeekTo(inStream, startPos + offset)) + RINOK(SzDecodeCopy(s3Size, inStream, tempBuf[2])) if ((tempSizes[0] & 3) != 0 || (tempSizes[1] & 3) != 0 || @@ -493,26 +525,22 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, p.destLim = outBuffer + outSize; Bcj2Dec_Init(&p); - RINOK(Bcj2Dec_Decode(&p)); + RINOK(Bcj2Dec_Decode(&p)) { unsigned i; for (i = 0; i < 4; i++) if (p.bufs[i] != p.lims[i]) return SZ_ERROR_DATA; - - if (!Bcj2Dec_IsFinished(&p)) - return SZ_ERROR_DATA; - - if (p.dest != p.destLim - || p.state != BCJ2_STREAM_MAIN) + if (p.dest != p.destLim || !Bcj2Dec_IsMaybeFinished(&p)) return SZ_ERROR_DATA; } } } - #ifndef _7Z_NO_METHODS_FILTERS +#if defined(Z7_USE_BRANCH_FILTER) else if (ci == 1) { +#if !defined(Z7_NO_METHODS_FILTERS) if (coder->MethodID == k_Delta) { if (coder->PropsSize != 1) @@ -522,31 +550,75 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, Delta_Init(state); Delta_Decode(state, (unsigned)(propsData[coder->PropsOffset]) + 1, outBuffer, outSize); } + continue; } - else +#endif + +#ifdef Z7_USE_FILTER_ARM64 + if (coder->MethodID == k_ARM64) + { + UInt32 pc = 0; + if (coder->PropsSize == 4) + { + pc = GetUi32(propsData + coder->PropsOffset); + if (pc & 3) + return SZ_ERROR_UNSUPPORTED; + } + else if (coder->PropsSize != 0) + return SZ_ERROR_UNSUPPORTED; + z7_BranchConv_ARM64_Dec(outBuffer, outSize, pc); + continue; + } +#endif + +#if !defined(Z7_NO_METHODS_FILTERS) + if (coder->MethodID == k_RISCV) + { + UInt32 pc = 0; + if (coder->PropsSize == 4) + { + pc = GetUi32(propsData + coder->PropsOffset); + if (pc & 1) + return SZ_ERROR_UNSUPPORTED; + } + else if (coder->PropsSize != 0) + return SZ_ERROR_UNSUPPORTED; + z7_BranchConv_RISCV_Dec(outBuffer, outSize, pc); + continue; + } +#endif + +#if !defined(Z7_NO_METHODS_FILTERS) || defined(Z7_USE_FILTER_ARMT) { if (coder->PropsSize != 0) return SZ_ERROR_UNSUPPORTED; + #define CASE_BRA_CONV(isa) case k_ ## isa: Z7_BRANCH_CONV_DEC(isa)(outBuffer, outSize, 0); break; // pc = 0; switch (coder->MethodID) { + #if !defined(Z7_NO_METHODS_FILTERS) case k_BCJ: { - UInt32 state; - x86_Convert_Init(state); - x86_Convert(outBuffer, outSize, 0, &state, 0); + UInt32 state = Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL; + z7_BranchConvSt_X86_Dec(outBuffer, outSize, 0, &state); // pc = 0 break; } - CASE_BRA_CONV(PPC) + case k_PPC: Z7_BRANCH_CONV_DEC_2(BranchConv_PPC)(outBuffer, outSize, 0); break; // pc = 0; + // CASE_BRA_CONV(PPC) CASE_BRA_CONV(IA64) CASE_BRA_CONV(SPARC) CASE_BRA_CONV(ARM) + #endif + #if !defined(Z7_NO_METHODS_FILTERS) || defined(Z7_USE_FILTER_ARMT) CASE_BRA_CONV(ARMT) + #endif default: return SZ_ERROR_UNSUPPORTED; } + continue; } - } - #endif +#endif + } // (c == 1) +#endif // Z7_USE_BRANCH_FILTER else return SZ_ERROR_UNSUPPORTED; } @@ -556,7 +628,7 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, - ILookInStream *inStream, UInt64 startPos, + ILookInStreamPtr inStream, UInt64 startPos, Byte *outBuffer, size_t outSize, ISzAllocPtr allocMain) { diff --git a/multiarc/src/formats/7z/C/7zFile.c b/multiarc/src/formats/7z/C/7zFile.c old mode 100755 new mode 100644 index 13d2efa47..ba5daa133 --- a/multiarc/src/formats/7z/C/7zFile.c +++ b/multiarc/src/formats/7z/C/7zFile.c @@ -1,5 +1,5 @@ /* 7zFile.c -- File IO -2021-04-29 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -268,7 +268,7 @@ WRes File_Write(CSzFile *p, const void *data, size_t *size) return errno; if (processed == 0) break; - data = (void *)((Byte *)data + (size_t)processed); + data = (const void *)((const Byte *)data + (size_t)processed); originalSize -= (size_t)processed; *size += (size_t)processed; } @@ -287,7 +287,8 @@ WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) DWORD moveMethod; UInt32 low = (UInt32)*pos; LONG high = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ - switch (origin) + // (int) to eliminate clang warning + switch ((int)origin) { case SZ_SEEK_SET: moveMethod = FILE_BEGIN; break; case SZ_SEEK_CUR: moveMethod = FILE_CURRENT; break; @@ -308,7 +309,7 @@ WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) int moveMethod; // = origin; - switch (origin) + switch ((int)origin) { case SZ_SEEK_SET: moveMethod = SEEK_SET; break; case SZ_SEEK_CUR: moveMethod = SEEK_CUR; break; @@ -387,10 +388,10 @@ WRes File_GetLength(CSzFile *p, UInt64 *length) /* ---------- FileSeqInStream ---------- */ -static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) +static SRes FileSeqInStream_Read(ISeqInStreamPtr pp, void *buf, size_t *size) { - CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); - WRes wres = File_Read(&p->file, buf, size); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CFileSeqInStream) + const WRes wres = File_Read(&p->file, buf, size); p->wres = wres; return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } @@ -403,18 +404,18 @@ void FileSeqInStream_CreateVTable(CFileSeqInStream *p) /* ---------- FileInStream ---------- */ -static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) +static SRes FileInStream_Read(ISeekInStreamPtr pp, void *buf, size_t *size) { - CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); - WRes wres = File_Read(&p->file, buf, size); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CFileInStream) + const WRes wres = File_Read(&p->file, buf, size); p->wres = wres; return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } -static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) +static SRes FileInStream_Seek(ISeekInStreamPtr pp, Int64 *pos, ESzSeek origin) { - CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); - WRes wres = File_Seek(&p->file, pos, origin); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CFileInStream) + const WRes wres = File_Seek(&p->file, pos, origin); p->wres = wres; return (wres == 0) ? SZ_OK : SZ_ERROR_READ; } @@ -428,10 +429,10 @@ void FileInStream_CreateVTable(CFileInStream *p) /* ---------- FileOutStream ---------- */ -static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) +static size_t FileOutStream_Write(ISeqOutStreamPtr pp, const void *data, size_t size) { - CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); - WRes wres = File_Write(&p->file, data, &size); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CFileOutStream) + const WRes wres = File_Write(&p->file, data, &size); p->wres = wres; return size; } diff --git a/multiarc/src/formats/7z/C/7zFile.h b/multiarc/src/formats/7z/C/7zFile.h old mode 100755 new mode 100644 index 788abb6b9..f5069cd9e --- a/multiarc/src/formats/7z/C/7zFile.h +++ b/multiarc/src/formats/7z/C/7zFile.h @@ -1,8 +1,8 @@ /* 7zFile.h -- File IO -2021-02-15 : Igor Pavlov : Public domain */ +2023-03-05 : Igor Pavlov : Public domain */ -#ifndef __7Z_FILE_H -#define __7Z_FILE_H +#ifndef ZIP7_INC_FILE_H +#define ZIP7_INC_FILE_H #ifdef _WIN32 #define USE_WINDOWS_FILE @@ -10,7 +10,8 @@ #endif #ifdef USE_WINDOWS_FILE -#include +#include "7zWindows.h" + #else // note: USE_FOPEN mode is limited to 32-bit file size // #define USE_FOPEN diff --git a/multiarc/src/formats/7z/C/7zStream.c b/multiarc/src/formats/7z/C/7zStream.c old mode 100755 new mode 100644 index 28a14604f..74e75b65a --- a/multiarc/src/formats/7z/C/7zStream.c +++ b/multiarc/src/formats/7z/C/7zStream.c @@ -1,5 +1,5 @@ /* 7zStream.c -- 7z Stream functions -2021-02-09 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,12 +7,33 @@ #include "7zTypes.h" -SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType) + +SRes SeqInStream_ReadMax(ISeqInStreamPtr stream, void *buf, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + const SRes res = ISeqInStream_Read(stream, buf, &cur); + *processedSize += cur; + buf = (void *)((Byte *)buf + cur); + size -= cur; + if (res != SZ_OK) + return res; + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + +/* +SRes SeqInStream_Read2(ISeqInStreamPtr stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(ISeqInStream_Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)) if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed); @@ -21,42 +42,44 @@ SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes return SZ_OK; } -SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size) +SRes SeqInStream_Read(ISeqInStreamPtr stream, void *buf, size_t size) { return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } +*/ + -SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) +SRes SeqInStream_ReadByte(ISeqInStreamPtr stream, Byte *buf) { size_t processed = 1; - RINOK(ISeqInStream_Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)) return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; } -SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) +SRes LookInStream_SeekTo(ILookInStreamPtr stream, UInt64 offset) { Int64 t = (Int64)offset; return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); } -SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size) +SRes LookInStream_LookRead(ILookInStreamPtr stream, void *buf, size_t *size) { const void *lookBuf; if (*size == 0) return SZ_OK; - RINOK(ILookInStream_Look(stream, &lookBuf, size)); + RINOK(ILookInStream_Look(stream, &lookBuf, size)) memcpy(buf, lookBuf, *size); return ILookInStream_Skip(stream, *size); } -SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType) +SRes LookInStream_Read2(ILookInStreamPtr stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(ILookInStream_Read(stream, buf, &processed)); + RINOK(ILookInStream_Read(stream, buf, &processed)) if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed); @@ -65,16 +88,16 @@ SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRe return SZ_OK; } -SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size) +SRes LookInStream_Read(ILookInStreamPtr stream, void *buf, size_t size) { return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -#define GET_LookToRead2 CLookToRead2 *p = CONTAINER_FROM_VTBL(pp, CLookToRead2, vt); +#define GET_LookToRead2 Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CLookToRead2) -static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf, size_t *size) +static SRes LookToRead2_Look_Lookahead(ILookInStreamPtr pp, const void **buf, size_t *size) { SRes res = SZ_OK; GET_LookToRead2 @@ -93,7 +116,7 @@ static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf return res; } -static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, size_t *size) +static SRes LookToRead2_Look_Exact(ILookInStreamPtr pp, const void **buf, size_t *size) { SRes res = SZ_OK; GET_LookToRead2 @@ -113,14 +136,14 @@ static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, si return res; } -static SRes LookToRead2_Skip(const ILookInStream *pp, size_t offset) +static SRes LookToRead2_Skip(ILookInStreamPtr pp, size_t offset) { GET_LookToRead2 p->pos += offset; return SZ_OK; } -static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) +static SRes LookToRead2_Read(ILookInStreamPtr pp, void *buf, size_t *size) { GET_LookToRead2 size_t rem = p->size - p->pos; @@ -134,7 +157,7 @@ static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) return SZ_OK; } -static SRes LookToRead2_Seek(const ILookInStream *pp, Int64 *pos, ESzSeek origin) +static SRes LookToRead2_Seek(ILookInStreamPtr pp, Int64 *pos, ESzSeek origin) { GET_LookToRead2 p->pos = p->size = 0; @@ -153,9 +176,9 @@ void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead) -static SRes SecToLook_Read(const ISeqInStream *pp, void *buf, size_t *size) +static SRes SecToLook_Read(ISeqInStreamPtr pp, void *buf, size_t *size) { - CSecToLook *p = CONTAINER_FROM_VTBL(pp, CSecToLook, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSecToLook) return LookInStream_LookRead(p->realStream, buf, size); } @@ -164,9 +187,9 @@ void SecToLook_CreateVTable(CSecToLook *p) p->vt.Read = SecToLook_Read; } -static SRes SecToRead_Read(const ISeqInStream *pp, void *buf, size_t *size) +static SRes SecToRead_Read(ISeqInStreamPtr pp, void *buf, size_t *size) { - CSecToRead *p = CONTAINER_FROM_VTBL(pp, CSecToRead, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSecToRead) return ILookInStream_Read(p->realStream, buf, size); } diff --git a/multiarc/src/formats/7z/C/7zTypes.h b/multiarc/src/formats/7z/C/7zTypes.h old mode 100755 new mode 100644 index c5065a7ce..c4a50982a --- a/multiarc/src/formats/7z/C/7zTypes.h +++ b/multiarc/src/formats/7z/C/7zTypes.h @@ -1,8 +1,8 @@ /* 7zTypes.h -- Basic types -2022-04-01 : Igor Pavlov : Public domain */ +2024-01-24 : Igor Pavlov : Public domain */ -#ifndef __7Z_TYPES_H -#define __7Z_TYPES_H +#ifndef ZIP7_7Z_TYPES_H +#define ZIP7_7Z_TYPES_H #ifdef _WIN32 /* #include */ @@ -52,6 +52,11 @@ typedef int SRes; #define MY_ALIGN(n) #endif #else + /* + // C11/C++11: + #include + #define MY_ALIGN(n) alignas(n) + */ #define MY_ALIGN(n) __attribute__ ((aligned(n))) #endif @@ -62,7 +67,7 @@ typedef int SRes; typedef unsigned WRes; #define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) -// #define MY_HRES_ERROR__INTERNAL_ERROR MY_SRes_HRESULT_FROM_WRes(ERROR_INTERNAL_ERROR) +// #define MY_HRES_ERROR_INTERNAL_ERROR MY_SRes_HRESULT_FROM_WRes(ERROR_INTERNAL_ERROR) #else // _WIN32 @@ -70,13 +75,13 @@ typedef unsigned WRes; typedef int WRes; // (FACILITY_ERRNO = 0x800) is 7zip's FACILITY constant to represent (errno) errors in HRESULT -#define MY__FACILITY_ERRNO 0x800 -#define MY__FACILITY_WIN32 7 -#define MY__FACILITY__WRes MY__FACILITY_ERRNO +#define MY_FACILITY_ERRNO 0x800 +#define MY_FACILITY_WIN32 7 +#define MY_FACILITY_WRes MY_FACILITY_ERRNO #define MY_HRESULT_FROM_errno_CONST_ERROR(x) ((HRESULT)( \ ( (HRESULT)(x) & 0x0000FFFF) \ - | (MY__FACILITY__WRes << 16) \ + | (MY_FACILITY_WRes << 16) \ | (HRESULT)0x80000000 )) #define MY_SRes_HRESULT_FROM_WRes(x) \ @@ -103,35 +108,23 @@ typedef int WRes; #define ERROR_REPARSE_TAG_MISMATCH 4394L */ -// if (MY__FACILITY__WRes != FACILITY_WIN32), -// we use FACILITY_WIN32 for COM errors: -#define E_OUTOFMEMORY ((HRESULT)0x8007000EL) -#define E_INVALIDARG ((HRESULT)0x80070057L) -#define MY__E_ERROR_NEGATIVE_SEEK ((HRESULT)0x80070083L) - -/* -// we can use FACILITY_ERRNO for some COM errors, that have errno equivalents: -#define E_OUTOFMEMORY MY_HRESULT_FROM_errno_CONST_ERROR(ENOMEM) -#define E_INVALIDARG MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) -#define MY__E_ERROR_NEGATIVE_SEEK MY_HRESULT_FROM_errno_CONST_ERROR(EINVAL) -*/ - +// we use errno equivalents for some WIN32 errors: #endif #ifndef RINOK -#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } +#define RINOK(x) { const int _result_ = (x); if (_result_ != 0) return _result_; } #endif #ifndef RINOK_WRes -#define RINOK_WRes(x) { WRes __result__ = (x); if (__result__ != 0) return __result__; } +#define RINOK_WRes(x) { const WRes _result_ = (x); if (_result_ != 0) return _result_; } #endif typedef unsigned char Byte; typedef short Int16; typedef unsigned short UInt16; -#ifdef _LZMA_UINT32_IS_ULONG +#ifdef Z7_DECL_Int32_AS_long typedef long Int32; typedef unsigned long UInt32; #else @@ -140,39 +133,51 @@ typedef unsigned int UInt32; #endif +#define MY_HRES_ERROR_INTERNAL_ERROR ((HRESULT)0x8007054FL) -#define MY_HRES_ERROR__INTERNAL_ERROR ((HRESULT)0x8007054FL) - - -#ifdef _SZ_NO_INT_64 - -/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. - NOTES: Some code will work incorrectly in that case! */ +#ifdef Z7_DECL_Int64_AS_long typedef long Int64; typedef unsigned long UInt64; #else -#if defined(_MSC_VER) || defined(__BORLANDC__) +#if (defined(_MSC_VER) || defined(__BORLANDC__)) && !defined(__clang__) typedef __int64 Int64; typedef unsigned __int64 UInt64; -#define UINT64_CONST(n) n +#else +#if defined(__clang__) || defined(__GNUC__) +#include +typedef int64_t Int64; +typedef uint64_t UInt64; #else typedef long long int Int64; typedef unsigned long long int UInt64; -#define UINT64_CONST(n) n ## ULL +// #define UINT64_CONST(n) n ## ULL +#endif #endif #endif -#ifdef _LZMA_NO_SYSTEM_SIZE_T -typedef UInt32 SizeT; +#define UINT64_CONST(n) n + + +#ifdef Z7_DECL_SizeT_AS_unsigned_int +typedef unsigned int SizeT; #else typedef size_t SizeT; #endif +/* +#if (defined(_MSC_VER) && _MSC_VER <= 1200) +typedef size_t MY_uintptr_t; +#else +#include +typedef uintptr_t MY_uintptr_t; +#endif +*/ + typedef int BoolInt; /* typedef BoolInt Bool; */ #define True 1 @@ -180,23 +185,23 @@ typedef int BoolInt; #ifdef _WIN32 -#define MY_STD_CALL __stdcall +#define Z7_STDCALL __stdcall #else -#define MY_STD_CALL +#define Z7_STDCALL #endif #ifdef _MSC_VER #if _MSC_VER >= 1300 -#define MY_NO_INLINE __declspec(noinline) +#define Z7_NO_INLINE __declspec(noinline) #else -#define MY_NO_INLINE +#define Z7_NO_INLINE #endif -#define MY_FORCE_INLINE __forceinline +#define Z7_FORCE_INLINE __forceinline -#define MY_CDECL __cdecl -#define MY_FAST_CALL __fastcall +#define Z7_CDECL __cdecl +#define Z7_FASTCALL __fastcall #else // _MSC_VER @@ -204,27 +209,25 @@ typedef int BoolInt; || (defined(__clang__) && (__clang_major__ >= 4)) \ || defined(__INTEL_COMPILER) \ || defined(__xlC__) -#define MY_NO_INLINE __attribute__((noinline)) -// #define MY_FORCE_INLINE __attribute__((always_inline)) inline +#define Z7_NO_INLINE __attribute__((noinline)) +#define Z7_FORCE_INLINE __attribute__((always_inline)) inline #else -#define MY_NO_INLINE +#define Z7_NO_INLINE +#define Z7_FORCE_INLINE #endif -#define MY_FORCE_INLINE - - -#define MY_CDECL +#define Z7_CDECL #if defined(_M_IX86) \ || defined(__i386__) -// #define MY_FAST_CALL __attribute__((fastcall)) -// #define MY_FAST_CALL __attribute__((cdecl)) -#define MY_FAST_CALL +// #define Z7_FASTCALL __attribute__((fastcall)) +// #define Z7_FASTCALL __attribute__((cdecl)) +#define Z7_FASTCALL #elif defined(MY_CPU_AMD64) -// #define MY_FAST_CALL __attribute__((ms_abi)) -#define MY_FAST_CALL +// #define Z7_FASTCALL __attribute__((ms_abi)) +#define Z7_FASTCALL #else -#define MY_FAST_CALL +#define Z7_FASTCALL #endif #endif // _MSC_VER @@ -232,41 +235,49 @@ typedef int BoolInt; /* The following interfaces use first parameter as pointer to structure */ -typedef struct IByteIn IByteIn; -struct IByteIn +// #define Z7_C_IFACE_CONST_QUAL +#define Z7_C_IFACE_CONST_QUAL const + +#define Z7_C_IFACE_DECL(a) \ + struct a ## _; \ + typedef Z7_C_IFACE_CONST_QUAL struct a ## _ * a ## Ptr; \ + typedef struct a ## _ a; \ + struct a ## _ + + +Z7_C_IFACE_DECL (IByteIn) { - Byte (*Read)(const IByteIn *p); /* reads one byte, returns 0 in case of EOF or error */ + Byte (*Read)(IByteInPtr p); /* reads one byte, returns 0 in case of EOF or error */ }; #define IByteIn_Read(p) (p)->Read(p) -typedef struct IByteOut IByteOut; -struct IByteOut +Z7_C_IFACE_DECL (IByteOut) { - void (*Write)(const IByteOut *p, Byte b); + void (*Write)(IByteOutPtr p, Byte b); }; #define IByteOut_Write(p, b) (p)->Write(p, b) -typedef struct ISeqInStream ISeqInStream; -struct ISeqInStream +Z7_C_IFACE_DECL (ISeqInStream) { - SRes (*Read)(const ISeqInStream *p, void *buf, size_t *size); + SRes (*Read)(ISeqInStreamPtr p, void *buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) < input(*size)) is allowed */ }; #define ISeqInStream_Read(p, buf, size) (p)->Read(p, buf, size) +/* try to read as much as avail in stream and limited by (*processedSize) */ +SRes SeqInStream_ReadMax(ISeqInStreamPtr stream, void *buf, size_t *processedSize); /* it can return SZ_ERROR_INPUT_EOF */ -SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size); -SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType); -SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf); +// SRes SeqInStream_Read(ISeqInStreamPtr stream, void *buf, size_t size); +// SRes SeqInStream_Read2(ISeqInStreamPtr stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(ISeqInStreamPtr stream, Byte *buf); -typedef struct ISeqOutStream ISeqOutStream; -struct ISeqOutStream +Z7_C_IFACE_DECL (ISeqOutStream) { - size_t (*Write)(const ISeqOutStream *p, const void *buf, size_t size); + size_t (*Write)(ISeqOutStreamPtr p, const void *buf, size_t size); /* Returns: result - the number of actually written bytes. (result < size) means error */ }; @@ -280,29 +291,26 @@ typedef enum } ESzSeek; -typedef struct ISeekInStream ISeekInStream; -struct ISeekInStream +Z7_C_IFACE_DECL (ISeekInStream) { - SRes (*Read)(const ISeekInStream *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ - SRes (*Seek)(const ISeekInStream *p, Int64 *pos, ESzSeek origin); + SRes (*Read)(ISeekInStreamPtr p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(ISeekInStreamPtr p, Int64 *pos, ESzSeek origin); }; #define ISeekInStream_Read(p, buf, size) (p)->Read(p, buf, size) #define ISeekInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) -typedef struct ILookInStream ILookInStream; -struct ILookInStream +Z7_C_IFACE_DECL (ILookInStream) { - SRes (*Look)(const ILookInStream *p, const void **buf, size_t *size); + SRes (*Look)(ILookInStreamPtr p, const void **buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) > input(*size)) is not allowed (output(*size) < input(*size)) is allowed */ - SRes (*Skip)(const ILookInStream *p, size_t offset); + SRes (*Skip)(ILookInStreamPtr p, size_t offset); /* offset must be <= output(*size) of Look */ - - SRes (*Read)(const ILookInStream *p, void *buf, size_t *size); + SRes (*Read)(ILookInStreamPtr p, void *buf, size_t *size); /* reads directly (without buffer). It's same as ISeqInStream::Read */ - SRes (*Seek)(const ILookInStream *p, Int64 *pos, ESzSeek origin); + SRes (*Seek)(ILookInStreamPtr p, Int64 *pos, ESzSeek origin); }; #define ILookInStream_Look(p, buf, size) (p)->Look(p, buf, size) @@ -311,19 +319,18 @@ struct ILookInStream #define ILookInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) -SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size); -SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset); +SRes LookInStream_LookRead(ILookInStreamPtr stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(ILookInStreamPtr stream, UInt64 offset); /* reads via ILookInStream::Read */ -SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType); -SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size); - +SRes LookInStream_Read2(ILookInStreamPtr stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(ILookInStreamPtr stream, void *buf, size_t size); typedef struct { ILookInStream vt; - const ISeekInStream *realStream; + ISeekInStreamPtr realStream; size_t pos; size_t size; /* it's data size */ @@ -335,13 +342,13 @@ typedef struct void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead); -#define LookToRead2_Init(p) { (p)->pos = (p)->size = 0; } +#define LookToRead2_INIT(p) { (p)->pos = (p)->size = 0; } typedef struct { ISeqInStream vt; - const ILookInStream *realStream; + ILookInStreamPtr realStream; } CSecToLook; void SecToLook_CreateVTable(CSecToLook *p); @@ -351,20 +358,19 @@ void SecToLook_CreateVTable(CSecToLook *p); typedef struct { ISeqInStream vt; - const ILookInStream *realStream; + ILookInStreamPtr realStream; } CSecToRead; void SecToRead_CreateVTable(CSecToRead *p); -typedef struct ICompressProgress ICompressProgress; - -struct ICompressProgress +Z7_C_IFACE_DECL (ICompressProgress) { - SRes (*Progress)(const ICompressProgress *p, UInt64 inSize, UInt64 outSize); + SRes (*Progress)(ICompressProgressPtr p, UInt64 inSize, UInt64 outSize); /* Returns: result. (result != SZ_OK) means break. Value (UInt64)(Int64)-1 for size means unknown value. */ }; + #define ICompressProgress_Progress(p, inSize, outSize) (p)->Progress(p, inSize, outSize) @@ -402,13 +408,13 @@ struct ISzAlloc -#ifndef MY_container_of +#ifndef Z7_container_of /* -#define MY_container_of(ptr, type, m) container_of(ptr, type, m) -#define MY_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m) -#define MY_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m))) -#define MY_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m)))) +#define Z7_container_of(ptr, type, m) container_of(ptr, type, m) +#define Z7_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m) +#define Z7_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m))) +#define Z7_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m)))) */ /* @@ -417,24 +423,64 @@ struct ISzAlloc GCC 4.8.1 : classes with non-public variable members" */ -#define MY_container_of(ptr, type, m) ((type *)(void *)((char *)(void *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) +#define Z7_container_of(ptr, type, m) \ + ((type *)(void *)((char *)(void *) \ + (1 ? (ptr) : &((type *)NULL)->m) - MY_offsetof(type, m))) -#endif - -#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(void *)(ptr)) +#define Z7_container_of_CONST(ptr, type, m) \ + ((const type *)(const void *)((const char *)(const void *) \ + (1 ? (ptr) : &((type *)NULL)->m) - MY_offsetof(type, m))) /* -#define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +#define Z7_container_of_NON_CONST_FROM_CONST(ptr, type, m) \ + ((type *)(void *)(const void *)((const char *)(const void *) \ + (1 ? (ptr) : &((type *)NULL)->m) - MY_offsetof(type, m))) */ -#define CONTAINER_FROM_VTBL(ptr, type, m) MY_container_of(ptr, type, m) -#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +#endif + +#define Z7_CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(void *)(ptr)) + +// #define Z7_CONTAINER_FROM_VTBL(ptr, type, m) Z7_CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +#define Z7_CONTAINER_FROM_VTBL(ptr, type, m) Z7_container_of(ptr, type, m) +// #define Z7_CONTAINER_FROM_VTBL(ptr, type, m) Z7_container_of_NON_CONST_FROM_CONST(ptr, type, m) + +#define Z7_CONTAINER_FROM_VTBL_CONST(ptr, type, m) Z7_container_of_CONST(ptr, type, m) + +#define Z7_CONTAINER_FROM_VTBL_CLS(ptr, type, m) Z7_CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) /* -#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL(ptr, type, m) +#define Z7_CONTAINER_FROM_VTBL_CLS(ptr, type, m) Z7_CONTAINER_FROM_VTBL(ptr, type, m) */ +#if defined (__clang__) || defined(__GNUC__) +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_CAST_QUAL \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#define Z7_DIAGNOSTIC_IGNORE_END_CAST_QUAL \ + _Pragma("GCC diagnostic pop") +#else +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_CAST_QUAL +#define Z7_DIAGNOSTIC_IGNORE_END_CAST_QUAL +#endif +#define Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR(ptr, type, m, p) \ + Z7_DIAGNOSTIC_IGNORE_BEGIN_CAST_QUAL \ + type *p = Z7_CONTAINER_FROM_VTBL(ptr, type, m); \ + Z7_DIAGNOSTIC_IGNORE_END_CAST_QUAL + +#define Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(type) \ + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR(pp, type, vt, p) + + +// #define ZIP7_DECLARE_HANDLE(name) typedef void *name; +#define Z7_DECLARE_HANDLE(name) struct name##_dummy{int unused;}; typedef struct name##_dummy *name; + + +#define Z7_memset_0_ARRAY(a) memset((a), 0, sizeof(a)) + +#ifndef Z7_ARRAY_SIZE +#define Z7_ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#endif -#define MY_memset_0_ARRAY(a) memset((a), 0, sizeof(a)) #ifdef _WIN32 @@ -463,3 +509,11 @@ struct ISzAlloc EXTERN_C_END #endif + +/* +#ifndef Z7_ST +#ifdef _7ZIP_ST +#define Z7_ST +#endif +#endif +*/ diff --git a/multiarc/src/formats/7z/C/7zVersion.h b/multiarc/src/formats/7z/C/7zVersion.h old mode 100755 new mode 100644 index 49ea81dd3..1ddef80b0 --- a/multiarc/src/formats/7z/C/7zVersion.h +++ b/multiarc/src/formats/7z/C/7zVersion.h @@ -1,7 +1,7 @@ -#define MY_VER_MAJOR 22 -#define MY_VER_MINOR 01 +#define MY_VER_MAJOR 24 +#define MY_VER_MINOR 8 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "22.01" +#define MY_VERSION_NUMBERS "24.08" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,12 +10,12 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2022-07-15" +#define MY_DATE "2024-08-11" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2022 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2024 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR diff --git a/multiarc/src/formats/7z/C/7zVersion.rc b/multiarc/src/formats/7z/C/7zVersion.rc old mode 100755 new mode 100644 diff --git a/multiarc/src/formats/7z/C/Aes.c b/multiarc/src/formats/7z/C/Aes.c old mode 100755 new mode 100644 index 27e32e622..abc5d24b4 --- a/multiarc/src/formats/7z/C/Aes.c +++ b/multiarc/src/formats/7z/C/Aes.c @@ -1,5 +1,5 @@ /* Aes.c -- AES encryption / decryption -2021-05-13 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -7,13 +7,15 @@ #include "Aes.h" AES_CODE_FUNC g_AesCbc_Decode; -#ifndef _SFX +#ifndef Z7_SFX AES_CODE_FUNC g_AesCbc_Encode; AES_CODE_FUNC g_AesCtr_Code; UInt32 g_Aes_SupportedFunctions_Flags; #endif +MY_ALIGN(64) static UInt32 T[256 * 4]; +MY_ALIGN(64) static const Byte Sbox[256] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, @@ -33,7 +35,9 @@ static const Byte Sbox[256] = { 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; +MY_ALIGN(64) static UInt32 D[256 * 4]; +MY_ALIGN(64) static Byte InvS[256]; #define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF) @@ -51,32 +55,62 @@ static Byte InvS[256]; #define DD(x) (D + (x << 8)) -// #define _SHOW_AES_STATUS +// #define Z7_SHOW_AES_STATUS #ifdef MY_CPU_X86_OR_AMD64 - #define USE_HW_AES -#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define USE_HW_AES - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check + + #if defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1110) #define USE_HW_AES + #if (__INTEL_COMPILER >= 1900) + #define USE_HW_VAES + #endif #endif + #elif defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400) + #define USE_HW_AES + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #define USE_HW_VAES + #endif #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 + #define USE_HW_AES + #define USE_HW_VAES + #endif + +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__ARM_FEATURE_AES) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_AES + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_AES #endif + #endif + #endif #endif #endif #ifdef USE_HW_AES -#ifdef _SHOW_AES_STATUS +// #pragma message("=== Aes.c USE_HW_AES === ") +#ifdef Z7_SHOW_AES_STATUS #include -#define _PRF(x) x +#define PRF(x) x #else -#define _PRF(x) +#define PRF(x) #endif #endif @@ -90,23 +124,23 @@ void AesGenTables(void) for (i = 0; i < 256; i++) { { - UInt32 a1 = Sbox[i]; - UInt32 a2 = xtime(a1); - UInt32 a3 = a2 ^ a1; + const UInt32 a1 = Sbox[i]; + const UInt32 a2 = xtime(a1); + const UInt32 a3 = a2 ^ a1; TT(0)[i] = Ui32(a2, a1, a1, a3); TT(1)[i] = Ui32(a3, a2, a1, a1); TT(2)[i] = Ui32(a1, a3, a2, a1); TT(3)[i] = Ui32(a1, a1, a3, a2); } { - UInt32 a1 = InvS[i]; - UInt32 a2 = xtime(a1); - UInt32 a4 = xtime(a2); - UInt32 a8 = xtime(a4); - UInt32 a9 = a8 ^ a1; - UInt32 aB = a8 ^ a2 ^ a1; - UInt32 aD = a8 ^ a4 ^ a1; - UInt32 aE = a8 ^ a4 ^ a2; + const UInt32 a1 = InvS[i]; + const UInt32 a2 = xtime(a1); + const UInt32 a4 = xtime(a2); + const UInt32 a8 = xtime(a4); + const UInt32 a9 = a8 ^ a1; + const UInt32 aB = a8 ^ a2 ^ a1; + const UInt32 aD = a8 ^ a4 ^ a1; + const UInt32 aE = a8 ^ a4 ^ a2; DD(0)[i] = Ui32(aE, a9, aD, aB); DD(1)[i] = Ui32(aB, aE, a9, aD); DD(2)[i] = Ui32(aD, aB, aE, a9); @@ -116,7 +150,7 @@ void AesGenTables(void) { AES_CODE_FUNC d = AesCbc_Decode; - #ifndef _SFX + #ifndef Z7_SFX AES_CODE_FUNC e = AesCbc_Encode; AES_CODE_FUNC c = AesCtr_Code; UInt32 flags = 0; @@ -126,31 +160,33 @@ void AesGenTables(void) if (CPU_IsSupported_AES()) { // #pragma message ("AES HW") - _PRF(printf("\n===AES HW\n")); + PRF(printf("\n===AES HW\n")); d = AesCbc_Decode_HW; - #ifndef _SFX + #ifndef Z7_SFX e = AesCbc_Encode_HW; c = AesCtr_Code_HW; flags = k_Aes_SupportedFunctions_HW; #endif #ifdef MY_CPU_X86_OR_AMD64 + #ifdef USE_HW_VAES if (CPU_IsSupported_VAES_AVX2()) { - _PRF(printf("\n===vaes avx2\n")); + PRF(printf("\n===vaes avx2\n")); d = AesCbc_Decode_HW_256; - #ifndef _SFX + #ifndef Z7_SFX c = AesCtr_Code_HW_256; flags |= k_Aes_SupportedFunctions_HW_256; #endif } #endif + #endif } #endif g_AesCbc_Decode = d; - #ifndef _SFX + #ifndef Z7_SFX g_AesCbc_Encode = e; g_AesCtr_Code = c; g_Aes_SupportedFunctions_Flags = flags; @@ -194,7 +230,7 @@ void AesGenTables(void) #define FD(i, x) InvS[gb(x, m[(i - x) & 3])] #define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i]; -void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) +void Z7_FASTCALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) { unsigned i, m; const UInt32 *wLim; @@ -230,7 +266,7 @@ void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) while (++w != wLim); } -void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) +void Z7_FASTCALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) { unsigned i, num; Aes_SetKey_Enc(w, key, keySize); @@ -251,7 +287,7 @@ void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) src and dest are pointers to 4 UInt32 words. src and dest can point to same block */ -// MY_FORCE_INLINE +// Z7_FORCE_INLINE static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) { UInt32 s[4]; @@ -265,17 +301,20 @@ static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) w += 4; for (;;) { - HT16(m, s, 0); + HT16(m, s, 0) if (--numRounds2 == 0) break; - HT16(s, m, 4); + HT16(s, m, 4) w += 8; } w += 4; - FT4(0); FT4(1); FT4(2); FT4(3); + FT4(0) + FT4(1) + FT4(2) + FT4(3) } -MY_FORCE_INLINE +Z7_FORCE_INLINE static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src) { UInt32 s[4]; @@ -289,12 +328,15 @@ static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src) for (;;) { w -= 8; - HD16(m, s, 4); + HD16(m, s, 4) if (--numRounds2 == 0) break; - HD16(s, m, 0); + HD16(s, m, 0) } - FD4(0); FD4(1); FD4(2); FD4(3); + FD4(0) + FD4(1) + FD4(2) + FD4(3) } void AesCbc_Init(UInt32 *p, const Byte *iv) @@ -304,7 +346,7 @@ void AesCbc_Init(UInt32 *p, const Byte *iv) p[i] = GetUi32(iv + i * 4); } -void MY_FAST_CALL AesCbc_Encode(UInt32 *p, Byte *data, size_t numBlocks) +void Z7_FASTCALL AesCbc_Encode(UInt32 *p, Byte *data, size_t numBlocks) { for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE) { @@ -315,14 +357,14 @@ void MY_FAST_CALL AesCbc_Encode(UInt32 *p, Byte *data, size_t numBlocks) Aes_Encode(p + 4, p, p); - SetUi32(data, p[0]); - SetUi32(data + 4, p[1]); - SetUi32(data + 8, p[2]); - SetUi32(data + 12, p[3]); + SetUi32(data, p[0]) + SetUi32(data + 4, p[1]) + SetUi32(data + 8, p[2]) + SetUi32(data + 12, p[3]) } } -void MY_FAST_CALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks) +void Z7_FASTCALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks) { UInt32 in[4], out[4]; for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE) @@ -334,10 +376,10 @@ void MY_FAST_CALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks) Aes_Decode(p + 4, out, in); - SetUi32(data, p[0] ^ out[0]); - SetUi32(data + 4, p[1] ^ out[1]); - SetUi32(data + 8, p[2] ^ out[2]); - SetUi32(data + 12, p[3] ^ out[3]); + SetUi32(data, p[0] ^ out[0]) + SetUi32(data + 4, p[1] ^ out[1]) + SetUi32(data + 8, p[2] ^ out[2]) + SetUi32(data + 12, p[3] ^ out[3]) p[0] = in[0]; p[1] = in[1]; @@ -346,7 +388,7 @@ void MY_FAST_CALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks) } } -void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) +void Z7_FASTCALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) { for (; numBlocks != 0; numBlocks--) { @@ -360,7 +402,7 @@ void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) for (i = 0; i < 4; i++, data += 4) { - UInt32 t = temp[i]; + const UInt32 t = temp[i]; #ifdef MY_CPU_LE_UNALIGN *((UInt32 *)(void *)data) ^= t; @@ -373,3 +415,15 @@ void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) } } } + +#undef xtime +#undef Ui32 +#undef gb0 +#undef gb1 +#undef gb2 +#undef gb3 +#undef gb +#undef TT +#undef DD +#undef USE_HW_AES +#undef PRF diff --git a/multiarc/src/formats/7z/C/Aes.h b/multiarc/src/formats/7z/C/Aes.h old mode 100755 new mode 100644 index 2aa225642..7f0182ac0 --- a/multiarc/src/formats/7z/C/Aes.h +++ b/multiarc/src/formats/7z/C/Aes.h @@ -1,8 +1,8 @@ /* Aes.h -- AES encryption / decryption -2018-04-28 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __AES_H -#define __AES_H +#ifndef ZIP7_INC_AES_H +#define ZIP7_INC_AES_H #include "7zTypes.h" @@ -20,19 +20,19 @@ void AesGenTables(void); /* aes - 16-byte aligned pointer to keyMode+roundKeys sequence */ /* keySize = 16 or 24 or 32 (bytes) */ -typedef void (MY_FAST_CALL *AES_SET_KEY_FUNC)(UInt32 *aes, const Byte *key, unsigned keySize); -void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *aes, const Byte *key, unsigned keySize); -void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *aes, const Byte *key, unsigned keySize); +typedef void (Z7_FASTCALL *AES_SET_KEY_FUNC)(UInt32 *aes, const Byte *key, unsigned keySize); +void Z7_FASTCALL Aes_SetKey_Enc(UInt32 *aes, const Byte *key, unsigned keySize); +void Z7_FASTCALL Aes_SetKey_Dec(UInt32 *aes, const Byte *key, unsigned keySize); /* ivAes - 16-byte aligned pointer to iv+keyMode+roundKeys sequence: UInt32[AES_NUM_IVMRK_WORDS] */ void AesCbc_Init(UInt32 *ivAes, const Byte *iv); /* iv size is AES_BLOCK_SIZE */ /* data - 16-byte aligned pointer to data */ /* numBlocks - the number of 16-byte blocks in data array */ -typedef void (MY_FAST_CALL *AES_CODE_FUNC)(UInt32 *ivAes, Byte *data, size_t numBlocks); +typedef void (Z7_FASTCALL *AES_CODE_FUNC)(UInt32 *ivAes, Byte *data, size_t numBlocks); extern AES_CODE_FUNC g_AesCbc_Decode; -#ifndef _SFX +#ifndef Z7_SFX extern AES_CODE_FUNC g_AesCbc_Encode; extern AES_CODE_FUNC g_AesCtr_Code; #define k_Aes_SupportedFunctions_HW (1 << 2) @@ -41,19 +41,19 @@ extern UInt32 g_Aes_SupportedFunctions_Flags; #endif -#define DECLARE__AES_CODE_FUNC(funcName) \ - void MY_FAST_CALL funcName(UInt32 *ivAes, Byte *data, size_t numBlocks); +#define Z7_DECLARE_AES_CODE_FUNC(funcName) \ + void Z7_FASTCALL funcName(UInt32 *ivAes, Byte *data, size_t numBlocks); -DECLARE__AES_CODE_FUNC (AesCbc_Encode) -DECLARE__AES_CODE_FUNC (AesCbc_Decode) -DECLARE__AES_CODE_FUNC (AesCtr_Code) +Z7_DECLARE_AES_CODE_FUNC (AesCbc_Encode) +Z7_DECLARE_AES_CODE_FUNC (AesCbc_Decode) +Z7_DECLARE_AES_CODE_FUNC (AesCtr_Code) -DECLARE__AES_CODE_FUNC (AesCbc_Encode_HW) -DECLARE__AES_CODE_FUNC (AesCbc_Decode_HW) -DECLARE__AES_CODE_FUNC (AesCtr_Code_HW) +Z7_DECLARE_AES_CODE_FUNC (AesCbc_Encode_HW) +Z7_DECLARE_AES_CODE_FUNC (AesCbc_Decode_HW) +Z7_DECLARE_AES_CODE_FUNC (AesCtr_Code_HW) -DECLARE__AES_CODE_FUNC (AesCbc_Decode_HW_256) -DECLARE__AES_CODE_FUNC (AesCtr_Code_HW_256) +Z7_DECLARE_AES_CODE_FUNC (AesCbc_Decode_HW_256) +Z7_DECLARE_AES_CODE_FUNC (AesCtr_Code_HW_256) EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/AesOpt.c b/multiarc/src/formats/7z/C/AesOpt.c old mode 100755 new mode 100644 index 99f3cf916..58769ea05 --- a/multiarc/src/formats/7z/C/AesOpt.c +++ b/multiarc/src/formats/7z/C/AesOpt.c @@ -1,48 +1,33 @@ /* AesOpt.c -- AES optimized code for x86 AES hardware instructions -2021-04-01 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "Aes.h" #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if defined(__APPLE__) -# include -# if !defined(MAC_OS_X_VERSION_10_12) -# define DONT_USE_INTEL_AES /* fixes build under ElCapitan */ -# endif -#endif - - #ifdef DONT_USE_INTEL_AES - #warning "Intel AES prohibited" - #elif defined(__clang__) - #if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 8) - #define USE_INTEL_AES - #define ATTRIB_AES __attribute__((__target__("aes"))) - #if (__clang_major__ >= 8) - #define USE_INTEL_VAES - #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2"))) - #endif - #endif - #elif defined(__GNUC__) - #if (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ > 8) - #define USE_INTEL_AES - #ifndef __AES__ - #define ATTRIB_AES __attribute__((__target__("aes"))) - #endif - #if (__GNUC__ >= 8) - #define USE_INTEL_VAES - #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx2"))) - #endif - #endif - #elif defined(__INTEL_COMPILER) + #if defined(__INTEL_COMPILER) #if (__INTEL_COMPILER >= 1110) #define USE_INTEL_AES #if (__INTEL_COMPILER >= 1900) #define USE_INTEL_VAES #endif #endif + #elif defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400) + #define USE_INTEL_AES + #if !defined(__AES__) + #define ATTRIB_AES __attribute__((__target__("aes"))) + #endif + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #define USE_INTEL_VAES + #if !defined(__AES__) || !defined(__VAES__) || !defined(__AVX__) || !defined(__AVX2__) + #define ATTRIB_VAES __attribute__((__target__("aes,vaes,avx,avx2"))) + #endif + #endif #elif defined(_MSC_VER) #if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) #define USE_INTEL_AES @@ -50,27 +35,40 @@ #define USE_INTEL_VAES #endif #endif + #ifndef USE_INTEL_AES + #define Z7_USE_AES_HW_STUB + #endif + #ifndef USE_INTEL_VAES + #define Z7_USE_VAES_HW_STUB + #endif #endif -#ifndef ATTRIB_AES - #define ATTRIB_AES -#endif -#ifndef ATTRIB_VAES - #define ATTRIB_VAES -#endif + #ifndef USE_INTEL_AES + // #define Z7_USE_AES_HW_STUB // for debug + #endif + #ifndef USE_INTEL_VAES + // #define Z7_USE_VAES_HW_STUB // for debug + #endif #ifdef USE_INTEL_AES #include -#ifndef USE_INTEL_VAES -#define AES_TYPE_keys __m128i -#define AES_TYPE_data __m128i +#if !defined(USE_INTEL_VAES) && defined(Z7_USE_VAES_HW_STUB) +#define AES_TYPE_keys UInt32 +#define AES_TYPE_data Byte +// #define AES_TYPE_keys __m128i +// #define AES_TYPE_data __m128i +#endif + +#ifndef ATTRIB_AES + #define ATTRIB_AES #endif #define AES_FUNC_START(name) \ - void MY_FAST_CALL name(__m128i *p, __m128i *data, size_t numBlocks) + void Z7_FASTCALL name(UInt32 *ivAes, Byte *data8, size_t numBlocks) + // void Z7_FASTCALL name(__m128i *p, __m128i *data, size_t numBlocks) #define AES_FUNC_START2(name) \ AES_FUNC_START (name); \ @@ -78,14 +76,14 @@ ATTRIB_AES \ AES_FUNC_START (name) #define MM_OP(op, dest, src) dest = op(dest, src); -#define MM_OP_m(op, src) MM_OP(op, m, src); - -#define MM_XOR( dest, src) MM_OP(_mm_xor_si128, dest, src); -#define AVX_XOR(dest, src) MM_OP(_mm256_xor_si256, dest, src); +#define MM_OP_m(op, src) MM_OP(op, m, src) +#define MM_XOR( dest, src) MM_OP(_mm_xor_si128, dest, src) AES_FUNC_START2 (AesCbc_Encode_HW) { + __m128i *p = (__m128i *)(void *)ivAes; + __m128i *data = (__m128i *)(void *)data8; __m128i m = *p; const __m128i k0 = p[2]; const __m128i k1 = p[3]; @@ -95,17 +93,17 @@ AES_FUNC_START2 (AesCbc_Encode_HW) UInt32 r = numRounds2; const __m128i *w = p + 4; __m128i temp = *data; - MM_XOR (temp, k0); - MM_XOR (m, temp); - MM_OP_m (_mm_aesenc_si128, k1); + MM_XOR (temp, k0) + MM_XOR (m, temp) + MM_OP_m (_mm_aesenc_si128, k1) do { - MM_OP_m (_mm_aesenc_si128, w[0]); - MM_OP_m (_mm_aesenc_si128, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]) + MM_OP_m (_mm_aesenc_si128, w[1]) w += 2; } while (--r); - MM_OP_m (_mm_aesenclast_si128, w[0]); + MM_OP_m (_mm_aesenclast_si128, w[0]) *data = m; } *p = m; @@ -113,14 +111,14 @@ AES_FUNC_START2 (AesCbc_Encode_HW) #define WOP_1(op) -#define WOP_2(op) WOP_1 (op) op (m1, 1); -#define WOP_3(op) WOP_2 (op) op (m2, 2); -#define WOP_4(op) WOP_3 (op) op (m3, 3); +#define WOP_2(op) WOP_1 (op) op (m1, 1) +#define WOP_3(op) WOP_2 (op) op (m2, 2) +#define WOP_4(op) WOP_3 (op) op (m3, 3) #ifdef MY_CPU_AMD64 -#define WOP_5(op) WOP_4 (op) op (m4, 4); -#define WOP_6(op) WOP_5 (op) op (m5, 5); -#define WOP_7(op) WOP_6 (op) op (m6, 6); -#define WOP_8(op) WOP_7 (op) op (m7, 7); +#define WOP_5(op) WOP_4 (op) op (m4, 4) +#define WOP_6(op) WOP_5 (op) op (m5, 5) +#define WOP_7(op) WOP_6 (op) op (m6, 6) +#define WOP_8(op) WOP_7 (op) op (m7, 7) #endif /* #define WOP_9(op) WOP_8 (op) op (m8, 8); @@ -139,21 +137,16 @@ AES_FUNC_START2 (AesCbc_Encode_HW) #define WOP_M1 WOP_4 #endif -#define WOP(op) op (m0, 0); WOP_M1(op) +#define WOP(op) op (m0, 0) WOP_M1(op) -#define DECLARE_VAR(reg, ii) __m128i reg +#define DECLARE_VAR(reg, ii) __m128i reg; #define LOAD_data( reg, ii) reg = data[ii]; #define STORE_data( reg, ii) data[ii] = reg; #if (NUM_WAYS > 1) -#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]); +#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]) #endif -#define AVX__DECLARE_VAR(reg, ii) __m256i reg -#define AVX__LOAD_data( reg, ii) reg = ((const __m256i *)(const void *)data)[ii]; -#define AVX__STORE_data( reg, ii) ((__m256i *)(void *)data)[ii] = reg; -#define AVX__XOR_data_M1(reg, ii) AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii])); - #define MM_OP_key(op, reg) MM_OP(op, reg, key); #define AES_DEC( reg, ii) MM_OP_key (_mm_aesdec_si128, reg) @@ -162,27 +155,13 @@ AES_FUNC_START2 (AesCbc_Encode_HW) #define AES_ENC_LAST( reg, ii) MM_OP_key (_mm_aesenclast_si128, reg) #define AES_XOR( reg, ii) MM_OP_key (_mm_xor_si128, reg) - -#define AVX__AES_DEC( reg, ii) MM_OP_key (_mm256_aesdec_epi128, reg) -#define AVX__AES_DEC_LAST( reg, ii) MM_OP_key (_mm256_aesdeclast_epi128, reg) -#define AVX__AES_ENC( reg, ii) MM_OP_key (_mm256_aesenc_epi128, reg) -#define AVX__AES_ENC_LAST( reg, ii) MM_OP_key (_mm256_aesenclast_epi128, reg) -#define AVX__AES_XOR( reg, ii) MM_OP_key (_mm256_xor_si256, reg) - -#define CTR_START(reg, ii) MM_OP (_mm_add_epi64, ctr, one); reg = ctr; -#define CTR_END( reg, ii) MM_XOR (data[ii], reg); - -#define AVX__CTR_START(reg, ii) MM_OP (_mm256_add_epi64, ctr2, two); reg = _mm256_xor_si256(ctr2, key); -#define AVX__CTR_END( reg, ii) AVX_XOR (((__m256i *)(void *)data)[ii], reg); +#define CTR_START(reg, ii) MM_OP (_mm_add_epi64, ctr, one) reg = ctr; +#define CTR_END( reg, ii) MM_XOR (data[ii], reg) #define WOP_KEY(op, n) { \ const __m128i key = w[n]; \ WOP(op); } -#define AVX__WOP_KEY(op, n) { \ - const __m256i key = w[n]; \ - WOP(op); } - #define WIDE_LOOP_START \ dataEnd = data + numBlocks; \ @@ -200,6 +179,40 @@ AES_FUNC_START2 (AesCbc_Encode_HW) for (; data < dataEnd; data++) + +#ifdef USE_INTEL_VAES + +#define AVX_XOR(dest, src) MM_OP(_mm256_xor_si256, dest, src) +#define AVX_DECLARE_VAR(reg, ii) __m256i reg; +#define AVX_LOAD_data( reg, ii) reg = ((const __m256i *)(const void *)data)[ii]; +#define AVX_STORE_data( reg, ii) ((__m256i *)(void *)data)[ii] = reg; +/* +AVX_XOR_data_M1() needs unaligned memory load +if (we don't use _mm256_loadu_si256() here) +{ + Most compilers with enabled optimizations generate fused AVX (LOAD + OP) + instruction that can load unaligned data. + But GCC and CLANG without -O2 or -O1 optimizations can generate separated + LOAD-ALIGNED (vmovdqa) instruction that will fail on execution. +} +Note: some compilers generate more instructions, if we use _mm256_loadu_si256() here. +v23.02: we use _mm256_loadu_si256() here, because we need compatibility with any compiler. +*/ +#define AVX_XOR_data_M1(reg, ii) AVX_XOR (reg, _mm256_loadu_si256(&(((const __m256i *)(const void *)(data - 1))[ii]))) +// for debug only: the following code will fail on execution, if compiled by some compilers: +// #define AVX_XOR_data_M1(reg, ii) AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii])) + +#define AVX_AES_DEC( reg, ii) MM_OP_key (_mm256_aesdec_epi128, reg) +#define AVX_AES_DEC_LAST( reg, ii) MM_OP_key (_mm256_aesdeclast_epi128, reg) +#define AVX_AES_ENC( reg, ii) MM_OP_key (_mm256_aesenc_epi128, reg) +#define AVX_AES_ENC_LAST( reg, ii) MM_OP_key (_mm256_aesenclast_epi128, reg) +#define AVX_AES_XOR( reg, ii) MM_OP_key (_mm256_xor_si256, reg) +#define AVX_CTR_START(reg, ii) MM_OP (_mm256_add_epi64, ctr2, two) reg = _mm256_xor_si256(ctr2, key); +#define AVX_CTR_END( reg, ii) AVX_XOR (((__m256i *)(void *)data)[ii], reg) +#define AVX_WOP_KEY(op, n) { \ + const __m256i key = w[n]; \ + WOP(op); } + #define NUM_AES_KEYS_MAX 15 #define WIDE_LOOP_START_AVX(OP) \ @@ -224,9 +237,14 @@ AES_FUNC_START2 (AesCbc_Encode_HW) /* MSVC for x86: If we don't call _mm256_zeroupper(), and -arch:IA32 is not specified, MSVC still can insert vzeroupper instruction. */ +#endif + + AES_FUNC_START2 (AesCbc_Decode_HW) { + __m128i *p = (__m128i *)(void *)ivAes; + __m128i *data = (__m128i *)(void *)data8; __m128i iv = *p; const __m128i *wStart = p + *(const UInt32 *)(p + 1) * 2 + 2 - 1; const __m128i *dataEnd; @@ -237,7 +255,7 @@ AES_FUNC_START2 (AesCbc_Decode_HW) const __m128i *w = wStart; WOP (DECLARE_VAR) - WOP (LOAD_data); + WOP (LOAD_data) WOP_KEY (AES_XOR, 1) do @@ -248,10 +266,10 @@ AES_FUNC_START2 (AesCbc_Decode_HW) while (w != p); WOP_KEY (AES_DEC_LAST, 0) - MM_XOR (m0, iv); + MM_XOR (m0, iv) WOP_M1 (XOR_data_M1) iv = data[NUM_WAYS - 1]; - WOP (STORE_data); + WOP (STORE_data) } WIDE_LOOP_END @@ -261,15 +279,15 @@ AES_FUNC_START2 (AesCbc_Decode_HW) __m128i m = _mm_xor_si128 (w[2], *data); do { - MM_OP_m (_mm_aesdec_si128, w[1]); - MM_OP_m (_mm_aesdec_si128, w[0]); + MM_OP_m (_mm_aesdec_si128, w[1]) + MM_OP_m (_mm_aesdec_si128, w[0]) w -= 2; } while (w != p); - MM_OP_m (_mm_aesdec_si128, w[1]); - MM_OP_m (_mm_aesdeclast_si128, w[0]); + MM_OP_m (_mm_aesdec_si128, w[1]) + MM_OP_m (_mm_aesdeclast_si128, w[0]) - MM_XOR (m, iv); + MM_XOR (m, iv) iv = *data; *data = m; } @@ -280,6 +298,8 @@ AES_FUNC_START2 (AesCbc_Decode_HW) AES_FUNC_START2 (AesCtr_Code_HW) { + __m128i *p = (__m128i *)(void *)ivAes; + __m128i *data = (__m128i *)(void *)data8; __m128i ctr = *p; UInt32 numRoundsMinus2 = *(const UInt32 *)(p + 1) * 2 - 1; const __m128i *dataEnd; @@ -292,7 +312,7 @@ AES_FUNC_START2 (AesCtr_Code_HW) const __m128i *w = p; UInt32 r = numRoundsMinus2; WOP (DECLARE_VAR) - WOP (CTR_START); + WOP (CTR_START) WOP_KEY (AES_XOR, 0) w += 1; do @@ -303,7 +323,7 @@ AES_FUNC_START2 (AesCtr_Code_HW) while (--r); WOP_KEY (AES_ENC_LAST, 0) - WOP (CTR_END); + WOP (CTR_END) } WIDE_LOOP_END @@ -312,19 +332,19 @@ AES_FUNC_START2 (AesCtr_Code_HW) UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1; const __m128i *w = p; __m128i m; - MM_OP (_mm_add_epi64, ctr, one); + MM_OP (_mm_add_epi64, ctr, one) m = _mm_xor_si128 (ctr, p[0]); w += 1; do { - MM_OP_m (_mm_aesenc_si128, w[0]); - MM_OP_m (_mm_aesenc_si128, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]) + MM_OP_m (_mm_aesenc_si128, w[1]) w += 2; } while (--numRounds2); - MM_OP_m (_mm_aesenc_si128, w[0]); - MM_OP_m (_mm_aesenclast_si128, w[1]); - MM_XOR (*data, m); + MM_OP_m (_mm_aesenc_si128, w[0]) + MM_OP_m (_mm_aesenclast_si128, w[1]) + MM_XOR (*data, m) } p[-2] = ctr; @@ -334,17 +354,61 @@ AES_FUNC_START2 (AesCtr_Code_HW) #ifdef USE_INTEL_VAES +/* +GCC before 2013-Jun: + : + #ifdef __AVX__ + #include + #endif +GCC after 2013-Jun: + : + #include +CLANG 3.8+: +{ + : + #if !defined(_MSC_VER) || defined(__AVX__) + #include + #endif + + if (the compiler is clang for Windows and if global arch is not set for __AVX__) + [ if (defined(_MSC_VER) && !defined(__AVX__)) ] + { + doesn't include + and we have 2 ways to fix it: + 1) we can define required __AVX__ before + or + 2) we can include after + } +} + +If we include manually for GCC/CLANG, it's +required that must be included before . +*/ + +/* #if defined(__clang__) && defined(_MSC_VER) -#define __SSE4_2__ -#define __AES__ #define __AVX__ #define __AVX2__ #define __VAES__ -#define __AVX512F__ -#define __AVX512VL__ #endif +*/ #include +#if defined(__clang__) && defined(_MSC_VER) + #if !defined(__AVX__) + #include + #endif + #if !defined(__AVX2__) + #include + #endif + #if !defined(__VAES__) + #include + #endif +#endif // __clang__ && _MSC_VER + +#ifndef ATTRIB_VAES + #define ATTRIB_VAES +#endif #define VAES_FUNC_START2(name) \ AES_FUNC_START (name); \ @@ -353,6 +417,8 @@ AES_FUNC_START (name) VAES_FUNC_START2 (AesCbc_Decode_HW_256) { + __m128i *p = (__m128i *)(void *)ivAes; + __m128i *data = (__m128i *)(void *)data8; __m128i iv = *p; const __m128i *dataEnd; UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1; @@ -362,22 +428,22 @@ VAES_FUNC_START2 (AesCbc_Decode_HW_256) { const __m256i *w = keys + numRounds - 2; - WOP (AVX__DECLARE_VAR) - WOP (AVX__LOAD_data); - AVX__WOP_KEY (AVX__AES_XOR, 1) + WOP (AVX_DECLARE_VAR) + WOP (AVX_LOAD_data) + AVX_WOP_KEY (AVX_AES_XOR, 1) do { - AVX__WOP_KEY (AVX__AES_DEC, 0) + AVX_WOP_KEY (AVX_AES_DEC, 0) w--; } while (w != keys); - AVX__WOP_KEY (AVX__AES_DEC_LAST, 0) + AVX_WOP_KEY (AVX_AES_DEC_LAST, 0) - AVX_XOR (m0, _mm256_setr_m128i(iv, data[0])); - WOP_M1 (AVX__XOR_data_M1) + AVX_XOR (m0, _mm256_setr_m128i(iv, data[0])) + WOP_M1 (AVX_XOR_data_M1) iv = data[NUM_WAYS * 2 - 1]; - WOP (AVX__STORE_data); + WOP (AVX_STORE_data) } WIDE_LOOP_END_AVX(;) @@ -387,15 +453,15 @@ VAES_FUNC_START2 (AesCbc_Decode_HW_256) __m128i m = _mm_xor_si128 (w[2], *data); do { - MM_OP_m (_mm_aesdec_si128, w[1]); - MM_OP_m (_mm_aesdec_si128, w[0]); + MM_OP_m (_mm_aesdec_si128, w[1]) + MM_OP_m (_mm_aesdec_si128, w[0]) w -= 2; } while (w != p); - MM_OP_m (_mm_aesdec_si128, w[1]); - MM_OP_m (_mm_aesdeclast_si128, w[0]); + MM_OP_m (_mm_aesdec_si128, w[1]) + MM_OP_m (_mm_aesdeclast_si128, w[0]) - MM_XOR (m, iv); + MM_XOR (m, iv) iv = *data; *data = m; } @@ -412,18 +478,20 @@ AVX2: _mm256_add_epi64 : vpaddq ymm, ymm, ymm _mm256_broadcastsi128_si256 : vbroadcasti128 */ -#define AVX__CTR_LOOP_START \ +#define AVX_CTR_LOOP_START \ ctr2 = _mm256_setr_m128i(_mm_sub_epi64(ctr, one), ctr); \ two = _mm256_setr_m128i(one, one); \ two = _mm256_add_epi64(two, two); \ // two = _mm256_setr_epi64x(2, 0, 2, 0); -#define AVX__CTR_LOOP_ENC \ +#define AVX_CTR_LOOP_ENC \ ctr = _mm256_extracti128_si256 (ctr2, 1); \ VAES_FUNC_START2 (AesCtr_Code_HW_256) { + __m128i *p = (__m128i *)(void *)ivAes; + __m128i *data = (__m128i *)(void *)data8; __m128i ctr = *p; UInt32 numRounds = *(const UInt32 *)(p + 1) * 2 + 1; const __m128i *dataEnd; @@ -431,44 +499,44 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256) __m256i ctr2, two; p += 2; - WIDE_LOOP_START_AVX (AVX__CTR_LOOP_START) + WIDE_LOOP_START_AVX (AVX_CTR_LOOP_START) { const __m256i *w = keys; UInt32 r = numRounds - 2; - WOP (AVX__DECLARE_VAR) - AVX__WOP_KEY (AVX__CTR_START, 0); + WOP (AVX_DECLARE_VAR) + AVX_WOP_KEY (AVX_CTR_START, 0) w += 1; do { - AVX__WOP_KEY (AVX__AES_ENC, 0) + AVX_WOP_KEY (AVX_AES_ENC, 0) w += 1; } while (--r); - AVX__WOP_KEY (AVX__AES_ENC_LAST, 0) + AVX_WOP_KEY (AVX_AES_ENC_LAST, 0) - WOP (AVX__CTR_END); + WOP (AVX_CTR_END) } - WIDE_LOOP_END_AVX (AVX__CTR_LOOP_ENC) + WIDE_LOOP_END_AVX (AVX_CTR_LOOP_ENC) SINGLE_LOOP { UInt32 numRounds2 = *(const UInt32 *)(p - 2 + 1) - 1; const __m128i *w = p; __m128i m; - MM_OP (_mm_add_epi64, ctr, one); + MM_OP (_mm_add_epi64, ctr, one) m = _mm_xor_si128 (ctr, p[0]); w += 1; do { - MM_OP_m (_mm_aesenc_si128, w[0]); - MM_OP_m (_mm_aesenc_si128, w[1]); + MM_OP_m (_mm_aesenc_si128, w[0]) + MM_OP_m (_mm_aesenc_si128, w[1]) w += 2; } while (--numRounds2); - MM_OP_m (_mm_aesenc_si128, w[0]); - MM_OP_m (_mm_aesenclast_si128, w[1]); - MM_XOR (*data, m); + MM_OP_m (_mm_aesenc_si128, w[0]) + MM_OP_m (_mm_aesenclast_si128, w[1]) + MM_XOR (*data, m) } p[-2] = ctr; @@ -480,13 +548,21 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256) /* no USE_INTEL_AES */ +#if defined(Z7_USE_AES_HW_STUB) +// We can compile this file with another C compiler, +// or we can compile asm version. +// So we can generate real code instead of this stub function. +// #if defined(_MSC_VER) #pragma message("AES HW_SW stub was used") +// #endif +#if !defined(USE_INTEL_VAES) && defined(Z7_USE_VAES_HW_STUB) #define AES_TYPE_keys UInt32 #define AES_TYPE_data Byte +#endif #define AES_FUNC_START(name) \ - void MY_FAST_CALL name(UInt32 *p, Byte *data, size_t numBlocks) \ + void Z7_FASTCALL name(UInt32 *p, Byte *data, size_t numBlocks) \ #define AES_COMPAT_STUB(name) \ AES_FUNC_START(name); \ @@ -496,51 +572,77 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256) AES_COMPAT_STUB (AesCbc_Encode) AES_COMPAT_STUB (AesCbc_Decode) AES_COMPAT_STUB (AesCtr_Code) +#endif // Z7_USE_AES_HW_STUB #endif // USE_INTEL_AES #ifndef USE_INTEL_VAES - +#if defined(Z7_USE_VAES_HW_STUB) +// #if defined(_MSC_VER) #pragma message("VAES HW_SW stub was used") +// #endif #define VAES_COMPAT_STUB(name) \ - void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \ - void MY_FAST_CALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks) \ + void Z7_FASTCALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \ + void Z7_FASTCALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks) \ { name((AES_TYPE_keys *)(void *)p, (AES_TYPE_data *)(void *)data, numBlocks); } VAES_COMPAT_STUB (AesCbc_Decode_HW) VAES_COMPAT_STUB (AesCtr_Code_HW) - +#endif #endif // ! USE_INTEL_VAES + + #elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) - #if defined(__ARM_NEON) || defined(__ARM_NEON__) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check + + #if defined(__ARM_FEATURE_AES) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_AES + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_AES #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_AES #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_AES #endif #endif - #endif + #ifdef USE_HW_AES // #pragma message("=== AES HW === ") +// __ARM_FEATURE_CRYPTO macro is deprecated in favor of the finer grained feature macro __ARM_FEATURE_AES #if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_AES) && \ + !defined(__ARM_FEATURE_CRYPTO) #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_AES __attribute__((__target__("crypto"))) +#else #define ATTRIB_AES __attribute__((__target__("+crypto"))) +#endif #else +#if defined(__clang__) + #define ATTRIB_AES __attribute__((__target__("armv8-a,aes"))) +#else #define ATTRIB_AES __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif #endif +#endif #else // _MSC_VER // for arm32 @@ -551,16 +653,65 @@ VAES_COMPAT_STUB (AesCtr_Code_HW) #define ATTRIB_AES #endif -#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else +/* + clang-17.0.1: error : Cannot select: intrinsic %llvm.arm.neon.aese + clang + 3.8.1 : __ARM_NEON : defined(__ARM_FEATURE_CRYPTO) + 7.0.1 : __ARM_NEON : __ARM_ARCH >= 8 && defined(__ARM_FEATURE_CRYPTO) + 11.?.0 : __ARM_NEON && __ARM_FP : __ARM_ARCH >= 8 && defined(__ARM_FEATURE_CRYPTO) + 13.0.1 : __ARM_NEON && __ARM_FP : __ARM_ARCH >= 8 && defined(__ARM_FEATURE_AES) + 16 : __ARM_NEON && __ARM_FP : __ARM_ARCH >= 8 +*/ +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_AES) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_AES 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + #include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_AES) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_AES + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") #endif +#endif // Z7_MSC_VER_ORIGINAL + typedef uint8x16_t v128; #define AES_FUNC_START(name) \ - void MY_FAST_CALL name(v128 *p, v128 *data, size_t numBlocks) + void Z7_FASTCALL name(UInt32 *ivAes, Byte *data8, size_t numBlocks) + // void Z7_FASTCALL name(v128 *p, v128 *data, size_t numBlocks) #define AES_FUNC_START2(name) \ AES_FUNC_START (name); \ @@ -568,18 +719,20 @@ ATTRIB_AES \ AES_FUNC_START (name) #define MM_OP(op, dest, src) dest = op(dest, src); -#define MM_OP_m(op, src) MM_OP(op, m, src); +#define MM_OP_m(op, src) MM_OP(op, m, src) #define MM_OP1_m(op) m = op(m); -#define MM_XOR( dest, src) MM_OP(veorq_u8, dest, src); -#define MM_XOR_m( src) MM_XOR(m, src); +#define MM_XOR( dest, src) MM_OP(veorq_u8, dest, src) +#define MM_XOR_m( src) MM_XOR(m, src) -#define AES_E_m(k) MM_OP_m (vaeseq_u8, k); -#define AES_E_MC_m(k) AES_E_m (k); MM_OP1_m(vaesmcq_u8); +#define AES_E_m(k) MM_OP_m (vaeseq_u8, k) +#define AES_E_MC_m(k) AES_E_m (k) MM_OP1_m(vaesmcq_u8) AES_FUNC_START2 (AesCbc_Encode_HW) { + v128 * const p = (v128*)(void*)ivAes; + v128 *data = (v128*)(void*)data8; v128 m = *p; const v128 k0 = p[2]; const v128 k1 = p[3]; @@ -597,7 +750,7 @@ AES_FUNC_START2 (AesCbc_Encode_HW) const v128 k_z0 = w[2]; for (; numBlocks != 0; numBlocks--, data++) { - MM_XOR_m (*data); + MM_XOR_m (*data) AES_E_MC_m (k0) AES_E_MC_m (k1) AES_E_MC_m (k2) @@ -617,8 +770,8 @@ AES_FUNC_START2 (AesCbc_Encode_HW) AES_E_MC_m (p[14]) } } - AES_E_m (k_z1); - MM_XOR_m (k_z0); + AES_E_m (k_z1) + MM_XOR_m (k_z0) *data = m; } *p = m; @@ -626,44 +779,44 @@ AES_FUNC_START2 (AesCbc_Encode_HW) #define WOP_1(op) -#define WOP_2(op) WOP_1 (op) op (m1, 1); -#define WOP_3(op) WOP_2 (op) op (m2, 2); -#define WOP_4(op) WOP_3 (op) op (m3, 3); -#define WOP_5(op) WOP_4 (op) op (m4, 4); -#define WOP_6(op) WOP_5 (op) op (m5, 5); -#define WOP_7(op) WOP_6 (op) op (m6, 6); -#define WOP_8(op) WOP_7 (op) op (m7, 7); +#define WOP_2(op) WOP_1 (op) op (m1, 1) +#define WOP_3(op) WOP_2 (op) op (m2, 2) +#define WOP_4(op) WOP_3 (op) op (m3, 3) +#define WOP_5(op) WOP_4 (op) op (m4, 4) +#define WOP_6(op) WOP_5 (op) op (m5, 5) +#define WOP_7(op) WOP_6 (op) op (m6, 6) +#define WOP_8(op) WOP_7 (op) op (m7, 7) #define NUM_WAYS 8 #define WOP_M1 WOP_8 -#define WOP(op) op (m0, 0); WOP_M1(op) +#define WOP(op) op (m0, 0) WOP_M1(op) -#define DECLARE_VAR(reg, ii) v128 reg +#define DECLARE_VAR(reg, ii) v128 reg; #define LOAD_data( reg, ii) reg = data[ii]; #define STORE_data( reg, ii) data[ii] = reg; #if (NUM_WAYS > 1) -#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]); +#define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]) #endif -#define MM_OP_key(op, reg) MM_OP (op, reg, key); +#define MM_OP_key(op, reg) MM_OP (op, reg, key) -#define AES_D_m(k) MM_OP_m (vaesdq_u8, k); -#define AES_D_IMC_m(k) AES_D_m (k); MM_OP1_m (vaesimcq_u8); +#define AES_D_m(k) MM_OP_m (vaesdq_u8, k) +#define AES_D_IMC_m(k) AES_D_m (k) MM_OP1_m (vaesimcq_u8) #define AES_XOR( reg, ii) MM_OP_key (veorq_u8, reg) #define AES_D( reg, ii) MM_OP_key (vaesdq_u8, reg) #define AES_E( reg, ii) MM_OP_key (vaeseq_u8, reg) -#define AES_D_IMC( reg, ii) AES_D (reg, ii); reg = vaesimcq_u8(reg) -#define AES_E_MC( reg, ii) AES_E (reg, ii); reg = vaesmcq_u8(reg) +#define AES_D_IMC( reg, ii) AES_D (reg, ii) reg = vaesimcq_u8(reg); +#define AES_E_MC( reg, ii) AES_E (reg, ii) reg = vaesmcq_u8(reg); -#define CTR_START(reg, ii) MM_OP (vaddq_u64, ctr, one); reg = vreinterpretq_u8_u64(ctr); -#define CTR_END( reg, ii) MM_XOR (data[ii], reg); +#define CTR_START(reg, ii) MM_OP (vaddq_u64, ctr, one) reg = vreinterpretq_u8_u64(ctr); +#define CTR_END( reg, ii) MM_XOR (data[ii], reg) #define WOP_KEY(op, n) { \ const v128 key = w[n]; \ - WOP(op); } + WOP(op) } #define WIDE_LOOP_START \ dataEnd = data + numBlocks; \ @@ -681,6 +834,8 @@ AES_FUNC_START2 (AesCbc_Encode_HW) AES_FUNC_START2 (AesCbc_Decode_HW) { + v128 *p = (v128*)(void*)ivAes; + v128 *data = (v128*)(void*)data8; v128 iv = *p; const v128 *wStart = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; const v128 *dataEnd; @@ -690,7 +845,7 @@ AES_FUNC_START2 (AesCbc_Decode_HW) { const v128 *w = wStart; WOP (DECLARE_VAR) - WOP (LOAD_data); + WOP (LOAD_data) WOP_KEY (AES_D_IMC, 2) do { @@ -701,10 +856,10 @@ AES_FUNC_START2 (AesCbc_Decode_HW) while (w != p); WOP_KEY (AES_D, 1) WOP_KEY (AES_XOR, 0) - MM_XOR (m0, iv); + MM_XOR (m0, iv) WOP_M1 (XOR_data_M1) iv = data[NUM_WAYS - 1]; - WOP (STORE_data); + WOP (STORE_data) } WIDE_LOOP_END @@ -715,14 +870,14 @@ AES_FUNC_START2 (AesCbc_Decode_HW) AES_D_IMC_m (w[2]) do { - AES_D_IMC_m (w[1]); - AES_D_IMC_m (w[0]); + AES_D_IMC_m (w[1]) + AES_D_IMC_m (w[0]) w -= 2; } while (w != p); - AES_D_m (w[1]); - MM_XOR_m (w[0]); - MM_XOR_m (iv); + AES_D_m (w[1]) + MM_XOR_m (w[0]) + MM_XOR_m (iv) iv = *data; *data = m; } @@ -733,10 +888,18 @@ AES_FUNC_START2 (AesCbc_Decode_HW) AES_FUNC_START2 (AesCtr_Code_HW) { + v128 *p = (v128*)(void*)ivAes; + v128 *data = (v128*)(void*)data8; uint64x2_t ctr = vreinterpretq_u64_u8(*p); const v128 *wEnd = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; const v128 *dataEnd; uint64x2_t one = vdupq_n_u64(0); + +// the bug in clang: +// __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); +#if defined(__clang__) && (__clang_major__ <= 9) +#pragma GCC diagnostic ignored "-Wvector-conversion" +#endif one = vsetq_lane_u64(1, one, 0); p += 2; @@ -744,7 +907,7 @@ AES_FUNC_START2 (AesCtr_Code_HW) { const v128 *w = p; WOP (DECLARE_VAR) - WOP (CTR_START); + WOP (CTR_START) do { WOP_KEY (AES_E_MC, 0) @@ -755,7 +918,7 @@ AES_FUNC_START2 (AesCtr_Code_HW) WOP_KEY (AES_E_MC, 0) WOP_KEY (AES_E, 1) WOP_KEY (AES_XOR, 2) - WOP (CTR_END); + WOP (CTR_END) } WIDE_LOOP_END @@ -763,18 +926,18 @@ AES_FUNC_START2 (AesCtr_Code_HW) { const v128 *w = p; v128 m; - CTR_START (m, 0); + CTR_START (m, 0) do { - AES_E_MC_m (w[0]); - AES_E_MC_m (w[1]); + AES_E_MC_m (w[0]) + AES_E_MC_m (w[1]) w += 2; } while (w != wEnd); - AES_E_MC_m (w[0]); - AES_E_m (w[1]); - MM_XOR_m (w[2]); - CTR_END (m, 0); + AES_E_MC_m (w[0]) + AES_E_m (w[1]) + MM_XOR_m (w[2]) + CTR_END (m, 0) } p[-2] = vreinterpretq_u8_u64(ctr); @@ -783,3 +946,12 @@ AES_FUNC_START2 (AesCtr_Code_HW) #endif // USE_HW_AES #endif // MY_CPU_ARM_OR_ARM64 + +#undef NUM_WAYS +#undef WOP_M1 +#undef WOP +#undef DECLARE_VAR +#undef LOAD_data +#undef STORE_data +#undef USE_INTEL_AES +#undef USE_HW_AES diff --git a/multiarc/src/formats/7z/C/Alloc.c b/multiarc/src/formats/7z/C/Alloc.c old mode 100755 new mode 100644 index d1af76c5c..63e1a121e --- a/multiarc/src/formats/7z/C/Alloc.c +++ b/multiarc/src/formats/7z/C/Alloc.c @@ -1,38 +1,53 @@ /* Alloc.c -- Memory allocation functions -2021-07-13 : Igor Pavlov : Public domain */ +2024-02-18 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include - #ifdef _WIN32 -#include +#include "7zWindows.h" #endif #include #include "Alloc.h" -/* #define _SZ_ALLOC_DEBUG */ +#if defined(Z7_LARGE_PAGES) && defined(_WIN32) && \ + (!defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0502) // < Win2003 (xp-64) + #define Z7_USE_DYN_GetLargePageMinimum +#endif + +// for debug: +#if 0 +#if defined(__CHERI__) && defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16) +// #pragma message("=== Z7_ALLOC_NO_OFFSET_ALLOCATOR === ") +#define Z7_ALLOC_NO_OFFSET_ALLOCATOR +#endif +#endif -/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ -#ifdef _SZ_ALLOC_DEBUG +// #define SZ_ALLOC_DEBUG +/* #define SZ_ALLOC_DEBUG */ +/* use SZ_ALLOC_DEBUG to debug alloc/free operations */ +#ifdef SZ_ALLOC_DEBUG + +#include #include -int g_allocCount = 0; -int g_allocCountMid = 0; -int g_allocCountBig = 0; +static int g_allocCount = 0; +#ifdef _WIN32 +static int g_allocCountMid = 0; +static int g_allocCountBig = 0; +#endif #define CONVERT_INT_TO_STR(charType, tempSize) \ - unsigned char temp[tempSize]; unsigned i = 0; \ - while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (char)('0' + (unsigned)(val % 10)); val /= 10; } \ *s++ = (charType)('0' + (unsigned)val); \ while (i != 0) { i--; *s++ = temp[i]; } \ *s = 0; static void ConvertUInt64ToString(UInt64 val, char *s) { - CONVERT_INT_TO_STR(char, 24); + CONVERT_INT_TO_STR(char, 24) } #define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) @@ -77,7 +92,7 @@ static void PrintAligned(const char *s, size_t align) Print(s); } -static void PrintLn() +static void PrintLn(void) { Print("\n"); } @@ -89,10 +104,10 @@ static void PrintHex(UInt64 v, size_t align) PrintAligned(s, align); } -static void PrintDec(UInt64 v, size_t align) +static void PrintDec(int v, size_t align) { char s[32]; - ConvertUInt64ToString(v, s); + ConvertUInt64ToString((unsigned)v, s); PrintAligned(s, align); } @@ -102,12 +117,19 @@ static void PrintAddr(void *p) } -#define PRINT_ALLOC(name, cnt, size, ptr) \ +#define PRINT_REALLOC(name, cnt, size, ptr) { \ + Print(name " "); \ + if (!ptr) PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#define PRINT_ALLOC(name, cnt, size, ptr) { \ Print(name " "); \ PrintDec(cnt++, 10); \ PrintHex(size, 10); \ PrintAddr(ptr); \ - PrintLn(); + PrintLn(); } #define PRINT_FREE(name, cnt, ptr) if (ptr) { \ Print(name " "); \ @@ -117,26 +139,45 @@ static void PrintAddr(void *p) #else +#ifdef _WIN32 #define PRINT_ALLOC(name, cnt, size, ptr) +#endif #define PRINT_FREE(name, cnt, ptr) #define Print(s) #define PrintLn() +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR #define PrintHex(v, align) +#endif #define PrintAddr(p) #endif +/* +by specification: + malloc(non_NULL, 0) : returns NULL or a unique pointer value that can later be successfully passed to free() + realloc(NULL, size) : the call is equivalent to malloc(size) + realloc(non_NULL, 0) : the call is equivalent to free(ptr) + +in main compilers: + malloc(0) : returns non_NULL + realloc(NULL, 0) : returns non_NULL + realloc(non_NULL, 0) : returns NULL +*/ + void *MyAlloc(size_t size) { if (size == 0) return NULL; - PRINT_ALLOC("Alloc ", g_allocCount, size, NULL); - #ifdef _SZ_ALLOC_DEBUG + // PRINT_ALLOC("Alloc ", g_allocCount, size, NULL) + #ifdef SZ_ALLOC_DEBUG { void *p = malloc(size); - // PRINT_ALLOC("Alloc ", g_allocCount, size, p); + if (p) + { + PRINT_ALLOC("Alloc ", g_allocCount, size, p) + } return p; } #else @@ -146,71 +187,107 @@ void *MyAlloc(size_t size) void MyFree(void *address) { - PRINT_FREE("Free ", g_allocCount, address); + PRINT_FREE("Free ", g_allocCount, address) free(address); } +void *MyRealloc(void *address, size_t size) +{ + if (size == 0) + { + MyFree(address); + return NULL; + } + // PRINT_REALLOC("Realloc ", g_allocCount, size, address) + #ifdef SZ_ALLOC_DEBUG + { + void *p = realloc(address, size); + if (p) + { + PRINT_REALLOC("Realloc ", g_allocCount, size, address) + } + return p; + } + #else + return realloc(address, size); + #endif +} + + #ifdef _WIN32 void *MidAlloc(size_t size) { if (size == 0) return NULL; - - PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); - + #ifdef SZ_ALLOC_DEBUG + { + void *p = VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); + if (p) + { + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, p) + } + return p; + } + #else return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); + #endif } void MidFree(void *address) { - PRINT_FREE("Free-Mid", g_allocCountMid, address); + PRINT_FREE("Free-Mid", g_allocCountMid, address) if (!address) return; VirtualFree(address, 0, MEM_RELEASE); } -#ifdef _7ZIP_LARGE_PAGES +#ifdef Z7_LARGE_PAGES #ifdef MEM_LARGE_PAGES - #define MY__MEM_LARGE_PAGES MEM_LARGE_PAGES + #define MY_MEM_LARGE_PAGES MEM_LARGE_PAGES #else - #define MY__MEM_LARGE_PAGES 0x20000000 + #define MY_MEM_LARGE_PAGES 0x20000000 #endif extern SIZE_T g_LargePageSize; SIZE_T g_LargePageSize = 0; -typedef SIZE_T (WINAPI *GetLargePageMinimumP)(VOID); +typedef SIZE_T (WINAPI *Func_GetLargePageMinimum)(VOID); -#endif // _7ZIP_LARGE_PAGES - -void SetLargePageSize() +void SetLargePageSize(void) { - #ifdef _7ZIP_LARGE_PAGES SIZE_T size; - GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) - GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (!largePageMinimum) +#ifdef Z7_USE_DYN_GetLargePageMinimum +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + + const + Func_GetLargePageMinimum fn = + (Func_GetLargePageMinimum) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), + "GetLargePageMinimum"); + if (!fn) return; - size = largePageMinimum(); + size = fn(); +#else + size = GetLargePageMinimum(); +#endif if (size == 0 || (size & (size - 1)) != 0) return; g_LargePageSize = size; - #endif } +#endif // Z7_LARGE_PAGES void *BigAlloc(size_t size) { if (size == 0) return NULL; - PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); - - #ifdef _7ZIP_LARGE_PAGES + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL) + + #ifdef Z7_LARGE_PAGES { SIZE_T ps = g_LargePageSize; if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) @@ -220,56 +297,43 @@ void *BigAlloc(size_t size) size2 = (size + ps) & ~ps; if (size2 >= size) { - void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MY__MEM_LARGE_PAGES, PAGE_READWRITE); - if (res) - return res; + void *p = VirtualAlloc(NULL, size2, MEM_COMMIT | MY_MEM_LARGE_PAGES, PAGE_READWRITE); + if (p) + { + PRINT_ALLOC("Alloc-BM ", g_allocCountMid, size2, p) + return p; + } } } } #endif - return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); + return MidAlloc(size); } void BigFree(void *address) { - PRINT_FREE("Free-Big", g_allocCountBig, address); - - if (!address) - return; - VirtualFree(address, 0, MEM_RELEASE); + PRINT_FREE("Free-Big", g_allocCountBig, address) + MidFree(address); } -#endif +#endif // _WIN32 -static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } +static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return MyAlloc(size); } +static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) MyFree(address); } const ISzAlloc g_Alloc = { SzAlloc, SzFree }; #ifdef _WIN32 -static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } -static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } -static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } -static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) MidFree(address); } +static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return BigAlloc(size); } +static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) BigFree(address); } const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; #endif -/* - uintptr_t : C99 (optional) - : unsupported in VS6 -*/ - -#ifdef _WIN32 - typedef UINT_PTR UIntPtr; -#else - /* - typedef uintptr_t UIntPtr; - */ - typedef ptrdiff_t UIntPtr; -#endif - +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR #define ADJUST_ALLOC_SIZE 0 /* @@ -280,14 +344,36 @@ const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; MyAlloc() can return address that is NOT multiple of sizeof(void *). */ - /* -#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) + uintptr_t : C99 (optional) + : unsupported in VS6 */ -#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) +typedef + #ifdef _WIN32 + UINT_PTR + #elif 1 + uintptr_t + #else + ptrdiff_t + #endif + MY_uintptr_t; + +#if 0 \ + || (defined(__CHERI__) \ + || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ > 8)) +// for 128-bit pointers (cheri): +#define MY_ALIGN_PTR_DOWN(p, align) \ + ((void *)((char *)(p) - ((size_t)(MY_uintptr_t)(p) & ((align) - 1)))) +#else +#define MY_ALIGN_PTR_DOWN(p, align) \ + ((void *)((((MY_uintptr_t)(p)) & ~((MY_uintptr_t)(align) - 1)))) +#endif +#endif -#if !defined(_WIN32) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) +#if !defined(_WIN32) \ + && (defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) \ + || defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) #define USE_posix_memalign #endif @@ -327,14 +413,13 @@ static int posix_memalign(void **ptr, size_t align, size_t size) #define ALLOC_ALIGN_SIZE ((size_t)1 << 7) -static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +void *z7_AlignedAlloc(size_t size) { - #ifndef USE_posix_memalign +#ifndef USE_posix_memalign void *p; void *pAligned; size_t newSize; - UNUSED_VAR(pp); /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned block to prevent cache line sharing with another allocated blocks */ @@ -359,10 +444,9 @@ static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) return pAligned; - #else +#else void *p; - UNUSED_VAR(pp); if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) return NULL; @@ -371,19 +455,37 @@ static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) return p; - #endif +#endif +} + + +void z7_AlignedFree(void *address) +{ +#ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); +#else + free(address); +#endif +} + + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + UNUSED_VAR(pp) + return z7_AlignedAlloc(size); } static void SzAlignedFree(ISzAllocPtr pp, void *address) { - UNUSED_VAR(pp); - #ifndef USE_posix_memalign + UNUSED_VAR(pp) +#ifndef USE_posix_memalign if (address) MyFree(((void **)address)[-1]); - #else +#else free(address); - #endif +#endif } @@ -391,17 +493,45 @@ const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; -#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) - /* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ -#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] -/* -#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] -*/ +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR +#if 1 + #define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + #define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +#else + // we can use this simplified code, + // if (CAlignOffsetAlloc::offset == (k * sizeof(void *)) + #define REAL_BLOCK_PTR_VAR(p) (((void **)(p))[-1]) +#endif +#endif + + +#if 0 +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR +#include +static void PrintPtr(const char *s, const void *p) +{ + const Byte *p2 = (const Byte *)&p; + unsigned i; + printf("%s %p ", s, p); + for (i = sizeof(p); i != 0;) + { + i--; + printf("%02x", p2[i]); + } + printf("\n"); +} +#endif +#endif + static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) { - CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); +#if defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) + UNUSED_VAR(pp) + return z7_AlignedAlloc(size); +#else + const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt); void *adr; void *pAligned; size_t newSize; @@ -429,6 +559,12 @@ static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; +#if 0 + printf("\nalignSize = %6x, offset=%6x, size=%8x \n", (unsigned)alignSize, (unsigned)p->offset, (unsigned)size); + PrintPtr("base", adr); + PrintPtr("alig", pAligned); +#endif + PrintLn(); Print("- Aligned: "); Print(" size="); PrintHex(size, 8); @@ -440,19 +576,25 @@ static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) REAL_BLOCK_PTR_VAR(pAligned) = adr; return pAligned; +#endif } static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) { +#if defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) + UNUSED_VAR(pp) + z7_AlignedFree(address); +#else if (address) { - CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt); PrintLn(); Print("- Aligned Free: "); PrintLn(); ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); } +#endif } diff --git a/multiarc/src/formats/7z/C/Alloc.h b/multiarc/src/formats/7z/C/Alloc.h old mode 100755 new mode 100644 index 3be2041eb..01bf6b7dd --- a/multiarc/src/formats/7z/C/Alloc.h +++ b/multiarc/src/formats/7z/C/Alloc.h @@ -1,31 +1,49 @@ /* Alloc.h -- Memory allocation functions -2021-07-13 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ -#ifndef __COMMON_ALLOC_H -#define __COMMON_ALLOC_H +#ifndef ZIP7_INC_ALLOC_H +#define ZIP7_INC_ALLOC_H #include "7zTypes.h" EXTERN_C_BEGIN +/* + MyFree(NULL) : is allowed, as free(NULL) + MyAlloc(0) : returns NULL : but malloc(0) is allowed to return NULL or non_NULL + MyRealloc(NULL, 0) : returns NULL : but realloc(NULL, 0) is allowed to return NULL or non_NULL +MyRealloc() is similar to realloc() for the following cases: + MyRealloc(non_NULL, 0) : returns NULL and always calls MyFree(ptr) + MyRealloc(NULL, non_ZERO) : returns NULL, if allocation failed + MyRealloc(non_NULL, non_ZERO) : returns NULL, if reallocation failed +*/ + void *MyAlloc(size_t size); void MyFree(void *address); +void *MyRealloc(void *address, size_t size); + +void *z7_AlignedAlloc(size_t size); +void z7_AlignedFree(void *p); #ifdef _WIN32 +#ifdef Z7_LARGE_PAGES void SetLargePageSize(void); +#endif void *MidAlloc(size_t size); void MidFree(void *address); void *BigAlloc(size_t size); void BigFree(void *address); +/* #define Z7_BIG_ALLOC_IS_ZERO_FILLED */ + #else -#define MidAlloc(size) MyAlloc(size) -#define MidFree(address) MyFree(address) -#define BigAlloc(size) MyAlloc(size) -#define BigFree(address) MyFree(address) +#define MidAlloc(size) z7_AlignedAlloc(size) +#define MidFree(address) z7_AlignedFree(address) +#define BigAlloc(size) z7_AlignedAlloc(size) +#define BigFree(address) z7_AlignedFree(address) #endif diff --git a/multiarc/src/formats/7z/C/Bcj2.c b/multiarc/src/formats/7z/C/Bcj2.c old mode 100755 new mode 100644 index c7b956708..7cb57ad62 --- a/multiarc/src/formats/7z/C/Bcj2.c +++ b/multiarc/src/formats/7z/C/Bcj2.c @@ -1,29 +1,24 @@ /* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2021-02-09 : Igor Pavlov : Public domain */ +2023-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "Bcj2.h" #include "CpuArch.h" -#define CProb UInt16 - #define kTopValue ((UInt32)1 << 24) -#define kNumModelBits 11 -#define kBitModelTotal (1 << kNumModelBits) +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) #define kNumMoveBits 5 -#define _IF_BIT_0 ttt = *prob; bound = (p->range >> kNumModelBits) * ttt; if (p->code < bound) -#define _UPDATE_0 p->range = bound; *prob = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); -#define _UPDATE_1 p->range -= bound; p->code -= bound; *prob = (CProb)(ttt - (ttt >> kNumMoveBits)); +// UInt32 bcj2_stats[256 + 2][2]; void Bcj2Dec_Init(CBcj2Dec *p) { unsigned i; - - p->state = BCJ2_DEC_STATE_OK; + p->state = BCJ2_STREAM_RC; // BCJ2_DEC_STATE_OK; p->ip = 0; - p->temp[3] = 0; + p->temp = 0; p->range = 0; p->code = 0; for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++) @@ -32,217 +27,248 @@ void Bcj2Dec_Init(CBcj2Dec *p) SRes Bcj2Dec_Decode(CBcj2Dec *p) { + UInt32 v = p->temp; + // const Byte *src; if (p->range <= 5) { - p->state = BCJ2_DEC_STATE_OK; + UInt32 code = p->code; + p->state = BCJ2_DEC_STATE_ERROR; /* for case if we return SZ_ERROR_DATA; */ for (; p->range != 5; p->range++) { - if (p->range == 1 && p->code != 0) + if (p->range == 1 && code != 0) return SZ_ERROR_DATA; - if (p->bufs[BCJ2_STREAM_RC] == p->lims[BCJ2_STREAM_RC]) { p->state = BCJ2_STREAM_RC; return SZ_OK; } - - p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; + code = (code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; + p->code = code; } - - if (p->code == 0xFFFFFFFF) + if (code == 0xffffffff) return SZ_ERROR_DATA; - - p->range = 0xFFFFFFFF; + p->range = 0xffffffff; } - else if (p->state >= BCJ2_DEC_STATE_ORIG_0) + // else { - while (p->state <= BCJ2_DEC_STATE_ORIG_3) + unsigned state = p->state; + // we check BCJ2_IS_32BIT_STREAM() here instead of check in the main loop + if (BCJ2_IS_32BIT_STREAM(state)) { - Byte *dest = p->dest; - if (dest == p->destLim) + const Byte *cur = p->bufs[state]; + if (cur == p->lims[state]) return SZ_OK; - *dest = p->temp[(size_t)p->state - BCJ2_DEC_STATE_ORIG_0]; - p->state++; - p->dest = dest + 1; + p->bufs[state] = cur + 4; + { + const UInt32 ip = p->ip + 4; + v = GetBe32a(cur) - ip; + p->ip = ip; + } + state = BCJ2_DEC_STATE_ORIG_0; } - } - - /* - if (BCJ2_IS_32BIT_STREAM(p->state)) - { - const Byte *cur = p->bufs[p->state]; - if (cur == p->lims[p->state]) - return SZ_OK; - p->bufs[p->state] = cur + 4; - + if ((unsigned)(state - BCJ2_DEC_STATE_ORIG_0) < 4) { - UInt32 val; - Byte *dest; - SizeT rem; - - p->ip += 4; - val = GetBe32(cur) - p->ip; - dest = p->dest; - rem = p->destLim - dest; - if (rem < 4) + Byte *dest = p->dest; + for (;;) { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; - p->dest = dest + rem; - p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; - return SZ_OK; + if (dest == p->destLim) + { + p->state = state; + p->temp = v; + return SZ_OK; + } + *dest++ = (Byte)v; + p->dest = dest; + if (++state == BCJ2_DEC_STATE_ORIG_3 + 1) + break; + v >>= 8; } - SetUi32(dest, val); - p->temp[3] = (Byte)(val >> 24); - p->dest = dest + 4; - p->state = BCJ2_DEC_STATE_OK; } } - */ + // src = p->bufs[BCJ2_STREAM_MAIN]; for (;;) { + /* if (BCJ2_IS_32BIT_STREAM(p->state)) p->state = BCJ2_DEC_STATE_OK; else + */ { if (p->range < kTopValue) { if (p->bufs[BCJ2_STREAM_RC] == p->lims[BCJ2_STREAM_RC]) { p->state = BCJ2_STREAM_RC; + p->temp = v; return SZ_OK; } p->range <<= 8; p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; } - { const Byte *src = p->bufs[BCJ2_STREAM_MAIN]; const Byte *srcLim; - Byte *dest; - SizeT num = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - src); - - if (num == 0) + Byte *dest = p->dest; { - p->state = BCJ2_STREAM_MAIN; - return SZ_OK; + const SizeT rem = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - src); + SizeT num = (SizeT)(p->destLim - dest); + if (num >= rem) + num = rem; + #define NUM_ITERS 4 + #if (NUM_ITERS & (NUM_ITERS - 1)) == 0 + num &= ~((SizeT)NUM_ITERS - 1); // if (NUM_ITERS == (1 << x)) + #else + num -= num % NUM_ITERS; // if (NUM_ITERS != (1 << x)) + #endif + srcLim = src + num; } - - dest = p->dest; - if (num > (SizeT)(p->destLim - dest)) + + #define NUM_SHIFT_BITS 24 + #define ONE_ITER(indx) { \ + const unsigned b = src[indx]; \ + *dest++ = (Byte)b; \ + v = (v << NUM_SHIFT_BITS) | b; \ + if (((b + (0x100 - 0xe8)) & 0xfe) == 0) break; \ + if (((v - (((UInt32)0x0f << (NUM_SHIFT_BITS)) + 0x80)) & \ + ((((UInt32)1 << (4 + NUM_SHIFT_BITS)) - 0x1) << 4)) == 0) break; \ + /* ++dest */; /* v = b; */ } + + if (src != srcLim) + for (;;) { - num = (SizeT)(p->destLim - dest); - if (num == 0) - { - p->state = BCJ2_DEC_STATE_ORIG; - return SZ_OK; - } + /* The dependency chain of 2-cycle for (v) calculation is not big problem here. + But we can remove dependency chain with v = b in the end of loop. */ + ONE_ITER(0) + #if (NUM_ITERS > 1) + ONE_ITER(1) + #if (NUM_ITERS > 2) + ONE_ITER(2) + #if (NUM_ITERS > 3) + ONE_ITER(3) + #if (NUM_ITERS > 4) + ONE_ITER(4) + #if (NUM_ITERS > 5) + ONE_ITER(5) + #if (NUM_ITERS > 6) + ONE_ITER(6) + #if (NUM_ITERS > 7) + ONE_ITER(7) + #endif + #endif + #endif + #endif + #endif + #endif + #endif + + src += NUM_ITERS; + if (src == srcLim) + break; } - - srcLim = src + num; - if (p->temp[3] == 0x0F && (src[0] & 0xF0) == 0x80) - *dest = src[0]; - else for (;;) + if (src == srcLim) + #if (NUM_ITERS > 1) + for (;;) + #endif { - Byte b = *src; - *dest = b; - if (b != 0x0F) + #if (NUM_ITERS > 1) + if (src == p->lims[BCJ2_STREAM_MAIN] || dest == p->destLim) + #endif { - if ((b & 0xFE) == 0xE8) - break; - dest++; - if (++src != srcLim) - continue; - break; + const SizeT num = (SizeT)(src - p->bufs[BCJ2_STREAM_MAIN]); + p->bufs[BCJ2_STREAM_MAIN] = src; + p->dest = dest; + p->ip += (UInt32)num; + /* state BCJ2_STREAM_MAIN has more priority than BCJ2_STATE_ORIG */ + p->state = + src == p->lims[BCJ2_STREAM_MAIN] ? + (unsigned)BCJ2_STREAM_MAIN : + (unsigned)BCJ2_DEC_STATE_ORIG; + p->temp = v; + return SZ_OK; } - dest++; - if (++src == srcLim) - break; - if ((*src & 0xF0) != 0x80) - continue; - *dest = *src; - break; + #if (NUM_ITERS > 1) + ONE_ITER(0) + src++; + #endif } - - num = (SizeT)(src - p->bufs[BCJ2_STREAM_MAIN]); - - if (src == srcLim) + { - p->temp[3] = src[-1]; - p->bufs[BCJ2_STREAM_MAIN] = src; + const SizeT num = (SizeT)(dest - p->dest); + p->dest = dest; // p->dest += num; + p->bufs[BCJ2_STREAM_MAIN] += num; // = src; p->ip += (UInt32)num; - p->dest += num; - p->state = - p->bufs[BCJ2_STREAM_MAIN] == - p->lims[BCJ2_STREAM_MAIN] ? - (unsigned)BCJ2_STREAM_MAIN : - (unsigned)BCJ2_DEC_STATE_ORIG; - return SZ_OK; } - { UInt32 bound, ttt; - CProb *prob; - Byte b = src[0]; - Byte prev = (Byte)(num == 0 ? p->temp[3] : src[-1]); - - p->temp[3] = b; - p->bufs[BCJ2_STREAM_MAIN] = src + 1; - num++; - p->ip += (UInt32)num; - p->dest += num; - - prob = p->probs + (unsigned)(b == 0xE8 ? 2 + (unsigned)prev : (b == 0xE9 ? 1 : 0)); - - _IF_BIT_0 + CBcj2Prob *prob; // unsigned index; + /* + prob = p->probs + (unsigned)((Byte)v == 0xe8 ? + 2 + (Byte)(v >> 8) : + ((v >> 5) & 1)); // ((Byte)v < 0xe8 ? 0 : 1)); + */ { - _UPDATE_0 + const unsigned c = ((v + 0x17) >> 6) & 1; + prob = p->probs + (unsigned) + (((0 - c) & (Byte)(v >> NUM_SHIFT_BITS)) + c + ((v >> 5) & 1)); + // (Byte) + // 8x->0 : e9->1 : xxe8->xx+2 + // 8x->0x100 : e9->0x101 : xxe8->xx + // (((0x100 - (e & ~v)) & (0x100 | (v >> 8))) + (e & v)); + // (((0x101 + (~e | v)) & (0x100 | (v >> 8))) + (e & v)); + } + ttt = *prob; + bound = (p->range >> kNumBitModelTotalBits) * ttt; + if (p->code < bound) + { + // bcj2_stats[prob - p->probs][0]++; + p->range = bound; + *prob = (CBcj2Prob)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); continue; } - _UPDATE_1 - + { + // bcj2_stats[prob - p->probs][1]++; + p->range -= bound; + p->code -= bound; + *prob = (CBcj2Prob)(ttt - (ttt >> kNumMoveBits)); + } } } } - { - UInt32 val; - unsigned cj = (p->temp[3] == 0xE8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; + /* (v == 0xe8 ? 0 : 1) uses setcc instruction with additional zero register usage in x64 MSVC. */ + // const unsigned cj = ((Byte)v == 0xe8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; + const unsigned cj = (((v + 0x57) >> 6) & 1) + BCJ2_STREAM_CALL; const Byte *cur = p->bufs[cj]; Byte *dest; SizeT rem; - if (cur == p->lims[cj]) { p->state = cj; break; } - - val = GetBe32(cur); + v = GetBe32a(cur); p->bufs[cj] = cur + 4; - - p->ip += 4; - val -= p->ip; + { + const UInt32 ip = p->ip + 4; + v -= ip; + p->ip = ip; + } dest = p->dest; rem = (SizeT)(p->destLim - dest); - if (rem < 4) { - p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; - p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; - p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; - p->temp[3] = (Byte)val; + if ((unsigned)rem > 0) { dest[0] = (Byte)v; v >>= 8; + if ((unsigned)rem > 1) { dest[1] = (Byte)v; v >>= 8; + if ((unsigned)rem > 2) { dest[2] = (Byte)v; v >>= 8; }}} + p->temp = v; p->dest = dest + rem; p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; break; } - - SetUi32(dest, val); - p->temp[3] = (Byte)(val >> 24); + SetUi32(dest, v) + v >>= 24; p->dest = dest + 4; } } @@ -252,6 +278,13 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) p->range <<= 8; p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; } - return SZ_OK; } + +#undef NUM_ITERS +#undef ONE_ITER +#undef NUM_SHIFT_BITS +#undef kTopValue +#undef kNumBitModelTotalBits +#undef kBitModelTotal +#undef kNumMoveBits diff --git a/multiarc/src/formats/7z/C/Bcj2.h b/multiarc/src/formats/7z/C/Bcj2.h old mode 100755 new mode 100644 index c27504fe4..4575545b6 --- a/multiarc/src/formats/7z/C/Bcj2.h +++ b/multiarc/src/formats/7z/C/Bcj2.h @@ -1,8 +1,8 @@ -/* Bcj2.h -- BCJ2 Converter for x86 code -2014-11-10 : Igor Pavlov : Public domain */ +/* Bcj2.h -- BCJ2 converter for x86 code (Branch CALL/JUMP variant2) +2023-03-02 : Igor Pavlov : Public domain */ -#ifndef __BCJ2_H -#define __BCJ2_H +#ifndef ZIP7_INC_BCJ2_H +#define ZIP7_INC_BCJ2_H #include "7zTypes.h" @@ -26,17 +26,18 @@ enum BCJ2_DEC_STATE_ORIG_3, BCJ2_DEC_STATE_ORIG, - BCJ2_DEC_STATE_OK + BCJ2_DEC_STATE_ERROR /* after detected data error */ }; enum { BCJ2_ENC_STATE_ORIG = BCJ2_NUM_STREAMS, - BCJ2_ENC_STATE_OK + BCJ2_ENC_STATE_FINISHED /* it's state after fully encoded stream */ }; -#define BCJ2_IS_32BIT_STREAM(s) ((s) == BCJ2_STREAM_CALL || (s) == BCJ2_STREAM_JUMP) +/* #define BCJ2_IS_32BIT_STREAM(s) ((s) == BCJ2_STREAM_CALL || (s) == BCJ2_STREAM_JUMP) */ +#define BCJ2_IS_32BIT_STREAM(s) ((unsigned)((unsigned)(s) - (unsigned)BCJ2_STREAM_CALL) < 2) /* CBcj2Dec / CBcj2Enc @@ -47,16 +48,46 @@ bufs sizes for BCJ2_STREAM_CALL and BCJ2_STREAM_JUMP must be multiply of 4: (BUF_SIZE(BCJ2_STREAM_JUMP) & 3) == 0 */ +// typedef UInt32 CBcj2Prob; +typedef UInt16 CBcj2Prob; + +/* +BCJ2 encoder / decoder internal requirements: + - If last bytes of stream contain marker (e8/e8/0f8x), then + there is also encoded symbol (0 : no conversion) in RC stream. + - One case of overlapped instructions is supported, + if last byte of converted instruction is (0f) and next byte is (8x): + marker [xx xx xx 0f] 8x + then the pair (0f 8x) is treated as marker. +*/ + +/* ---------- BCJ2 Decoder ---------- */ + /* CBcj2Dec: -dest is allowed to overlap with bufs[BCJ2_STREAM_MAIN], with the following conditions: +(dest) is allowed to overlap with bufs[BCJ2_STREAM_MAIN], with the following conditions: bufs[BCJ2_STREAM_MAIN] >= dest && - bufs[BCJ2_STREAM_MAIN] - dest >= tempReserv + + bufs[BCJ2_STREAM_MAIN] - dest >= BUF_SIZE(BCJ2_STREAM_CALL) + BUF_SIZE(BCJ2_STREAM_JUMP) - tempReserv = 0 : for first call of Bcj2Dec_Decode - tempReserv = 4 : for any other calls of Bcj2Dec_Decode - overlap with offset = 1 is not allowed + reserve = bufs[BCJ2_STREAM_MAIN] - dest - + ( BUF_SIZE(BCJ2_STREAM_CALL) + + BUF_SIZE(BCJ2_STREAM_JUMP) ) + and additional conditions: + if (it's first call of Bcj2Dec_Decode() after Bcj2Dec_Init()) + { + (reserve != 1) : if (ver < v23.00) + } + else // if there are more than one calls of Bcj2Dec_Decode() after Bcj2Dec_Init()) + { + (reserve >= 6) : if (ver < v23.00) + (reserve >= 4) : if (ver >= v23.00) + We need that (reserve) because after first call of Bcj2Dec_Decode(), + CBcj2Dec::temp can contain up to 4 bytes for writing to (dest). + } + (reserve == 0) is allowed, if we decode full stream via single call of Bcj2Dec_Decode(). + (reserve == 0) also is allowed in case of multi-call, if we use fixed buffers, + and (reserve) is calculated from full (final) sizes of all streams before first call. */ typedef struct @@ -68,21 +99,65 @@ typedef struct unsigned state; /* BCJ2_STREAM_MAIN has more priority than BCJ2_STATE_ORIG */ - UInt32 ip; - Byte temp[4]; + UInt32 ip; /* property of starting base for decoding */ + UInt32 temp; /* Byte temp[4]; */ UInt32 range; UInt32 code; - UInt16 probs[2 + 256]; + CBcj2Prob probs[2 + 256]; } CBcj2Dec; + +/* Note: + Bcj2Dec_Init() sets (CBcj2Dec::ip = 0) + if (ip != 0) property is required, the caller must set CBcj2Dec::ip after Bcj2Dec_Init() +*/ void Bcj2Dec_Init(CBcj2Dec *p); -/* Returns: SZ_OK or SZ_ERROR_DATA */ + +/* Bcj2Dec_Decode(): + returns: + SZ_OK + SZ_ERROR_DATA : if data in 5 starting bytes of BCJ2_STREAM_RC stream are not correct +*/ SRes Bcj2Dec_Decode(CBcj2Dec *p); -#define Bcj2Dec_IsFinished(_p_) ((_p_)->code == 0) +/* To check that decoding was finished you can compare + sizes of processed streams with sizes known from another sources. + You must do at least one mandatory check from the two following options: + - the check for size of processed output (ORIG) stream. + - the check for size of processed input (MAIN) stream. + additional optional checks: + - the checks for processed sizes of all input streams (MAIN, CALL, JUMP, RC) + - the checks Bcj2Dec_IsMaybeFinished*() + also before actual decoding you can check that the + following condition is met for stream sizes: + ( size(ORIG) == size(MAIN) + size(CALL) + size(JUMP) ) +*/ +/* (state == BCJ2_STREAM_MAIN) means that decoder is ready for + additional input data in BCJ2_STREAM_MAIN stream. + Note that (state == BCJ2_STREAM_MAIN) is allowed for non-finished decoding. +*/ +#define Bcj2Dec_IsMaybeFinished_state_MAIN(_p_) ((_p_)->state == BCJ2_STREAM_MAIN) +/* if the stream decoding was finished correctly, then range decoder + part of CBcj2Dec also was finished, and then (CBcj2Dec::code == 0). + Note that (CBcj2Dec::code == 0) is allowed for non-finished decoding. +*/ +#define Bcj2Dec_IsMaybeFinished_code(_p_) ((_p_)->code == 0) + +/* use Bcj2Dec_IsMaybeFinished() only as additional check + after at least one mandatory check from the two following options: + - the check for size of processed output (ORIG) stream. + - the check for size of processed input (MAIN) stream. +*/ +#define Bcj2Dec_IsMaybeFinished(_p_) ( \ + Bcj2Dec_IsMaybeFinished_state_MAIN(_p_) && \ + Bcj2Dec_IsMaybeFinished_code(_p_)) + + + +/* ---------- BCJ2 Encoder ---------- */ typedef enum { @@ -91,6 +166,91 @@ typedef enum BCJ2_ENC_FINISH_MODE_END_STREAM } EBcj2Enc_FinishMode; +/* + BCJ2_ENC_FINISH_MODE_CONTINUE: + process non finished encoding. + It notifies the encoder that additional further calls + can provide more input data (src) than provided by current call. + In that case the CBcj2Enc encoder still can move (src) pointer + up to (srcLim), but CBcj2Enc encoder can store some of the last + processed bytes (up to 4 bytes) from src to internal CBcj2Enc::temp[] buffer. + at return: + (CBcj2Enc::src will point to position that includes + processed data and data copied to (temp[]) buffer) + That data from (temp[]) buffer will be used in further calls. + + BCJ2_ENC_FINISH_MODE_END_BLOCK: + finish encoding of current block (ended at srcLim) without RC flushing. + at return: if (CBcj2Enc::state == BCJ2_ENC_STATE_ORIG) && + CBcj2Enc::src == CBcj2Enc::srcLim) + : it shows that block encoding was finished. And the encoder is + ready for new (src) data or for stream finish operation. + finished block means + { + CBcj2Enc has completed block encoding up to (srcLim). + (1 + 4 bytes) or (2 + 4 bytes) CALL/JUMP cortages will + not cross block boundary at (srcLim). + temporary CBcj2Enc buffer for (ORIG) src data is empty. + 3 output uncompressed streams (MAIN, CALL, JUMP) were flushed. + RC stream was not flushed. And RC stream will cross block boundary. + } + Note: some possible implementation of BCJ2 encoder could + write branch marker (e8/e8/0f8x) in one call of Bcj2Enc_Encode(), + and it could calculate symbol for RC in another call of Bcj2Enc_Encode(). + BCJ2 encoder uses ip/fileIp/fileSize/relatLimit values to calculate RC symbol. + And these CBcj2Enc variables can have different values in different Bcj2Enc_Encode() calls. + So caller must finish each block with BCJ2_ENC_FINISH_MODE_END_BLOCK + to ensure that RC symbol is calculated and written in proper block. + + BCJ2_ENC_FINISH_MODE_END_STREAM + finish encoding of stream (ended at srcLim) fully including RC flushing. + at return: if (CBcj2Enc::state == BCJ2_ENC_STATE_FINISHED) + : it shows that stream encoding was finished fully, + and all output streams were flushed fully. + also Bcj2Enc_IsFinished() can be called. +*/ + + +/* + 32-bit relative offset in JUMP/CALL commands is + - (mod 4 GiB) for 32-bit x86 code + - signed Int32 for 64-bit x86-64 code + BCJ2 encoder also does internal relative to absolute address conversions. + And there are 2 possible ways to do it: + before v23: we used 32-bit variables and (mod 4 GiB) conversion + since v23: we use 64-bit variables and (signed Int32 offset) conversion. + The absolute address condition for conversion in v23: + ((UInt64)((Int64)ip64 - (Int64)fileIp64 + 5 + (Int32)offset) < (UInt64)fileSize64) + note that if (fileSize64 > 2 GiB). there is difference between + old (mod 4 GiB) way (v22) and new (signed Int32 offset) way (v23). + And new (v23) way is more suitable to encode 64-bit x86-64 code for (fileSize64 > 2 GiB) cases. +*/ + +/* +// for old (v22) way for conversion: +typedef UInt32 CBcj2Enc_ip_unsigned; +typedef Int32 CBcj2Enc_ip_signed; +#define BCJ2_ENC_FileSize_MAX ((UInt32)1 << 31) +*/ +typedef UInt64 CBcj2Enc_ip_unsigned; +typedef Int64 CBcj2Enc_ip_signed; + +/* maximum size of file that can be used for conversion condition */ +#define BCJ2_ENC_FileSize_MAX ((CBcj2Enc_ip_unsigned)0 - 2) + +/* default value of fileSize64_minus1 variable that means + that absolute address limitation will not be used */ +#define BCJ2_ENC_FileSizeField_UNLIMITED ((CBcj2Enc_ip_unsigned)0 - 1) + +/* calculate value that later can be set to CBcj2Enc::fileSize64_minus1 */ +#define BCJ2_ENC_GET_FileSizeField_VAL_FROM_FileSize(fileSize) \ + ((CBcj2Enc_ip_unsigned)(fileSize) - 1) + +/* set CBcj2Enc::fileSize64_minus1 variable from size of file */ +#define Bcj2Enc_SET_FileSize(p, fileSize) \ + (p)->fileSize64_minus1 = BCJ2_ENC_GET_FileSizeField_VAL_FROM_FileSize(fileSize); + + typedef struct { Byte *bufs[BCJ2_NUM_STREAMS]; @@ -101,45 +261,71 @@ typedef struct unsigned state; EBcj2Enc_FinishMode finishMode; - Byte prevByte; + Byte context; + Byte flushRem; + Byte isFlushState; Byte cache; UInt32 range; UInt64 low; UInt64 cacheSize; + + // UInt32 context; // for marker version, it can include marker flag. - UInt32 ip; - - /* 32-bit ralative offset in JUMP/CALL commands is - - (mod 4 GB) in 32-bit mode - - signed Int32 in 64-bit mode - We use (mod 4 GB) check for fileSize. - Use fileSize up to 2 GB, if you want to support 32-bit and 64-bit code conversion. */ - UInt32 fileIp; - UInt32 fileSize; /* (fileSize <= ((UInt32)1 << 31)), 0 means no_limit */ - UInt32 relatLimit; /* (relatLimit <= ((UInt32)1 << 31)), 0 means desable_conversion */ + /* (ip64) and (fileIp64) correspond to virtual source stream position + that doesn't include data in temp[] */ + CBcj2Enc_ip_unsigned ip64; /* current (ip) position */ + CBcj2Enc_ip_unsigned fileIp64; /* start (ip) position of current file */ + CBcj2Enc_ip_unsigned fileSize64_minus1; /* size of current file (for conversion limitation) */ + UInt32 relatLimit; /* (relatLimit <= ((UInt32)1 << 31)) : 0 means disable_conversion */ + // UInt32 relatExcludeBits; UInt32 tempTarget; - unsigned tempPos; - Byte temp[4 * 2]; - - unsigned flushPos; - - UInt16 probs[2 + 256]; + unsigned tempPos; /* the number of bytes that were copied to temp[] buffer + (tempPos <= 4) outside of Bcj2Enc_Encode() */ + // Byte temp[4]; // for marker version + Byte temp[8]; + CBcj2Prob probs[2 + 256]; } CBcj2Enc; void Bcj2Enc_Init(CBcj2Enc *p); -void Bcj2Enc_Encode(CBcj2Enc *p); -#define Bcj2Enc_Get_InputData_Size(p) ((SizeT)((p)->srcLim - (p)->src) + (p)->tempPos) -#define Bcj2Enc_IsFinished(p) ((p)->flushPos == 5) +/* +Bcj2Enc_Encode(): at exit: + p->State < BCJ2_NUM_STREAMS : we need more buffer space for output stream + (bufs[p->State] == lims[p->State]) + p->State == BCJ2_ENC_STATE_ORIG : we need more data in input src stream + (src == srcLim) + p->State == BCJ2_ENC_STATE_FINISHED : after fully encoded stream +*/ +void Bcj2Enc_Encode(CBcj2Enc *p); -#define BCJ2_RELAT_LIMIT_NUM_BITS 26 -#define BCJ2_RELAT_LIMIT ((UInt32)1 << BCJ2_RELAT_LIMIT_NUM_BITS) +/* Bcj2Enc encoder can look ahead for up 4 bytes of source stream. + CBcj2Enc::tempPos : is the number of bytes that were copied from input stream to temp[] buffer. + (CBcj2Enc::src) after Bcj2Enc_Encode() is starting position after + fully processed data and after data copied to temp buffer. + So if the caller needs to get real number of fully processed input + bytes (without look ahead data in temp buffer), + the caller must subtruct (CBcj2Enc::tempPos) value from processed size + value that is calculated based on current (CBcj2Enc::src): + cur_processed_pos = Calc_Big_Processed_Pos(enc.src)) - + Bcj2Enc_Get_AvailInputSize_in_Temp(&enc); +*/ +/* get the size of input data that was stored in temp[] buffer: */ +#define Bcj2Enc_Get_AvailInputSize_in_Temp(p) ((p)->tempPos) -/* limit for CBcj2Enc::fileSize variable */ -#define BCJ2_FileSize_MAX ((UInt32)1 << 31) +#define Bcj2Enc_IsFinished(p) ((p)->flushRem == 0) + +/* Note : the decoder supports overlapping of marker (0f 80). + But we can eliminate such overlapping cases by setting + the limit for relative offset conversion as + CBcj2Enc::relatLimit <= (0x0f << 24) == (240 MiB) +*/ +/* default value for CBcj2Enc::relatLimit */ +#define BCJ2_ENC_RELAT_LIMIT_DEFAULT ((UInt32)0x0f << 24) +#define BCJ2_ENC_RELAT_LIMIT_MAX ((UInt32)1 << 31) +// #define BCJ2_RELAT_EXCLUDE_NUM_BITS 5 EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/Bcj2Enc.c b/multiarc/src/formats/7z/C/Bcj2Enc.c old mode 100755 new mode 100644 index 682362a15..79460bb15 --- a/multiarc/src/formats/7z/C/Bcj2Enc.c +++ b/multiarc/src/formats/7z/C/Bcj2Enc.c @@ -1,60 +1,62 @@ -/* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2021-02-09 : Igor Pavlov : Public domain */ +/* Bcj2Enc.c -- BCJ2 Encoder converter for x86 code (Branch CALL/JUMP variant2) +2023-04-02 : Igor Pavlov : Public domain */ #include "Precomp.h" /* #define SHOW_STAT */ - #ifdef SHOW_STAT #include -#define PRF(x) x +#define PRF2(s) printf("%s ip=%8x tempPos=%d src= %8x\n", s, (unsigned)p->ip64, p->tempPos, (unsigned)(p->srcLim - p->src)); #else -#define PRF(x) +#define PRF2(s) #endif -#include - #include "Bcj2.h" #include "CpuArch.h" -#define CProb UInt16 - #define kTopValue ((UInt32)1 << 24) -#define kNumModelBits 11 -#define kBitModelTotal (1 << kNumModelBits) +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) #define kNumMoveBits 5 void Bcj2Enc_Init(CBcj2Enc *p) { unsigned i; - - p->state = BCJ2_ENC_STATE_OK; + p->state = BCJ2_ENC_STATE_ORIG; p->finishMode = BCJ2_ENC_FINISH_MODE_CONTINUE; - - p->prevByte = 0; - + p->context = 0; + p->flushRem = 5; + p->isFlushState = 0; p->cache = 0; - p->range = 0xFFFFFFFF; + p->range = 0xffffffff; p->low = 0; p->cacheSize = 1; - - p->ip = 0; - - p->fileIp = 0; - p->fileSize = 0; - p->relatLimit = BCJ2_RELAT_LIMIT; - + p->ip64 = 0; + p->fileIp64 = 0; + p->fileSize64_minus1 = BCJ2_ENC_FileSizeField_UNLIMITED; + p->relatLimit = BCJ2_ENC_RELAT_LIMIT_DEFAULT; + // p->relatExcludeBits = 0; p->tempPos = 0; - - p->flushPos = 0; - for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++) p->probs[i] = kBitModelTotal >> 1; } -static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +// Z7_NO_INLINE +Z7_FORCE_INLINE +static BoolInt Bcj2_RangeEnc_ShiftLow(CBcj2Enc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) + const UInt32 low = (UInt32)p->low; + const unsigned high = (unsigned) + #if defined(Z7_MSC_VER_ORIGINAL) \ + && defined(MY_CPU_X86) \ + && defined(MY_CPU_LE) \ + && !defined(MY_CPU_64BIT) + // we try to rid of __aullshr() call in MSVS-x86 + (((const UInt32 *)&p->low)[1]); // [1] : for little-endian only + #else + (p->low >> 32); + #endif + if (low < (UInt32)0xff000000 || high != 0) { Byte *buf = p->bufs[BCJ2_STREAM_RC]; do @@ -65,247 +67,440 @@ static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) p->bufs[BCJ2_STREAM_RC] = buf; return True; } - *buf++ = (Byte)(p->cache + (Byte)(p->low >> 32)); - p->cache = 0xFF; + *buf++ = (Byte)(p->cache + high); + p->cache = 0xff; } while (--p->cacheSize); p->bufs[BCJ2_STREAM_RC] = buf; - p->cache = (Byte)((UInt32)p->low >> 24); + p->cache = (Byte)(low >> 24); } p->cacheSize++; - p->low = (UInt32)p->low << 8; + p->low = low << 8; return False; } -static void Bcj2Enc_Encode_2(CBcj2Enc *p) -{ - if (BCJ2_IS_32BIT_STREAM(p->state)) + +/* +We can use 2 alternative versions of code: +1) non-marker version: + Byte CBcj2Enc::context + Byte temp[8]; + Last byte of marker (e8/e9/[0f]8x) can be written to temp[] buffer. + Encoder writes last byte of marker (e8/e9/[0f]8x) to dest, only in conjunction + with writing branch symbol to range coder in same Bcj2Enc_Encode_2() call. + +2) marker version: + UInt32 CBcj2Enc::context + Byte CBcj2Enc::temp[4]; + MARKER_FLAG in CBcj2Enc::context shows that CBcj2Enc::context contains finded marker. + it's allowed that + one call of Bcj2Enc_Encode_2() writes last byte of marker (e8/e9/[0f]8x) to dest, + and another call of Bcj2Enc_Encode_2() does offset conversion. + So different values of (fileIp) and (fileSize) are possible + in these different Bcj2Enc_Encode_2() calls. + +Also marker version requires additional if((v & MARKER_FLAG) == 0) check in main loop. +So we use non-marker version. +*/ + +/* + Corner cases with overlap in multi-block. + before v23: there was one corner case, where converted instruction + could start in one sub-stream and finish in next sub-stream. + If multi-block (solid) encoding is used, + and BCJ2_ENC_FINISH_MODE_END_BLOCK is used for each sub-stream. + and (0f) is last byte of previous sub-stream + and (8x) is first byte of current sub-stream + then (0f 8x) pair is treated as marker by BCJ2 encoder and decoder. + BCJ2 encoder can converts 32-bit offset for that (0f 8x) cortage, + if that offset meets limit requirements. + If encoder allows 32-bit offset conversion for such overlap case, + then the data in 3 uncompressed BCJ2 streams for some sub-stream + can depend from data of previous sub-stream. + That corner case is not big problem, and it's rare case. + Since v23.00 we do additional check to prevent conversions in such overlap cases. +*/ + +/* + Bcj2Enc_Encode_2() output variables at exit: { - Byte *cur = p->bufs[p->state]; - if (cur == p->lims[p->state]) - return; - SetBe32(cur, p->tempTarget); - p->bufs[p->state] = cur + 4; + if (Bcj2Enc_Encode_2() exits with (p->state == BCJ2_ENC_STATE_ORIG)) + { + it means that encoder needs more input data. + if (p->srcLim == p->src) at exit, then + { + (p->finishMode != BCJ2_ENC_FINISH_MODE_END_STREAM) + all input data were read and processed, and we are ready for + new input data. + } + else + { + (p->srcLim != p->src) + (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) + The encoder have found e8/e9/0f_8x marker, + and p->src points to last byte of that marker, + Bcj2Enc_Encode_2() needs more input data to get totally + 5 bytes (last byte of marker and 32-bit branch offset) + as continuous array starting from p->src. + (p->srcLim - p->src < 5) requirement is met after exit. + So non-processed resedue from p->src to p->srcLim is always less than 5 bytes. + } + } } +*/ - p->state = BCJ2_ENC_STATE_ORIG; - - for (;;) +Z7_NO_INLINE +static void Bcj2Enc_Encode_2(CBcj2Enc *p) +{ + if (!p->isFlushState) { - if (p->range < kTopValue) + const Byte *src; + UInt32 v; { - if (RangeEnc_ShiftLow(p)) - return; - p->range <<= 8; + const unsigned state = p->state; + if (BCJ2_IS_32BIT_STREAM(state)) + { + Byte *cur = p->bufs[state]; + if (cur == p->lims[state]) + return; + SetBe32a(cur, p->tempTarget) + p->bufs[state] = cur + 4; + } } + p->state = BCJ2_ENC_STATE_ORIG; // for main reason of exit + src = p->src; + v = p->context; + + // #define WRITE_CONTEXT p->context = v; // for marker version + #define WRITE_CONTEXT p->context = (Byte)v; + #define WRITE_CONTEXT_AND_SRC p->src = src; WRITE_CONTEXT + for (;;) { + // const Byte *src; + // UInt32 v; + CBcj2Enc_ip_unsigned ip; + if (p->range < kTopValue) + { + // to reduce register pressure and code size: we save and restore local variables. + WRITE_CONTEXT_AND_SRC + if (Bcj2_RangeEnc_ShiftLow(p)) + return; + p->range <<= 8; + src = p->src; + v = p->context; + } + // src = p->src; + // #define MARKER_FLAG ((UInt32)1 << 17) + // if ((v & MARKER_FLAG) == 0) // for marker version { - const Byte *src = p->src; const Byte *srcLim; - Byte *dest; - SizeT num = (SizeT)(p->srcLim - src); - - if (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) + Byte *dest = p->bufs[BCJ2_STREAM_MAIN]; { - if (num <= 4) - return; - num -= 4; + const SizeT remSrc = (SizeT)(p->srcLim - src); + SizeT rem = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest); + if (rem >= remSrc) + rem = remSrc; + srcLim = src + rem; } - else if (num == 0) - break; - - dest = p->bufs[BCJ2_STREAM_MAIN]; - if (num > (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest)) + /* p->context contains context of previous byte: + bits [0 : 7] : src[-1], if (src) was changed in this call + bits [8 : 31] : are undefined for non-marker version + */ + // v = p->context; + #define NUM_SHIFT_BITS 24 + #define CONV_FLAG ((UInt32)1 << 16) + #define ONE_ITER { \ + b = src[0]; \ + *dest++ = (Byte)b; \ + v = (v << NUM_SHIFT_BITS) | b; \ + if (((b + (0x100 - 0xe8)) & 0xfe) == 0) break; \ + if (((v - (((UInt32)0x0f << (NUM_SHIFT_BITS)) + 0x80)) & \ + ((((UInt32)1 << (4 + NUM_SHIFT_BITS)) - 0x1) << 4)) == 0) break; \ + src++; if (src == srcLim) { break; } } + + if (src != srcLim) + for (;;) { - num = (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest); - if (num == 0) - { - p->state = BCJ2_STREAM_MAIN; - return; - } + /* clang can generate ineffective code with setne instead of two jcc instructions. + we can use 2 iterations and external (unsigned b) to avoid that ineffective code genaration. */ + unsigned b; + ONE_ITER + ONE_ITER } - - srcLim = src + num; + + ip = p->ip64 + (CBcj2Enc_ip_unsigned)(SizeT)(dest - p->bufs[BCJ2_STREAM_MAIN]); + p->bufs[BCJ2_STREAM_MAIN] = dest; + p->ip64 = ip; - if (p->prevByte == 0x0F && (src[0] & 0xF0) == 0x80) - *dest = src[0]; - else for (;;) + if (src == srcLim) { - Byte b = *src; - *dest = b; - if (b != 0x0F) + WRITE_CONTEXT_AND_SRC + if (src != p->srcLim) { - if ((b & 0xFE) == 0xE8) - break; - dest++; - if (++src != srcLim) - continue; - break; + p->state = BCJ2_STREAM_MAIN; + return; } - dest++; - if (++src == srcLim) - break; - if ((*src & 0xF0) != 0x80) - continue; - *dest = *src; + /* (p->src == p->srcLim) + (p->state == BCJ2_ENC_STATE_ORIG) */ + if (p->finishMode != BCJ2_ENC_FINISH_MODE_END_STREAM) + return; + /* (p->finishMode == BCJ2_ENC_FINISH_MODE_END_STREAM */ + // (p->flushRem == 5); + p->isFlushState = 1; break; } - - num = (SizeT)(src - p->src); - - if (src == srcLim) - { - p->prevByte = src[-1]; - p->bufs[BCJ2_STREAM_MAIN] = dest; - p->src = src; - p->ip += (UInt32)num; - continue; - } - + src++; + // p->src = src; + } + // ip = p->ip; // for marker version + /* marker was found */ + /* (v) contains marker that was found: + bits [NUM_SHIFT_BITS : NUM_SHIFT_BITS + 7] + : value of src[-2] : xx/xx/0f + bits [0 : 7] : value of src[-1] : e8/e9/8x + */ + { { - Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - BoolInt needConvert; - - p->bufs[BCJ2_STREAM_MAIN] = dest + 1; - p->ip += (UInt32)num + 1; - src++; - - needConvert = False; - + #if NUM_SHIFT_BITS != 24 + v &= ~(UInt32)CONV_FLAG; + #endif + // UInt32 relat = 0; if ((SizeT)(p->srcLim - src) >= 4) { - UInt32 relatVal = GetUi32(src); - if ((p->fileSize == 0 || (UInt32)(p->ip + 4 + relatVal - p->fileIp) < p->fileSize) - && ((relatVal + p->relatLimit) >> 1) < p->relatLimit) - needConvert = True; + /* + if (relat != 0 || (Byte)v != 0xe8) + BoolInt isBigOffset = True; + */ + const UInt32 relat = GetUi32(src); + /* + #define EXCLUDE_FLAG ((UInt32)1 << 4) + #define NEED_CONVERT(rel) ((((rel) + EXCLUDE_FLAG) & (0 - EXCLUDE_FLAG * 2)) != 0) + if (p->relatExcludeBits != 0) + { + const UInt32 flag = (UInt32)1 << (p->relatExcludeBits - 1); + isBigOffset = (((relat + flag) & (0 - flag * 2)) != 0); + } + // isBigOffset = False; // for debug + */ + ip -= p->fileIp64; + // Use the following if check, if (ip) is 64-bit: + if (ip > (((v + 0x20) >> 5) & 1)) // 23.00 : we eliminate milti-block overlap for (Of 80) and (e8/e9) + if ((CBcj2Enc_ip_unsigned)((CBcj2Enc_ip_signed)ip + 4 + (Int32)relat) <= p->fileSize64_minus1) + if (((UInt32)(relat + p->relatLimit) >> 1) < p->relatLimit) + v |= CONV_FLAG; } - + else if (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) { - UInt32 bound; - unsigned ttt; - Byte b = src[-1]; - CProb *prob = p->probs + (unsigned)(b == 0xE8 ? 2 + (unsigned)context : (b == 0xE9 ? 1 : 0)); - - ttt = *prob; - bound = (p->range >> kNumModelBits) * ttt; - - if (!needConvert) + // (p->srcLim - src < 4) + // /* + // for non-marker version + p->ip64--; // p->ip = ip - 1; + p->bufs[BCJ2_STREAM_MAIN]--; + src--; + v >>= NUM_SHIFT_BITS; + // (0 < p->srcLim - p->src <= 4) + // */ + // v |= MARKER_FLAG; // for marker version + /* (p->state == BCJ2_ENC_STATE_ORIG) */ + WRITE_CONTEXT_AND_SRC + return; + } + { + const unsigned c = ((v + 0x17) >> 6) & 1; + CBcj2Prob *prob = p->probs + (unsigned) + (((0 - c) & (Byte)(v >> NUM_SHIFT_BITS)) + c + ((v >> 5) & 1)); + /* + ((Byte)v == 0xe8 ? 2 + ((Byte)(v >> 8)) : + ((Byte)v < 0xe8 ? 0 : 1)); // ((v >> 5) & 1)); + */ + const unsigned ttt = *prob; + const UInt32 bound = (p->range >> kNumBitModelTotalBits) * ttt; + if ((v & CONV_FLAG) == 0) { + // static int yyy = 0; yyy++; printf("\n!needConvert = %d\n", yyy); + // v = (Byte)v; // for marker version p->range = bound; - *prob = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); - p->src = src; - p->prevByte = b; + *prob = (CBcj2Prob)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + // WRITE_CONTEXT_AND_SRC continue; } - p->low += bound; p->range -= bound; - *prob = (CProb)(ttt - (ttt >> kNumMoveBits)); - + *prob = (CBcj2Prob)(ttt - (ttt >> kNumMoveBits)); + } + // p->context = src[3]; + { + // const unsigned cj = ((Byte)v == 0xe8 ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP); + const unsigned cj = (((v + 0x57) >> 6) & 1) + BCJ2_STREAM_CALL; + ip = p->ip64; + v = GetUi32(src); // relat + ip += 4; + p->ip64 = ip; + src += 4; + // p->src = src; { - UInt32 relatVal = GetUi32(src); - UInt32 absVal; - p->ip += 4; - absVal = p->ip + relatVal; - p->prevByte = src[3]; - src += 4; - p->src = src; + const UInt32 absol = (UInt32)ip + v; + Byte *cur = p->bufs[cj]; + v >>= 24; + // WRITE_CONTEXT + if (cur == p->lims[cj]) { - unsigned cj = (b == 0xE8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; - Byte *cur = p->bufs[cj]; - if (cur == p->lims[cj]) - { - p->state = cj; - p->tempTarget = absVal; - return; - } - SetBe32(cur, absVal); - p->bufs[cj] = cur + 4; + p->state = cj; + p->tempTarget = absol; + WRITE_CONTEXT_AND_SRC + return; } + SetBe32a(cur, absol) + p->bufs[cj] = cur + 4; } } } } - } + } // end of loop } - if (p->finishMode != BCJ2_ENC_FINISH_MODE_END_STREAM) - return; - - for (; p->flushPos < 5; p->flushPos++) - if (RangeEnc_ShiftLow(p)) + for (; p->flushRem != 0; p->flushRem--) + if (Bcj2_RangeEnc_ShiftLow(p)) return; - p->state = BCJ2_ENC_STATE_OK; + p->state = BCJ2_ENC_STATE_FINISHED; } +/* +BCJ2 encoder needs look ahead for up to 4 bytes in (src) buffer. +So base function Bcj2Enc_Encode_2() + in BCJ2_ENC_FINISH_MODE_CONTINUE mode can return with + (p->state == BCJ2_ENC_STATE_ORIG && p->src < p->srcLim) +Bcj2Enc_Encode() solves that look ahead problem by using p->temp[] buffer. + so if (p->state == BCJ2_ENC_STATE_ORIG) after Bcj2Enc_Encode(), + then (p->src == p->srcLim). + And the caller's code is simpler with Bcj2Enc_Encode(). +*/ + +Z7_NO_INLINE void Bcj2Enc_Encode(CBcj2Enc *p) { - PRF(printf("\n")); - PRF(printf("---- ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); - + PRF2("\n----") if (p->tempPos != 0) { + /* extra: number of bytes that were copied from (src) to (temp) buffer in this call */ unsigned extra = 0; - + /* We will touch only minimal required number of bytes in input (src) stream. + So we will add input bytes from (src) stream to temp[] with step of 1 byte. + We don't add new bytes to temp[] before Bcj2Enc_Encode_2() call + in first loop iteration because + - previous call of Bcj2Enc_Encode() could use another (finishMode), + - previous call could finish with (p->state != BCJ2_ENC_STATE_ORIG). + the case with full temp[] buffer (p->tempPos == 4) is possible here. + */ for (;;) { + // (0 < p->tempPos <= 5) // in non-marker version + /* p->src : the current src data position including extra bytes + that were copied to temp[] buffer in this call */ const Byte *src = p->src; const Byte *srcLim = p->srcLim; - EBcj2Enc_FinishMode finishMode = p->finishMode; - - p->src = p->temp; - p->srcLim = p->temp + p->tempPos; + const EBcj2Enc_FinishMode finishMode = p->finishMode; if (src != srcLim) + { + /* if there are some src data after the data copied to temp[], + then we use MODE_CONTINUE for temp data */ p->finishMode = BCJ2_ENC_FINISH_MODE_CONTINUE; - - PRF(printf(" ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); - + } + p->src = p->temp; + p->srcLim = p->temp + p->tempPos; + PRF2(" ") Bcj2Enc_Encode_2(p); - { - unsigned num = (unsigned)(p->src - p->temp); - unsigned tempPos = p->tempPos - num; + const unsigned num = (unsigned)(p->src - p->temp); + const unsigned tempPos = p->tempPos - num; unsigned i; p->tempPos = tempPos; for (i = 0; i < tempPos; i++) - p->temp[i] = p->temp[(size_t)i + num]; - + p->temp[i] = p->temp[(SizeT)i + num]; + // tempPos : number of bytes in temp buffer p->src = src; p->srcLim = srcLim; p->finishMode = finishMode; - - if (p->state != BCJ2_ENC_STATE_ORIG || src == srcLim) + if (p->state != BCJ2_ENC_STATE_ORIG) + { + // (p->tempPos <= 4) // in non-marker version + /* if (the reason of exit from Bcj2Enc_Encode_2() + is not BCJ2_ENC_STATE_ORIG), + then we exit from Bcj2Enc_Encode() with same reason */ + // optional code begin : we rollback (src) and tempPos, if it's possible: + if (extra >= tempPos) + extra = tempPos; + p->src = src - extra; + p->tempPos = tempPos - extra; + // optional code end : rollback of (src) and tempPos return; - + } + /* (p->tempPos <= 4) + (p->state == BCJ2_ENC_STATE_ORIG) + so encoder needs more data than in temp[] */ + if (src == srcLim) + return; // src buffer has no more input data. + /* (src != srcLim) + so we can provide more input data from src for Bcj2Enc_Encode_2() */ if (extra >= tempPos) { - p->src = src - tempPos; + /* (extra >= tempPos) means that temp buffer contains + only data from src buffer of this call. + So now we can encode without temp buffer */ + p->src = src - tempPos; // rollback (src) p->tempPos = 0; break; } - - p->temp[tempPos] = src[0]; + // we append one additional extra byte from (src) to temp[] buffer: + p->temp[tempPos] = *src; p->tempPos = tempPos + 1; + // (0 < p->tempPos <= 5) // in non-marker version p->src = src + 1; extra++; } } } - PRF(printf("++++ ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); - + PRF2("++++") + // (p->tempPos == 0) Bcj2Enc_Encode_2(p); + PRF2("====") if (p->state == BCJ2_ENC_STATE_ORIG) { const Byte *src = p->src; - unsigned rem = (unsigned)(p->srcLim - src); - unsigned i; - for (i = 0; i < rem; i++) - p->temp[i] = src[i]; - p->tempPos = rem; - p->src = src + rem; + const Byte *srcLim = p->srcLim; + const unsigned rem = (unsigned)(srcLim - src); + /* (rem <= 4) here. + if (p->src != p->srcLim), then + - we copy non-processed bytes from (p->src) to temp[] buffer, + - we set p->src equal to p->srcLim. + */ + if (rem) + { + unsigned i = 0; + p->src = srcLim; + p->tempPos = rem; + // (0 < p->tempPos <= 4) + do + p->temp[i] = src[i]; + while (++i != rem); + } + // (p->tempPos <= 4) + // (p->src == p->srcLim) } } + +#undef PRF2 +#undef CONV_FLAG +#undef MARKER_FLAG +#undef WRITE_CONTEXT +#undef WRITE_CONTEXT_AND_SRC +#undef ONE_ITER +#undef NUM_SHIFT_BITS +#undef kTopValue +#undef kNumBitModelTotalBits +#undef kBitModelTotal +#undef kNumMoveBits diff --git a/multiarc/src/formats/7z/C/Blake2.h b/multiarc/src/formats/7z/C/Blake2.h old mode 100755 new mode 100644 index 14f3cb64e..801ea7ac5 --- a/multiarc/src/formats/7z/C/Blake2.h +++ b/multiarc/src/formats/7z/C/Blake2.h @@ -1,47 +1,104 @@ -/* Blake2.h -- BLAKE2 Hash -2015-06-30 : Igor Pavlov : Public domain -2015 : Samuel Neves : Public domain */ +/* Blake2.h -- BLAKE2sp Hash +2024-01-17 : Igor Pavlov : Public domain */ -#ifndef __BLAKE2_H -#define __BLAKE2_H +#ifndef ZIP7_INC_BLAKE2_H +#define ZIP7_INC_BLAKE2_H #include "7zTypes.h" -EXTERN_C_BEGIN +#if 0 +#include "Compiler.h" +#include "CpuArch.h" +#if defined(MY_CPU_X86_OR_AMD64) +#if defined(__SSE2__) \ + || defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) +#include // SSE2 +#endif -#define BLAKE2S_BLOCK_SIZE 64 -#define BLAKE2S_DIGEST_SIZE 32 -#define BLAKE2SP_PARALLEL_DEGREE 8 +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) +#include +#if defined(__clang__) +#include +#include +#endif +#endif // avx2 +#endif // MY_CPU_X86_OR_AMD64 +#endif // 0 -typedef struct -{ - UInt32 h[8]; - UInt32 t[2]; - UInt32 f[2]; - Byte buf[BLAKE2S_BLOCK_SIZE]; - UInt32 bufPos; - UInt32 lastNode_f1; - UInt32 dummy[2]; /* for sizeof(CBlake2s) alignment */ -} CBlake2s; - -/* You need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() */ -/* -void Blake2s_Init0(CBlake2s *p); -void Blake2s_Update(CBlake2s *p, const Byte *data, size_t size); -void Blake2s_Final(CBlake2s *p, Byte *digest); -*/ +EXTERN_C_BEGIN +#define Z7_BLAKE2S_BLOCK_SIZE 64 +#define Z7_BLAKE2S_DIGEST_SIZE 32 +#define Z7_BLAKE2SP_PARALLEL_DEGREE 8 +#define Z7_BLAKE2SP_NUM_STRUCT_WORDS 16 +#if 1 || defined(Z7_BLAKE2SP_USE_FUNCTIONS) +typedef void (Z7_FASTCALL *Z7_BLAKE2SP_FUNC_COMPRESS)(UInt32 *states, const Byte *data, const Byte *end); +typedef void (Z7_FASTCALL *Z7_BLAKE2SP_FUNC_INIT)(UInt32 *states); +#endif + +// it's required that CBlake2sp is aligned for 32-bytes, +// because the code can use unaligned access with sse and avx256. +// but 64-bytes alignment can be better. +MY_ALIGN(64) typedef struct { - CBlake2s S[BLAKE2SP_PARALLEL_DEGREE]; - unsigned bufPos; -} CBlake2sp; + union + { +#if 0 +#if defined(MY_CPU_X86_OR_AMD64) +#if defined(__SSE2__) \ + || defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) + __m128i _pad_align_128bit[4]; +#endif // sse2 +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + __m256i _pad_align_256bit[2]; +#endif // avx2 +#endif // x86 +#endif // 0 + void * _pad_align_ptr[8]; + UInt32 _pad_align_32bit[16]; + struct + { + unsigned cycPos; + unsigned _pad_unused; +#if 1 || defined(Z7_BLAKE2SP_USE_FUNCTIONS) + Z7_BLAKE2SP_FUNC_COMPRESS func_Compress_Fast; + Z7_BLAKE2SP_FUNC_COMPRESS func_Compress_Single; + Z7_BLAKE2SP_FUNC_INIT func_Init; + Z7_BLAKE2SP_FUNC_INIT func_Final; +#endif + } header; + } u; + // MY_ALIGN(64) + UInt32 states[Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2SP_NUM_STRUCT_WORDS]; + // MY_ALIGN(64) + UInt32 buf32[Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2SP_NUM_STRUCT_WORDS * 2]; +} CBlake2sp; +BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo); void Blake2sp_Init(CBlake2sp *p); +void Blake2sp_InitState(CBlake2sp *p); void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size); void Blake2sp_Final(CBlake2sp *p, Byte *digest); +void z7_Black2sp_Prepare(void); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/Blake2s.c b/multiarc/src/formats/7z/C/Blake2s.c old mode 100755 new mode 100644 index 3c56a8b82..abb907d76 --- a/multiarc/src/formats/7z/C/Blake2s.c +++ b/multiarc/src/formats/7z/C/Blake2s.c @@ -1,244 +1,2672 @@ -/* Blake2s.c -- BLAKE2s and BLAKE2sp Hash -2021-02-09 : Igor Pavlov : Public domain -2015 : Samuel Neves : Public domain */ +/* Blake2s.c -- BLAKE2sp Hash +2024-05-18 : Igor Pavlov : Public domain +2015-2019 : Samuel Neves : original code : CC0 1.0 Universal (CC0 1.0). */ +#include "Precomp.h" + +// #include #include #include "Blake2.h" -#include "CpuArch.h" #include "RotateDefs.h" +#include "Compiler.h" +#include "CpuArch.h" + +/* + if defined(__AVX512F__) && defined(__AVX512VL__) + { + we define Z7_BLAKE2S_USE_AVX512_ALWAYS, + but the compiler can use avx512 for any code. + } + else if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + { we use avx512 only for sse* and avx* branches of code. } +*/ +// #define Z7_BLAKE2S_USE_AVX512_ALWAYS // for debug + +#if defined(__SSE2__) + #define Z7_BLAKE2S_USE_VECTORS +#elif defined(MY_CPU_X86_OR_AMD64) + #if defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) + #define Z7_BLAKE2S_USE_VECTORS + #endif +#endif + +#ifdef Z7_BLAKE2S_USE_VECTORS + +#define Z7_BLAKE2SP_USE_FUNCTIONS + +// define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED, if CBlake2sp can be non aligned for 32-bytes. +// #define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED + +// SSSE3 : for _mm_shuffle_epi8 (pshufb) that improves the performance for 5-15%. +#if defined(__SSSE3__) + #define Z7_BLAKE2S_USE_SSSE3 +#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000) + #define Z7_BLAKE2S_USE_SSSE3 +#endif + +#ifdef Z7_BLAKE2S_USE_SSSE3 +/* SSE41 : for _mm_insert_epi32 (pinsrd) + it can slightly reduce code size and improves the performance in some cases. + it's used only for last 512-1024 bytes, if FAST versions (2 or 3) of vector algos are used. + it can be used for all blocks in another algos (4+). +*/ +#if defined(__SSE4_1__) + #define Z7_BLAKE2S_USE_SSE41 +#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000) + #define Z7_BLAKE2S_USE_SSE41 +#endif +#endif // SSSE3 + +#if defined(__GNUC__) || defined(__clang__) +#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__)) + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("avx512vl,avx512f"))) +#else + #if defined(Z7_BLAKE2S_USE_SSE41) + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse4.1"))) + #elif defined(Z7_BLAKE2S_USE_SSSE3) + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("ssse3"))) + #else + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse2"))) + #endif +#endif +#endif + + +#if defined(__AVX2__) + #define Z7_BLAKE2S_USE_AVX2 +#else + #if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) + #define Z7_BLAKE2S_USE_AVX2 + #ifdef Z7_BLAKE2S_USE_AVX2 +#if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) && !(defined(__AVX512F__) && defined(__AVX512VL__)) + #define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx512vl,avx512f"))) +#else + #define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx2"))) +#endif + #endif + #elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + #if (Z7_MSC_VER_ORIGINAL == 1900) + #pragma warning(disable : 4752) // found Intel(R) Advanced Vector Extensions; consider using /arch:AVX + #endif + #define Z7_BLAKE2S_USE_AVX2 + #endif +#endif + +#ifdef Z7_BLAKE2S_USE_SSE41 +#include // SSE4.1 +#elif defined(Z7_BLAKE2S_USE_SSSE3) +#include // SSSE3 +#else +#include // SSE2 +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2 +#include +#if defined(__clang__) +#include +#include +#endif +#endif // avx2 + + +#if defined(__AVX512F__) && defined(__AVX512VL__) + // && defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL > 1930) + #ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS + #define Z7_BLAKE2S_USE_AVX512_ALWAYS + #endif + // #pragma message ("=== Blake2s AVX512") +#endif + + +#define Z7_BLAKE2S_USE_V128_FAST +// for speed optimization for small messages: +// #define Z7_BLAKE2S_USE_V128_WAY2 + +#ifdef Z7_BLAKE2S_USE_AVX2 + +// for debug: +// gather is slow +// #define Z7_BLAKE2S_USE_GATHER + + #define Z7_BLAKE2S_USE_AVX2_FAST +// for speed optimization for small messages: +// #define Z7_BLAKE2S_USE_AVX2_WAY2 +// #define Z7_BLAKE2S_USE_AVX2_WAY4 +#if defined(Z7_BLAKE2S_USE_AVX2_WAY2) || \ + defined(Z7_BLAKE2S_USE_AVX2_WAY4) + #define Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#endif +#endif + + #define Z7_BLAKE2SP_ALGO_DEFAULT 0 + #define Z7_BLAKE2SP_ALGO_SCALAR 1 +#ifdef Z7_BLAKE2S_USE_V128_FAST + #define Z7_BLAKE2SP_ALGO_V128_FAST 2 +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + #define Z7_BLAKE2SP_ALGO_V256_FAST 3 +#endif + #define Z7_BLAKE2SP_ALGO_V128_WAY1 4 +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + #define Z7_BLAKE2SP_ALGO_V128_WAY2 5 +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + #define Z7_BLAKE2SP_ALGO_V256_WAY2 6 +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + #define Z7_BLAKE2SP_ALGO_V256_WAY4 7 +#endif + +#endif // Z7_BLAKE2S_USE_VECTORS + + + -#define rotr32 rotrFixed +#define BLAKE2S_FINAL_FLAG (~(UInt32)0) +#define NSW Z7_BLAKE2SP_NUM_STRUCT_WORDS +#define SUPER_BLOCK_SIZE (Z7_BLAKE2S_BLOCK_SIZE * Z7_BLAKE2SP_PARALLEL_DEGREE) +#define SUPER_BLOCK_MASK (SUPER_BLOCK_SIZE - 1) -#define BLAKE2S_NUM_ROUNDS 10 -#define BLAKE2S_FINAL_FLAG (~(UInt32)0) +#define V_INDEX_0_0 0 +#define V_INDEX_1_0 1 +#define V_INDEX_2_0 2 +#define V_INDEX_3_0 3 +#define V_INDEX_0_1 4 +#define V_INDEX_1_1 5 +#define V_INDEX_2_1 6 +#define V_INDEX_3_1 7 +#define V_INDEX_0_2 8 +#define V_INDEX_1_2 9 +#define V_INDEX_2_2 10 +#define V_INDEX_3_2 11 +#define V_INDEX_0_3 12 +#define V_INDEX_1_3 13 +#define V_INDEX_2_3 14 +#define V_INDEX_3_3 15 +#define V_INDEX_4_0 0 +#define V_INDEX_5_0 1 +#define V_INDEX_6_0 2 +#define V_INDEX_7_0 3 +#define V_INDEX_7_1 4 +#define V_INDEX_4_1 5 +#define V_INDEX_5_1 6 +#define V_INDEX_6_1 7 +#define V_INDEX_6_2 8 +#define V_INDEX_7_2 9 +#define V_INDEX_4_2 10 +#define V_INDEX_5_2 11 +#define V_INDEX_5_3 12 +#define V_INDEX_6_3 13 +#define V_INDEX_7_3 14 +#define V_INDEX_4_3 15 +#define V(row, col) v[V_INDEX_ ## row ## _ ## col] + +#define k_Blake2s_IV_0 0x6A09E667UL +#define k_Blake2s_IV_1 0xBB67AE85UL +#define k_Blake2s_IV_2 0x3C6EF372UL +#define k_Blake2s_IV_3 0xA54FF53AUL +#define k_Blake2s_IV_4 0x510E527FUL +#define k_Blake2s_IV_5 0x9B05688CUL +#define k_Blake2s_IV_6 0x1F83D9ABUL +#define k_Blake2s_IV_7 0x5BE0CD19UL + +#define KIV(n) (k_Blake2s_IV_## n) + +#ifdef Z7_BLAKE2S_USE_VECTORS +MY_ALIGN(16) static const UInt32 k_Blake2s_IV[8] = { - 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, - 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL + KIV(0), KIV(1), KIV(2), KIV(3), KIV(4), KIV(5), KIV(6), KIV(7) }; +#endif -static const Byte k_Blake2s_Sigma[BLAKE2S_NUM_ROUNDS][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; +#define STATE_T(s) ((s) + 8) +#define STATE_F(s) ((s) + 10) +#ifdef Z7_BLAKE2S_USE_VECTORS -static void Blake2s_Init0(CBlake2s *p) +#define LOAD_128(p) _mm_load_si128 ((const __m128i *)(const void *)(p)) +#define LOADU_128(p) _mm_loadu_si128((const __m128i *)(const void *)(p)) +#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED + // here we use unaligned load and stores + // use this branch if CBlake2sp can be unaligned for 16 bytes + #define STOREU_128(p, r) _mm_storeu_si128((__m128i *)(void *)(p), r) + #define LOAD_128_FROM_STRUCT(p) LOADU_128(p) + #define STORE_128_TO_STRUCT(p, r) STOREU_128(p, r) +#else + // here we use aligned load and stores + // use this branch if CBlake2sp is aligned for 16 bytes + #define STORE_128(p, r) _mm_store_si128((__m128i *)(void *)(p), r) + #define LOAD_128_FROM_STRUCT(p) LOAD_128(p) + #define STORE_128_TO_STRUCT(p, r) STORE_128(p, r) +#endif + +#endif // Z7_BLAKE2S_USE_VECTORS + + +#if 0 +static void PrintState(const UInt32 *s, unsigned num) +{ + unsigned i; + printf("\n"); + for (i = 0; i < num; i++) + printf(" %08x", (unsigned)s[i]); +} +static void PrintStates2(const UInt32 *s, unsigned x, unsigned y) { unsigned i; - for (i = 0; i < 8; i++) - p->h[i] = k_Blake2s_IV[i]; - p->t[0] = 0; - p->t[1] = 0; - p->f[0] = 0; - p->f[1] = 0; - p->bufPos = 0; - p->lastNode_f1 = 0; + for (i = 0; i < y; i++) + PrintState(s + i * x, x); + printf("\n"); } +#endif + + +#define REP8_MACRO(m) { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) } + +#define BLAKE2S_NUM_ROUNDS 10 + +#if defined(Z7_BLAKE2S_USE_VECTORS) +#define ROUNDS_LOOP(mac) \ + { unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) mac(r) } +#endif +/* +#define ROUNDS_LOOP_2(mac) \ + { unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r += 2) { mac(r) mac(r + 1) } } +*/ +#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS) +#define ROUNDS_LOOP_UNROLLED(m) \ + { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) } +#endif +#define SIGMA_TABLE(M) \ + M( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ), \ + M( 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ), \ + M( 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ), \ + M( 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ), \ + M( 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ), \ + M( 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ), \ + M( 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ), \ + M( 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ), \ + M( 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ), \ + M( 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 ) -static void Blake2s_Compress(CBlake2s *p) +#define SIGMA_TABLE_MULT(m, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + { a0*m,a1*m,a2*m,a3*m,a4*m,a5*m,a6*m,a7*m,a8*m,a9*m,a10*m,a11*m,a12*m,a13*m,a14*m,a15*m } +#define SIGMA_TABLE_MULT_4( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + SIGMA_TABLE_MULT(4, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) + +// MY_ALIGN(32) +MY_ALIGN(16) +static const Byte k_Blake2s_Sigma_4[BLAKE2S_NUM_ROUNDS][16] = + { SIGMA_TABLE(SIGMA_TABLE_MULT_4) }; + +#define GET_SIGMA_PTR(p, index) \ + ((const void *)((const Byte *)(const void *)(p) + (index))) + +#define GET_STATE_TABLE_PTR_FROM_BYTE_POS(s, pos) \ + ((UInt32 *)(void *)((Byte *)(void *)(s) + (pos))) + + +#ifdef Z7_BLAKE2S_USE_VECTORS + + +#if 0 + // use loading constants from memory + // is faster for some compilers. + #define KK4(n) KIV(n), KIV(n), KIV(n), KIV(n) +MY_ALIGN(64) +static const UInt32 k_Blake2s_IV_WAY4[]= { - UInt32 m[16]; - UInt32 v[16]; - + KK4(0), KK4(1), KK4(2), KK4(3), KK4(4), KK4(5), KK4(6), KK4(7) +}; + #define GET_128_IV_WAY4(i) LOAD_128(k_Blake2s_IV_WAY4 + 4 * (i)) +#else + // use constant generation: + #define GET_128_IV_WAY4(i) _mm_set1_epi32((Int32)KIV(i)) +#endif + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#define GET_CONST_128_FROM_ARRAY32(k) \ + _mm_set_epi32((Int32)(k)[3], (Int32)(k)[2], (Int32)(k)[1], (Int32)(k)[0]) +#endif + + +#if 0 +#define k_r8 _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1) +#define k_r16 _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2) +#define k_inc _mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE) +#define k_iv0_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0) +#define k_iv4_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4) +#else +#if defined(Z7_BLAKE2S_USE_SSSE3) && \ + !defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) +MY_ALIGN(16) static const Byte k_r8_arr [16] = { 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 }; +MY_ALIGN(16) static const Byte k_r16_arr[16] = { 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 }; +#define k_r8 LOAD_128(k_r8_arr) +#define k_r16 LOAD_128(k_r16_arr) +#endif +MY_ALIGN(16) static const UInt32 k_inc_arr[4] = { Z7_BLAKE2S_BLOCK_SIZE, 0, 0, 0 }; +#define k_inc LOAD_128(k_inc_arr) +#define k_iv0_128 LOAD_128(k_Blake2s_IV + 0) +#define k_iv4_128 LOAD_128(k_Blake2s_IV + 4) +#endif + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW + +#ifdef Z7_BLAKE2S_USE_AVX2 +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000) + #define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1) +#else + #define MY_mm256_set_m128i _mm256_set_m128i +#endif + +#define SET_FROM_128(a) MY_mm256_set_m128i(a, a) + +#ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS +MY_ALIGN(32) static const Byte k_r8_arr_256 [32] = +{ + 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12, + 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 +}; +MY_ALIGN(32) static const Byte k_r16_arr_256[32] = +{ + 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13, + 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +}; +#define k_r8_256 LOAD_256(k_r8_arr_256) +#define k_r16_256 LOAD_256(k_r16_arr_256) +#endif + +// #define k_r8_256 SET_FROM_128(_mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)) +// #define k_r16_256 SET_FROM_128(_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)) +// #define k_inc_256 SET_FROM_128(_mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)) +// #define k_iv0_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0)) +#define k_iv4_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4)) +#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#endif + + +/* +IPC(TP) ports: +1 p__5 : skl- : SSE : shufps : _mm_shuffle_ps +2 p_15 : icl+ +1 p__5 : nhm-bdw : SSE : xorps : _mm_xor_ps +3 p015 : skl+ + +3 p015 : SSE2 : pxor : _mm_xor_si128 +2 p_15: snb-bdw : SSE2 : padd : _mm_add_epi32 +2 p0_5: mrm-wsm : +3 p015 : skl+ + +2 p_15 : ivb-,icl+ : SSE2 : punpcklqdq, punpckhqdq, punpckldq, punpckhdq +2 p_15 : : SSE2 : pshufd : _mm_shuffle_epi32 +2 p_15 : : SSE2 : pshuflw : _mm_shufflelo_epi16 +2 p_15 : : SSE2 : psrldq : +2 p_15 : : SSE3 : pshufb : _mm_shuffle_epi8 +2 p_15 : : SSE4 : pblendw : _mm_blend_epi16 +1 p__5 : hsw-skl : * + +1 p0 : SSE2 : pslld (i8) : _mm_slli_si128 +2 p01 : skl+ : + +2 p_15 : ivb- : SSE3 : palignr +1 p__5 : hsw+ + +2 p_15 + p23 : ivb-, icl+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, m32, i8) +1 p__5 + p23 : hsw-skl +1 p_15 + p5 : ivb-, ice+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, r32, i8) +0.5 2*p5 : hsw-skl + +2 p23 : SSE2 : movd (m32) +3 p23A : adl : +1 p5: : SSE2 : movd (r32) +*/ + +#if 0 && defined(__XOP__) +// we must debug and test __XOP__ instruction +#include +#include + #define LOAD_ROTATE_CONSTS + #define MM_ROR_EPI32(r, c) _mm_roti_epi32(r, -(c)) + #define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED +#elif 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + #define LOAD_ROTATE_CONSTS + #define MM_ROR_EPI32(r, c) _mm_ror_epi32(r, c) + #define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED +#else + +// MSVC_1937+ uses "orps" instruction for _mm_or_si128(). +// But "orps" has low throughput: TP=1 for bdw-nhm. +// So it can be better to use _mm_add_epi32()/"paddd" (TP=2 for bdw-nhm) instead of "xorps". +// But "orps" is fast for modern cpus (skl+). +// So we are default with "or" version: +#if 0 || 0 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1937 + // minor optimization for some old cpus, if "xorps" is slow. + #define MM128_EPI32_OR_or_ADD _mm_add_epi32 +#else + #define MM128_EPI32_OR_or_ADD _mm_or_si128 +#endif + + #define MM_ROR_EPI32_VIA_SHIFT(r, c)( \ + MM128_EPI32_OR_or_ADD( \ + _mm_srli_epi32((r), (c)), \ + _mm_slli_epi32((r), 32-(c)))) + #if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41) + #define LOAD_ROTATE_CONSTS \ + const __m128i r8 = k_r8; \ + const __m128i r16 = k_r16; + #define MM_ROR_EPI32(r, c) ( \ + ( 8==(c)) ? _mm_shuffle_epi8(r,r8) \ + : (16==(c)) ? _mm_shuffle_epi8(r,r16) \ + : MM_ROR_EPI32_VIA_SHIFT(r, c)) + #else + #define LOAD_ROTATE_CONSTS + #define MM_ROR_EPI32(r, c) ( \ + (16==(c)) ? _mm_shufflehi_epi16(_mm_shufflelo_epi16(r, 0xb1), 0xb1) \ + : MM_ROR_EPI32_VIA_SHIFT(r, c)) + #endif +#endif + +/* +we have 3 main ways to load 4 32-bit integers to __m128i: + 1) SSE2: _mm_set_epi32() + 2) SSE2: _mm_unpacklo_epi64() / _mm_unpacklo_epi32 / _mm_cvtsi32_si128() + 3) SSE41: _mm_insert_epi32() and _mm_cvtsi32_si128() +good compiler for _mm_set_epi32() generates these instructions: +{ + movd xmm, [m32]; vpunpckldq; vpunpckldq; vpunpcklqdq; +} +good new compiler generates one instruction +{ + for _mm_insert_epi32() : { pinsrd xmm, [m32], i } + for _mm_cvtsi32_si128() : { movd xmm, [m32] } +} +but vc2010 generates slow pair of instructions: +{ + for _mm_insert_epi32() : { mov r32, [m32]; pinsrd xmm, r32, i } + for _mm_cvtsi32_si128() : { mov r32, [m32]; movd xmm, r32 } +} +_mm_insert_epi32() (pinsrd) code reduces xmm register pressure +in comparison with _mm_set_epi32() (movd + vpunpckld) code. +Note that variant with "movd xmm, r32" can be more slow, +but register pressure can be more important. +So we can force to "pinsrd" always. +*/ +// #if !defined(Z7_MSC_VER_ORIGINAL) || Z7_MSC_VER_ORIGINAL > 1600 || defined(MY_CPU_X86) +#ifdef Z7_BLAKE2S_USE_SSE41 + /* _mm_set_epi32() can be more effective for GCC and CLANG + _mm_insert_epi32() is more effective for MSVC */ + #if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) + #define Z7_BLAKE2S_USE_INSERT_INSTRUCTION + #endif +#endif // USE_SSE41 +// #endif + +#ifdef Z7_BLAKE2S_USE_INSERT_INSTRUCTION + // for SSE4.1 +#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ + _mm_insert_epi32( \ + _mm_insert_epi32( \ + _mm_insert_epi32( \ + _mm_cvtsi32_si128( \ + *(const Int32 *)p0), \ + *(const Int32 *)p1, 1), \ + *(const Int32 *)p2, 2), \ + *(const Int32 *)p3, 3) +#elif 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) +/* MSVC 1400 implements _mm_set_epi32() via slow memory write/read. + Also _mm_unpacklo_epi32 is more effective for another MSVC compilers. + But _mm_set_epi32() is more effective for GCC and CLANG. + So we use _mm_unpacklo_epi32 for MSVC only */ +#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ + _mm_unpacklo_epi64( \ + _mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p0), \ + _mm_cvtsi32_si128(*(const Int32 *)p1)), \ + _mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p2), \ + _mm_cvtsi32_si128(*(const Int32 *)p3))) +#else +#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ + _mm_set_epi32( \ + *(const Int32 *)p3, \ + *(const Int32 *)p2, \ + *(const Int32 *)p1, \ + *(const Int32 *)p0) +#endif + +#define SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) \ + MM_LOAD_EPI32_FROM_4_POINTERS( \ + GET_SIGMA_PTR(input, i0), \ + GET_SIGMA_PTR(input, i1), \ + GET_SIGMA_PTR(input, i2), \ + GET_SIGMA_PTR(input, i3)) + +#define SET_ROW_FROM_SIGMA(input, sigma_index) \ + SET_ROW_FROM_SIGMA_BASE(input, \ + sigma[(sigma_index) ], \ + sigma[(sigma_index) + 2 * 1], \ + sigma[(sigma_index) + 2 * 2], \ + sigma[(sigma_index) + 2 * 3]) \ + + +#define ADD_128(a, b) _mm_add_epi32(a, b) +#define XOR_128(a, b) _mm_xor_si128(a, b) + +#define D_ADD_128(dest, src) dest = ADD_128(dest, src) +#define D_XOR_128(dest, src) dest = XOR_128(dest, src) +#define D_ROR_128(dest, shift) dest = MM_ROR_EPI32(dest, shift) +#define D_ADD_EPI64_128(dest, src) dest = _mm_add_epi64(dest, src) + + +#define AXR(a, b, d, shift) \ + D_ADD_128(a, b); \ + D_XOR_128(d, a); \ + D_ROR_128(d, shift); + +#define AXR2(a, b, c, d, input, sigma_index, shift1, shift2) \ + a = _mm_add_epi32 (a, SET_ROW_FROM_SIGMA(input, sigma_index)); \ + AXR(a, b, d, shift1) \ + AXR(c, d, b, shift2) + +#define ROTATE_WORDS_TO_RIGHT(a, n) \ + a = _mm_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3)); + +#define AXR4(a, b, c, d, input, sigma_index) \ + AXR2(a, b, c, d, input, sigma_index, 16, 12) \ + AXR2(a, b, c, d, input, sigma_index + 1, 8, 7) \ + +#define RR2(a, b, c, d, input) \ + { \ + AXR4(a, b, c, d, input, 0) \ + ROTATE_WORDS_TO_RIGHT(b, 1) \ + ROTATE_WORDS_TO_RIGHT(c, 2) \ + ROTATE_WORDS_TO_RIGHT(d, 3) \ + AXR4(a, b, c, d, input, 8) \ + ROTATE_WORDS_TO_RIGHT(b, 3) \ + ROTATE_WORDS_TO_RIGHT(c, 2) \ + ROTATE_WORDS_TO_RIGHT(d, 1) \ + } + + +/* +Way1: +per 64 bytes block: +10 rounds * 4 iters * (7 + 2) = 360 cycles = if pslld TP=1 + * (7 + 1) = 320 cycles = if pslld TP=2 (skl+) +additional operations per 7_op_iter : +4 movzx byte mem +1 movd mem +3 pinsrd mem +1.5 pshufd +*/ + +static +#if 0 || 0 && (defined(Z7_BLAKE2S_USE_V128_WAY2) || \ + defined(Z7_BLAKE2S_USE_V256_WAY2)) + Z7_NO_INLINE +#else + Z7_FORCE_INLINE +#endif +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2s_Compress_V128_Way1(UInt32 * const s, const Byte * const input) +{ + __m128i a, b, c, d; + __m128i f0, f1; + + LOAD_ROTATE_CONSTS + d = LOAD_128_FROM_STRUCT(STATE_T(s)); + c = k_iv0_128; + a = f0 = LOAD_128_FROM_STRUCT(s); + b = f1 = LOAD_128_FROM_STRUCT(s + 4); + D_ADD_EPI64_128(d, k_inc); + STORE_128_TO_STRUCT (STATE_T(s), d); + D_XOR_128(d, k_iv4_128); + +#define RR(r) { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \ + RR2(a, b, c, d, input) } + + ROUNDS_LOOP(RR) +#undef RR + + STORE_128_TO_STRUCT(s , XOR_128(f0, XOR_128(a, c))); + STORE_128_TO_STRUCT(s + 4, XOR_128(f1, XOR_128(b, d))); +} + + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_V128_Way1(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + do { - unsigned i; + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Blake2s_Compress_V128_Way1(s, data); + data += Z7_BLAKE2S_BLOCK_SIZE; + pos += Z7_BLAKE2S_BLOCK_SIZE; + pos &= SUPER_BLOCK_MASK; + } + while (data != end); +} + + +#if defined(Z7_BLAKE2S_USE_V128_WAY2) || \ + defined(Z7_BLAKE2S_USE_AVX2_WAY2) +#if 1 + #define Z7_BLAKE2S_CompressSingleBlock(s, data) \ + Blake2sp_Compress2_V128_Way1(s, data, \ + (const Byte *)(const void *)(data) + Z7_BLAKE2S_BLOCK_SIZE) +#else + #define Z7_BLAKE2S_CompressSingleBlock Blake2s_Compress_V128_Way1 +#endif +#endif + + +#if (defined(Z7_BLAKE2S_USE_AVX2_WAY_SLOW) || \ + defined(Z7_BLAKE2S_USE_V128_WAY2)) && \ + !defined(Z7_BLAKE2S_USE_GATHER) +#define AXR2_LOAD_INDEXES(sigma_index) \ + const unsigned i0 = sigma[(sigma_index)]; \ + const unsigned i1 = sigma[(sigma_index) + 2 * 1]; \ + const unsigned i2 = sigma[(sigma_index) + 2 * 2]; \ + const unsigned i3 = sigma[(sigma_index) + 2 * 3]; \ + +#define SET_ROW_FROM_SIGMA_W(input) \ + SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) +#endif + + +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + +#if 1 || !defined(Z7_BLAKE2S_USE_SSE41) +/* we use SET_ROW_FROM_SIGMA_BASE, that uses + (SSE4) _mm_insert_epi32(), if Z7_BLAKE2S_USE_INSERT_INSTRUCTION is defined + (SSE2) _mm_set_epi32() + MSVC can be faster for this branch: +*/ +#define AXR2_W(sigma_index, shift1, shift2) \ + { \ + AXR2_LOAD_INDEXES(sigma_index) \ + a0 = _mm_add_epi32(a0, SET_ROW_FROM_SIGMA_W(data)); \ + a1 = _mm_add_epi32(a1, SET_ROW_FROM_SIGMA_W(data + Z7_BLAKE2S_BLOCK_SIZE)); \ + AXR(a0, b0, d0, shift1) \ + AXR(a1, b1, d1, shift1) \ + AXR(c0, d0, b0, shift2) \ + AXR(c1, d1, b1, shift2) \ + } +#else +/* we use interleaved _mm_insert_epi32(): + GCC can be faster for this branch: +*/ +#define AXR2_W_PRE_INSERT(sigma_index, i) \ + { const unsigned ii = sigma[(sigma_index) + i * 2]; \ + t0 = _mm_insert_epi32(t0, *(const Int32 *)GET_SIGMA_PTR(data, ii), i); \ + t1 = _mm_insert_epi32(t1, *(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii), i); \ + } +#define AXR2_W(sigma_index, shift1, shift2) \ + { __m128i t0, t1; \ + { const unsigned ii = sigma[sigma_index]; \ + t0 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, ii)); \ + t1 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii)); \ + } \ + AXR2_W_PRE_INSERT(sigma_index, 1) \ + AXR2_W_PRE_INSERT(sigma_index, 2) \ + AXR2_W_PRE_INSERT(sigma_index, 3) \ + a0 = _mm_add_epi32(a0, t0); \ + a1 = _mm_add_epi32(a1, t1); \ + AXR(a0, b0, d0, shift1) \ + AXR(a1, b1, d1, shift1) \ + AXR(c0, d0, b0, shift2) \ + AXR(c1, d1, b1, shift2) \ + } +#endif + + +#define AXR4_W(sigma_index) \ + AXR2_W(sigma_index, 16, 12) \ + AXR2_W(sigma_index + 1, 8, 7) \ + +#define WW(r) \ + { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \ + AXR4_W(0) \ + ROTATE_WORDS_TO_RIGHT(b0, 1) \ + ROTATE_WORDS_TO_RIGHT(b1, 1) \ + ROTATE_WORDS_TO_RIGHT(c0, 2) \ + ROTATE_WORDS_TO_RIGHT(c1, 2) \ + ROTATE_WORDS_TO_RIGHT(d0, 3) \ + ROTATE_WORDS_TO_RIGHT(d1, 3) \ + AXR4_W(8) \ + ROTATE_WORDS_TO_RIGHT(b0, 3) \ + ROTATE_WORDS_TO_RIGHT(b1, 3) \ + ROTATE_WORDS_TO_RIGHT(c0, 2) \ + ROTATE_WORDS_TO_RIGHT(c1, 2) \ + ROTATE_WORDS_TO_RIGHT(d0, 1) \ + ROTATE_WORDS_TO_RIGHT(d1, 1) \ + } + + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_V128_Way2(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + end -= Z7_BLAKE2S_BLOCK_SIZE; + + if (data != end) + { + LOAD_ROTATE_CONSTS + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + __m128i a0, b0, c0, d0; + __m128i a1, b1, c1, d1; + { + const __m128i inc = k_inc; + const __m128i temp = k_iv4_128; + d0 = LOAD_128_FROM_STRUCT (STATE_T(s)); + d1 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); + D_ADD_EPI64_128(d0, inc); + D_ADD_EPI64_128(d1, inc); + STORE_128_TO_STRUCT (STATE_T(s ), d0); + STORE_128_TO_STRUCT (STATE_T(s + NSW), d1); + D_XOR_128(d0, temp); + D_XOR_128(d1, temp); + } + c1 = c0 = k_iv0_128; + a0 = LOAD_128_FROM_STRUCT(s); + b0 = LOAD_128_FROM_STRUCT(s + 4); + a1 = LOAD_128_FROM_STRUCT(s + NSW); + b1 = LOAD_128_FROM_STRUCT(s + NSW + 4); + + ROUNDS_LOOP (WW) + +#undef WW + + D_XOR_128(a0, c0); + D_XOR_128(b0, d0); + D_XOR_128(a1, c1); + D_XOR_128(b1, d1); + + D_XOR_128(a0, LOAD_128_FROM_STRUCT(s)); + D_XOR_128(b0, LOAD_128_FROM_STRUCT(s + 4)); + D_XOR_128(a1, LOAD_128_FROM_STRUCT(s + NSW)); + D_XOR_128(b1, LOAD_128_FROM_STRUCT(s + NSW + 4)); + + STORE_128_TO_STRUCT(s, a0); + STORE_128_TO_STRUCT(s + 4, b0); + STORE_128_TO_STRUCT(s + NSW, a1); + STORE_128_TO_STRUCT(s + NSW + 4, b1); + + data += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos &= SUPER_BLOCK_MASK; + } + while (data < end); + if (data != end) + return; + } + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Z7_BLAKE2S_CompressSingleBlock(s, data); + } +} +#endif // Z7_BLAKE2S_USE_V128_WAY2 + + +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + #define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way2 +#else + #define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way1 +#endif + + + +#ifdef Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED + #define ROT_128_8(x) MM_ROR_EPI32(x, 8) + #define ROT_128_16(x) MM_ROR_EPI32(x, 16) + #define ROT_128_7(x) MM_ROR_EPI32(x, 7) + #define ROT_128_12(x) MM_ROR_EPI32(x, 12) +#else +#if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41) + #define ROT_128_8(x) _mm_shuffle_epi8(x, r8) // k_r8 + #define ROT_128_16(x) _mm_shuffle_epi8(x, r16) // k_r16 +#else + #define ROT_128_8(x) MM_ROR_EPI32_VIA_SHIFT(x, 8) + #define ROT_128_16(x) MM_ROR_EPI32_VIA_SHIFT(x, 16) +#endif + #define ROT_128_7(x) MM_ROR_EPI32_VIA_SHIFT(x, 7) + #define ROT_128_12(x) MM_ROR_EPI32_VIA_SHIFT(x, 12) +#endif + + +#if 1 +// this branch can provide similar speed on x86* in most cases, +// because [base + index*4] provides same speed as [base + index]. +// but some compilers can generate different code with this branch, that can be faster sometimes. +// this branch uses additional table of 10*16=160 bytes. +#define SIGMA_TABLE_MULT_16( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + SIGMA_TABLE_MULT(16, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) +MY_ALIGN(16) +static const Byte k_Blake2s_Sigma_16[BLAKE2S_NUM_ROUNDS][16] = + { SIGMA_TABLE(SIGMA_TABLE_MULT_16) }; +#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_16[r]; +#define GET_SIGMA_VAL_128(n) (sigma[n]) +#else +#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_4[r]; +#define GET_SIGMA_VAL_128(n) (4 * (size_t)sigma[n]) +#endif + + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST +#if 1 +#define SIGMA_TABLE_MULT_32( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + SIGMA_TABLE_MULT(32, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) +MY_ALIGN(64) +static const UInt16 k_Blake2s_Sigma_32[BLAKE2S_NUM_ROUNDS][16] = + { SIGMA_TABLE(SIGMA_TABLE_MULT_32) }; +#define GET_SIGMA_PTR_256(r) const UInt16 * const sigma = k_Blake2s_Sigma_32[r]; +#define GET_SIGMA_VAL_256(n) (sigma[n]) +#else +#define GET_SIGMA_PTR_256(r) const Byte * const sigma = k_Blake2s_Sigma_4[r]; +#define GET_SIGMA_VAL_256(n) (8 * (size_t)sigma[n]) +#endif +#endif // Z7_BLAKE2S_USE_AVX2_FAST + + +#define D_ROT_128_7(dest) dest = ROT_128_7(dest) +#define D_ROT_128_8(dest) dest = ROT_128_8(dest) +#define D_ROT_128_12(dest) dest = ROT_128_12(dest) +#define D_ROT_128_16(dest) dest = ROT_128_16(dest) + +#define OP_L(a, i) D_ADD_128 (V(a, 0), \ + LOAD_128((const Byte *)(w) + GET_SIGMA_VAL_128(2*(a)+(i)))); + +#define OP_0(a) OP_L(a, 0) +#define OP_7(a) OP_L(a, 1) + +#define OP_1(a) D_ADD_128 (V(a, 0), V(a, 1)); +#define OP_2(a) D_XOR_128 (V(a, 3), V(a, 0)); +#define OP_4(a) D_ADD_128 (V(a, 2), V(a, 3)); +#define OP_5(a) D_XOR_128 (V(a, 1), V(a, 2)); + +#define OP_3(a) D_ROT_128_16 (V(a, 3)); +#define OP_6(a) D_ROT_128_12 (V(a, 1)); +#define OP_8(a) D_ROT_128_8 (V(a, 3)); +#define OP_9(a) D_ROT_128_7 (V(a, 1)); + + +// for 32-bit x86 : interleave mode works slower, because of register pressure. + +#if 0 || 1 && (defined(MY_CPU_X86) \ + || defined(__GNUC__) && !defined(__clang__)) +// non-inteleaved version: +// is fast for x86 32-bit. +// is fast for GCC x86-64. + +#define V4G(a) \ + OP_0 (a) \ + OP_1 (a) \ + OP_2 (a) \ + OP_3 (a) \ + OP_4 (a) \ + OP_5 (a) \ + OP_6 (a) \ + OP_7 (a) \ + OP_1 (a) \ + OP_2 (a) \ + OP_8 (a) \ + OP_4 (a) \ + OP_5 (a) \ + OP_9 (a) \ + +#define V4R \ +{ \ + V4G (0) \ + V4G (1) \ + V4G (2) \ + V4G (3) \ + V4G (4) \ + V4G (5) \ + V4G (6) \ + V4G (7) \ +} + +#elif 0 || 1 && defined(MY_CPU_X86) + +#define OP_INTER_2(op, a,b) \ + op (a) \ + op (b) \ + +#define V4G(a,b) \ + OP_INTER_2 (OP_0, a,b) \ + OP_INTER_2 (OP_1, a,b) \ + OP_INTER_2 (OP_2, a,b) \ + OP_INTER_2 (OP_3, a,b) \ + OP_INTER_2 (OP_4, a,b) \ + OP_INTER_2 (OP_5, a,b) \ + OP_INTER_2 (OP_6, a,b) \ + OP_INTER_2 (OP_7, a,b) \ + OP_INTER_2 (OP_1, a,b) \ + OP_INTER_2 (OP_2, a,b) \ + OP_INTER_2 (OP_8, a,b) \ + OP_INTER_2 (OP_4, a,b) \ + OP_INTER_2 (OP_5, a,b) \ + OP_INTER_2 (OP_9, a,b) \ + +#define V4R \ +{ \ + V4G (0, 1) \ + V4G (2, 3) \ + V4G (4, 5) \ + V4G (6, 7) \ +} + +#else +// iterleave-4 version is fast for x64 (MSVC/CLANG) + +#define OP_INTER_4(op, a,b,c,d) \ + op (a) \ + op (b) \ + op (c) \ + op (d) \ + +#define V4G(a,b,c,d) \ + OP_INTER_4 (OP_0, a,b,c,d) \ + OP_INTER_4 (OP_1, a,b,c,d) \ + OP_INTER_4 (OP_2, a,b,c,d) \ + OP_INTER_4 (OP_3, a,b,c,d) \ + OP_INTER_4 (OP_4, a,b,c,d) \ + OP_INTER_4 (OP_5, a,b,c,d) \ + OP_INTER_4 (OP_6, a,b,c,d) \ + OP_INTER_4 (OP_7, a,b,c,d) \ + OP_INTER_4 (OP_1, a,b,c,d) \ + OP_INTER_4 (OP_2, a,b,c,d) \ + OP_INTER_4 (OP_8, a,b,c,d) \ + OP_INTER_4 (OP_4, a,b,c,d) \ + OP_INTER_4 (OP_5, a,b,c,d) \ + OP_INTER_4 (OP_9, a,b,c,d) \ + +#define V4R \ +{ \ + V4G (0, 1, 2, 3) \ + V4G (4, 5, 6, 7) \ +} + +#endif + +#define V4_ROUND(r) { GET_SIGMA_PTR_128(r); V4R } + + +#define V4_LOAD_MSG_1(w, m, i) \ +{ \ + __m128i m0, m1, m2, m3; \ + __m128i t0, t1, t2, t3; \ + m0 = LOADU_128((m) + ((i) + 0 * 4) * 16); \ + m1 = LOADU_128((m) + ((i) + 1 * 4) * 16); \ + m2 = LOADU_128((m) + ((i) + 2 * 4) * 16); \ + m3 = LOADU_128((m) + ((i) + 3 * 4) * 16); \ + t0 = _mm_unpacklo_epi32(m0, m1); \ + t1 = _mm_unpackhi_epi32(m0, m1); \ + t2 = _mm_unpacklo_epi32(m2, m3); \ + t3 = _mm_unpackhi_epi32(m2, m3); \ + w[(i) * 4 + 0] = _mm_unpacklo_epi64(t0, t2); \ + w[(i) * 4 + 1] = _mm_unpackhi_epi64(t0, t2); \ + w[(i) * 4 + 2] = _mm_unpacklo_epi64(t1, t3); \ + w[(i) * 4 + 3] = _mm_unpackhi_epi64(t1, t3); \ +} + +#define V4_LOAD_MSG(w, m) \ +{ \ + V4_LOAD_MSG_1 (w, m, 0) \ + V4_LOAD_MSG_1 (w, m, 1) \ + V4_LOAD_MSG_1 (w, m, 2) \ + V4_LOAD_MSG_1 (w, m, 3) \ +} + +#define V4_LOAD_UNPACK_PAIR_128(src32, i, d0, d1) \ +{ \ + const __m128i v0 = LOAD_128_FROM_STRUCT((src32) + (i ) * 4); \ + const __m128i v1 = LOAD_128_FROM_STRUCT((src32) + (i + 1) * 4); \ + d0 = _mm_unpacklo_epi32(v0, v1); \ + d1 = _mm_unpackhi_epi32(v0, v1); \ +} + +#define V4_UNPACK_PAIR_128(dest32, i, s0, s1) \ +{ \ + STORE_128_TO_STRUCT((dest32) + i * 4 , _mm_unpacklo_epi64(s0, s1)); \ + STORE_128_TO_STRUCT((dest32) + i * 4 + 16, _mm_unpackhi_epi64(s0, s1)); \ +} + +#define V4_UNPACK_STATE(dest32, src32) \ +{ \ + __m128i t0, t1, t2, t3, t4, t5, t6, t7; \ + V4_LOAD_UNPACK_PAIR_128(src32, 0, t0, t1) \ + V4_LOAD_UNPACK_PAIR_128(src32, 2, t2, t3) \ + V4_LOAD_UNPACK_PAIR_128(src32, 4, t4, t5) \ + V4_LOAD_UNPACK_PAIR_128(src32, 6, t6, t7) \ + V4_UNPACK_PAIR_128(dest32, 0, t0, t2) \ + V4_UNPACK_PAIR_128(dest32, 8, t1, t3) \ + V4_UNPACK_PAIR_128(dest32, 1, t4, t6) \ + V4_UNPACK_PAIR_128(dest32, 9, t5, t7) \ +} + + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_V128_Fast(UInt32 *s_items, const Byte *data, const Byte *end) +{ + // PrintStates2(s_items, 8, 16); + size_t pos = 0; + pos /= 2; + do + { +#if defined(Z7_BLAKE2S_USE_SSSE3) && \ + !defined(Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED) + const __m128i r8 = k_r8; + const __m128i r16 = k_r16; +#endif + __m128i w[16]; + __m128i v[16]; + UInt32 *s; + V4_LOAD_MSG(w, data) + s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + { + __m128i ctr = LOAD_128_FROM_STRUCT(s + 64); + D_ADD_EPI64_128 (ctr, k_inc); + STORE_128_TO_STRUCT(s + 64, ctr); + v[12] = XOR_128 (GET_128_IV_WAY4(4), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0))); + v[13] = XOR_128 (GET_128_IV_WAY4(5), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1))); + } + v[ 8] = GET_128_IV_WAY4(0); + v[ 9] = GET_128_IV_WAY4(1); + v[10] = GET_128_IV_WAY4(2); + v[11] = GET_128_IV_WAY4(3); + v[14] = GET_128_IV_WAY4(6); + v[15] = GET_128_IV_WAY4(7); - for (i = 0; i < 16; i++) - m[i] = GetUi32(p->buf + i * sizeof(m[i])); +#define LOAD_STATE_128_FROM_STRUCT(i) \ + v[i] = LOAD_128_FROM_STRUCT(s + (i) * 4); + +#define UPDATE_STATE_128_IN_STRUCT(i) \ + STORE_128_TO_STRUCT(s + (i) * 4, XOR_128( \ + XOR_128(v[i], v[(i) + 8]), \ + LOAD_128_FROM_STRUCT(s + (i) * 4))); - for (i = 0; i < 8; i++) - v[i] = p->h[i]; + REP8_MACRO (LOAD_STATE_128_FROM_STRUCT) + ROUNDS_LOOP (V4_ROUND) + REP8_MACRO (UPDATE_STATE_128_IN_STRUCT) + + data += Z7_BLAKE2S_BLOCK_SIZE * 4; + pos += Z7_BLAKE2S_BLOCK_SIZE * 4 / 2; + pos &= SUPER_BLOCK_SIZE / 2 - 1; } + while (data != end); +} - v[ 8] = k_Blake2s_IV[0]; - v[ 9] = k_Blake2s_IV[1]; - v[10] = k_Blake2s_IV[2]; - v[11] = k_Blake2s_IV[3]; - - v[12] = p->t[0] ^ k_Blake2s_IV[4]; - v[13] = p->t[1] ^ k_Blake2s_IV[5]; - v[14] = p->f[0] ^ k_Blake2s_IV[6]; - v[15] = p->f[1] ^ k_Blake2s_IV[7]; - #define G(r,i,a,b,c,d) \ - a += b + m[sigma[2*i+0]]; d ^= a; d = rotr32(d, 16); c += d; b ^= c; b = rotr32(b, 12); \ - a += b + m[sigma[2*i+1]]; d ^= a; d = rotr32(d, 8); c += d; b ^= c; b = rotr32(b, 7); \ +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Final_V128_Fast(UInt32 *states) +{ + const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64); + // printf("\nBlake2sp_Compress2_V128_Fast_Final4\n"); + // PrintStates2(states, 8, 16); + { + ptrdiff_t pos = 8 * 4; + do + { + UInt32 *src32 = states + (size_t)(pos * 1); + UInt32 *dest32 = states + (size_t)(pos * 2); + V4_UNPACK_STATE(dest32, src32) + pos -= 8 * 4; + } + while (pos >= 0); + } + { + unsigned k; + for (k = 0; k < 8; k++) + { + UInt32 *s = states + (size_t)k * 16; + STORE_128_TO_STRUCT (STATE_T(s), ctr); + } + } + // PrintStates2(states, 8, 16); +} + + + +#ifdef Z7_BLAKE2S_USE_AVX2 + +#define ADD_256(a, b) _mm256_add_epi32(a, b) +#define XOR_256(a, b) _mm256_xor_si256(a, b) + +#if 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + #define MM256_ROR_EPI32 _mm256_ror_epi32 + #define Z7_MM256_ROR_EPI32_IS_SUPPORTED +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + #define LOAD_ROTATE_CONSTS_256 +#endif +#else +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + #define LOAD_ROTATE_CONSTS_256 \ + const __m256i r8 = k_r8_256; \ + const __m256i r16 = k_r16_256; +#endif // AVX2_WAY2 + + #define MM256_ROR_EPI32(r, c) ( \ + ( 8==(c)) ? _mm256_shuffle_epi8(r,r8) \ + : (16==(c)) ? _mm256_shuffle_epi8(r,r16) \ + : _mm256_or_si256( \ + _mm256_srli_epi32((r), (c)), \ + _mm256_slli_epi32((r), 32-(c)))) +#endif // WAY_SLOW +#endif + + +#define D_ADD_256(dest, src) dest = ADD_256(dest, src) +#define D_XOR_256(dest, src) dest = XOR_256(dest, src) + +#define LOADU_256(p) _mm256_loadu_si256((const __m256i *)(const void *)(p)) + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + +#ifdef Z7_MM256_ROR_EPI32_IS_SUPPORTED +#define ROT_256_16(x) MM256_ROR_EPI32((x), 16) +#define ROT_256_12(x) MM256_ROR_EPI32((x), 12) +#define ROT_256_8(x) MM256_ROR_EPI32((x), 8) +#define ROT_256_7(x) MM256_ROR_EPI32((x), 7) +#else +#define ROTATE8 _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, \ + 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1) +#define ROTATE16 _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, \ + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2) +#define ROT_256_16(x) _mm256_shuffle_epi8((x), ROTATE16) +#define ROT_256_12(x) _mm256_or_si256(_mm256_srli_epi32((x), 12), _mm256_slli_epi32((x), 20)) +#define ROT_256_8(x) _mm256_shuffle_epi8((x), ROTATE8) +#define ROT_256_7(x) _mm256_or_si256(_mm256_srli_epi32((x), 7), _mm256_slli_epi32((x), 25)) +#endif + +#define D_ROT_256_7(dest) dest = ROT_256_7(dest) +#define D_ROT_256_8(dest) dest = ROT_256_8(dest) +#define D_ROT_256_12(dest) dest = ROT_256_12(dest) +#define D_ROT_256_16(dest) dest = ROT_256_16(dest) + +#define LOAD_256(p) _mm256_load_si256((const __m256i *)(const void *)(p)) +#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED + #define STOREU_256(p, r) _mm256_storeu_si256((__m256i *)(void *)(p), r) + #define LOAD_256_FROM_STRUCT(p) LOADU_256(p) + #define STORE_256_TO_STRUCT(p, r) STOREU_256(p, r) +#else + // if struct is aligned for 32-bytes + #define STORE_256(p, r) _mm256_store_si256((__m256i *)(void *)(p), r) + #define LOAD_256_FROM_STRUCT(p) LOAD_256(p) + #define STORE_256_TO_STRUCT(p, r) STORE_256(p, r) +#endif + +#endif // Z7_BLAKE2S_USE_AVX2_FAST + + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW + +#if 0 + #define DIAG_PERM2(s) \ + { \ + const __m256i a = LOAD_256_FROM_STRUCT((s) ); \ + const __m256i b = LOAD_256_FROM_STRUCT((s) + NSW); \ + STORE_256_TO_STRUCT((s ), _mm256_permute2x128_si256(a, b, 0x20)); \ + STORE_256_TO_STRUCT((s + NSW), _mm256_permute2x128_si256(a, b, 0x31)); \ + } +#else + #define DIAG_PERM2(s) \ + { \ + const __m128i a = LOAD_128_FROM_STRUCT((s) + 4); \ + const __m128i b = LOAD_128_FROM_STRUCT((s) + NSW); \ + STORE_128_TO_STRUCT((s) + NSW, a); \ + STORE_128_TO_STRUCT((s) + 4 , b); \ + } +#endif + #define DIAG_PERM8(s_items) \ + { \ + DIAG_PERM2(s_items) \ + DIAG_PERM2(s_items + NSW * 2) \ + DIAG_PERM2(s_items + NSW * 4) \ + DIAG_PERM2(s_items + NSW * 6) \ + } + + +#define AXR256(a, b, d, shift) \ + D_ADD_256(a, b); \ + D_XOR_256(d, a); \ + d = MM256_ROR_EPI32(d, shift); \ + + + +#ifdef Z7_BLAKE2S_USE_GATHER + + #define TABLE_GATHER_256_4(a0,a1,a2,a3) \ + a0,a1,a2,a3, a0+16,a1+16,a2+16,a3+16 + #define TABLE_GATHER_256( \ + a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + { TABLE_GATHER_256_4(a0,a2,a4,a6), \ + TABLE_GATHER_256_4(a1,a3,a5,a7), \ + TABLE_GATHER_256_4(a8,a10,a12,a14), \ + TABLE_GATHER_256_4(a9,a11,a13,a15) } +MY_ALIGN(64) +static const UInt32 k_Blake2s_Sigma_gather256[BLAKE2S_NUM_ROUNDS][16 * 2] = + { SIGMA_TABLE(TABLE_GATHER_256) }; + #define GET_SIGMA(r) \ + const UInt32 * const sigma = k_Blake2s_Sigma_gather256[r]; + #define AXR2_LOAD_INDEXES_AVX(sigma_index) \ + const __m256i i01234567 = LOAD_256(sigma + (sigma_index)); + #define SET_ROW_FROM_SIGMA_AVX(in) \ + _mm256_i32gather_epi32((const void *)(in), i01234567, 4) + #define SIGMA_INTERLEAVE 8 + #define SIGMA_HALF_ROW_SIZE 16 + +#else // !Z7_BLAKE2S_USE_GATHER + + #define GET_SIGMA(r) \ + const Byte * const sigma = k_Blake2s_Sigma_4[r]; + #define AXR2_LOAD_INDEXES_AVX(sigma_index) \ + AXR2_LOAD_INDEXES(sigma_index) + #define SET_ROW_FROM_SIGMA_AVX(in) \ + MY_mm256_set_m128i( \ + SET_ROW_FROM_SIGMA_W((in) + Z7_BLAKE2S_BLOCK_SIZE), \ + SET_ROW_FROM_SIGMA_W(in)) + #define SIGMA_INTERLEAVE 1 + #define SIGMA_HALF_ROW_SIZE 8 +#endif // !Z7_BLAKE2S_USE_GATHER - #define R(r) \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ +#define ROTATE_WORDS_TO_RIGHT_256(a, n) \ + a = _mm256_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3)); + + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + +#define AXR2_A(sigma_index, shift1, shift2) \ + AXR2_LOAD_INDEXES_AVX(sigma_index) \ + D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \ + AXR256(a0, b0, d0, shift1) \ + AXR256(c0, d0, b0, shift2) \ + +#define AXR4_A(sigma_index) \ + { AXR2_A(sigma_index, 16, 12) } \ + { AXR2_A(sigma_index + SIGMA_INTERLEAVE, 8, 7) } + +#define EE1(r) \ + { GET_SIGMA(r) \ + AXR4_A(0) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 1) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 3) \ + AXR4_A(SIGMA_HALF_ROW_SIZE) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 3) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 1) \ + } + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_AVX2_Way2(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + end -= Z7_BLAKE2S_BLOCK_SIZE; + + if (data != end) { - unsigned r; - for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) + LOAD_ROTATE_CONSTS_256 + DIAG_PERM8(s_items) + do { - const Byte *sigma = k_Blake2s_Sigma[r]; - R(r); + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + __m256i a0, b0, c0, d0; + { + const __m128i inc = k_inc; + __m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s)); + __m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); + D_ADD_EPI64_128(d0_128, inc); + D_ADD_EPI64_128(d1_128, inc); + STORE_128_TO_STRUCT (STATE_T(s ), d0_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW), d1_128); + d0 = MY_mm256_set_m128i(d1_128, d0_128); + D_XOR_256(d0, k_iv4_256); + } + c0 = SET_FROM_128(k_iv0_128); + a0 = LOAD_256_FROM_STRUCT(s + NSW * 0); + b0 = LOAD_256_FROM_STRUCT(s + NSW * 1); + + ROUNDS_LOOP (EE1) + + D_XOR_256(a0, c0); + D_XOR_256(b0, d0); + + D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0)); + D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1)); + + STORE_256_TO_STRUCT(s + NSW * 0, a0); + STORE_256_TO_STRUCT(s + NSW * 1, b0); + + data += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos &= SUPER_BLOCK_MASK; } - /* R(0); R(1); R(2); R(3); R(4); R(5); R(6); R(7); R(8); R(9); */ + while (data < end); + DIAG_PERM8(s_items) + if (data != end) + return; + } + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Z7_BLAKE2S_CompressSingleBlock(s, data); } +} - #undef G - #undef R +#endif // Z7_BLAKE2S_USE_AVX2_WAY2 + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + +#define AXR2_X(sigma_index, shift1, shift2) \ + AXR2_LOAD_INDEXES_AVX(sigma_index) \ + D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \ + D_ADD_256( a1, SET_ROW_FROM_SIGMA_AVX((data) + Z7_BLAKE2S_BLOCK_SIZE * 2)); \ + AXR256(a0, b0, d0, shift1) \ + AXR256(a1, b1, d1, shift1) \ + AXR256(c0, d0, b0, shift2) \ + AXR256(c1, d1, b1, shift2) \ + +#define AXR4_X(sigma_index) \ + { AXR2_X(sigma_index, 16, 12) } \ + { AXR2_X(sigma_index + SIGMA_INTERLEAVE, 8, 7) } + +#define EE2(r) \ + { GET_SIGMA(r) \ + AXR4_X(0) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 1) \ + ROTATE_WORDS_TO_RIGHT_256(b1, 1) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(c1, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 3) \ + ROTATE_WORDS_TO_RIGHT_256(d1, 3) \ + AXR4_X(SIGMA_HALF_ROW_SIZE) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 3) \ + ROTATE_WORDS_TO_RIGHT_256(b1, 3) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(c1, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 1) \ + ROTATE_WORDS_TO_RIGHT_256(d1, 1) \ + } + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_AVX2_Way4(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + + if ((size_t)(end - data) >= Z7_BLAKE2S_BLOCK_SIZE * 4) { - unsigned i; - for (i = 0; i < 8; i++) - p->h[i] ^= v[i] ^ v[i + 8]; +#ifndef Z7_MM256_ROR_EPI32_IS_SUPPORTED + const __m256i r8 = k_r8_256; + const __m256i r16 = k_r16_256; +#endif + end -= Z7_BLAKE2S_BLOCK_SIZE * 3; + DIAG_PERM8(s_items) + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + __m256i a0, b0, c0, d0; + __m256i a1, b1, c1, d1; + { + const __m128i inc = k_inc; + __m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s)); + __m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); + __m128i d2_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 2)); + __m128i d3_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 3)); + D_ADD_EPI64_128(d0_128, inc); + D_ADD_EPI64_128(d1_128, inc); + D_ADD_EPI64_128(d2_128, inc); + D_ADD_EPI64_128(d3_128, inc); + STORE_128_TO_STRUCT (STATE_T(s ), d0_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW * 1), d1_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW * 2), d2_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW * 3), d3_128); + d0 = MY_mm256_set_m128i(d1_128, d0_128); + d1 = MY_mm256_set_m128i(d3_128, d2_128); + D_XOR_256(d0, k_iv4_256); + D_XOR_256(d1, k_iv4_256); + } + c1 = c0 = SET_FROM_128(k_iv0_128); + a0 = LOAD_256_FROM_STRUCT(s + NSW * 0); + b0 = LOAD_256_FROM_STRUCT(s + NSW * 1); + a1 = LOAD_256_FROM_STRUCT(s + NSW * 2); + b1 = LOAD_256_FROM_STRUCT(s + NSW * 3); + + ROUNDS_LOOP (EE2) + + D_XOR_256(a0, c0); + D_XOR_256(b0, d0); + D_XOR_256(a1, c1); + D_XOR_256(b1, d1); + + D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0)); + D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1)); + D_XOR_256(a1, LOAD_256_FROM_STRUCT(s + NSW * 2)); + D_XOR_256(b1, LOAD_256_FROM_STRUCT(s + NSW * 3)); + + STORE_256_TO_STRUCT(s + NSW * 0, a0); + STORE_256_TO_STRUCT(s + NSW * 1, b0); + STORE_256_TO_STRUCT(s + NSW * 2, a1); + STORE_256_TO_STRUCT(s + NSW * 3, b1); + + data += Z7_BLAKE2S_BLOCK_SIZE * 4; + pos += Z7_BLAKE2S_BLOCK_SIZE * 4; + pos &= SUPER_BLOCK_MASK; + } + while (data < end); + DIAG_PERM8(s_items) + end += Z7_BLAKE2S_BLOCK_SIZE * 3; + } + if (data == end) + return; + // Z7_BLAKE2S_Compress2_V128(s_items, data, end, pos); + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Z7_BLAKE2S_CompressSingleBlock(s, data); + data += Z7_BLAKE2S_BLOCK_SIZE; + pos += Z7_BLAKE2S_BLOCK_SIZE; + pos &= SUPER_BLOCK_MASK; } + while (data != end); } +#endif // Z7_BLAKE2S_USE_AVX2_WAY4 +#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW + -#define Blake2s_Increment_Counter(S, inc) \ - { p->t[0] += (inc); p->t[1] += (p->t[0] < (inc)); } +// --------------------------------------------------------- -#define Blake2s_Set_LastBlock(p) \ - { p->f[0] = BLAKE2S_FINAL_FLAG; p->f[1] = p->lastNode_f1; } +#ifdef Z7_BLAKE2S_USE_AVX2_FAST +#define OP256_L(a, i) D_ADD_256 (V(a, 0), \ + LOAD_256((const Byte *)(w) + GET_SIGMA_VAL_256(2*(a)+(i)))); + +#define OP256_0(a) OP256_L(a, 0) +#define OP256_7(a) OP256_L(a, 1) + +#define OP256_1(a) D_ADD_256 (V(a, 0), V(a, 1)); +#define OP256_2(a) D_XOR_256 (V(a, 3), V(a, 0)); +#define OP256_4(a) D_ADD_256 (V(a, 2), V(a, 3)); +#define OP256_5(a) D_XOR_256 (V(a, 1), V(a, 2)); + +#define OP256_3(a) D_ROT_256_16 (V(a, 3)); +#define OP256_6(a) D_ROT_256_12 (V(a, 1)); +#define OP256_8(a) D_ROT_256_8 (V(a, 3)); +#define OP256_9(a) D_ROT_256_7 (V(a, 1)); + + +#if 0 || 1 && defined(MY_CPU_X86) + +#define V8_G(a) \ + OP256_0 (a) \ + OP256_1 (a) \ + OP256_2 (a) \ + OP256_3 (a) \ + OP256_4 (a) \ + OP256_5 (a) \ + OP256_6 (a) \ + OP256_7 (a) \ + OP256_1 (a) \ + OP256_2 (a) \ + OP256_8 (a) \ + OP256_4 (a) \ + OP256_5 (a) \ + OP256_9 (a) \ + +#define V8R { \ + V8_G (0); \ + V8_G (1); \ + V8_G (2); \ + V8_G (3); \ + V8_G (4); \ + V8_G (5); \ + V8_G (6); \ + V8_G (7); \ +} + +#else + +#define OP256_INTER_4(op, a,b,c,d) \ + op (a) \ + op (b) \ + op (c) \ + op (d) \ + +#define V8_G(a,b,c,d) \ + OP256_INTER_4 (OP256_0, a,b,c,d) \ + OP256_INTER_4 (OP256_1, a,b,c,d) \ + OP256_INTER_4 (OP256_2, a,b,c,d) \ + OP256_INTER_4 (OP256_3, a,b,c,d) \ + OP256_INTER_4 (OP256_4, a,b,c,d) \ + OP256_INTER_4 (OP256_5, a,b,c,d) \ + OP256_INTER_4 (OP256_6, a,b,c,d) \ + OP256_INTER_4 (OP256_7, a,b,c,d) \ + OP256_INTER_4 (OP256_1, a,b,c,d) \ + OP256_INTER_4 (OP256_2, a,b,c,d) \ + OP256_INTER_4 (OP256_8, a,b,c,d) \ + OP256_INTER_4 (OP256_4, a,b,c,d) \ + OP256_INTER_4 (OP256_5, a,b,c,d) \ + OP256_INTER_4 (OP256_9, a,b,c,d) \ + +#define V8R { \ + V8_G (0, 1, 2, 3) \ + V8_G (4, 5, 6, 7) \ +} +#endif + +#define V8_ROUND(r) { GET_SIGMA_PTR_256(r); V8R } + + +// for debug: +// #define Z7_BLAKE2S_PERMUTE_WITH_GATHER +#if defined(Z7_BLAKE2S_PERMUTE_WITH_GATHER) +// gather instruction is slow. +#define V8_LOAD_MSG(w, m) \ +{ \ + unsigned i; \ + for (i = 0; i < 16; ++i) { \ + w[i] = _mm256_i32gather_epi32( \ + (const void *)((m) + i * sizeof(UInt32)),\ + _mm256_set_epi32(0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00), \ + sizeof(UInt32)); \ + } \ +} +#else // !Z7_BLAKE2S_PERMUTE_WITH_GATHER + +#define V8_LOAD_MSG_2(w, a0, a1) \ +{ \ + (w)[0] = _mm256_permute2x128_si256(a0, a1, 0x20); \ + (w)[4] = _mm256_permute2x128_si256(a0, a1, 0x31); \ +} + +#define V8_LOAD_MSG_4(w, z0, z1, z2, z3) \ +{ \ + __m256i s0, s1, s2, s3; \ + s0 = _mm256_unpacklo_epi64(z0, z1); \ + s1 = _mm256_unpackhi_epi64(z0, z1); \ + s2 = _mm256_unpacklo_epi64(z2, z3); \ + s3 = _mm256_unpackhi_epi64(z2, z3); \ + V8_LOAD_MSG_2((w) + 0, s0, s2) \ + V8_LOAD_MSG_2((w) + 1, s1, s3) \ +} + +#define V8_LOAD_MSG_0(t0, t1, m) \ +{ \ + __m256i m0, m1; \ + m0 = LOADU_256(m); \ + m1 = LOADU_256((m) + 2 * 32); \ + t0 = _mm256_unpacklo_epi32(m0, m1); \ + t1 = _mm256_unpackhi_epi32(m0, m1); \ +} + +#define V8_LOAD_MSG_8(w, m) \ +{ \ + __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ + V8_LOAD_MSG_0(t0, t4, (m) + 0 * 4 * 32) \ + V8_LOAD_MSG_0(t1, t5, (m) + 1 * 4 * 32) \ + V8_LOAD_MSG_0(t2, t6, (m) + 2 * 4 * 32) \ + V8_LOAD_MSG_0(t3, t7, (m) + 3 * 4 * 32) \ + V8_LOAD_MSG_4((w) , t0, t1, t2, t3) \ + V8_LOAD_MSG_4((w) + 2, t4, t5, t6, t7) \ +} + +#define V8_LOAD_MSG(w, m) \ +{ \ + V8_LOAD_MSG_8(w, m) \ + V8_LOAD_MSG_8((w) + 8, (m) + 32) \ +} + +#endif // !Z7_BLAKE2S_PERMUTE_WITH_GATHER + + +#define V8_PERM_PAIR_STORE(u, a0, a2) \ +{ \ + STORE_256_TO_STRUCT((u), _mm256_permute2x128_si256(a0, a2, 0x20)); \ + STORE_256_TO_STRUCT((u) + 8, _mm256_permute2x128_si256(a0, a2, 0x31)); \ +} -static void Blake2s_Update(CBlake2s *p, const Byte *data, size_t size) +#define V8_UNPACK_STORE_4(u, z0, z1, z2, z3) \ +{ \ + __m256i s0, s1, s2, s3; \ + s0 = _mm256_unpacklo_epi64(z0, z1); \ + s1 = _mm256_unpackhi_epi64(z0, z1); \ + s2 = _mm256_unpacklo_epi64(z2, z3); \ + s3 = _mm256_unpackhi_epi64(z2, z3); \ + V8_PERM_PAIR_STORE(u + 0, s0, s2) \ + V8_PERM_PAIR_STORE(u + 2, s1, s3) \ +} + +#define V8_UNPACK_STORE_0(src32, d0, d1) \ +{ \ + const __m256i v0 = LOAD_256_FROM_STRUCT ((src32) ); \ + const __m256i v1 = LOAD_256_FROM_STRUCT ((src32) + 8); \ + d0 = _mm256_unpacklo_epi32(v0, v1); \ + d1 = _mm256_unpackhi_epi32(v0, v1); \ +} + +#define V8_UNPACK_STATE(dest32, src32) \ +{ \ + __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ + V8_UNPACK_STORE_0 ((src32) + 16 * 0, t0, t4) \ + V8_UNPACK_STORE_0 ((src32) + 16 * 1, t1, t5) \ + V8_UNPACK_STORE_0 ((src32) + 16 * 2, t2, t6) \ + V8_UNPACK_STORE_0 ((src32) + 16 * 3, t3, t7) \ + V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) , t0, t1, t2, t3) \ + V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) + 4, t4, t5, t6, t7) \ +} + + + +#define V8_LOAD_STATE_256_FROM_STRUCT(i) \ + v[i] = LOAD_256_FROM_STRUCT(s_items + (i) * 8); + +#if 0 || 0 && defined(MY_CPU_X86) +#define Z7_BLAKE2S_AVX2_FAST_USE_STRUCT +#endif + +#ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT +// this branch doesn't use (iv) array +// so register pressure can be lower. +// it can be faster sometimes +#define V8_LOAD_STATE_256(i) V8_LOAD_STATE_256_FROM_STRUCT(i) +#define V8_UPDATE_STATE_256(i) \ +{ \ + STORE_256_TO_STRUCT(s_items + (i) * 8, XOR_256( \ + XOR_256(v[i], v[(i) + 8]), \ + LOAD_256_FROM_STRUCT(s_items + (i) * 8))); \ +} +#else +// it uses more variables (iv) registers +// it's better for gcc +// maybe that branch is better, if register pressure will be lower (avx512) +#define V8_LOAD_STATE_256(i) { iv[i] = v[i]; } +#define V8_UPDATE_STATE_256(i) { v[i] = XOR_256(XOR_256(v[i], v[i + 8]), iv[i]); } +#define V8_STORE_STATE_256(i) { STORE_256_TO_STRUCT(s_items + (i) * 8, v[i]); } +#endif + + +#if 0 + // use loading constants from memory + #define KK8(n) KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n) +MY_ALIGN(64) +static const UInt32 k_Blake2s_IV_WAY8[]= { - while (size != 0) - { - unsigned pos = (unsigned)p->bufPos; - unsigned rem = BLAKE2S_BLOCK_SIZE - pos; + KK8(0), KK8(1), KK8(2), KK8(3), KK8(4), KK8(5), KK8(6), KK8(7) +}; + #define GET_256_IV_WAY8(i) LOAD_256(k_Blake2s_IV_WAY8 + 8 * (i)) +#else + // use constant generation: + #define GET_256_IV_WAY8(i) _mm256_set1_epi32((Int32)KIV(i)) +#endif + - if (size <= rem) +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_AVX2_Fast(UInt32 *s_items, const Byte *data, const Byte *end) +{ +#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + __m256i v[16]; +#endif + + // PrintStates2(s_items, 8, 16); + +#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + REP8_MACRO (V8_LOAD_STATE_256_FROM_STRUCT) +#endif + + do + { + __m256i w[16]; +#ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + __m256i v[16]; +#else + __m256i iv[8]; +#endif + V8_LOAD_MSG(w, data) { - memcpy(p->buf + pos, data, size); - p->bufPos += (UInt32)size; - return; + // we use load/store ctr inside loop to reduce register pressure: +#if 1 || 1 && defined(MY_CPU_X86) + const __m256i ctr = _mm256_add_epi64( + LOAD_256_FROM_STRUCT(s_items + 64), + _mm256_set_epi32( + 0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE, + 0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)); + STORE_256_TO_STRUCT(s_items + 64, ctr); +#else + const UInt64 ctr64 = *(const UInt64 *)(const void *)(s_items + 64) + + Z7_BLAKE2S_BLOCK_SIZE; + const __m256i ctr = _mm256_set_epi64x(0, (Int64)ctr64, 0, (Int64)ctr64); + *(UInt64 *)(void *)(s_items + 64) = ctr64; +#endif + v[12] = XOR_256 (GET_256_IV_WAY8(4), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0))); + v[13] = XOR_256 (GET_256_IV_WAY8(5), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1))); } + v[ 8] = GET_256_IV_WAY8(0); + v[ 9] = GET_256_IV_WAY8(1); + v[10] = GET_256_IV_WAY8(2); + v[11] = GET_256_IV_WAY8(3); + v[14] = GET_256_IV_WAY8(6); + v[15] = GET_256_IV_WAY8(7); - memcpy(p->buf + pos, data, rem); - Blake2s_Increment_Counter(S, BLAKE2S_BLOCK_SIZE); - Blake2s_Compress(p); - p->bufPos = 0; - data += rem; - size -= rem; + REP8_MACRO (V8_LOAD_STATE_256) + ROUNDS_LOOP (V8_ROUND) + REP8_MACRO (V8_UPDATE_STATE_256) + data += SUPER_BLOCK_SIZE; } + while (data != end); + +#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + REP8_MACRO (V8_STORE_STATE_256) +#endif } -static void Blake2s_Final(CBlake2s *p, Byte *digest) +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Final_AVX2_Fast(UInt32 *states) { - unsigned i; + const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64); + // PrintStates2(states, 8, 16); + V8_UNPACK_STATE(states, states) + // PrintStates2(states, 8, 16); + { + unsigned k; + for (k = 0; k < 8; k++) + { + UInt32 *s = states + (size_t)k * 16; + STORE_128_TO_STRUCT (STATE_T(s), ctr); + } + } + // PrintStates2(states, 8, 16); + // printf("\nafter V8_UNPACK_STATE \n"); +} + +#endif // Z7_BLAKE2S_USE_AVX2_FAST +#endif // avx2 +#endif // vector + + +/* +#define Blake2s_Increment_Counter(s, inc) \ + { STATE_T(s)[0] += (inc); STATE_T(s)[1] += (STATE_T(s)[0] < (inc)); } +#define Blake2s_Increment_Counter_Small(s, inc) \ + { STATE_T(s)[0] += (inc); } +*/ + +#define Blake2s_Set_LastBlock(s) \ + { STATE_F(s)[0] = BLAKE2S_FINAL_FLAG; /* STATE_F(s)[1] = p->u.header.lastNode_f1; */ } + + +#if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL >= 1600 + // good for vs2022 + #define LOOP_8(mac) { unsigned kkk; for (kkk = 0; kkk < 8; kkk++) mac(kkk) } +#else + // good for Z7_BLAKE2S_UNROLL for GCC9 (arm*/x86*) and MSC_VER_1400-x64. + #define LOOP_8(mac) { REP8_MACRO(mac) } +#endif + + +static +Z7_FORCE_INLINE +// Z7_NO_INLINE +void +Z7_FASTCALL +Blake2s_Compress(UInt32 *s, const Byte *input) +{ + UInt32 m[16]; + UInt32 v[16]; + { + unsigned i; + for (i = 0; i < 16; i++) + m[i] = GetUi32(input + i * 4); + } + +#define INIT_v_FROM_s(i) v[i] = s[i]; + + LOOP_8(INIT_v_FROM_s) + + // Blake2s_Increment_Counter(s, Z7_BLAKE2S_BLOCK_SIZE) + { + const UInt32 t0 = STATE_T(s)[0] + Z7_BLAKE2S_BLOCK_SIZE; + const UInt32 t1 = STATE_T(s)[1] + (t0 < Z7_BLAKE2S_BLOCK_SIZE); + STATE_T(s)[0] = t0; + STATE_T(s)[1] = t1; + v[12] = t0 ^ KIV(4); + v[13] = t1 ^ KIV(5); + } + // v[12] = STATE_T(s)[0] ^ KIV(4); + // v[13] = STATE_T(s)[1] ^ KIV(5); + v[14] = STATE_F(s)[0] ^ KIV(6); + v[15] = STATE_F(s)[1] ^ KIV(7); + + v[ 8] = KIV(0); + v[ 9] = KIV(1); + v[10] = KIV(2); + v[11] = KIV(3); + // PrintStates2((const UInt32 *)v, 1, 16); + + #define ADD_SIGMA(a, index) V(a, 0) += *(const UInt32 *)GET_SIGMA_PTR(m, sigma[index]); + #define ADD32M(dest, src, a) V(a, dest) += V(a, src); + #define XOR32M(dest, src, a) V(a, dest) ^= V(a, src); + #define RTR32M(dest, shift, a) V(a, dest) = rotrFixed(V(a, dest), shift); + +// big interleaving can provides big performance gain, if scheduler queues are small. +#if 0 || 1 && defined(MY_CPU_X86) + // interleave-1: for small register number (x86-32bit) + #define G2(index, a, x, y) \ + ADD_SIGMA (a, (index) + 2 * 0) \ + ADD32M (0, 1, a) \ + XOR32M (3, 0, a) \ + RTR32M (3, x, a) \ + ADD32M (2, 3, a) \ + XOR32M (1, 2, a) \ + RTR32M (1, y, a) \ + + #define G(a) \ + G2(a * 2 , a, 16, 12) \ + G2(a * 2 + 1, a, 8, 7) \ + + #define R2 \ + G(0) \ + G(1) \ + G(2) \ + G(3) \ + G(4) \ + G(5) \ + G(6) \ + G(7) \ + +#elif 0 || 1 && defined(MY_CPU_X86_OR_AMD64) + // interleave-2: is good if the number of registers is not big (x86-64). + + #define REP2(mac, dest, src, a, b) \ + mac(dest, src, a) \ + mac(dest, src, b) + + #define G2(index, a, b, x, y) \ + ADD_SIGMA (a, (index) + 2 * 0) \ + ADD_SIGMA (b, (index) + 2 * 1) \ + REP2 (ADD32M, 0, 1, a, b) \ + REP2 (XOR32M, 3, 0, a, b) \ + REP2 (RTR32M, 3, x, a, b) \ + REP2 (ADD32M, 2, 3, a, b) \ + REP2 (XOR32M, 1, 2, a, b) \ + REP2 (RTR32M, 1, y, a, b) \ + + #define G(a, b) \ + G2(a * 2 , a, b, 16, 12) \ + G2(a * 2 + 1, a, b, 8, 7) \ + + #define R2 \ + G(0, 1) \ + G(2, 3) \ + G(4, 5) \ + G(6, 7) \ + +#else + // interleave-4: + // it has big register pressure for x86/x64. + // and MSVC compilers for x86/x64 are slow for this branch. + // but if we have big number of registers, this branch can be faster. + + #define REP4(mac, dest, src, a, b, c, d) \ + mac(dest, src, a) \ + mac(dest, src, b) \ + mac(dest, src, c) \ + mac(dest, src, d) - Blake2s_Increment_Counter(S, (UInt32)p->bufPos); - Blake2s_Set_LastBlock(p); - memset(p->buf + p->bufPos, 0, BLAKE2S_BLOCK_SIZE - p->bufPos); - Blake2s_Compress(p); + #define G2(index, a, b, c, d, x, y) \ + ADD_SIGMA (a, (index) + 2 * 0) \ + ADD_SIGMA (b, (index) + 2 * 1) \ + ADD_SIGMA (c, (index) + 2 * 2) \ + ADD_SIGMA (d, (index) + 2 * 3) \ + REP4 (ADD32M, 0, 1, a, b, c, d) \ + REP4 (XOR32M, 3, 0, a, b, c, d) \ + REP4 (RTR32M, 3, x, a, b, c, d) \ + REP4 (ADD32M, 2, 3, a, b, c, d) \ + REP4 (XOR32M, 1, 2, a, b, c, d) \ + REP4 (RTR32M, 1, y, a, b, c, d) \ - for (i = 0; i < 8; i++) - SetUi32(digest + sizeof(p->h[i]) * i, p->h[i]); + #define G(a, b, c, d) \ + G2(a * 2 , a, b, c, d, 16, 12) \ + G2(a * 2 + 1, a, b, c, d, 8, 7) \ + + #define R2 \ + G(0, 1, 2, 3) \ + G(4, 5, 6, 7) \ + +#endif + + #define R(r) { const Byte *sigma = k_Blake2s_Sigma_4[r]; R2 } + + // Z7_BLAKE2S_UNROLL gives 5-6 KB larger code, but faster: + // 20-40% faster for (x86/x64) VC2010+/GCC/CLANG. + // 30-60% faster for (arm64-arm32) GCC. + // 5-11% faster for (arm64) CLANG-MAC. + // so Z7_BLAKE2S_UNROLL is good optimization, if there is no vector branch. + // But if there is vectors branch (for x86*), this scalar code will be unused mostly. + // So we want smaller code (without unrolling) in that case (x86*). +#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS) + #define Z7_BLAKE2S_UNROLL +#endif + +#ifdef Z7_BLAKE2S_UNROLL + ROUNDS_LOOP_UNROLLED (R) +#else + ROUNDS_LOOP (R) +#endif + + #undef G + #undef G2 + #undef R + #undef R2 + + // printf("\n v after: \n"); + // PrintStates2((const UInt32 *)v, 1, 16); +#define XOR_s_PAIR_v(i) s[i] ^= v[i] ^ v[i + 8]; + + LOOP_8(XOR_s_PAIR_v) + // printf("\n s after:\n"); + // PrintStates2((const UInt32 *)s, 1, 16); } -/* ---------- BLAKE2s ---------- */ +static +Z7_NO_INLINE +void +Z7_FASTCALL +Blake2sp_Compress2(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + // PrintStates2(s_items, 8, 16); + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Blake2s_Compress(s, data); + data += Z7_BLAKE2S_BLOCK_SIZE; + pos += Z7_BLAKE2S_BLOCK_SIZE; + pos &= SUPER_BLOCK_MASK; + } + while (data != end); +} + -/* we need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() */ +#ifdef Z7_BLAKE2S_USE_VECTORS + +static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = Blake2sp_Compress2; +static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = Blake2sp_Compress2; +static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Init; +static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Final; +static unsigned g_z7_Blake2sp_SupportedFlags; + + #define Z7_BLAKE2SP_Compress_Fast(p) (p)->u.header.func_Compress_Fast + #define Z7_BLAKE2SP_Compress_Single(p) (p)->u.header.func_Compress_Single +#else + #define Z7_BLAKE2SP_Compress_Fast(p) Blake2sp_Compress2 + #define Z7_BLAKE2SP_Compress_Single(p) Blake2sp_Compress2 +#endif // Z7_BLAKE2S_USE_VECTORS + + +#if 1 && defined(MY_CPU_LE) + #define GET_DIGEST(_s, _digest) \ + { memcpy(_digest, _s, Z7_BLAKE2S_DIGEST_SIZE); } +#else + #define GET_DIGEST(_s, _digest) \ + { unsigned _i; for (_i = 0; _i < 8; _i++) \ + { SetUi32((_digest) + 4 * _i, (_s)[_i]) } \ + } +#endif + + +/* ---------- BLAKE2s ---------- */ /* +// we need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() typedef struct { Byte digest_length; Byte key_length; - Byte fanout; - Byte depth; + Byte fanout; // = 1 : in sequential mode + Byte depth; // = 1 : in sequential mode UInt32 leaf_length; - Byte node_offset[6]; - Byte node_depth; - Byte inner_length; + Byte node_offset[6]; // 0 for the first, leftmost, leaf, or in sequential mode + Byte node_depth; // 0 for the leaves, or in sequential mode + Byte inner_length; // [0, 32], 0 in sequential mode Byte salt[BLAKE2S_SALTBYTES]; Byte personal[BLAKE2S_PERSONALBYTES]; } CBlake2sParam; */ +#define k_Blake2sp_IV_0 \ + (KIV(0) ^ (Z7_BLAKE2S_DIGEST_SIZE | ((UInt32)Z7_BLAKE2SP_PARALLEL_DEGREE << 16) | ((UInt32)2 << 24))) +#define k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth) \ + (KIV(3) ^ ((UInt32)(node_depth) << 16) ^ ((UInt32)Z7_BLAKE2S_DIGEST_SIZE << 24)) -static void Blake2sp_Init_Spec(CBlake2s *p, unsigned node_offset, unsigned node_depth) +Z7_FORCE_INLINE +static void Blake2sp_Init_Spec(UInt32 *s, unsigned node_offset, unsigned node_depth) { - Blake2s_Init0(p); - - p->h[0] ^= (BLAKE2S_DIGEST_SIZE | ((UInt32)BLAKE2SP_PARALLEL_DEGREE << 16) | ((UInt32)2 << 24)); - p->h[2] ^= ((UInt32)node_offset); - p->h[3] ^= ((UInt32)node_depth << 16) | ((UInt32)BLAKE2S_DIGEST_SIZE << 24); - /* - P->digest_length = BLAKE2S_DIGEST_SIZE; - P->key_length = 0; - P->fanout = BLAKE2SP_PARALLEL_DEGREE; - P->depth = 2; - P->leaf_length = 0; - store48(P->node_offset, node_offset); - P->node_depth = node_depth; - P->inner_length = BLAKE2S_DIGEST_SIZE; - */ + s[0] = k_Blake2sp_IV_0; + s[1] = KIV(1); + s[2] = KIV(2) ^ (UInt32)node_offset; + s[3] = k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth); + s[4] = KIV(4); + s[5] = KIV(5); + s[6] = KIV(6); + s[7] = KIV(7); + + STATE_T(s)[0] = 0; + STATE_T(s)[1] = 0; + STATE_F(s)[0] = 0; + STATE_F(s)[1] = 0; +} + + +#ifdef Z7_BLAKE2S_USE_V128_FAST + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_InitState_V128_Fast(UInt32 *states) +{ +#define STORE_128_PAIR_INIT_STATES_2(i, t0, t1) \ + { STORE_128_TO_STRUCT(states + 0 + 4 * (i), (t0)); \ + STORE_128_TO_STRUCT(states + 32 + 4 * (i), (t1)); \ + } +#define STORE_128_PAIR_INIT_STATES_1(i, mac) \ + { const __m128i t = mac; \ + STORE_128_PAIR_INIT_STATES_2(i, t, t) \ + } +#define STORE_128_PAIR_INIT_STATES_IV(i) \ + STORE_128_PAIR_INIT_STATES_1(i, GET_128_IV_WAY4(i)) + + STORE_128_PAIR_INIT_STATES_1 (0, _mm_set1_epi32((Int32)k_Blake2sp_IV_0)) + STORE_128_PAIR_INIT_STATES_IV (1) + { + const __m128i t = GET_128_IV_WAY4(2); + STORE_128_PAIR_INIT_STATES_2 (2, + XOR_128(t, _mm_set_epi32(3, 2, 1, 0)), + XOR_128(t, _mm_set_epi32(7, 6, 5, 4))) + } + STORE_128_PAIR_INIT_STATES_1 (3, _mm_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0))) + STORE_128_PAIR_INIT_STATES_IV (4) + STORE_128_PAIR_INIT_STATES_IV (5) + STORE_128_PAIR_INIT_STATES_IV (6) + STORE_128_PAIR_INIT_STATES_IV (7) + STORE_128_PAIR_INIT_STATES_1 (16, _mm_set_epi32(0, 0, 0, 0)) + // printf("\n== exit Blake2sp_InitState_V128_Fast ctr=%d\n", states[64]); +} + +#endif // Z7_BLAKE2S_USE_V128_FAST + + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_InitState_AVX2_Fast(UInt32 *states) +{ +#define STORE_256_INIT_STATES(i, t) \ + STORE_256_TO_STRUCT(states + 8 * (i), t); +#define STORE_256_INIT_STATES_IV(i) \ + STORE_256_INIT_STATES(i, GET_256_IV_WAY8(i)) + + STORE_256_INIT_STATES (0, _mm256_set1_epi32((Int32)k_Blake2sp_IV_0)) + STORE_256_INIT_STATES_IV (1) + STORE_256_INIT_STATES (2, XOR_256( GET_256_IV_WAY8(2), + _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0))) + STORE_256_INIT_STATES (3, _mm256_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0))) + STORE_256_INIT_STATES_IV (4) + STORE_256_INIT_STATES_IV (5) + STORE_256_INIT_STATES_IV (6) + STORE_256_INIT_STATES_IV (7) + STORE_256_INIT_STATES (8, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 0)) + // printf("\n== exit Blake2sp_InitState_AVX2_Fast\n"); } +#endif // Z7_BLAKE2S_USE_AVX2_FAST + + + +Z7_NO_INLINE +void Blake2sp_InitState(CBlake2sp *p) +{ + size_t i; + // memset(p->states, 0, sizeof(p->states)); // for debug + p->u.header.cycPos = 0; +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + if (p->u.header.func_Init) + { + p->u.header.func_Init(p->states); + return; + } +#endif + for (i = 0; i < Z7_BLAKE2SP_PARALLEL_DEGREE; i++) + Blake2sp_Init_Spec(p->states + i * NSW, (unsigned)i, 0); +} void Blake2sp_Init(CBlake2sp *p) { - unsigned i; - - p->bufPos = 0; +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + p->u.header.func_Compress_Fast = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast; +#else + NULL; +#endif + + p->u.header.func_Compress_Single = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_COMPRESS_Single; +#else + NULL; +#endif - for (i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) - Blake2sp_Init_Spec(&p->S[i], i, 0); + p->u.header.func_Init = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_INIT_Init; +#else + NULL; +#endif - p->S[BLAKE2SP_PARALLEL_DEGREE - 1].lastNode_f1 = BLAKE2S_FINAL_FLAG; + p->u.header.func_Final = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_INIT_Final; +#else + NULL; +#endif +#endif + + Blake2sp_InitState(p); } void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size) { - unsigned pos = p->bufPos; - while (size != 0) + size_t pos; + // printf("\nsize = 0x%6x, cycPos = %5u data = %p\n", (unsigned)size, (unsigned)p->u.header.cycPos, data); + if (size == 0) + return; + pos = p->u.header.cycPos; + // pos < SUPER_BLOCK_SIZE * 2 : is expected + // pos == SUPER_BLOCK_SIZE * 2 : is not expected, but is supported also + { + const size_t pos2 = pos & SUPER_BLOCK_MASK; + if (pos2) + { + const size_t rem = SUPER_BLOCK_SIZE - pos2; + if (rem > size) + { + p->u.header.cycPos = (unsigned)(pos + size); + // cycPos < SUPER_BLOCK_SIZE * 2 + memcpy((Byte *)(void *)p->buf32 + pos, data, size); + /* to simpilify the code here we don't try to process first superblock, + if (cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) */ + return; + } + // (rem <= size) + memcpy((Byte *)(void *)p->buf32 + pos, data, rem); + pos += rem; + data += rem; + size -= rem; + } + } + + // pos <= SUPER_BLOCK_SIZE * 2 + // pos % SUPER_BLOCK_SIZE == 0 + if (pos) + { + /* pos == SUPER_BLOCK_SIZE || + pos == SUPER_BLOCK_SIZE * 2 */ + size_t end = pos; + if (size > SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE + || (end -= SUPER_BLOCK_SIZE)) + { + Z7_BLAKE2SP_Compress_Fast(p)(p->states, + (const Byte *)(const void *)p->buf32, + (const Byte *)(const void *)p->buf32 + end); + if (pos -= end) + memcpy(p->buf32, (const Byte *)(const void *)p->buf32 + + SUPER_BLOCK_SIZE, SUPER_BLOCK_SIZE); + } + } + + // pos == 0 || (pos == SUPER_BLOCK_SIZE && size <= SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE) + if (size > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) + { + // pos == 0 + const Byte *end; + const size_t size2 = (size - (SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE + 1)) + & ~(size_t)SUPER_BLOCK_MASK; + size -= size2; + // size < SUPER_BLOCK_SIZE * 2 + end = data + size2; + Z7_BLAKE2SP_Compress_Fast(p)(p->states, data, end); + data = end; + } + + if (size != 0) { - unsigned index = pos / BLAKE2S_BLOCK_SIZE; - unsigned rem = BLAKE2S_BLOCK_SIZE - (pos & (BLAKE2S_BLOCK_SIZE - 1)); - if (rem > size) - rem = (unsigned)size; - Blake2s_Update(&p->S[index], data, rem); - size -= rem; - data += rem; - pos += rem; - pos &= (BLAKE2S_BLOCK_SIZE * BLAKE2SP_PARALLEL_DEGREE - 1); + memcpy((Byte *)(void *)p->buf32 + pos, data, size); + pos += size; } - p->bufPos = pos; + p->u.header.cycPos = (unsigned)pos; + // cycPos < SUPER_BLOCK_SIZE * 2 } void Blake2sp_Final(CBlake2sp *p, Byte *digest) { - CBlake2s R; - unsigned i; + // UInt32 * const R_states = p->states; + // printf("\nBlake2sp_Final \n"); +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + if (p->u.header.func_Final) + p->u.header.func_Final(p->states); +#endif + // printf("\n=====\nBlake2sp_Final \n"); + // PrintStates(p->states, 32); + + // (p->u.header.cycPos == SUPER_BLOCK_SIZE) can be processed in any branch: + if (p->u.header.cycPos <= SUPER_BLOCK_SIZE) + { + unsigned pos; + memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, + 0, SUPER_BLOCK_SIZE - p->u.header.cycPos); + STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; + for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos); + Blake2s_Set_LastBlock(s) + if (pos + Z7_BLAKE2S_BLOCK_SIZE > p->u.header.cycPos) + { + UInt32 delta = Z7_BLAKE2S_BLOCK_SIZE; + if (pos < p->u.header.cycPos) + delta -= p->u.header.cycPos & (Z7_BLAKE2S_BLOCK_SIZE - 1); + // 0 < delta <= Z7_BLAKE2S_BLOCK_SIZE + { + const UInt32 v = STATE_T(s)[0]; + STATE_T(s)[1] -= v < delta; // (v < delta) is same condition here as (v == 0) + STATE_T(s)[0] = v - delta; + } + } + } + // PrintStates(p->states, 16); + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (Byte *)(void *)p->buf32, + (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE); + // PrintStates(p->states, 16); + } + else + { + // (p->u.header.cycPos > SUPER_BLOCK_SIZE) + unsigned pos; + for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos); + if (pos + SUPER_BLOCK_SIZE >= p->u.header.cycPos) + Blake2s_Set_LastBlock(s) + } + if (p->u.header.cycPos <= SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) + STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; + + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (Byte *)(void *)p->buf32, + (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE); + + // if (p->u.header.cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE; + STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; + + // if (p->u.header.cycPos != SUPER_BLOCK_SIZE) + { + pos = SUPER_BLOCK_SIZE; + for (;;) + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos & SUPER_BLOCK_MASK); + Blake2s_Set_LastBlock(s) + pos += Z7_BLAKE2S_BLOCK_SIZE; + if (pos >= p->u.header.cycPos) + { + if (pos != p->u.header.cycPos) + { + const UInt32 delta = pos - p->u.header.cycPos; + const UInt32 v = STATE_T(s)[0]; + STATE_T(s)[1] -= v < delta; + STATE_T(s)[0] = v - delta; + memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, 0, delta); + } + break; + } + } + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE, + (Byte *)(void *)p->buf32 + pos); + } + } + + { + size_t pos; + for (pos = 0; pos < SUPER_BLOCK_SIZE / 2; pos += Z7_BLAKE2S_BLOCK_SIZE / 2) + { + const UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, (pos * 2)); + Byte *dest = (Byte *)(void *)p->buf32 + pos; + GET_DIGEST(s, dest) + } + } + Blake2sp_Init_Spec(p->states, 0, 1); + { + size_t pos; + for (pos = 0; pos < (Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2S_DIGEST_SIZE) + - Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) + { + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (const Byte *)(const void *)p->buf32 + pos, + (const Byte *)(const void *)p->buf32 + pos + Z7_BLAKE2S_BLOCK_SIZE); + } + } + // Blake2s_Final(p->states, 0, digest, p, (Byte *)(void *)p->buf32 + i); + Blake2s_Set_LastBlock(p->states) + STATE_F(p->states)[1] = BLAKE2S_FINAL_FLAG; + { + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE, + (const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE); + } + GET_DIGEST(p->states, digest) + // printf("\n Blake2sp_Final 555 numDataInBufs = %5u\n", (unsigned)p->u.header.numDataInBufs); +} + + +BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo) +{ + // printf("\n========== setfunction = %d ======== \n", algo); +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + Z7_BLAKE2SP_FUNC_COMPRESS func = NULL; + Z7_BLAKE2SP_FUNC_COMPRESS func_Single = NULL; + Z7_BLAKE2SP_FUNC_INIT func_Final = NULL; + Z7_BLAKE2SP_FUNC_INIT func_Init = NULL; +#else + UNUSED_VAR(p) +#endif + +#ifdef Z7_BLAKE2S_USE_VECTORS - Blake2sp_Init_Spec(&R, 0, 1); - R.lastNode_f1 = BLAKE2S_FINAL_FLAG; + func = func_Single = Blake2sp_Compress2; - for (i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) + if (algo != Z7_BLAKE2SP_ALGO_SCALAR) { - Byte hash[BLAKE2S_DIGEST_SIZE]; - Blake2s_Final(&p->S[i], hash); - Blake2s_Update(&R, hash, BLAKE2S_DIGEST_SIZE); + // printf("\n========== setfunction NON-SCALER ======== \n"); + if (algo == Z7_BLAKE2SP_ALGO_DEFAULT) + { + func = g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast; + func_Single = g_Z7_BLAKE2SP_FUNC_COMPRESS_Single; + func_Init = g_Z7_BLAKE2SP_FUNC_INIT_Init; + func_Final = g_Z7_BLAKE2SP_FUNC_INIT_Final; + } + else + { + if ((g_z7_Blake2sp_SupportedFlags & (1u << algo)) == 0) + return False; + +#ifdef Z7_BLAKE2S_USE_AVX2 + + func_Single = +#if defined(Z7_BLAKE2S_USE_AVX2_WAY2) + Blake2sp_Compress2_AVX2_Way2; +#else + Z7_BLAKE2S_Compress2_V128; +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + if (algo == Z7_BLAKE2SP_ALGO_V256_FAST) + { + func = Blake2sp_Compress2_AVX2_Fast; + func_Final = Blake2sp_Final_AVX2_Fast; + func_Init = Blake2sp_InitState_AVX2_Fast; + } + else +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + if (algo == Z7_BLAKE2SP_ALGO_V256_WAY2) + func = Blake2sp_Compress2_AVX2_Way2; + else +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + if (algo == Z7_BLAKE2SP_ALGO_V256_WAY4) + { + func_Single = func = Blake2sp_Compress2_AVX2_Way4; + } + else +#endif +#endif // avx2 + { + if (algo == Z7_BLAKE2SP_ALGO_V128_FAST) + { + func = Blake2sp_Compress2_V128_Fast; + func_Final = Blake2sp_Final_V128_Fast; + func_Init = Blake2sp_InitState_V128_Fast; + func_Single = Z7_BLAKE2S_Compress2_V128; + } + else +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + if (algo == Z7_BLAKE2SP_ALGO_V128_WAY2) + func = func_Single = Blake2sp_Compress2_V128_Way2; + else +#endif + { + if (algo != Z7_BLAKE2SP_ALGO_V128_WAY1) + return False; + func = func_Single = Blake2sp_Compress2_V128_Way1; + } + } + } } +#else // !VECTORS + if (algo > 1) // Z7_BLAKE2SP_ALGO_SCALAR + return False; +#endif // !VECTORS - Blake2s_Final(&R, digest); +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + p->u.header.func_Compress_Fast = func; + p->u.header.func_Compress_Single = func_Single; + p->u.header.func_Final = func_Final; + p->u.header.func_Init = func_Init; +#endif + // printf("\n p->u.header.func_Compress = %p", p->u.header.func_Compress); + return True; } + + +void z7_Black2sp_Prepare(void) +{ +#ifdef Z7_BLAKE2S_USE_VECTORS + unsigned flags = 0; // (1u << Z7_BLAKE2SP_ALGO_V128_SCALAR); + + Z7_BLAKE2SP_FUNC_COMPRESS func_Fast = Blake2sp_Compress2; + Z7_BLAKE2SP_FUNC_COMPRESS func_Single = Blake2sp_Compress2; + Z7_BLAKE2SP_FUNC_INIT func_Init = NULL; + Z7_BLAKE2SP_FUNC_INIT func_Final = NULL; + +#if defined(MY_CPU_X86_OR_AMD64) + #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + // optional check + #if 0 || !(defined(__AVX512F__) && defined(__AVX512VL__)) + if (CPU_IsSupported_AVX512F_AVX512VL()) + #endif + #elif defined(Z7_BLAKE2S_USE_SSE41) + if (CPU_IsSupported_SSE41()) + #elif defined(Z7_BLAKE2S_USE_SSSE3) + if (CPU_IsSupported_SSSE3()) + #elif !defined(MY_CPU_AMD64) + if (CPU_IsSupported_SSE2()) + #endif +#endif + { + #if defined(Z7_BLAKE2S_USE_SSE41) + // printf("\n========== Blake2s SSE41 128-bit\n"); + #elif defined(Z7_BLAKE2S_USE_SSSE3) + // printf("\n========== Blake2s SSSE3 128-bit\n"); + #else + // printf("\n========== Blake2s SSE2 128-bit\n"); + #endif + // func_Fast = f_vector = Blake2sp_Compress2_V128_Way2; + // printf("\n========== Blake2sp_Compress2_V128_Way2\n"); + func_Fast = + func_Single = Z7_BLAKE2S_Compress2_V128; + flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY1); +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY2); +#endif +#ifdef Z7_BLAKE2S_USE_V128_FAST + flags |= (1u << Z7_BLAKE2SP_ALGO_V128_FAST); + func_Fast = Blake2sp_Compress2_V128_Fast; + func_Init = Blake2sp_InitState_V128_Fast; + func_Final = Blake2sp_Final_V128_Fast; +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2 +#if defined(MY_CPU_X86_OR_AMD64) + + #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + #if 0 + if (CPU_IsSupported_AVX512F_AVX512VL()) + #endif + #else + if (CPU_IsSupported_AVX2()) + #endif +#endif + { + // #pragma message ("=== Blake2s AVX2") + // printf("\n========== Blake2s AVX2\n"); + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + func_Single = Blake2sp_Compress2_AVX2_Way2; + flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY2); +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY4); +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + flags |= (1u << Z7_BLAKE2SP_ALGO_V256_FAST); + func_Fast = Blake2sp_Compress2_AVX2_Fast; + func_Init = Blake2sp_InitState_AVX2_Fast; + func_Final = Blake2sp_Final_AVX2_Fast; +#elif defined(Z7_BLAKE2S_USE_AVX2_WAY4) + func_Fast = Blake2sp_Compress2_AVX2_Way4; +#elif defined(Z7_BLAKE2S_USE_AVX2_WAY2) + func_Fast = Blake2sp_Compress2_AVX2_Way2; +#endif + } // avx2 +#endif // avx2 + } // sse* + g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = func_Fast; + g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = func_Single; + g_Z7_BLAKE2SP_FUNC_INIT_Init = func_Init; + g_Z7_BLAKE2SP_FUNC_INIT_Final = func_Final; + g_z7_Blake2sp_SupportedFlags = flags; + // printf("\nflags=%x\n", flags); +#endif // vectors +} + +/* +#ifdef Z7_BLAKE2S_USE_VECTORS +void align_test2(CBlake2sp *sp); +void align_test2(CBlake2sp *sp) +{ + __m128i a = LOAD_128(sp->states); + D_XOR_128(a, LOAD_128(sp->states + 4)); + STORE_128(sp->states, a); +} +void align_test2(void); +void align_test2(void) +{ + CBlake2sp sp; + Blake2sp_Init(&sp); + Blake2sp_Update(&sp, NULL, 0); +} +#endif +*/ diff --git a/multiarc/src/formats/7z/C/Bra.c b/multiarc/src/formats/7z/C/Bra.c old mode 100755 new mode 100644 index 3b854d9ca..e61edf8f1 --- a/multiarc/src/formats/7z/C/Bra.c +++ b/multiarc/src/formats/7z/C/Bra.c @@ -1,230 +1,709 @@ -/* Bra.c -- Converters for RISC code -2021-02-09 : Igor Pavlov : Public domain */ +/* Bra.c -- Branch converters for RISC code +2024-01-20 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "CpuArch.h" #include "Bra.h" +#include "RotateDefs.h" +#include "CpuArch.h" + +#if defined(MY_CPU_SIZEOF_POINTER) \ + && ( MY_CPU_SIZEOF_POINTER == 4 \ + || MY_CPU_SIZEOF_POINTER == 8) + #define BR_CONV_USE_OPT_PC_PTR +#endif + +#ifdef BR_CONV_USE_OPT_PC_PTR +#define BR_PC_INIT pc -= (UInt32)(SizeT)p; +#define BR_PC_GET (pc + (UInt32)(SizeT)p) +#else +#define BR_PC_INIT pc += (UInt32)size; +#define BR_PC_GET (pc - (UInt32)(SizeT)(lim - p)) +// #define BR_PC_INIT +// #define BR_PC_GET (pc + (UInt32)(SizeT)(p - data)) +#endif + +#define BR_CONVERT_VAL(v, c) if (encoding) v += c; else v -= c; +// #define BR_CONVERT_VAL(v, c) if (!encoding) c = (UInt32)0 - c; v += c; + +#define Z7_BRANCH_CONV(name) z7_ ## name + +#define Z7_BRANCH_FUNC_MAIN(name) \ +static \ +Z7_FORCE_INLINE \ +Z7_ATTRIB_NO_VECTOR \ +Byte *Z7_BRANCH_CONV(name)(Byte *p, SizeT size, UInt32 pc, int encoding) + +#define Z7_BRANCH_FUNC_IMP(name, m, encoding) \ +Z7_NO_INLINE \ +Z7_ATTRIB_NO_VECTOR \ +Byte *m(name)(Byte *data, SizeT size, UInt32 pc) \ + { return Z7_BRANCH_CONV(name)(data, size, pc, encoding); } \ + +#ifdef Z7_EXTRACT_ONLY +#define Z7_BRANCH_FUNCS_IMP(name) \ + Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_DEC_2, 0) +#else +#define Z7_BRANCH_FUNCS_IMP(name) \ + Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_DEC_2, 0) \ + Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_ENC_2, 1) +#endif -SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +#if defined(__clang__) +#define BR_EXTERNAL_FOR +#define BR_NEXT_ITERATION continue; +#else +#define BR_EXTERNAL_FOR for (;;) +#define BR_NEXT_ITERATION break; +#endif + +#if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 1000) \ + // GCC is not good for __builtin_expect() here + /* || defined(_MSC_VER) && (_MSC_VER >= 1920) */ + // #define Z7_unlikely [[unlikely]] + // #define Z7_LIKELY(x) (__builtin_expect((x), 1)) + #define Z7_UNLIKELY(x) (__builtin_expect((x), 0)) + // #define Z7_likely [[likely]] +#else + // #define Z7_LIKELY(x) (x) + #define Z7_UNLIKELY(x) (x) + // #define Z7_likely +#endif + + +Z7_BRANCH_FUNC_MAIN(BranchConv_ARM64) { - Byte *p; + // Byte *p = data; const Byte *lim; - size &= ~(size_t)3; - ip += 4; - p = data; - lim = data + size; + const UInt32 flag = (UInt32)1 << (24 - 4); + const UInt32 mask = ((UInt32)1 << 24) - (flag << 1); + size &= ~(SizeT)3; + // if (size == 0) return p; + lim = p + size; + BR_PC_INIT + pc -= 4; // because (p) will point to next instruction + + BR_EXTERNAL_FOR + { + // Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + for (;;) + { + UInt32 v; + if Z7_UNLIKELY(p == lim) + return p; + v = GetUi32a(p); + p += 4; + if Z7_UNLIKELY(((v - 0x94000000) & 0xfc000000) == 0) + { + UInt32 c = BR_PC_GET >> 2; + BR_CONVERT_VAL(v, c) + v &= 0x03ffffff; + v |= 0x94000000; + SetUi32a(p - 4, v) + BR_NEXT_ITERATION + } + // v = rotlFixed(v, 8); v += (flag << 8) - 0x90; if Z7_UNLIKELY((v & ((mask << 8) + 0x9f)) == 0) + v -= 0x90000000; if Z7_UNLIKELY((v & 0x9f000000) == 0) + { + UInt32 z, c; + // v = rotrFixed(v, 8); + v += flag; if Z7_UNLIKELY(v & mask) continue; + z = (v & 0xffffffe0) | (v >> 26); + c = (BR_PC_GET >> (12 - 3)) & ~(UInt32)7; + BR_CONVERT_VAL(z, c) + v &= 0x1f; + v |= 0x90000000; + v |= z << 26; + v |= 0x00ffffe0 & ((z & (((flag << 1) - 1))) - flag); + SetUi32a(p - 4, v) + } + } + } +} +Z7_BRANCH_FUNCS_IMP(BranchConv_ARM64) - if (encoding) +Z7_BRANCH_FUNC_MAIN(BranchConv_ARM) +{ + // Byte *p = data; + const Byte *lim; + size &= ~(SizeT)3; + lim = p + size; + BR_PC_INIT + /* in ARM: branch offset is relative to the +2 instructions from current instruction. + (p) will point to next instruction */ + pc += 8 - 4; + for (;;) { for (;;) { - if (p >= lim) - return (SizeT)(p - data); - p += 4; - if (p[-1] == 0xEB) - break; + if Z7_UNLIKELY(p >= lim) { return p; } p += 4; if Z7_UNLIKELY(p[-1] == 0xeb) break; + if Z7_UNLIKELY(p >= lim) { return p; } p += 4; if Z7_UNLIKELY(p[-1] == 0xeb) break; } { - UInt32 v = GetUi32(p - 4); - v <<= 2; - v += ip + (UInt32)(p - data); - v >>= 2; - v &= 0x00FFFFFF; - v |= 0xEB000000; - SetUi32(p - 4, v); + UInt32 v = GetUi32a(p - 4); + UInt32 c = BR_PC_GET >> 2; + BR_CONVERT_VAL(v, c) + v &= 0x00ffffff; + v |= 0xeb000000; + SetUi32a(p - 4, v) } } +} +Z7_BRANCH_FUNCS_IMP(BranchConv_ARM) + +Z7_BRANCH_FUNC_MAIN(BranchConv_PPC) +{ + // Byte *p = data; + const Byte *lim; + size &= ~(SizeT)3; + lim = p + size; + BR_PC_INIT + pc -= 4; // because (p) will point to next instruction + for (;;) { + UInt32 v; for (;;) { - if (p >= lim) - return (SizeT)(p - data); + if Z7_UNLIKELY(p == lim) + return p; + // v = GetBe32a(p); + v = *(UInt32 *)(void *)p; p += 4; - if (p[-1] == 0xEB) - break; + // if ((v & 0xfc000003) == 0x48000001) break; + // if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) break; + if Z7_UNLIKELY( + ((v - Z7_CONV_BE_TO_NATIVE_CONST32(0x48000001)) + & Z7_CONV_BE_TO_NATIVE_CONST32(0xfc000003)) == 0) break; } { - UInt32 v = GetUi32(p - 4); - v <<= 2; - v -= ip + (UInt32)(p - data); - v >>= 2; - v &= 0x00FFFFFF; - v |= 0xEB000000; - SetUi32(p - 4, v); + v = Z7_CONV_NATIVE_TO_BE_32(v); + { + UInt32 c = BR_PC_GET; + BR_CONVERT_VAL(v, c) + } + v &= 0x03ffffff; + v |= 0x48000000; + SetBe32a(p - 4, v) } } } +Z7_BRANCH_FUNCS_IMP(BranchConv_PPC) + +#ifdef Z7_CPU_FAST_ROTATE_SUPPORTED +#define BR_SPARC_USE_ROTATE +#endif -SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +Z7_BRANCH_FUNC_MAIN(BranchConv_SPARC) { - Byte *p; + // Byte *p = data; const Byte *lim; - size &= ~(size_t)1; - p = data; - lim = data + size - 4; - - if (encoding) - + const UInt32 flag = (UInt32)1 << 22; + size &= ~(SizeT)3; + lim = p + size; + BR_PC_INIT + pc -= 4; // because (p) will point to next instruction for (;;) { - UInt32 b1; + UInt32 v; for (;;) { - UInt32 b3; - if (p > lim) - return (SizeT)(p - data); - b1 = p[1]; - b3 = p[3]; - p += 2; - b1 ^= 8; - if ((b3 & b1) >= 0xF8) + if Z7_UNLIKELY(p == lim) + return p; + /* // the code without GetBe32a(): + { const UInt32 v = GetUi16a(p) & 0xc0ff; p += 4; if (v == 0x40 || v == 0xc07f) break; } + */ + v = GetBe32a(p); + p += 4; + #ifdef BR_SPARC_USE_ROTATE + v = rotlFixed(v, 2); + v += (flag << 2) - 1; + if Z7_UNLIKELY((v & (3 - (flag << 3))) == 0) + #else + v += (UInt32)5 << 29; + v ^= (UInt32)7 << 29; + v += flag; + if Z7_UNLIKELY((v & (0 - (flag << 1))) == 0) + #endif break; } { - UInt32 v = - ((UInt32)b1 << 19) - + (((UInt32)p[1] & 0x7) << 8) - + (((UInt32)p[-2] << 11)) - + (p[0]); - - p += 2; + // UInt32 v = GetBe32a(p - 4); + #ifndef BR_SPARC_USE_ROTATE + v <<= 2; + #endif { - UInt32 cur = (ip + (UInt32)(p - data)) >> 1; - v += cur; + UInt32 c = BR_PC_GET; + BR_CONVERT_VAL(v, c) } - - p[-4] = (Byte)(v >> 11); - p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); - p[-2] = (Byte)v; - p[-1] = (Byte)(0xF8 | (v >> 8)); + v &= (flag << 3) - 1; + #ifdef BR_SPARC_USE_ROTATE + v -= (flag << 2) - 1; + v = rotrFixed(v, 2); + #else + v -= (flag << 2); + v >>= 2; + v |= (UInt32)1 << 30; + #endif + SetBe32a(p - 4, v) } } +} +Z7_BRANCH_FUNCS_IMP(BranchConv_SPARC) + + +Z7_BRANCH_FUNC_MAIN(BranchConv_ARMT) +{ + // Byte *p = data; + Byte *lim; + size &= ~(SizeT)1; + // if (size == 0) return p; + if (size <= 2) return p; + size -= 2; + lim = p + size; + BR_PC_INIT + /* in ARM: branch offset is relative to the +2 instructions from current instruction. + (p) will point to the +2 instructions from current instruction */ + // pc += 4 - 4; + // if (encoding) pc -= 0xf800 << 1; else pc += 0xf800 << 1; + // #define ARMT_TAIL_PROC { goto armt_tail; } + #define ARMT_TAIL_PROC { return p; } - for (;;) + do { - UInt32 b1; + /* in MSVC 32-bit x86 compilers: + UInt32 version : it loads value from memory with movzx + Byte version : it loads value to 8-bit register (AL/CL) + movzx version is slightly faster in some cpus + */ + unsigned b1; + // Byte / unsigned + b1 = p[1]; + // optimized version to reduce one (p >= lim) check: + // unsigned a1 = p[1]; b1 = p[3]; p += 2; if Z7_LIKELY((b1 & (a1 ^ 8)) < 0xf8) for (;;) { - UInt32 b3; - if (p > lim) - return (SizeT)(p - data); - b1 = p[1]; - b3 = p[3]; - p += 2; - b1 ^= 8; - if ((b3 & b1) >= 0xF8) - break; + unsigned b3; // Byte / UInt32 + /* (Byte)(b3) normalization can use low byte computations in MSVC. + It gives smaller code, and no loss of speed in some compilers/cpus. + But new MSVC 32-bit x86 compilers use more slow load + from memory to low byte register in that case. + So we try to use full 32-bit computations for faster code. + */ + // if (p >= lim) { ARMT_TAIL_PROC } b3 = b1 + 8; b1 = p[3]; p += 2; if ((b3 & b1) >= 0xf8) break; + if Z7_UNLIKELY(p >= lim) { ARMT_TAIL_PROC } b3 = p[3]; p += 2; if Z7_UNLIKELY((b3 & (b1 ^ 8)) >= 0xf8) break; + if Z7_UNLIKELY(p >= lim) { ARMT_TAIL_PROC } b1 = p[3]; p += 2; if Z7_UNLIKELY((b1 & (b3 ^ 8)) >= 0xf8) break; } { + /* we can adjust pc for (0xf800) to rid of (& 0x7FF) operation. + But gcc/clang for arm64 can use bfi instruction for full code here */ UInt32 v = - ((UInt32)b1 << 19) + ((UInt32)GetUi16a(p - 2) << 11) | + ((UInt32)GetUi16a(p) & 0x7FF); + /* + UInt32 v = + ((UInt32)p[1 - 2] << 19) + (((UInt32)p[1] & 0x7) << 8) + (((UInt32)p[-2] << 11)) + (p[0]); - + */ p += 2; { - UInt32 cur = (ip + (UInt32)(p - data)) >> 1; - v -= cur; + UInt32 c = BR_PC_GET >> 1; + BR_CONVERT_VAL(v, c) } - + SetUi16a(p - 4, (UInt16)(((v >> 11) & 0x7ff) | 0xf000)) + SetUi16a(p - 2, (UInt16)(v | 0xf800)) /* - SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); - SetUi16(p - 2, (UInt16)(v | 0xF800)); - */ - p[-4] = (Byte)(v >> 11); - p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-3] = (Byte)(0xf0 | ((v >> 19) & 0x7)); p[-2] = (Byte)v; - p[-1] = (Byte)(0xF8 | (v >> 8)); + p[-1] = (Byte)(0xf8 | (v >> 8)); + */ } } + while (p < lim); + return p; + // armt_tail: + // if ((Byte)((lim[1] & 0xf8)) != 0xf0) { lim += 2; } return lim; + // return (Byte *)(lim + ((Byte)((lim[1] ^ 0xf0) & 0xf8) == 0 ? 0 : 2)); + // return (Byte *)(lim + (((lim[1] ^ ~0xfu) & ~7u) == 0 ? 0 : 2)); + // return (Byte *)(lim + 2 - (((((unsigned)lim[1] ^ 8) + 8) >> 7) & 2)); } +Z7_BRANCH_FUNCS_IMP(BranchConv_ARMT) -SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +// #define BR_IA64_NO_INLINE + +Z7_BRANCH_FUNC_MAIN(BranchConv_IA64) { - Byte *p; + // Byte *p = data; const Byte *lim; - size &= ~(size_t)3; - ip -= 4; - p = data; - lim = data + size; - + size &= ~(SizeT)15; + lim = p + size; + pc -= 1 << 4; + pc >>= 4 - 1; + // pc -= 1 << 1; + for (;;) { + unsigned m; for (;;) { - if (p >= lim) - return (SizeT)(p - data); - p += 4; - /* if ((v & 0xFC000003) == 0x48000001) */ - if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) + if Z7_UNLIKELY(p == lim) + return p; + m = (unsigned)((UInt32)0x334b0000 >> (*p & 0x1e)); + p += 16; + pc += 1 << 1; + if (m &= 3) break; } { - UInt32 v = GetBe32(p - 4); - if (encoding) - v += ip + (UInt32)(p - data); - else - v -= ip + (UInt32)(p - data); - v &= 0x03FFFFFF; - v |= 0x48000000; - SetBe32(p - 4, v); + p += (ptrdiff_t)m * 5 - 20; // negative value is expected here. + do + { + const UInt32 t = + #if defined(MY_CPU_X86_OR_AMD64) + // we use 32-bit load here to reduce code size on x86: + GetUi32(p); + #else + GetUi16(p); + #endif + UInt32 z = GetUi32(p + 1) >> m; + p += 5; + if (((t >> m) & (0x70 << 1)) == 0 + && ((z - (0x5000000 << 1)) & (0xf000000 << 1)) == 0) + { + UInt32 v = (UInt32)((0x8fffff << 1) | 1) & z; + z ^= v; + #ifdef BR_IA64_NO_INLINE + v |= (v & ((UInt32)1 << (23 + 1))) >> 3; + { + UInt32 c = pc; + BR_CONVERT_VAL(v, c) + } + v &= (0x1fffff << 1) | 1; + #else + { + if (encoding) + { + // pc &= ~(0xc00000 << 1); // we just need to clear at least 2 bits + pc &= (0x1fffff << 1) | 1; + v += pc; + } + else + { + // pc |= 0xc00000 << 1; // we need to set at least 2 bits + pc |= ~(UInt32)((0x1fffff << 1) | 1); + v -= pc; + } + } + v &= ~(UInt32)(0x600000 << 1); + #endif + v += (0x700000 << 1); + v &= (0x8fffff << 1) | 1; + z |= v; + z <<= m; + SetUi32(p + 1 - 5, z) + } + m++; + } + while (m &= 3); // while (m < 4); } } } +Z7_BRANCH_FUNCS_IMP(BranchConv_IA64) + + +#define BR_CONVERT_VAL_ENC(v) v += BR_PC_GET; +#define BR_CONVERT_VAL_DEC(v) v -= BR_PC_GET; +#if 1 && defined(MY_CPU_LE_UNALIGN) + #define RISCV_USE_UNALIGNED_LOAD +#endif -SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +#ifdef RISCV_USE_UNALIGNED_LOAD + #define RISCV_GET_UI32(p) GetUi32(p) + #define RISCV_SET_UI32(p, v) { SetUi32(p, v) } +#else + #define RISCV_GET_UI32(p) \ + ((UInt32)GetUi16a(p) + \ + ((UInt32)GetUi16a((p) + 2) << 16)) + #define RISCV_SET_UI32(p, v) { \ + SetUi16a(p, (UInt16)(v)) \ + SetUi16a((p) + 2, (UInt16)(v >> 16)) } +#endif + +#if 1 && defined(MY_CPU_LE) + #define RISCV_USE_16BIT_LOAD +#endif + +#ifdef RISCV_USE_16BIT_LOAD + #define RISCV_LOAD_VAL(p) GetUi16a(p) +#else + #define RISCV_LOAD_VAL(p) (*(p)) +#endif + +#define RISCV_INSTR_SIZE 2 +#define RISCV_STEP_1 (4 + RISCV_INSTR_SIZE) +#define RISCV_STEP_2 4 +#define RISCV_REG_VAL (2 << 7) +#define RISCV_CMD_VAL 3 +#if 1 + // for code size optimization: + #define RISCV_DELTA_7F 0x7f +#else + #define RISCV_DELTA_7F 0 +#endif + +#define RISCV_CHECK_1(v, b) \ + (((((b) - RISCV_CMD_VAL) ^ ((v) << 8)) & (0xf8000 + RISCV_CMD_VAL)) == 0) + +#if 1 + #define RISCV_CHECK_2(v, r) \ + ((((v) - ((RISCV_CMD_VAL << 12) | RISCV_REG_VAL | 8)) \ + << 18) \ + < ((r) & 0x1d)) +#else + // this branch gives larger code, because + // compilers generate larger code for big constants. + #define RISCV_CHECK_2(v, r) \ + ((((v) - ((RISCV_CMD_VAL << 12) | RISCV_REG_VAL)) \ + & ((RISCV_CMD_VAL << 12) | RISCV_REG_VAL)) \ + < ((r) & 0x1d)) +#endif + + +#define RISCV_SCAN_LOOP \ + Byte *lim; \ + size &= ~(SizeT)(RISCV_INSTR_SIZE - 1); \ + if (size <= 6) return p; \ + size -= 6; \ + lim = p + size; \ + BR_PC_INIT \ + for (;;) \ + { \ + UInt32 a, v; \ + /* Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE */ \ + for (;;) \ + { \ + if Z7_UNLIKELY(p >= lim) { return p; } \ + a = (RISCV_LOAD_VAL(p) ^ 0x10u) + 1; \ + if ((a & 0x77) == 0) break; \ + a = (RISCV_LOAD_VAL(p + RISCV_INSTR_SIZE) ^ 0x10u) + 1; \ + p += RISCV_INSTR_SIZE * 2; \ + if ((a & 0x77) == 0) \ + { \ + p -= RISCV_INSTR_SIZE; \ + if Z7_UNLIKELY(p >= lim) { return p; } \ + break; \ + } \ + } +// (xx6f ^ 10) + 1 = xx7f + 1 = xx80 : JAL +// (xxef ^ 10) + 1 = xxff + 1 = xx00 + 100 : JAL +// (xx17 ^ 10) + 1 = xx07 + 1 = xx08 : AUIPC +// (xx97 ^ 10) + 1 = xx87 + 1 = xx88 : AUIPC + +Byte * Z7_BRANCH_CONV_ENC(RISCV)(Byte *p, SizeT size, UInt32 pc) { - Byte *p; - const Byte *lim; - size &= ~(size_t)3; - ip -= 4; - p = data; - lim = data + size; + RISCV_SCAN_LOOP + v = a; + a = RISCV_GET_UI32(p); +#ifndef RISCV_USE_16BIT_LOAD + v += (UInt32)p[1] << 8; +#endif - for (;;) - { - for (;;) + if ((v & 8) == 0) // JAL { - if (p >= lim) - return (SizeT)(p - data); - /* - v = GetBe32(p); - p += 4; - m = v + ((UInt32)5 << 29); - m ^= (UInt32)7 << 29; - m += (UInt32)1 << 22; - if ((m & ((UInt32)0x1FF << 23)) == 0) - break; - */ + if ((v - (0x100 /* - RISCV_DELTA_7F */)) & 0xd80) + { + p += RISCV_INSTR_SIZE; + continue; + } + { + v = ((a & 1u << 31) >> 11) + | ((a & 0x3ff << 21) >> 20) + | ((a & 1 << 20) >> 9) + | (a & 0xff << 12); + BR_CONVERT_VAL_ENC(v) + // ((v & 1) == 0) + // v: bits [1 : 20] contain offset bits +#if 0 && defined(RISCV_USE_UNALIGNED_LOAD) + a &= 0xfff; + a |= ((UInt32)(v << 23)) + | ((UInt32)(v << 7) & ((UInt32)0xff << 16)) + | ((UInt32)(v >> 5) & ((UInt32)0xf0 << 8)); + RISCV_SET_UI32(p, a) +#else // aligned +#if 0 + SetUi16a(p, (UInt16)(((v >> 5) & 0xf000) | (a & 0xfff))) +#else + p[1] = (Byte)(((v >> 13) & 0xf0) | ((a >> 8) & 0xf)); +#endif + +#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + v <<= 15; + v = Z7_BSWAP32(v); + SetUi16a(p + 2, (UInt16)v) +#else + p[2] = (Byte)(v >> 9); + p[3] = (Byte)(v >> 1); +#endif +#endif // aligned + } p += 4; - if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || - (p[-4] == 0x7F && (p[-3] >= 0xC0))) - break; + continue; + } // JAL + + { + // AUIPC + if (v & 0xe80) // (not x0) and (not x2) + { + const UInt32 b = RISCV_GET_UI32(p + 4); + if (RISCV_CHECK_1(v, b)) + { + { + const UInt32 temp = (b << 12) | (0x17 + RISCV_REG_VAL); + RISCV_SET_UI32(p, temp) + } + a &= 0xfffff000; + { +#if 1 + const int t = -1 >> 1; + if (t != -1) + a += (b >> 20) - ((b >> 19) & 0x1000); // arithmetic right shift emulation + else +#endif + a += (UInt32)((Int32)b >> 20); // arithmetic right shift (sign-extension). + } + BR_CONVERT_VAL_ENC(a) +#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + a = Z7_BSWAP32(a); + RISCV_SET_UI32(p + 4, a) +#else + SetBe32(p + 4, a) +#endif + p += 8; + } + else + p += RISCV_STEP_1; + } + else + { + UInt32 r = a >> 27; + if (RISCV_CHECK_2(v, r)) + { + v = RISCV_GET_UI32(p + 4); + r = (r << 7) + 0x17 + (v & 0xfffff000); + a = (a >> 12) | (v << 20); + RISCV_SET_UI32(p, r) + RISCV_SET_UI32(p + 4, a) + p += 8; + } + else + p += RISCV_STEP_2; + } } + } // for +} + + +Byte * Z7_BRANCH_CONV_DEC(RISCV)(Byte *p, SizeT size, UInt32 pc) +{ + RISCV_SCAN_LOOP +#ifdef RISCV_USE_16BIT_LOAD + if ((a & 8) == 0) { - UInt32 v = GetBe32(p - 4); - v <<= 2; - if (encoding) - v += ip + (UInt32)(p - data); +#else + v = a; + a += (UInt32)p[1] << 8; + if ((v & 8) == 0) + { +#endif + // JAL + a -= 0x100 - RISCV_DELTA_7F; + if (a & 0xd80) + { + p += RISCV_INSTR_SIZE; + continue; + } + { + const UInt32 a_old = (a + (0xef - RISCV_DELTA_7F)) & 0xfff; +#if 0 // unaligned + a = GetUi32(p); + v = (UInt32)(a >> 23) & ((UInt32)0xff << 1) + | (UInt32)(a >> 7) & ((UInt32)0xff << 9) +#elif 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + v = GetUi16a(p + 2); + v = Z7_BSWAP32(v) >> 15 +#else + v = (UInt32)p[3] << 1 + | (UInt32)p[2] << 9 +#endif + | (UInt32)((a & 0xf000) << 5); + BR_CONVERT_VAL_DEC(v) + a = a_old + | (v << 11 & 1u << 31) + | (v << 20 & 0x3ff << 21) + | (v << 9 & 1 << 20) + | (v & 0xff << 12); + RISCV_SET_UI32(p, a) + } + p += 4; + continue; + } // JAL + + { + // AUIPC + v = a; +#if 1 && defined(RISCV_USE_UNALIGNED_LOAD) + a = GetUi32(p); +#else + a |= (UInt32)GetUi16a(p + 2) << 16; +#endif + if ((v & 0xe80) == 0) // x0/x2 + { + const UInt32 r = a >> 27; + if (RISCV_CHECK_2(v, r)) + { + UInt32 b; +#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + b = RISCV_GET_UI32(p + 4); + b = Z7_BSWAP32(b); +#else + b = GetBe32(p + 4); +#endif + v = a >> 12; + BR_CONVERT_VAL_DEC(b) + a = (r << 7) + 0x17; + a += (b + 0x800) & 0xfffff000; + v |= b << 20; + RISCV_SET_UI32(p, a) + RISCV_SET_UI32(p + 4, v) + p += 8; + } + else + p += RISCV_STEP_2; + } else - v -= ip + (UInt32)(p - data); - - v &= 0x01FFFFFF; - v -= (UInt32)1 << 24; - v ^= 0xFF000000; - v >>= 2; - v |= 0x40000000; - SetBe32(p - 4, v); + { + const UInt32 b = RISCV_GET_UI32(p + 4); + if (!RISCV_CHECK_1(v, b)) + p += RISCV_STEP_1; + else + { + v = (a & 0xfffff000) | (b >> 20); + a = (b << 12) | (0x17 + RISCV_REG_VAL); + RISCV_SET_UI32(p, a) + RISCV_SET_UI32(p + 4, v) + p += 8; + } + } } - } + } // for } diff --git a/multiarc/src/formats/7z/C/Bra.h b/multiarc/src/formats/7z/C/Bra.h old mode 100755 new mode 100644 index 855e37a6b..b47112ced --- a/multiarc/src/formats/7z/C/Bra.h +++ b/multiarc/src/formats/7z/C/Bra.h @@ -1,64 +1,105 @@ /* Bra.h -- Branch converters for executables -2013-01-18 : Igor Pavlov : Public domain */ +2024-01-20 : Igor Pavlov : Public domain */ -#ifndef __BRA_H -#define __BRA_H +#ifndef ZIP7_INC_BRA_H +#define ZIP7_INC_BRA_H #include "7zTypes.h" EXTERN_C_BEGIN +/* #define PPC BAD_PPC_11 // for debug */ + +#define Z7_BRANCH_CONV_DEC_2(name) z7_ ## name ## _Dec +#define Z7_BRANCH_CONV_ENC_2(name) z7_ ## name ## _Enc +#define Z7_BRANCH_CONV_DEC(name) Z7_BRANCH_CONV_DEC_2(BranchConv_ ## name) +#define Z7_BRANCH_CONV_ENC(name) Z7_BRANCH_CONV_ENC_2(BranchConv_ ## name) +#define Z7_BRANCH_CONV_ST_DEC(name) z7_BranchConvSt_ ## name ## _Dec +#define Z7_BRANCH_CONV_ST_ENC(name) z7_BranchConvSt_ ## name ## _Enc + +#define Z7_BRANCH_CONV_DECL(name) Byte * name(Byte *data, SizeT size, UInt32 pc) +#define Z7_BRANCH_CONV_ST_DECL(name) Byte * name(Byte *data, SizeT size, UInt32 pc, UInt32 *state) + +typedef Z7_BRANCH_CONV_DECL( (*z7_Func_BranchConv)); +typedef Z7_BRANCH_CONV_ST_DECL((*z7_Func_BranchConvSt)); + +#define Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL 0 +Z7_BRANCH_CONV_ST_DECL (Z7_BRANCH_CONV_ST_DEC(X86)); +Z7_BRANCH_CONV_ST_DECL (Z7_BRANCH_CONV_ST_ENC(X86)); + +#define Z7_BRANCH_FUNCS_DECL(name) \ +Z7_BRANCH_CONV_DECL (Z7_BRANCH_CONV_DEC_2(name)); \ +Z7_BRANCH_CONV_DECL (Z7_BRANCH_CONV_ENC_2(name)); + +Z7_BRANCH_FUNCS_DECL (BranchConv_ARM64) +Z7_BRANCH_FUNCS_DECL (BranchConv_ARM) +Z7_BRANCH_FUNCS_DECL (BranchConv_ARMT) +Z7_BRANCH_FUNCS_DECL (BranchConv_PPC) +Z7_BRANCH_FUNCS_DECL (BranchConv_SPARC) +Z7_BRANCH_FUNCS_DECL (BranchConv_IA64) +Z7_BRANCH_FUNCS_DECL (BranchConv_RISCV) + /* -These functions convert relative addresses to absolute addresses -in CALL instructions to increase the compression ratio. - - In: - data - data buffer - size - size of data - ip - current virtual Instruction Pinter (IP) value - state - state variable for x86 converter - encoding - 0 (for decoding), 1 (for encoding) - - Out: - state - state variable for x86 converter +These functions convert data that contain CPU instructions. +Each such function converts relative addresses to absolute addresses in some +branch instructions: CALL (in all converters) and JUMP (X86 converter only). +Such conversion allows to increase compression ratio, if we compress that data. + +There are 2 types of converters: + Byte * Conv_RISC (Byte *data, SizeT size, UInt32 pc); + Byte * ConvSt_X86(Byte *data, SizeT size, UInt32 pc, UInt32 *state); +Each Converter supports 2 versions: one for encoding +and one for decoding (_Enc/_Dec postfixes in function name). - Returns: - The number of processed bytes. If you call these functions with multiple calls, - you must start next call with first byte after block of processed bytes. +In params: + data : data buffer + size : size of data + pc : current virtual Program Counter (Instruction Pointer) value +In/Out param: + state : pointer to state variable (for X86 converter only) + +Return: + The pointer to position in (data) buffer after last byte that was processed. + If the caller calls converter again, it must call it starting with that position. + But the caller is allowed to move data in buffer. So pointer to + current processed position also will be changed for next call. + Also the caller must increase internal (pc) value for next call. +Each converter has some characteristics: Endian, Alignment, LookAhead. Type Endian Alignment LookAhead - x86 little 1 4 + X86 little 1 4 ARMT little 2 2 + RISCV little 2 6 ARM little 4 0 + ARM64 little 4 0 PPC big 4 0 SPARC big 4 0 IA64 little 16 0 - size must be >= Alignment + LookAhead, if it's not last block. - If (size < Alignment + LookAhead), converter returns 0. - - Example: + (data) must be aligned for (Alignment). + processed size can be calculated as: + SizeT processed = Conv(data, size, pc) - data; + if (processed == 0) + it means that converter needs more data for processing. + If (size < Alignment + LookAhead) + then (processed == 0) is allowed. - UInt32 ip = 0; - for () - { - ; size must be >= Alignment + LookAhead, if it's not last block - SizeT processed = Convert(data, size, ip, 1); - data += processed; - size -= processed; - ip += processed; - } +Example code for conversion in loop: + UInt32 pc = 0; + size = 0; + for (;;) + { + size += Load_more_input_data(data + size); + SizeT processed = Conv(data, size, pc) - data; + if (processed == 0 && no_more_input_data_after_size) + break; // we stop convert loop + data += processed; + size -= processed; + pc += processed; + } */ -#define x86_Convert_Init(state) { state = 0; } -SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding); -SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); - EXTERN_C_END #endif diff --git a/multiarc/src/formats/7z/C/Bra86.c b/multiarc/src/formats/7z/C/Bra86.c old mode 100755 new mode 100644 index 10a0fbd16..d81f392ae --- a/multiarc/src/formats/7z/C/Bra86.c +++ b/multiarc/src/formats/7z/C/Bra86.c @@ -1,82 +1,187 @@ -/* Bra86.c -- Converter for x86 code (BCJ) -2021-02-09 : Igor Pavlov : Public domain */ +/* Bra86.c -- Branch converter for X86 code (BCJ) +2023-04-02 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "Bra.h" +#include "CpuArch.h" -#define Test86MSByte(b) ((((b) + 1) & 0xFE) == 0) -SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding) +#if defined(MY_CPU_SIZEOF_POINTER) \ + && ( MY_CPU_SIZEOF_POINTER == 4 \ + || MY_CPU_SIZEOF_POINTER == 8) + #define BR_CONV_USE_OPT_PC_PTR +#endif + +#ifdef BR_CONV_USE_OPT_PC_PTR +#define BR_PC_INIT pc -= (UInt32)(SizeT)p; // (MY_uintptr_t) +#define BR_PC_GET (pc + (UInt32)(SizeT)p) +#else +#define BR_PC_INIT pc += (UInt32)size; +#define BR_PC_GET (pc - (UInt32)(SizeT)(lim - p)) +// #define BR_PC_INIT +// #define BR_PC_GET (pc + (UInt32)(SizeT)(p - data)) +#endif + +#define BR_CONVERT_VAL(v, c) if (encoding) v += c; else v -= c; +// #define BR_CONVERT_VAL(v, c) if (!encoding) c = (UInt32)0 - c; v += c; + +#define Z7_BRANCH_CONV_ST(name) z7_BranchConvSt_ ## name + +#define BR86_NEED_CONV_FOR_MS_BYTE(b) ((((b) + 1) & 0xfe) == 0) + +#ifdef MY_CPU_LE_UNALIGN + #define BR86_PREPARE_BCJ_SCAN const UInt32 v = GetUi32(p) ^ 0xe8e8e8e8; + #define BR86_IS_BCJ_BYTE(n) ((v & ((UInt32)0xfe << (n) * 8)) == 0) +#else + #define BR86_PREPARE_BCJ_SCAN + // bad for MSVC X86 (partial write to byte reg): + #define BR86_IS_BCJ_BYTE(n) ((p[n - 4] & 0xfe) == 0xe8) + // bad for old MSVC (partial write to byte reg): + // #define BR86_IS_BCJ_BYTE(n) (((*p ^ 0xe8) & 0xfe) == 0) +#endif + +static +Z7_FORCE_INLINE +Z7_ATTRIB_NO_VECTOR +Byte *Z7_BRANCH_CONV_ST(X86)(Byte *p, SizeT size, UInt32 pc, UInt32 *state, int encoding) { - SizeT pos = 0; - UInt32 mask = *state & 7; if (size < 5) - return 0; - size -= 4; - ip += 5; + return p; + { + // Byte *p = data; + const Byte *lim = p + size - 4; + unsigned mask = (unsigned)*state; // & 7; +#ifdef BR_CONV_USE_OPT_PC_PTR + /* if BR_CONV_USE_OPT_PC_PTR is defined: we need to adjust (pc) for (+4), + because call/jump offset is relative to the next instruction. + if BR_CONV_USE_OPT_PC_PTR is not defined : we don't need to adjust (pc) for (+4), + because BR_PC_GET uses (pc - (lim - p)), and lim was adjusted for (-4) before. + */ + pc += 4; +#endif + BR_PC_INIT + goto start; - for (;;) + for (;; mask |= 4) { - Byte *p = data + pos; - const Byte *limit = data + size; - for (; p < limit; p++) - if ((*p & 0xFE) == 0xE8) - break; - + // cont: mask |= 4; + start: + if (p >= lim) + goto fin; { - SizeT d = (SizeT)(p - data) - pos; - pos = (SizeT)(p - data); - if (p >= limit) - { - *state = (d > 2 ? 0 : mask >> (unsigned)d); - return pos; - } - if (d > 2) - mask = 0; - else - { - mask >>= (unsigned)d; - if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) - { - mask = (mask >> 1) | 4; - pos++; - continue; - } - } + BR86_PREPARE_BCJ_SCAN + p += 4; + if (BR86_IS_BCJ_BYTE(0)) { goto m0; } mask >>= 1; + if (BR86_IS_BCJ_BYTE(1)) { goto m1; } mask >>= 1; + if (BR86_IS_BCJ_BYTE(2)) { goto m2; } mask = 0; + if (BR86_IS_BCJ_BYTE(3)) { goto a3; } } + goto main_loop; - if (Test86MSByte(p[4])) + m0: p--; + m1: p--; + m2: p--; + if (mask == 0) + goto a3; + if (p > lim) + goto fin_p; + + // if (((0x17u >> mask) & 1) == 0) + if (mask > 4 || mask == 3) + { + mask >>= 1; + continue; // goto cont; + } + mask >>= 1; + if (BR86_NEED_CONV_FOR_MS_BYTE(p[mask])) + continue; // goto cont; + // if (!BR86_NEED_CONV_FOR_MS_BYTE(p[3])) continue; // goto cont; { - UInt32 v = ((UInt32)p[4] << 24) | ((UInt32)p[3] << 16) | ((UInt32)p[2] << 8) | ((UInt32)p[1]); - UInt32 cur = ip + (UInt32)pos; - pos += 5; - if (encoding) - v += cur; - else - v -= cur; - if (mask != 0) + UInt32 v = GetUi32(p); + UInt32 c; + v += (1 << 24); if (v & 0xfe000000) continue; // goto cont; + c = BR_PC_GET; + BR_CONVERT_VAL(v, c) { - unsigned sh = (mask & 6) << 2; - if (Test86MSByte((Byte)(v >> sh))) + mask <<= 3; + if (BR86_NEED_CONV_FOR_MS_BYTE(v >> mask)) { - v ^= (((UInt32)0x100 << sh) - 1); - if (encoding) - v += cur; - else - v -= cur; + v ^= (((UInt32)0x100 << mask) - 1); + #ifdef MY_CPU_X86 + // for X86 : we can recalculate (c) to reduce register pressure + c = BR_PC_GET; + #endif + BR_CONVERT_VAL(v, c) } mask = 0; } - p[1] = (Byte)v; - p[2] = (Byte)(v >> 8); - p[3] = (Byte)(v >> 16); - p[4] = (Byte)(0 - ((v >> 24) & 1)); + // v = (v & ((1 << 24) - 1)) - (v & (1 << 24)); + v &= (1 << 25) - 1; v -= (1 << 24); + SetUi32(p, v) + p += 4; + goto main_loop; } - else + + main_loop: + if (p >= lim) + goto fin; + for (;;) { - mask = (mask >> 1) | 4; - pos++; + BR86_PREPARE_BCJ_SCAN + p += 4; + if (BR86_IS_BCJ_BYTE(0)) { goto a0; } + if (BR86_IS_BCJ_BYTE(1)) { goto a1; } + if (BR86_IS_BCJ_BYTE(2)) { goto a2; } + if (BR86_IS_BCJ_BYTE(3)) { goto a3; } + if (p >= lim) + goto fin; + } + + a0: p--; + a1: p--; + a2: p--; + a3: + if (p > lim) + goto fin_p; + // if (!BR86_NEED_CONV_FOR_MS_BYTE(p[3])) continue; // goto cont; + { + UInt32 v = GetUi32(p); + UInt32 c; + v += (1 << 24); if (v & 0xfe000000) continue; // goto cont; + c = BR_PC_GET; + BR_CONVERT_VAL(v, c) + // v = (v & ((1 << 24) - 1)) - (v & (1 << 24)); + v &= (1 << 25) - 1; v -= (1 << 24); + SetUi32(p, v) + p += 4; + goto main_loop; } } + +fin_p: + p--; +fin: + // the following processing for tail is optional and can be commented + /* + lim += 4; + for (; p < lim; p++, mask >>= 1) + if ((*p & 0xfe) == 0xe8) + break; + */ + *state = (UInt32)mask; + return p; + } } + + +#define Z7_BRANCH_CONV_ST_FUNC_IMP(name, m, encoding) \ +Z7_NO_INLINE \ +Z7_ATTRIB_NO_VECTOR \ +Byte *m(name)(Byte *data, SizeT size, UInt32 pc, UInt32 *state) \ + { return Z7_BRANCH_CONV_ST(name)(data, size, pc, state, encoding); } + +Z7_BRANCH_CONV_ST_FUNC_IMP(X86, Z7_BRANCH_CONV_ST_DEC, 0) +#ifndef Z7_EXTRACT_ONLY +Z7_BRANCH_CONV_ST_FUNC_IMP(X86, Z7_BRANCH_CONV_ST_ENC, 1) +#endif diff --git a/multiarc/src/formats/7z/C/BraIA64.c b/multiarc/src/formats/7z/C/BraIA64.c old mode 100755 new mode 100644 index d1dbc62c5..9dfe3e289 --- a/multiarc/src/formats/7z/C/BraIA64.c +++ b/multiarc/src/formats/7z/C/BraIA64.c @@ -1,53 +1,14 @@ /* BraIA64.c -- Converter for IA-64 code -2017-01-26 : Igor Pavlov : Public domain */ +2023-02-20 : Igor Pavlov : Public domain */ #include "Precomp.h" -#include "CpuArch.h" -#include "Bra.h" +// the code was moved to Bra.c -SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - SizeT i; - if (size < 16) - return 0; - size -= 16; - i = 0; - do - { - unsigned m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3; - if (m) - { - m++; - do - { - Byte *p = data + (i + (size_t)m * 5 - 8); - if (((p[3] >> m) & 15) == 5 - && (((p[-1] | ((UInt32)p[0] << 8)) >> m) & 0x70) == 0) - { - unsigned raw = GetUi32(p); - unsigned v = raw >> m; - v = (v & 0xFFFFF) | ((v & (1 << 23)) >> 3); - - v <<= 4; - if (encoding) - v += ip + (UInt32)i; - else - v -= ip + (UInt32)i; - v >>= 4; - - v &= 0x1FFFFF; - v += 0x700000; - v &= 0x8FFFFF; - raw &= ~((UInt32)0x8FFFFF << m); - raw |= (v << m); - SetUi32(p, raw); - } - } - while (++m <= 4); - } - i += 16; - } - while (i <= size); - return i; -} +#ifdef _MSC_VER +#pragma warning(disable : 4206) // nonstandard extension used : translation unit is empty +#endif + +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wempty-translation-unit" +#endif diff --git a/multiarc/src/formats/7z/C/BwtSort.c b/multiarc/src/formats/7z/C/BwtSort.c old mode 100755 new mode 100644 index 3eb57efad..05ad6de8b --- a/multiarc/src/formats/7z/C/BwtSort.c +++ b/multiarc/src/formats/7z/C/BwtSort.c @@ -1,5 +1,5 @@ /* BwtSort.c -- BWT block sorting -2021-04-01 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,8 +8,6 @@ /* #define BLOCK_SORT_USE_HEAP_SORT */ -#define NO_INLINE MY_FAST_CALL - /* Don't change it !!! */ #define kNumHashBytes 2 #define kNumHashValues (1 << (kNumHashBytes * 8)) @@ -60,7 +58,10 @@ SortGroup - is recursive Range-Sort function with HeapSort optimization for smal returns: 1 - if there are groups, 0 - no more groups */ -static UInt32 NO_INLINE SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 groupSize, int NumRefBits, UInt32 *Indices +static +UInt32 +Z7_FASTCALL +SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt32 groupOffset, UInt32 groupSize, int NumRefBits, UInt32 *Indices #ifndef BLOCK_SORT_USE_HEAP_SORT , UInt32 left, UInt32 range #endif @@ -72,7 +73,7 @@ static UInt32 NO_INLINE SortGroup(UInt32 BlockSize, UInt32 NumSortedBytes, UInt3 { /* #ifndef BLOCK_SORT_EXTERNAL_FLAGS - SetFinishedGroupSize(ind2, 1); + SetFinishedGroupSize(ind2, 1) #endif */ return 0; @@ -463,7 +464,7 @@ UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize) Indices[(size_t)(i - finishedGroupSize) + 1] &= kIndexMask; { UInt32 newGroupSize = groupSize + finishedGroupSize; - SetFinishedGroupSize(Indices + i - finishedGroupSize, newGroupSize); + SetFinishedGroupSize(Indices + i - finishedGroupSize, newGroupSize) finishedGroupSize = newGroupSize; } i += groupSize; diff --git a/multiarc/src/formats/7z/C/BwtSort.h b/multiarc/src/formats/7z/C/BwtSort.h old mode 100755 new mode 100644 index 7e989a992..a34b243a3 --- a/multiarc/src/formats/7z/C/BwtSort.h +++ b/multiarc/src/formats/7z/C/BwtSort.h @@ -1,8 +1,8 @@ /* BwtSort.h -- BWT block sorting -2013-01-18 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ -#ifndef __BWT_SORT_H -#define __BWT_SORT_H +#ifndef ZIP7_INC_BWT_SORT_H +#define ZIP7_INC_BWT_SORT_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/Compiler.h b/multiarc/src/formats/7z/C/Compiler.h old mode 100755 new mode 100644 index a9816fa5a..f016f09a7 --- a/multiarc/src/formats/7z/C/Compiler.h +++ b/multiarc/src/formats/7z/C/Compiler.h @@ -1,12 +1,105 @@ -/* Compiler.h -2021-01-05 : Igor Pavlov : Public domain */ +/* Compiler.h : Compiler specific defines and pragmas +2024-01-22 : Igor Pavlov : Public domain */ -#ifndef __7Z_COMPILER_H -#define __7Z_COMPILER_H +#ifndef ZIP7_INC_COMPILER_H +#define ZIP7_INC_COMPILER_H - #ifdef __clang__ - #pragma clang diagnostic ignored "-Wunused-private-field" +#if defined(__clang__) +# define Z7_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) +#endif +#if defined(__clang__) && defined(__apple_build_version__) +# define Z7_APPLE_CLANG_VERSION Z7_CLANG_VERSION +#elif defined(__clang__) +# define Z7_LLVM_CLANG_VERSION Z7_CLANG_VERSION +#elif defined(__GNUC__) +# define Z7_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#endif + +#ifdef _MSC_VER +#if !defined(__clang__) && !defined(__GNUC__) +#define Z7_MSC_VER_ORIGINAL _MSC_VER +#endif +#endif + +#if defined(__MINGW32__) || defined(__MINGW64__) +#define Z7_MINGW +#endif + +#if defined(__LCC__) && (defined(__MCST__) || defined(__e2k__)) +#define Z7_MCST_LCC +#define Z7_MCST_LCC_VERSION (__LCC__ * 100 + __LCC_MINOR__) +#endif + +/* +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + #define Z7_COMPILER_AVX2_SUPPORTED #endif +#endif +*/ + +// #pragma GCC diagnostic ignored "-Wunknown-pragmas" + +#ifdef __clang__ +// padding size of '' with 4 bytes to alignment boundary +#pragma GCC diagnostic ignored "-Wpadded" + +#if defined(Z7_LLVM_CLANG_VERSION) && (__clang_major__ == 13) \ + && (defined(__FreeBSD__) || defined(__DragonFly__)) +// freebsd: +#pragma GCC diagnostic ignored "-Wexcess-padding" +#endif + +#if __clang_major__ >= 16 +#pragma GCC diagnostic ignored "-Wunsafe-buffer-usage" +#endif + +#if __clang_major__ == 13 +#if defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16) +// cheri +#pragma GCC diagnostic ignored "-Wcapability-to-integer-cast" +#endif +#endif + +#if __clang_major__ == 13 + // for + #pragma GCC diagnostic ignored "-Wreserved-identifier" +#endif + +#endif // __clang__ + +#if defined(_WIN32) && defined(__clang__) && __clang_major__ >= 16 +// #pragma GCC diagnostic ignored "-Wcast-function-type-strict" +#define Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION \ + _Pragma("GCC diagnostic ignored \"-Wcast-function-type-strict\"") +#else +#define Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION +#endif + +typedef void (*Z7_void_Function)(void); +#if defined(__clang__) || defined(__GNUC__) +#define Z7_CAST_FUNC_C (Z7_void_Function) +#elif defined(_MSC_VER) && _MSC_VER > 1920 +#define Z7_CAST_FUNC_C (void *) +// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' +#else +#define Z7_CAST_FUNC_C +#endif +/* +#if (defined(__GNUC__) && (__GNUC__ >= 8)) || defined(__clang__) + // #pragma GCC diagnostic ignored "-Wcast-function-type" +#endif +*/ +#ifdef __GNUC__ +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40000) && (Z7_GCC_VERSION < 70000) +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif +#endif + #ifdef _MSC_VER @@ -17,24 +110,124 @@ #pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int #endif - #if _MSC_VER >= 1300 - #pragma warning(disable : 4996) // This function or variable may be unsafe - #else - #pragma warning(disable : 4511) // copy constructor could not be generated - #pragma warning(disable : 4512) // assignment operator could not be generated - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4702) // unreachable code - #pragma warning(disable : 4710) // not inlined - #pragma warning(disable : 4714) // function marked as __forceinline not inlined - #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information - #endif +#if defined(_MSC_VER) && _MSC_VER >= 1800 +#pragma warning(disable : 4464) // relative include path contains '..' +#endif - #ifdef __clang__ - #pragma clang diagnostic ignored "-Wdeprecated-declarations" - #pragma clang diagnostic ignored "-Wmicrosoft-exception-spec" - // #pragma clang diagnostic ignored "-Wreserved-id-macro" - #endif +// == 1200 : -O1 : for __forceinline +// >= 1900 : -O1 : for printf +#pragma warning(disable : 4710) // function not inlined + +#if _MSC_VER < 1900 +// winnt.h: 'Int64ShllMod32' +#pragma warning(disable : 4514) // unreferenced inline function has been removed +#endif + +#if _MSC_VER < 1300 +// #pragma warning(disable : 4702) // unreachable code +// Bra.c : -O1: +#pragma warning(disable : 4714) // function marked as __forceinline not inlined +#endif + +/* +#if _MSC_VER > 1400 && _MSC_VER <= 1900 +// strcat: This function or variable may be unsafe +// sysinfoapi.h: kit10: GetVersion was declared deprecated +#pragma warning(disable : 4996) +#endif +*/ + +#if _MSC_VER > 1200 +// -Wall warnings + +#pragma warning(disable : 4711) // function selected for automatic inline expansion +#pragma warning(disable : 4820) // '2' bytes padding added after data member + +#if _MSC_VER >= 1400 && _MSC_VER < 1920 +// 1400: string.h: _DBG_MEMCPY_INLINE_ +// 1600 - 191x : smmintrin.h __cplusplus' +// is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' +#pragma warning(disable : 4668) + +// 1400 - 1600 : WinDef.h : 'FARPROC' : +// 1900 - 191x : immintrin.h: _readfsbase_u32 +// no function prototype given : converting '()' to '(void)' +#pragma warning(disable : 4255) +#endif + +#if _MSC_VER >= 1914 +// Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#pragma warning(disable : 5045) +#endif + +#endif // _MSC_VER > 1200 +#endif // _MSC_VER + + +#if defined(__clang__) && (__clang_major__ >= 4) + #define Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + _Pragma("clang loop unroll(disable)") \ + _Pragma("clang loop vectorize(disable)") + #define Z7_ATTRIB_NO_VECTORIZE +#elif defined(__GNUC__) && (__GNUC__ >= 5) \ + && (!defined(Z7_MCST_LCC_VERSION) || (Z7_MCST_LCC_VERSION >= 12610)) + #define Z7_ATTRIB_NO_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) + // __attribute__((optimize("no-unroll-loops"))); + #define Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE +#elif defined(_MSC_VER) && (_MSC_VER >= 1920) + #define Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + _Pragma("loop( no_vector )") + #define Z7_ATTRIB_NO_VECTORIZE +#else + #define Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + #define Z7_ATTRIB_NO_VECTORIZE +#endif + +#if defined(MY_CPU_X86_OR_AMD64) && ( \ + defined(__clang__) && (__clang_major__ >= 4) \ + || defined(__GNUC__) && (__GNUC__ >= 5)) + #define Z7_ATTRIB_NO_SSE __attribute__((__target__("no-sse"))) +#else + #define Z7_ATTRIB_NO_SSE +#endif + +#define Z7_ATTRIB_NO_VECTOR \ + Z7_ATTRIB_NO_VECTORIZE \ + Z7_ATTRIB_NO_SSE + + +#if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 1000) \ + /* || defined(_MSC_VER) && (_MSC_VER >= 1920) */ + // GCC is not good for __builtin_expect() + #define Z7_LIKELY(x) (__builtin_expect((x), 1)) + #define Z7_UNLIKELY(x) (__builtin_expect((x), 0)) + // #define Z7_unlikely [[unlikely]] + // #define Z7_likely [[likely]] +#else + #define Z7_LIKELY(x) (x) + #define Z7_UNLIKELY(x) (x) + // #define Z7_likely +#endif + + +#if (defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30600)) + +#if (Z7_CLANG_VERSION < 130000) +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wreserved-id-macro\"") +#else +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wreserved-macro-identifier\"") +#endif +#define Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER \ + _Pragma("GCC diagnostic pop") +#else +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#define Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER #endif #define UNUSED_VAR(x) (void)x; diff --git a/multiarc/src/formats/7z/C/CpuArch.c b/multiarc/src/formats/7z/C/CpuArch.c old mode 100755 new mode 100644 index 0b80ba96d..4255206b4 --- a/multiarc/src/formats/7z/C/CpuArch.c +++ b/multiarc/src/formats/7z/C/CpuArch.c @@ -1,187 +1,318 @@ /* CpuArch.c -- CPU specific code -2021-07-13 : Igor Pavlov : Public domain */ +2024-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include + #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if (defined(_MSC_VER) && !defined(MY_CPU_AMD64)) || defined(__GNUC__) -#define USE_ASM +#undef NEED_CHECK_FOR_CPUID +#if !defined(MY_CPU_AMD64) +#define NEED_CHECK_FOR_CPUID #endif -#if !defined(USE_ASM) && _MSC_VER >= 1500 -#include +/* + cpuid instruction supports (subFunction) parameter in ECX, + that is used only with some specific (function) parameter values. + But we always use only (subFunction==0). +*/ +/* + __cpuid(): MSVC and GCC/CLANG use same function/macro name + but parameters are different. + We use MSVC __cpuid() parameters style for our z7_x86_cpuid() function. +*/ + +#if defined(__GNUC__) /* && (__GNUC__ >= 10) */ \ + || defined(__clang__) /* && (__clang_major__ >= 10) */ + +/* there was some CLANG/GCC compilers that have issues with + rbx(ebx) handling in asm blocks in -fPIC mode (__PIC__ is defined). + compiler's contains the macro __cpuid() that is similar to our code. + The history of __cpuid() changes in CLANG/GCC: + GCC: + 2007: it preserved ebx for (__PIC__ && __i386__) + 2013: it preserved rbx and ebx for __PIC__ + 2014: it doesn't preserves rbx and ebx anymore + we suppose that (__GNUC__ >= 5) fixed that __PIC__ ebx/rbx problem. + CLANG: + 2014+: it preserves rbx, but only for 64-bit code. No __PIC__ check. + Why CLANG cares about 64-bit mode only, and doesn't care about ebx (in 32-bit)? + Do we need __PIC__ test for CLANG or we must care about rbx even if + __PIC__ is not defined? +*/ + +#define ASM_LN "\n" + +#if defined(MY_CPU_AMD64) && defined(__PIC__) \ + && ((defined (__GNUC__) && (__GNUC__ < 5)) || defined(__clang__)) + +#define x86_cpuid_MACRO(p, func) { \ + __asm__ __volatile__ ( \ + ASM_LN "mov %%rbx, %q1" \ + ASM_LN "cpuid" \ + ASM_LN "xchg %%rbx, %q1" \ + : "=a" ((p)[0]), "=&r" ((p)[1]), "=c" ((p)[2]), "=d" ((p)[3]) : "0" (func), "2"(0)); } + + /* "=&r" selects free register. It can select even rbx, if that register is free. + "=&D" for (RDI) also works, but the code can be larger with "=&D" + "2"(0) means (subFunction = 0), + 2 is (zero-based) index in the output constraint list "=c" (ECX). */ + +#elif defined(MY_CPU_X86) && defined(__PIC__) \ + && ((defined (__GNUC__) && (__GNUC__ < 5)) || defined(__clang__)) + +#define x86_cpuid_MACRO(p, func) { \ + __asm__ __volatile__ ( \ + ASM_LN "mov %%ebx, %k1" \ + ASM_LN "cpuid" \ + ASM_LN "xchg %%ebx, %k1" \ + : "=a" ((p)[0]), "=&r" ((p)[1]), "=c" ((p)[2]), "=d" ((p)[3]) : "0" (func), "2"(0)); } + +#else + +#define x86_cpuid_MACRO(p, func) { \ + __asm__ __volatile__ ( \ + ASM_LN "cpuid" \ + : "=a" ((p)[0]), "=b" ((p)[1]), "=c" ((p)[2]), "=d" ((p)[3]) : "0" (func), "2"(0)); } + #endif -#if defined(USE_ASM) && !defined(MY_CPU_AMD64) -static UInt32 CheckFlag(UInt32 flag) -{ - #ifdef _MSC_VER - __asm pushfd; - __asm pop EAX; - __asm mov EDX, EAX; - __asm xor EAX, flag; - __asm push EAX; - __asm popfd; - __asm pushfd; - __asm pop EAX; - __asm xor EAX, EDX; - __asm push EDX; - __asm popfd; - __asm and flag, EAX; - #else + +void Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) +{ + x86_cpuid_MACRO(p, func) +} + + +Z7_NO_INLINE +UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void) +{ + #if defined(NEED_CHECK_FOR_CPUID) + #define EFALGS_CPUID_BIT 21 + UInt32 a; __asm__ __volatile__ ( - "pushf\n\t" - "pop %%EAX\n\t" - "movl %%EAX,%%EDX\n\t" - "xorl %0,%%EAX\n\t" - "push %%EAX\n\t" - "popf\n\t" - "pushf\n\t" - "pop %%EAX\n\t" - "xorl %%EDX,%%EAX\n\t" - "push %%EDX\n\t" - "popf\n\t" - "andl %%EAX, %0\n\t": - "=c" (flag) : "c" (flag) : - "%eax", "%edx"); + ASM_LN "pushf" + ASM_LN "pushf" + ASM_LN "pop %0" + // ASM_LN "movl %0, %1" + // ASM_LN "xorl $0x200000, %0" + ASM_LN "btc %1, %0" + ASM_LN "push %0" + ASM_LN "popf" + ASM_LN "pushf" + ASM_LN "pop %0" + ASM_LN "xorl (%%esp), %0" + + ASM_LN "popf" + ASM_LN + : "=&r" (a) // "=a" + : "i" (EFALGS_CPUID_BIT) + ); + if ((a & (1 << EFALGS_CPUID_BIT)) == 0) + return 0; + #endif + { + UInt32 p[4]; + x86_cpuid_MACRO(p, 0) + return p[0]; + } +} + +#undef ASM_LN + +#elif !defined(_MSC_VER) + +/* +// for gcc/clang and other: we can try to use __cpuid macro: +#include +void Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) +{ + __cpuid(func, p[0], p[1], p[2], p[3]); +} +UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void) +{ + return (UInt32)__get_cpuid_max(0, NULL); +} +*/ +// for unsupported cpuid: +void Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) +{ + UNUSED_VAR(func) + p[0] = p[1] = p[2] = p[3] = 0; +} +UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void) +{ + return 0; +} + +#else // _MSC_VER + +#if !defined(MY_CPU_AMD64) + +UInt32 __declspec(naked) Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void) +{ + #if defined(NEED_CHECK_FOR_CPUID) + #define EFALGS_CPUID_BIT 21 + __asm pushfd + __asm pushfd + /* + __asm pop eax + // __asm mov edx, eax + __asm btc eax, EFALGS_CPUID_BIT + __asm push eax + */ + __asm btc dword ptr [esp], EFALGS_CPUID_BIT + __asm popfd + __asm pushfd + __asm pop eax + // __asm xor eax, edx + __asm xor eax, [esp] + // __asm push edx + __asm popfd + __asm and eax, (1 shl EFALGS_CPUID_BIT) + __asm jz end_func #endif - return flag; + __asm push ebx + __asm xor eax, eax // func + __asm xor ecx, ecx // subFunction (optional) for (func == 0) + __asm cpuid + __asm pop ebx + #if defined(NEED_CHECK_FOR_CPUID) + end_func: + #endif + __asm ret 0 } -#define CHECK_CPUID_IS_SUPPORTED if (CheckFlag(1 << 18) == 0 || CheckFlag(1 << 21) == 0) return False; -#else -#define CHECK_CPUID_IS_SUPPORTED -#endif -#ifndef USE_ASM - #ifdef _MSC_VER +void __declspec(naked) Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) +{ + UNUSED_VAR(p) + UNUSED_VAR(func) + __asm push ebx + __asm push edi + __asm mov edi, ecx // p + __asm mov eax, edx // func + __asm xor ecx, ecx // subfunction (optional) for (func == 0) + __asm cpuid + __asm mov [edi ], eax + __asm mov [edi + 4], ebx + __asm mov [edi + 8], ecx + __asm mov [edi + 12], edx + __asm pop edi + __asm pop ebx + __asm ret 0 +} + +#else // MY_CPU_AMD64 + #if _MSC_VER >= 1600 - #define MY__cpuidex __cpuidex + #include + #define MY_cpuidex __cpuidex #else - /* - __cpuid (function == 4) requires subfunction number in ECX. + __cpuid (func == (0 or 7)) requires subfunction number in ECX. MSDN: The __cpuid intrinsic clears the ECX register before calling the cpuid instruction. __cpuid() in new MSVC clears ECX. - __cpuid() in old MSVC (14.00) doesn't clear ECX - We still can use __cpuid for low (function) values that don't require ECX, - but __cpuid() in old MSVC will be incorrect for some function values: (function == 4). + __cpuid() in old MSVC (14.00) x64 doesn't clear ECX + We still can use __cpuid for low (func) values that don't require ECX, + but __cpuid() in old MSVC will be incorrect for some func values: (func == 7). So here we use the hack for old MSVC to send (subFunction) in ECX register to cpuid instruction, - where ECX value is first parameter for FAST_CALL / NO_INLINE function, - So the caller of MY__cpuidex_HACK() sets ECX as subFunction, and + where ECX value is first parameter for FASTCALL / NO_INLINE func, + So the caller of MY_cpuidex_HACK() sets ECX as subFunction, and old MSVC for __cpuid() doesn't change ECX and cpuid instruction gets (subFunction) value. - DON'T remove MY_NO_INLINE and MY_FAST_CALL for MY__cpuidex_HACK() !!! +DON'T remove Z7_NO_INLINE and Z7_FASTCALL for MY_cpuidex_HACK(): !!! */ - static -MY_NO_INLINE -void MY_FAST_CALL MY__cpuidex_HACK(UInt32 subFunction, int *CPUInfo, UInt32 function) +Z7_NO_INLINE void Z7_FASTCALL MY_cpuidex_HACK(Int32 subFunction, Int32 func, Int32 *CPUInfo) { - UNUSED_VAR(subFunction); - __cpuid(CPUInfo, function); + UNUSED_VAR(subFunction) + __cpuid(CPUInfo, func); } - - #define MY__cpuidex(info, func, func2) MY__cpuidex_HACK(func2, info, func) - #pragma message("======== MY__cpuidex_HACK WAS USED ========") - #endif - #else - #define MY__cpuidex(info, func, func2) __cpuid(info, func) - #pragma message("======== (INCORRECT ?) cpuid WAS USED ========") - #endif + #define MY_cpuidex(info, func, func2) MY_cpuidex_HACK(func2, func, info) + #pragma message("======== MY_cpuidex_HACK WAS USED ========") + #endif // _MSC_VER >= 1600 + +#if !defined(MY_CPU_AMD64) +/* inlining for __cpuid() in MSVC x86 (32-bit) produces big ineffective code, + so we disable inlining here */ +Z7_NO_INLINE #endif - - - - -void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) +void Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) { - #ifdef USE_ASM - - #ifdef _MSC_VER - - UInt32 a2, b2, c2, d2; - __asm xor EBX, EBX; - __asm xor ECX, ECX; - __asm xor EDX, EDX; - __asm mov EAX, function; - __asm cpuid; - __asm mov a2, EAX; - __asm mov b2, EBX; - __asm mov c2, ECX; - __asm mov d2, EDX; - - *a = a2; - *b = b2; - *c = c2; - *d = d2; - - #else - - __asm__ __volatile__ ( - #if defined(MY_CPU_AMD64) && defined(__PIC__) - "mov %%rbx, %%rdi;" - "cpuid;" - "xchg %%rbx, %%rdi;" - : "=a" (*a) , - "=D" (*b) , - #elif defined(MY_CPU_X86) && defined(__PIC__) - "mov %%ebx, %%edi;" - "cpuid;" - "xchgl %%ebx, %%edi;" - : "=a" (*a) , - "=D" (*b) , - #else - "cpuid" - : "=a" (*a) , - "=b" (*b) , - #endif - "=c" (*c) , - "=d" (*d) - : "0" (function), "c"(0) ) ; - - #endif - - #else + MY_cpuidex((Int32 *)p, (Int32)func, 0); +} - int CPUInfo[4]; +Z7_NO_INLINE +UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void) +{ + Int32 a[4]; + MY_cpuidex(a, 0, 0); + return a[0]; +} - MY__cpuidex(CPUInfo, (int)function, 0); +#endif // MY_CPU_AMD64 +#endif // _MSC_VER - *a = (UInt32)CPUInfo[0]; - *b = (UInt32)CPUInfo[1]; - *c = (UInt32)CPUInfo[2]; - *d = (UInt32)CPUInfo[3]; +#if defined(NEED_CHECK_FOR_CPUID) +#define CHECK_CPUID_IS_SUPPORTED { if (z7_x86_cpuid_GetMaxFunc() == 0) return 0; } +#else +#define CHECK_CPUID_IS_SUPPORTED +#endif +#undef NEED_CHECK_FOR_CPUID - #endif -} -BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) +static +BoolInt x86cpuid_Func_1(UInt32 *p) { CHECK_CPUID_IS_SUPPORTED - MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); - MyCPUID(1, &p->ver, &p->b, &p->c, &p->d); + z7_x86_cpuid(p, 1); return True; } -static const UInt32 kVendors[][3] = +/* +static const UInt32 kVendors[][1] = { - { 0x756E6547, 0x49656E69, 0x6C65746E}, - { 0x68747541, 0x69746E65, 0x444D4163}, - { 0x746E6543, 0x48727561, 0x736C7561} + { 0x756E6547 }, // , 0x49656E69, 0x6C65746E }, + { 0x68747541 }, // , 0x69746E65, 0x444D4163 }, + { 0x746E6543 } // , 0x48727561, 0x736C7561 } }; +*/ + +/* +typedef struct +{ + UInt32 maxFunc; + UInt32 vendor[3]; + UInt32 ver; + UInt32 b; + UInt32 c; + UInt32 d; +} Cx86cpuid; + +enum +{ + CPU_FIRM_INTEL, + CPU_FIRM_AMD, + CPU_FIRM_VIA +}; +int x86cpuid_GetFirm(const Cx86cpuid *p); +#define x86cpuid_ver_GetFamily(ver) (((ver >> 16) & 0xff0) | ((ver >> 8) & 0xf)) +#define x86cpuid_ver_GetModel(ver) (((ver >> 12) & 0xf0) | ((ver >> 4) & 0xf)) +#define x86cpuid_ver_GetStepping(ver) (ver & 0xf) int x86cpuid_GetFirm(const Cx86cpuid *p) { unsigned i; - for (i = 0; i < sizeof(kVendors) / sizeof(kVendors[i]); i++) + for (i = 0; i < sizeof(kVendors) / sizeof(kVendors[0]); i++) { const UInt32 *v = kVendors[i]; - if (v[0] == p->vendor[0] && - v[1] == p->vendor[1] && - v[2] == p->vendor[2]) + if (v[0] == p->vendor[0] + // && v[1] == p->vendor[1] + // && v[2] == p->vendor[2] + ) return (int)i; } return -1; @@ -190,41 +321,55 @@ int x86cpuid_GetFirm(const Cx86cpuid *p) BoolInt CPU_Is_InOrder() { Cx86cpuid p; - int firm; UInt32 family, model; if (!x86cpuid_CheckAndRead(&p)) return True; - family = x86cpuid_GetFamily(p.ver); - model = x86cpuid_GetModel(p.ver); - - firm = x86cpuid_GetFirm(&p); + family = x86cpuid_ver_GetFamily(p.ver); + model = x86cpuid_ver_GetModel(p.ver); - switch (firm) + switch (x86cpuid_GetFirm(&p)) { case CPU_FIRM_INTEL: return (family < 6 || (family == 6 && ( - /* In-Order Atom CPU */ - model == 0x1C /* 45 nm, N4xx, D4xx, N5xx, D5xx, 230, 330 */ - || model == 0x26 /* 45 nm, Z6xx */ - || model == 0x27 /* 32 nm, Z2460 */ - || model == 0x35 /* 32 nm, Z2760 */ - || model == 0x36 /* 32 nm, N2xxx, D2xxx */ + // In-Order Atom CPU + model == 0x1C // 45 nm, N4xx, D4xx, N5xx, D5xx, 230, 330 + || model == 0x26 // 45 nm, Z6xx + || model == 0x27 // 32 nm, Z2460 + || model == 0x35 // 32 nm, Z2760 + || model == 0x36 // 32 nm, N2xxx, D2xxx ))); case CPU_FIRM_AMD: return (family < 5 || (family == 5 && (model < 6 || model == 0xA))); case CPU_FIRM_VIA: return (family < 6 || (family == 6 && model < 0xF)); } - return True; + return False; // v23 : unknown processors are not In-Order } +*/ + +#ifdef _WIN32 +#include "7zWindows.h" +#endif #if !defined(MY_CPU_AMD64) && defined(_WIN32) -#include -static BoolInt CPU_Sys_Is_SSE_Supported() + +/* for legacy SSE ia32: there is no user-space cpu instruction to check + that OS supports SSE register storing/restoring on context switches. + So we need some OS-specific function to check that it's safe to use SSE registers. +*/ + +Z7_FORCE_INLINE +static BoolInt CPU_Sys_Is_SSE_Supported(void) { - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (!GetVersionEx(&vi)) - return False; - return (vi.dwMajorVersion >= 5); +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4996) // `GetVersion': was declared deprecated +#endif + /* low byte is major version of Windows + We suppose that any Windows version since + Windows2000 (major == 5) supports SSE registers */ + return (Byte)GetVersion() >= 5; +#if defined(_MSC_VER) + #pragma warning(pop) +#endif } #define CHECK_SYS_SSE_SUPPORT if (!CPU_Sys_Is_SSE_Supported()) return False; #else @@ -232,117 +377,347 @@ static BoolInt CPU_Sys_Is_SSE_Supported() #endif -static UInt32 X86_CPUID_ECX_Get_Flags() +#if !defined(MY_CPU_AMD64) + +BoolInt CPU_IsSupported_CMOV(void) { - Cx86cpuid p; + UInt32 a[4]; + if (!x86cpuid_Func_1(&a[0])) + return 0; + return (BoolInt)(a[3] >> 15) & 1; +} + +BoolInt CPU_IsSupported_SSE(void) +{ + UInt32 a[4]; CHECK_SYS_SSE_SUPPORT - if (!x86cpuid_CheckAndRead(&p)) + if (!x86cpuid_Func_1(&a[0])) return 0; - return p.c; + return (BoolInt)(a[3] >> 25) & 1; } -BoolInt CPU_IsSupported_AES() +BoolInt CPU_IsSupported_SSE2(void) { - return (X86_CPUID_ECX_Get_Flags() >> 25) & 1; + UInt32 a[4]; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_Func_1(&a[0])) + return 0; + return (BoolInt)(a[3] >> 26) & 1; } -BoolInt CPU_IsSupported_SSSE3() +#endif + + +static UInt32 x86cpuid_Func_1_ECX(void) { - return (X86_CPUID_ECX_Get_Flags() >> 9) & 1; + UInt32 a[4]; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_Func_1(&a[0])) + return 0; + return a[2]; } -BoolInt CPU_IsSupported_SSE41() +BoolInt CPU_IsSupported_AES(void) { - return (X86_CPUID_ECX_Get_Flags() >> 19) & 1; + return (BoolInt)(x86cpuid_Func_1_ECX() >> 25) & 1; } -BoolInt CPU_IsSupported_SHA() +BoolInt CPU_IsSupported_SSSE3(void) +{ + return (BoolInt)(x86cpuid_Func_1_ECX() >> 9) & 1; +} + +BoolInt CPU_IsSupported_SSE41(void) +{ + return (BoolInt)(x86cpuid_Func_1_ECX() >> 19) & 1; +} + +BoolInt CPU_IsSupported_SHA(void) { - Cx86cpuid p; CHECK_SYS_SSE_SUPPORT - if (!x86cpuid_CheckAndRead(&p)) - return False; - if (p.maxFunc < 7) + if (z7_x86_cpuid_GetMaxFunc() < 7) return False; { - UInt32 d[4] = { 0 }; - MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); - return (d[1] >> 29) & 1; + UInt32 d[4]; + z7_x86_cpuid(d, 7); + return (BoolInt)(d[1] >> 29) & 1; } } -// #include +/* +MSVC: _xgetbv() intrinsic is available since VS2010SP1. + MSVC also defines (_XCR_XFEATURE_ENABLED_MASK) macro in + that we can use or check. + For any 32-bit x86 we can use asm code in MSVC, + but MSVC asm code is huge after compilation. + So _xgetbv() is better + +ICC: _xgetbv() intrinsic is available (in what version of ICC?) + ICC defines (__GNUC___) and it supports gnu assembler + also ICC supports MASM style code with -use-msasm switch. + but ICC doesn't support __attribute__((__target__)) + +GCC/CLANG 9: + _xgetbv() is macro that works via __builtin_ia32_xgetbv() + and we need __attribute__((__target__("xsave")). + But with __target__("xsave") the function will be not + inlined to function that has no __target__("xsave") attribute. + If we want _xgetbv() call inlining, then we should use asm version + instead of calling _xgetbv(). + Note:intrinsic is broke before GCC 8.2: + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85684 +*/ -#ifdef _WIN32 -#include +#if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1100) \ + || defined(_MSC_VER) && (_MSC_VER >= 1600) && (_MSC_FULL_VER >= 160040219) \ + || defined(__GNUC__) && (__GNUC__ >= 9) \ + || defined(__clang__) && (__clang_major__ >= 9) +// we define ATTRIB_XGETBV, if we want to use predefined _xgetbv() from compiler +#if defined(__INTEL_COMPILER) +#define ATTRIB_XGETBV +#elif defined(__GNUC__) || defined(__clang__) +// we don't define ATTRIB_XGETBV here, because asm version is better for inlining. +// #define ATTRIB_XGETBV __attribute__((__target__("xsave"))) +#else +#define ATTRIB_XGETBV +#endif +#endif + +#if defined(ATTRIB_XGETBV) +#include #endif -BoolInt CPU_IsSupported_AVX2() + +// XFEATURE_ENABLED_MASK/XCR0 +#define MY_XCR_XFEATURE_ENABLED_MASK 0 + +#if defined(ATTRIB_XGETBV) +ATTRIB_XGETBV +#endif +static UInt64 x86_xgetbv_0(UInt32 num) { - Cx86cpuid p; - CHECK_SYS_SSE_SUPPORT +#if defined(ATTRIB_XGETBV) + { + return + #if (defined(_MSC_VER)) + _xgetbv(num); + #else + __builtin_ia32_xgetbv( + #if !defined(__clang__) + (int) + #endif + num); + #endif + } + +#elif defined(__GNUC__) || defined(__clang__) || defined(__SUNPRO_CC) + + UInt32 a, d; + #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) + __asm__ + ( + "xgetbv" + : "=a"(a), "=d"(d) : "c"(num) : "cc" + ); + #else // is old gcc + __asm__ + ( + ".byte 0x0f, 0x01, 0xd0" "\n\t" + : "=a"(a), "=d"(d) : "c"(num) : "cc" + ); + #endif + return ((UInt64)d << 32) | a; + // return a; + +#elif defined(_MSC_VER) && !defined(MY_CPU_AMD64) + + UInt32 a, d; + __asm { + push eax + push edx + push ecx + mov ecx, num; + // xor ecx, ecx // = MY_XCR_XFEATURE_ENABLED_MASK + _emit 0x0f + _emit 0x01 + _emit 0xd0 + mov a, eax + mov d, edx + pop ecx + pop edx + pop eax + } + return ((UInt64)d << 32) | a; + // return a; + +#else // it's unknown compiler + // #error "Need xgetbv function" + UNUSED_VAR(num) + // for MSVC-X64 we could call external function from external file. + /* Actually we had checked OSXSAVE/AVX in cpuid before. + So it's expected that OS supports at least AVX and below. */ + // if (num != MY_XCR_XFEATURE_ENABLED_MASK) return 0; // if not XCR0 + return + // (1 << 0) | // x87 + (1 << 1) // SSE + | (1 << 2); // AVX + +#endif +} +#ifdef _WIN32 +/* + Windows versions do not know about new ISA extensions that + can be introduced. But we still can use new extensions, + even if Windows doesn't report about supporting them, + But we can use new extensions, only if Windows knows about new ISA extension + that changes the number or size of registers: SSE, AVX/XSAVE, AVX512 + So it's enough to check + MY_PF_AVX_INSTRUCTIONS_AVAILABLE + instead of + MY_PF_AVX2_INSTRUCTIONS_AVAILABLE +*/ +#define MY_PF_XSAVE_ENABLED 17 +// #define MY_PF_SSSE3_INSTRUCTIONS_AVAILABLE 36 +// #define MY_PF_SSE4_1_INSTRUCTIONS_AVAILABLE 37 +// #define MY_PF_SSE4_2_INSTRUCTIONS_AVAILABLE 38 +// #define MY_PF_AVX_INSTRUCTIONS_AVAILABLE 39 +// #define MY_PF_AVX2_INSTRUCTIONS_AVAILABLE 40 +// #define MY_PF_AVX512F_INSTRUCTIONS_AVAILABLE 41 +#endif + +BoolInt CPU_IsSupported_AVX(void) +{ #ifdef _WIN32 - #define MY__PF_XSAVE_ENABLED 17 - if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + if (!IsProcessorFeaturePresent(MY_PF_XSAVE_ENABLED)) return False; + /* PF_AVX_INSTRUCTIONS_AVAILABLE probably is supported starting from + some latest Win10 revisions. But we need AVX in older Windows also. + So we don't use the following check: */ + /* + if (!IsProcessorFeaturePresent(MY_PF_AVX_INSTRUCTIONS_AVAILABLE)) + return False; + */ #endif - if (!x86cpuid_CheckAndRead(&p)) + /* + OS must use new special XSAVE/XRSTOR instructions to save + AVX registers when it required for context switching. + At OS statring: + OS sets CR4.OSXSAVE flag to signal the processor that OS supports the XSAVE extensions. + Also OS sets bitmask in XCR0 register that defines what + registers will be processed by XSAVE instruction: + XCR0.SSE[bit 0] - x87 registers and state + XCR0.SSE[bit 1] - SSE registers and state + XCR0.AVX[bit 2] - AVX registers and state + CR4.OSXSAVE is reflected to CPUID.1:ECX.OSXSAVE[bit 27]. + So we can read that bit in user-space. + XCR0 is available for reading in user-space by new XGETBV instruction. + */ + { + const UInt32 c = x86cpuid_Func_1_ECX(); + if (0 == (1 + & (c >> 28) // AVX instructions are supported by hardware + & (c >> 27))) // OSXSAVE bit: XSAVE and related instructions are enabled by OS. + return False; + } + + /* also we can check + CPUID.1:ECX.XSAVE [bit 26] : that shows that + XSAVE, XRESTOR, XSETBV, XGETBV instructions are supported by hardware. + But that check is redundant, because if OSXSAVE bit is set, then XSAVE is also set */ + + /* If OS have enabled XSAVE extension instructions (OSXSAVE == 1), + in most cases we expect that OS also will support storing/restoring + for AVX and SSE states at least. + But to be ensure for that we call user-space instruction + XGETBV(0) to get XCR0 value that contains bitmask that defines + what exact states(registers) OS have enabled for storing/restoring. + */ + + { + const UInt32 bm = (UInt32)x86_xgetbv_0(MY_XCR_XFEATURE_ENABLED_MASK); + // printf("\n=== XGetBV=0x%x\n", bm); + return 1 + & (BoolInt)(bm >> 1) // SSE state is supported (set by OS) for storing/restoring + & (BoolInt)(bm >> 2); // AVX state is supported (set by OS) for storing/restoring + } + // since Win7SP1: we can use GetEnabledXStateFeatures(); +} + + +BoolInt CPU_IsSupported_AVX2(void) +{ + if (!CPU_IsSupported_AVX()) return False; - if (p.maxFunc < 7) + if (z7_x86_cpuid_GetMaxFunc() < 7) return False; { - UInt32 d[4] = { 0 }; - MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + UInt32 d[4]; + z7_x86_cpuid(d, 7); // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); return 1 - & (d[1] >> 5); // avx2 + & (BoolInt)(d[1] >> 5); // avx2 } } -BoolInt CPU_IsSupported_VAES_AVX2() +#if 0 +BoolInt CPU_IsSupported_AVX512F_AVX512VL(void) { - Cx86cpuid p; - CHECK_SYS_SSE_SUPPORT - - #ifdef _WIN32 - #define MY__PF_XSAVE_ENABLED 17 - if (!IsProcessorFeaturePresent(MY__PF_XSAVE_ENABLED)) + if (!CPU_IsSupported_AVX()) return False; - #endif + if (z7_x86_cpuid_GetMaxFunc() < 7) + return False; + { + UInt32 d[4]; + BoolInt v; + z7_x86_cpuid(d, 7); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + v = 1 + & (BoolInt)(d[1] >> 16) // avx512f + & (BoolInt)(d[1] >> 31); // avx512vl + if (!v) + return False; + } + { + const UInt32 bm = (UInt32)x86_xgetbv_0(MY_XCR_XFEATURE_ENABLED_MASK); + // printf("\n=== XGetBV=0x%x\n", bm); + return 1 + & (BoolInt)(bm >> 5) // OPMASK + & (BoolInt)(bm >> 6) // ZMM upper 256-bit + & (BoolInt)(bm >> 7); // ZMM16 ... ZMM31 + } +} +#endif - if (!x86cpuid_CheckAndRead(&p)) +BoolInt CPU_IsSupported_VAES_AVX2(void) +{ + if (!CPU_IsSupported_AVX()) return False; - if (p.maxFunc < 7) + if (z7_x86_cpuid_GetMaxFunc() < 7) return False; { - UInt32 d[4] = { 0 }; - MyCPUID(7, &d[0], &d[1], &d[2], &d[3]); + UInt32 d[4]; + z7_x86_cpuid(d, 7); // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); return 1 - & (d[1] >> 5) // avx2 + & (BoolInt)(d[1] >> 5) // avx2 // & (d[1] >> 31) // avx512vl - & (d[2] >> 9); // vaes // VEX-256/EVEX + & (BoolInt)(d[2] >> 9); // vaes // VEX-256/EVEX } } -BoolInt CPU_IsSupported_PageGB() +BoolInt CPU_IsSupported_PageGB(void) { - Cx86cpuid cpuid; - if (!x86cpuid_CheckAndRead(&cpuid)) - return False; + CHECK_CPUID_IS_SUPPORTED { - UInt32 d[4] = { 0 }; - MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + UInt32 d[4]; + z7_x86_cpuid(d, 0x80000000); if (d[0] < 0x80000001) return False; - } - { - UInt32 d[4] = { 0 }; - MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); - return (d[3] >> 26) & 1; + z7_x86_cpuid(d, 0x80000001); + return (BoolInt)(d[3] >> 26) & 1; } } @@ -351,11 +726,11 @@ BoolInt CPU_IsSupported_PageGB() #ifdef _WIN32 -#include +#include "7zWindows.h" -BoolInt CPU_IsSupported_CRC32() { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } -BoolInt CPU_IsSupported_CRYPTO() { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } -BoolInt CPU_IsSupported_NEON() { return IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRC32(void) { return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_CRYPTO(void) { return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } +BoolInt CPU_IsSupported_NEON(void) { return IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE) ? 1 : 0; } #else @@ -378,28 +753,27 @@ static void Print_sysctlbyname(const char *name) } } */ +/* + Print_sysctlbyname("hw.pagesize"); + Print_sysctlbyname("machdep.cpu.brand_string"); +*/ -static BoolInt My_sysctlbyname_Get_BoolInt(const char *name) +static BoolInt z7_sysctlbyname_Get_BoolInt(const char *name) { UInt32 val = 0; - if (My_sysctlbyname_Get_UInt32(name, &val) == 0 && val == 1) + if (z7_sysctlbyname_Get_UInt32(name, &val) == 0 && val == 1) return 1; return 0; } - /* - Print_sysctlbyname("hw.pagesize"); - Print_sysctlbyname("machdep.cpu.brand_string"); - */ - BoolInt CPU_IsSupported_CRC32(void) { - return My_sysctlbyname_Get_BoolInt("hw.optional.armv8_crc32"); + return z7_sysctlbyname_Get_BoolInt("hw.optional.armv8_crc32"); } BoolInt CPU_IsSupported_NEON(void) { - return My_sysctlbyname_Get_BoolInt("hw.optional.neon"); + return z7_sysctlbyname_Get_BoolInt("hw.optional.neon"); } #ifdef MY_CPU_ARM64 @@ -415,40 +789,72 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; } #else // __APPLE__ -#if !defined(__UCLIBC__) -# include +#if defined(__GLIBC__) && (__GLIBC__ * 100 + __GLIBC_MINOR__ >= 216) + #define Z7_GETAUXV_AVAILABLE +#else +// #pragma message("=== is not NEW GLIBC === ") + #if defined __has_include + #if __has_include () +// #pragma message("=== sys/auxv.h is avail=== ") + #define Z7_GETAUXV_AVAILABLE + #endif + #endif +#endif +#ifdef Z7_GETAUXV_AVAILABLE +// #pragma message("=== Z7_GETAUXV_AVAILABLE === ") +#include #define USE_HWCAP +#endif #ifdef USE_HWCAP +#if defined(__FreeBSD__) || defined(__DragonFly__) +static unsigned long MY_getauxval(int aux) +{ + unsigned long val; + if (elf_aux_info(aux, &val, sizeof(val))) + return 0; + return val; +} +#else +#define MY_getauxval getauxval + #if defined __has_include + #if __has_include () #include + #endif + #endif +#endif #define MY_HWCAP_CHECK_FUNC_2(name1, name2) \ - BoolInt CPU_IsSupported_ ## name1() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name2)) ? 1 : 0; } + BoolInt CPU_IsSupported_ ## name1(void) { return (MY_getauxval(AT_HWCAP) & (HWCAP_ ## name2)); } #ifdef MY_CPU_ARM64 #define MY_HWCAP_CHECK_FUNC(name) \ MY_HWCAP_CHECK_FUNC_2(name, name) +#if 1 || defined(__ARM_NEON) + BoolInt CPU_IsSupported_NEON(void) { return True; } +#else MY_HWCAP_CHECK_FUNC_2(NEON, ASIMD) +#endif // MY_HWCAP_CHECK_FUNC (ASIMD) #elif defined(MY_CPU_ARM) #define MY_HWCAP_CHECK_FUNC(name) \ - BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP2) & (HWCAP2_ ## name)) ? 1 : 0; } + BoolInt CPU_IsSupported_ ## name(void) { return (MY_getauxval(AT_HWCAP2) & (HWCAP2_ ## name)); } MY_HWCAP_CHECK_FUNC_2(NEON, NEON) #endif #else // USE_HWCAP #define MY_HWCAP_CHECK_FUNC(name) \ - BoolInt CPU_IsSupported_ ## name() { return 0; } + BoolInt CPU_IsSupported_ ## name(void) { return 0; } +#if defined(__ARM_NEON) + BoolInt CPU_IsSupported_NEON(void) { return True; } +#else MY_HWCAP_CHECK_FUNC(NEON) +#endif #endif // USE_HWCAP -#else // !defined(__UCLIBC__) - #define MY_HWCAP_CHECK_FUNC(name) \ - BoolInt CPU_IsSupported_ ## name() { return 0; } -#endif // !defined(__UCLIBC__) MY_HWCAP_CHECK_FUNC (CRC32) MY_HWCAP_CHECK_FUNC (SHA1) @@ -466,15 +872,15 @@ MY_HWCAP_CHECK_FUNC (AES) #include -int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize) +int z7_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize) { return sysctlbyname(name, buf, bufSize, NULL, 0); } -int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val) +int z7_sysctlbyname_Get_UInt32(const char *name, UInt32 *val) { size_t bufSize = sizeof(*val); - int res = My_sysctlbyname_Get(name, val, &bufSize); + const int res = z7_sysctlbyname_Get(name, val, &bufSize); if (res == 0 && bufSize != sizeof(*val)) return EFAULT; return res; diff --git a/multiarc/src/formats/7z/C/CpuArch.h b/multiarc/src/formats/7z/C/CpuArch.h old mode 100755 new mode 100644 index 4856fbb12..683cfaa86 --- a/multiarc/src/formats/7z/C/CpuArch.h +++ b/multiarc/src/formats/7z/C/CpuArch.h @@ -1,8 +1,8 @@ /* CpuArch.h -- CPU specific code -2022-07-15 : Igor Pavlov : Public domain */ +2024-06-17 : Igor Pavlov : Public domain */ -#ifndef __CPU_ARCH_H -#define __CPU_ARCH_H +#ifndef ZIP7_INC_CPU_ARCH_H +#define ZIP7_INC_CPU_ARCH_H #include "7zTypes.h" @@ -20,6 +20,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. MY_CPU_64BIT doesn't mean that (sizeof(void *) == 8) */ +#if !defined(_M_ARM64EC) #if defined(_M_X64) \ || defined(_M_AMD64) \ || defined(__x86_64__) \ @@ -35,6 +36,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif #define MY_CPU_64BIT #endif +#endif #if defined(_M_IX86) \ @@ -47,11 +49,26 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #if defined(_M_ARM64) \ + || defined(_M_ARM64EC) \ || defined(__AARCH64EL__) \ || defined(__AARCH64EB__) \ || defined(__aarch64__) #define MY_CPU_ARM64 - #define MY_CPU_NAME "arm64" +#if defined(__ILP32__) \ + || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 4) + #define MY_CPU_NAME "arm64-32" + #define MY_CPU_SIZEOF_POINTER 4 +#elif defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16) + #define MY_CPU_NAME "arm64-128" + #define MY_CPU_SIZEOF_POINTER 16 +#else +#if defined(_M_ARM64EC) + #define MY_CPU_NAME "arm64ec" +#else + #define MY_CPU_NAME "arm64" +#endif + #define MY_CPU_SIZEOF_POINTER 8 +#endif #define MY_CPU_64BIT #endif @@ -68,8 +85,10 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #define MY_CPU_ARM #if defined(__thumb__) || defined(__THUMBEL__) || defined(_M_ARMT) + #define MY_CPU_ARMT #define MY_CPU_NAME "armt" #else + #define MY_CPU_ARM32 #define MY_CPU_NAME "arm" #endif /* #define MY_CPU_32BIT */ @@ -103,6 +122,8 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. || defined(__PPC__) \ || defined(_POWER) +#define MY_CPU_PPC_OR_PPC64 + #if defined(__ppc64__) \ || defined(__powerpc64__) \ || defined(_LP64) \ @@ -123,8 +144,36 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif +#if defined(__sparc__) \ + || defined(__sparc) + #define MY_CPU_SPARC + #if defined(__LP64__) \ + || defined(_LP64) \ + || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 8) + #define MY_CPU_NAME "sparcv9" + #define MY_CPU_SIZEOF_POINTER 8 + #define MY_CPU_64BIT + #elif defined(__sparc_v9__) \ + || defined(__sparcv9) + #define MY_CPU_64BIT + #if defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 4) + #define MY_CPU_NAME "sparcv9-32" + #else + #define MY_CPU_NAME "sparcv9m" + #endif + #elif defined(__sparc_v8__) \ + || defined(__sparcv8) + #define MY_CPU_NAME "sparcv8" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "sparc" + #endif +#endif + + #if defined(__riscv) \ || defined(__riscv__) + #define MY_CPU_RISCV #if __riscv_xlen == 32 #define MY_CPU_NAME "riscv32" #elif __riscv_xlen == 64 @@ -135,6 +184,39 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif +#if defined(__loongarch__) + #define MY_CPU_LOONGARCH + #if defined(__loongarch64) || defined(__loongarch_grlen) && (__loongarch_grlen == 64) + #define MY_CPU_64BIT + #endif + #if defined(__loongarch64) + #define MY_CPU_NAME "loongarch64" + #define MY_CPU_LOONGARCH64 + #else + #define MY_CPU_NAME "loongarch" + #endif +#endif + + +// #undef MY_CPU_NAME +// #undef MY_CPU_SIZEOF_POINTER +// #define __e2k__ +// #define __SIZEOF_POINTER__ 4 +#if defined(__e2k__) + #define MY_CPU_E2K + #if defined(__ILP32__) || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 4) + #define MY_CPU_NAME "e2k-32" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "e2k" + #if defined(__LP64__) || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 8) + #define MY_CPU_SIZEOF_POINTER 8 + #endif + #endif + #define MY_CPU_64BIT +#endif + + #if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) #define MY_CPU_X86_OR_AMD64 #endif @@ -165,6 +247,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. || defined(MY_CPU_ARM_LE) \ || defined(MY_CPU_ARM64_LE) \ || defined(MY_CPU_IA64_LE) \ + || defined(_LITTLE_ENDIAN) \ || defined(__LITTLE_ENDIAN__) \ || defined(__ARMEL__) \ || defined(__THUMBEL__) \ @@ -197,6 +280,9 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #error Stop_Compiling_Bad_Endian #endif +#if !defined(MY_CPU_LE) && !defined(MY_CPU_BE) + #error Stop_Compiling_CPU_ENDIAN_must_be_detected_at_compile_time +#endif #if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) #error Stop_Compiling_Bad_32_64_BIT @@ -238,6 +324,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #ifndef MY_CPU_NAME + // #define MY_CPU_IS_UNKNOWN #ifdef MY_CPU_LE #define MY_CPU_NAME "LE" #elif defined(MY_CPU_BE) @@ -253,15 +340,121 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. +#ifdef __has_builtin + #define Z7_has_builtin(x) __has_builtin(x) +#else + #define Z7_has_builtin(x) 0 +#endif + + +#define Z7_BSWAP32_CONST(v) \ + ( (((UInt32)(v) << 24) ) \ + | (((UInt32)(v) << 8) & (UInt32)0xff0000) \ + | (((UInt32)(v) >> 8) & (UInt32)0xff00 ) \ + | (((UInt32)(v) >> 24) )) + + +#if defined(_MSC_VER) && (_MSC_VER >= 1300) + +#include + +/* Note: these macros will use bswap instruction (486), that is unsupported in 386 cpu */ + +#pragma intrinsic(_byteswap_ushort) +#pragma intrinsic(_byteswap_ulong) +#pragma intrinsic(_byteswap_uint64) + +#define Z7_BSWAP16(v) _byteswap_ushort(v) +#define Z7_BSWAP32(v) _byteswap_ulong (v) +#define Z7_BSWAP64(v) _byteswap_uint64(v) +#define Z7_CPU_FAST_BSWAP_SUPPORTED + +/* GCC can generate slow code that calls function for __builtin_bswap32() for: + - GCC for RISCV, if Zbb/XTHeadBb extension is not used. + - GCC for SPARC. + The code from CLANG for SPARC also is not fastest. + So we don't define Z7_CPU_FAST_BSWAP_SUPPORTED in some cases. +*/ +#elif (!defined(MY_CPU_RISCV) || defined (__riscv_zbb) || defined(__riscv_xtheadbb)) \ + && !defined(MY_CPU_SPARC) \ + && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && Z7_has_builtin(__builtin_bswap16)) \ + ) + +#define Z7_BSWAP16(v) __builtin_bswap16(v) +#define Z7_BSWAP32(v) __builtin_bswap32(v) +#define Z7_BSWAP64(v) __builtin_bswap64(v) +#define Z7_CPU_FAST_BSWAP_SUPPORTED + +#else + +#define Z7_BSWAP16(v) ((UInt16) \ + ( ((UInt32)(v) << 8) \ + | ((UInt32)(v) >> 8) \ + )) + +#define Z7_BSWAP32(v) Z7_BSWAP32_CONST(v) + +#define Z7_BSWAP64(v) \ + ( ( ( (UInt64)(v) ) << 8 * 7 ) \ + | ( ( (UInt64)(v) & ((UInt32)0xff << 8 * 1) ) << 8 * 5 ) \ + | ( ( (UInt64)(v) & ((UInt32)0xff << 8 * 2) ) << 8 * 3 ) \ + | ( ( (UInt64)(v) & ((UInt32)0xff << 8 * 3) ) << 8 * 1 ) \ + | ( ( (UInt64)(v) >> 8 * 1 ) & ((UInt32)0xff << 8 * 3) ) \ + | ( ( (UInt64)(v) >> 8 * 3 ) & ((UInt32)0xff << 8 * 2) ) \ + | ( ( (UInt64)(v) >> 8 * 5 ) & ((UInt32)0xff << 8 * 1) ) \ + | ( ( (UInt64)(v) >> 8 * 7 ) ) \ + ) + +#endif + + + #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ - || defined(MY_CPU_ARM64) + || defined(MY_CPU_ARM64) \ + || defined(MY_CPU_RISCV) && defined(__riscv_misaligned_fast) \ + || defined(MY_CPU_E2K) && defined(__iset__) && (__iset__ >= 6) #define MY_CPU_LE_UNALIGN #define MY_CPU_LE_UNALIGN_64 #elif defined(__ARM_FEATURE_UNALIGNED) - /* gcc9 for 32-bit arm can use LDRD instruction that requires 32-bit alignment. - So we can't use unaligned 64-bit operations. */ - #define MY_CPU_LE_UNALIGN +/* === ALIGNMENT on 32-bit arm and LDRD/STRD/LDM/STM instructions. + Description of problems: +problem-1 : 32-bit ARM architecture: + multi-access (pair of 32-bit accesses) instructions (LDRD/STRD/LDM/STM) + require 32-bit (WORD) alignment (by 32-bit ARM architecture). + So there is "Alignment fault exception", if data is not aligned for 32-bit. + +problem-2 : 32-bit kernels and arm64 kernels: + 32-bit linux kernels provide fixup for these "paired" instruction "Alignment fault exception". + So unaligned paired-access instructions work via exception handler in kernel in 32-bit linux. + + But some arm64 kernels do not handle these faults in 32-bit programs. + So we have unhandled exception for such instructions. + Probably some new arm64 kernels have fixed it, and unaligned + paired-access instructions work in new kernels? + +problem-3 : compiler for 32-bit arm: + Compilers use LDRD/STRD/LDM/STM for UInt64 accesses + and for another cases where two 32-bit accesses are fused + to one multi-access instruction. + So UInt64 variables must be aligned for 32-bit, and each + 32-bit access must be aligned for 32-bit, if we want to + avoid "Alignment fault" exception (handled or unhandled). + +problem-4 : performace: + Even if unaligned access is handled by kernel, it will be slow. + So if we allow unaligned access, we can get fast unaligned + single-access, and slow unaligned paired-access. + + We don't allow unaligned access on 32-bit arm, because compiler + genarates paired-access instructions that require 32-bit alignment, + and some arm64 kernels have no handler for these instructions. + Also unaligned paired-access instructions will be slow, if kernel handles them. +*/ + // it must be disabled: + // #define MY_CPU_LE_UNALIGN #endif #endif @@ -272,13 +465,11 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #define GetUi32(p) (*(const UInt32 *)(const void *)(p)) #ifdef MY_CPU_LE_UNALIGN_64 #define GetUi64(p) (*(const UInt64 *)(const void *)(p)) +#define SetUi64(p, v) { *(UInt64 *)(void *)(p) = (v); } #endif #define SetUi16(p, v) { *(UInt16 *)(void *)(p) = (v); } #define SetUi32(p, v) { *(UInt32 *)(void *)(p) = (v); } -#ifdef MY_CPU_LE_UNALIGN_64 -#define SetUi64(p, v) { *(UInt64 *)(void *)(p) = (v); } -#endif #else @@ -305,51 +496,26 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif -#ifndef MY_CPU_LE_UNALIGN_64 - +#ifndef GetUi64 #define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) +#endif +#ifndef SetUi64 #define SetUi64(p, v) { Byte *_ppp2_ = (Byte *)(p); UInt64 _vvv2_ = (v); \ - SetUi32(_ppp2_ , (UInt32)_vvv2_); \ - SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); } - + SetUi32(_ppp2_ , (UInt32)_vvv2_) \ + SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)) } #endif +#if defined(MY_CPU_LE_UNALIGN) && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) +#define GetBe32(p) Z7_BSWAP32 (*(const UInt32 *)(const void *)(p)) +#define SetBe32(p, v) { (*(UInt32 *)(void *)(p)) = Z7_BSWAP32(v); } -#ifdef __has_builtin - #define MY__has_builtin(x) __has_builtin(x) -#else - #define MY__has_builtin(x) 0 +#if defined(MY_CPU_LE_UNALIGN_64) +#define GetBe64(p) Z7_BSWAP64 (*(const UInt64 *)(const void *)(p)) #endif -#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ defined(_MSC_VER) && (_MSC_VER >= 1300) - -/* Note: we use bswap instruction, that is unsupported in 386 cpu */ - -#include - -#pragma intrinsic(_byteswap_ushort) -#pragma intrinsic(_byteswap_ulong) -#pragma intrinsic(_byteswap_uint64) - -/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ -#define GetBe32(p) _byteswap_ulong (*(const UInt32 *)(const void *)(p)) -#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const void *)(p)) - -#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) - -#elif defined(MY_CPU_LE_UNALIGN) && ( \ - (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ - || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) - -/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const void *)(p)) */ -#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const void *)(p)) -#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const void *)(p)) - -#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = __builtin_bswap32(v) - #else #define GetBe32(p) ( \ @@ -358,8 +524,6 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. ((UInt32)((const Byte *)(p))[2] << 8) | \ ((const Byte *)(p))[3] ) -#define GetBe64(p) (((UInt64)GetBe32(p) << 32) | GetBe32(((const Byte *)(p)) + 4)) - #define SetBe32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ _ppp_[0] = (Byte)(_vvv_ >> 24); \ _ppp_[1] = (Byte)(_vvv_ >> 16); \ @@ -368,50 +532,88 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif +#ifndef GetBe64 +#define GetBe64(p) (((UInt64)GetBe32(p) << 32) | GetBe32(((const Byte *)(p)) + 4)) +#endif #ifndef GetBe16 - #define GetBe16(p) ( (UInt16) ( \ ((UInt16)((const Byte *)(p))[0] << 8) | \ ((const Byte *)(p))[1] )) +#endif + +#if defined(MY_CPU_BE) +#define Z7_CONV_BE_TO_NATIVE_CONST32(v) (v) +#define Z7_CONV_LE_TO_NATIVE_CONST32(v) Z7_BSWAP32_CONST(v) +#define Z7_CONV_NATIVE_TO_BE_32(v) (v) +#elif defined(MY_CPU_LE) +#define Z7_CONV_BE_TO_NATIVE_CONST32(v) Z7_BSWAP32_CONST(v) +#define Z7_CONV_LE_TO_NATIVE_CONST32(v) (v) +#define Z7_CONV_NATIVE_TO_BE_32(v) Z7_BSWAP32(v) +#else +#error Stop_Compiling_Unknown_Endian_CONV #endif +#if defined(MY_CPU_BE) -#ifdef MY_CPU_X86_OR_AMD64 +#define GetBe64a(p) (*(const UInt64 *)(const void *)(p)) +#define GetBe32a(p) (*(const UInt32 *)(const void *)(p)) +#define GetBe16a(p) (*(const UInt16 *)(const void *)(p)) +#define SetBe32a(p, v) { *(UInt32 *)(void *)(p) = (v); } +#define SetBe16a(p, v) { *(UInt16 *)(void *)(p) = (v); } -typedef struct -{ - UInt32 maxFunc; - UInt32 vendor[3]; - UInt32 ver; - UInt32 b; - UInt32 c; - UInt32 d; -} Cx86cpuid; +#define GetUi64a(p) GetUi64(p) +#define GetUi32a(p) GetUi32(p) +#define GetUi16a(p) GetUi16(p) +#define SetUi32a(p, v) SetUi32(p, v) +#define SetUi16a(p, v) SetUi16(p, v) -enum -{ - CPU_FIRM_INTEL, - CPU_FIRM_AMD, - CPU_FIRM_VIA -}; +#elif defined(MY_CPU_LE) -void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); +#define GetUi64a(p) (*(const UInt64 *)(const void *)(p)) +#define GetUi32a(p) (*(const UInt32 *)(const void *)(p)) +#define GetUi16a(p) (*(const UInt16 *)(const void *)(p)) +#define SetUi32a(p, v) { *(UInt32 *)(void *)(p) = (v); } +#define SetUi16a(p, v) { *(UInt16 *)(void *)(p) = (v); } -BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); -int x86cpuid_GetFirm(const Cx86cpuid *p); +#define GetBe64a(p) GetBe64(p) +#define GetBe32a(p) GetBe32(p) +#define GetBe16a(p) GetBe16(p) +#define SetBe32a(p, v) SetBe32(p, v) +#define SetBe16a(p, v) SetBe16(p, v) -#define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) -#define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) -#define x86cpuid_GetStepping(ver) (ver & 0xF) +#else +#error Stop_Compiling_Unknown_Endian_CPU_a +#endif -BoolInt CPU_Is_InOrder(void); + +#if defined(MY_CPU_X86_OR_AMD64) \ + || defined(MY_CPU_ARM_OR_ARM64) \ + || defined(MY_CPU_PPC_OR_PPC64) + #define Z7_CPU_FAST_ROTATE_SUPPORTED +#endif + + +#ifdef MY_CPU_X86_OR_AMD64 + +void Z7_FASTCALL z7_x86_cpuid(UInt32 a[4], UInt32 function); +UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void); +#if defined(MY_CPU_AMD64) +#define Z7_IF_X86_CPUID_SUPPORTED +#else +#define Z7_IF_X86_CPUID_SUPPORTED if (z7_x86_cpuid_GetMaxFunc()) +#endif BoolInt CPU_IsSupported_AES(void); +BoolInt CPU_IsSupported_AVX(void); BoolInt CPU_IsSupported_AVX2(void); +BoolInt CPU_IsSupported_AVX512F_AVX512VL(void); BoolInt CPU_IsSupported_VAES_AVX2(void); +BoolInt CPU_IsSupported_CMOV(void); +BoolInt CPU_IsSupported_SSE(void); +BoolInt CPU_IsSupported_SSE2(void); BoolInt CPU_IsSupported_SSSE3(void); BoolInt CPU_IsSupported_SSE41(void); BoolInt CPU_IsSupported_SHA(void); @@ -436,8 +638,8 @@ BoolInt CPU_IsSupported_AES(void); #endif #if defined(__APPLE__) -int My_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize); -int My_sysctlbyname_Get_UInt32(const char *name, UInt32 *val); +int z7_sysctlbyname_Get(const char *name, void *buf, size_t *bufSize); +int z7_sysctlbyname_Get_UInt32(const char *name, UInt32 *val); #endif EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/Delta.c b/multiarc/src/formats/7z/C/Delta.c old mode 100755 new mode 100644 diff --git a/multiarc/src/formats/7z/C/Delta.h b/multiarc/src/formats/7z/C/Delta.h old mode 100755 new mode 100644 index 2fa54ad67..706095417 --- a/multiarc/src/formats/7z/C/Delta.h +++ b/multiarc/src/formats/7z/C/Delta.h @@ -1,8 +1,8 @@ /* Delta.h -- Delta converter -2013-01-18 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ -#ifndef __DELTA_H -#define __DELTA_H +#ifndef ZIP7_INC_DELTA_H +#define ZIP7_INC_DELTA_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/DllSecur.c b/multiarc/src/formats/7z/C/DllSecur.c old mode 100755 new mode 100644 index dce0c96c0..bbbfc0a76 --- a/multiarc/src/formats/7z/C/DllSecur.c +++ b/multiarc/src/formats/7z/C/DllSecur.c @@ -1,114 +1,99 @@ /* DllSecur.c -- DLL loading security -2022-07-15 : Igor Pavlov : Public domain */ +2023-12-03 : Igor Pavlov : Public domain */ #include "Precomp.h" #ifdef _WIN32 -#include +#include "7zWindows.h" #include "DllSecur.h" #ifndef UNDER_CE -#if defined(__GNUC__) && (__GNUC__ >= 8) - #pragma GCC diagnostic ignored "-Wcast-function-type" -#endif +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); #define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400 #define MY_LOAD_LIBRARY_SEARCH_SYSTEM32 0x800 +#define DELIM "\0" + static const char * const g_Dlls = + "userenv" + DELIM "setupapi" + DELIM "apphelp" + DELIM "propsys" + DELIM "dwmapi" + DELIM "cryptbase" + DELIM "oleacc" + DELIM "clbcatq" + DELIM "version" #ifndef _CONSOLE - "UXTHEME\0" + DELIM "uxtheme" #endif - "USERENV\0" - "SETUPAPI\0" - "APPHELP\0" - "PROPSYS\0" - "DWMAPI\0" - "CRYPTBASE\0" - "OLEACC\0" - "CLBCATQ\0" - "VERSION\0" - ; + DELIM; + +#endif +#ifdef __clang__ + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +#if defined (_MSC_VER) && _MSC_VER >= 1900 +// sysinfoapi.h: kit10: GetVersion was declared deprecated +#pragma warning(disable : 4996) #endif -// #define MY_CAST_FUNC (void(*)()) -#define MY_CAST_FUNC +#define IF_NON_VISTA_SET_DLL_DIRS_AND_RETURN \ + if ((UInt16)GetVersion() != 6) { \ + const \ + Func_SetDefaultDllDirectories setDllDirs = \ + (Func_SetDefaultDllDirectories) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), \ + "SetDefaultDllDirectories"); \ + if (setDllDirs) if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) return; } -void My_SetDefaultDllDirectories() +void My_SetDefaultDllDirectories(void) { #ifndef UNDER_CE - - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) - { - Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) - MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); - if (setDllDirs) - if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) - return; - } - + IF_NON_VISTA_SET_DLL_DIRS_AND_RETURN #endif } -void LoadSecurityDlls() +void LoadSecurityDlls(void) { #ifndef UNDER_CE - - wchar_t buf[MAX_PATH + 100]; - - { - // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ??? - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) - { - Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) - MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); - if (setDllDirs) - if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) - return; - } - } - - { - unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); - if (len == 0 || len > MAX_PATH) - return; - } + // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ??? + IF_NON_VISTA_SET_DLL_DIRS_AND_RETURN { + wchar_t buf[MAX_PATH + 100]; const char *dll; - unsigned pos = (unsigned)lstrlenW(buf); - + unsigned pos = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (pos == 0 || pos > MAX_PATH) + return; if (buf[pos - 1] != '\\') buf[pos++] = '\\'; - - for (dll = g_Dlls; dll[0] != 0;) + for (dll = g_Dlls; *dll != 0;) { - unsigned k = 0; + wchar_t *dest = &buf[pos]; for (;;) { - char c = *dll++; - buf[pos + k] = (Byte)c; - k++; + const char c = *dll++; if (c == 0) break; + *dest++ = (Byte)c; } - - lstrcatW(buf, L".dll"); + dest[0] = '.'; + dest[1] = 'd'; + dest[2] = 'l'; + dest[3] = 'l'; + dest[4] = 0; + // lstrcatW(buf, L".dll"); LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); } } - #endif } -#endif +#endif // _WIN32 diff --git a/multiarc/src/formats/7z/C/DllSecur.h b/multiarc/src/formats/7z/C/DllSecur.h old mode 100755 new mode 100644 index 64ff26cd9..9fa415382 --- a/multiarc/src/formats/7z/C/DllSecur.h +++ b/multiarc/src/formats/7z/C/DllSecur.h @@ -1,8 +1,8 @@ /* DllSecur.h -- DLL loading for security -2018-02-19 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ -#ifndef __DLL_SECUR_H -#define __DLL_SECUR_H +#ifndef ZIP7_INC_DLL_SECUR_H +#define ZIP7_INC_DLL_SECUR_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/HuffEnc.c b/multiarc/src/formats/7z/C/HuffEnc.c old mode 100755 new mode 100644 index f3c2996d0..996da302c --- a/multiarc/src/formats/7z/C/HuffEnc.c +++ b/multiarc/src/formats/7z/C/HuffEnc.c @@ -1,5 +1,5 @@ /* HuffEnc.c -- functions for Huffman encoding -2021-02-09 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,7 +8,7 @@ #define kMaxLen 16 #define NUM_BITS 10 -#define MASK (((unsigned)1 << NUM_BITS) - 1) +#define MASK ((1u << NUM_BITS) - 1) #define NUM_COUNTERS 64 @@ -106,7 +106,7 @@ void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 numSymb p[--e] &= MASK; lenCounters[1] = 2; - while (e > 0) + while (e != 0) { UInt32 len = (p[p[--e] >> NUM_BITS] >> NUM_BITS) + 1; p[e] = (p[e] & MASK) | (len << NUM_BITS); @@ -146,3 +146,9 @@ void Huffman_Generate(const UInt32 *freqs, UInt32 *p, Byte *lens, UInt32 numSymb } } } + +#undef kMaxLen +#undef NUM_BITS +#undef MASK +#undef NUM_COUNTERS +#undef HUFFMAN_SPEED_OPT diff --git a/multiarc/src/formats/7z/C/HuffEnc.h b/multiarc/src/formats/7z/C/HuffEnc.h old mode 100755 new mode 100644 index 92b6878de..cbc5d11f9 --- a/multiarc/src/formats/7z/C/HuffEnc.h +++ b/multiarc/src/formats/7z/C/HuffEnc.h @@ -1,8 +1,8 @@ /* HuffEnc.h -- Huffman encoding -2013-01-18 : Igor Pavlov : Public domain */ +2023-03-05 : Igor Pavlov : Public domain */ -#ifndef __HUFF_ENC_H -#define __HUFF_ENC_H +#ifndef ZIP7_INC_HUFF_ENC_H +#define ZIP7_INC_HUFF_ENC_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/LzFind.c b/multiarc/src/formats/7z/C/LzFind.c old mode 100755 new mode 100644 index 1b73c2848..1ce404648 --- a/multiarc/src/formats/7z/C/LzFind.c +++ b/multiarc/src/formats/7z/C/LzFind.c @@ -1,5 +1,5 @@ /* LzFind.c -- Match finder for LZ algorithms -2021-11-29 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -17,7 +17,7 @@ #define kEmptyHashValue 0 #define kMaxValForNormalize ((UInt32)0) -// #define kMaxValForNormalize ((UInt32)(1 << 20) + 0xFFF) // for debug +// #define kMaxValForNormalize ((UInt32)(1 << 20) + 0xfff) // for debug // #define kNormalizeAlign (1 << 7) // alignment for speculated accesses @@ -67,10 +67,10 @@ static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) { - if (!p->directInput) + // if (!p->directInput) { - ISzAlloc_Free(alloc, p->bufferBase); - p->bufferBase = NULL; + ISzAlloc_Free(alloc, p->bufBase); + p->bufBase = NULL; } } @@ -79,7 +79,7 @@ static int LzInWindow_Create2(CMatchFinder *p, UInt32 blockSize, ISzAllocPtr all { if (blockSize == 0) return 0; - if (!p->bufferBase || p->blockSize != blockSize) + if (!p->bufBase || p->blockSize != blockSize) { // size_t blockSizeT; LzInWindow_Free(p, alloc); @@ -101,19 +101,25 @@ static int LzInWindow_Create2(CMatchFinder *p, UInt32 blockSize, ISzAllocPtr all #endif */ - p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, blockSize); - // printf("\nbufferBase = %p\n", p->bufferBase); + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, blockSize); + // printf("\nbufferBase = %p\n", p->bufBase); // return 0; // for debug } - return (p->bufferBase != NULL); + return (p->bufBase != NULL); } -static const Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } +static const Byte *MatchFinder_GetPointerToCurrentPos(void *p) +{ + return ((CMatchFinder *)p)->buffer; +} -static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return GET_AVAIL_BYTES(p); } +static UInt32 MatchFinder_GetNumAvailableBytes(void *p) +{ + return GET_AVAIL_BYTES((CMatchFinder *)p); +} -MY_NO_INLINE +Z7_NO_INLINE static void MatchFinder_ReadBlock(CMatchFinder *p) { if (p->streamEndWasReached || p->result != SZ_OK) @@ -127,8 +133,8 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) UInt32 curSize = 0xFFFFFFFF - GET_AVAIL_BYTES(p); if (curSize > p->directInputRem) curSize = (UInt32)p->directInputRem; - p->directInputRem -= curSize; p->streamPos += curSize; + p->directInputRem -= curSize; if (p->directInputRem == 0) p->streamEndWasReached = 1; return; @@ -136,8 +142,8 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) for (;;) { - Byte *dest = p->buffer + GET_AVAIL_BYTES(p); - size_t size = (size_t)(p->bufferBase + p->blockSize - dest); + const Byte *dest = p->buffer + GET_AVAIL_BYTES(p); + size_t size = (size_t)(p->bufBase + p->blockSize - dest); if (size == 0) { /* we call ReadBlock() after NeedMove() and MoveBlock(). @@ -153,7 +159,14 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) // #define kRead 3 // if (size > kRead) size = kRead; // for debug - p->result = ISeqInStream_Read(p->stream, dest, &size); + /* + // we need cast (Byte *)dest. + #ifdef __clang__ + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + */ + p->result = ISeqInStream_Read(p->stream, + p->bufBase + (dest - p->bufBase), &size); if (p->result != SZ_OK) return; if (size == 0) @@ -173,14 +186,14 @@ static void MatchFinder_ReadBlock(CMatchFinder *p) -MY_NO_INLINE +Z7_NO_INLINE void MatchFinder_MoveBlock(CMatchFinder *p) { - const size_t offset = (size_t)(p->buffer - p->bufferBase) - p->keepSizeBefore; + const size_t offset = (size_t)(p->buffer - p->bufBase) - p->keepSizeBefore; const size_t keepBefore = (offset & (kBlockMoveAlign - 1)) + p->keepSizeBefore; - p->buffer = p->bufferBase + keepBefore; - memmove(p->bufferBase, - p->bufferBase + (offset & ~((size_t)kBlockMoveAlign - 1)), + p->buffer = p->bufBase + keepBefore; + memmove(p->bufBase, + p->bufBase + (offset & ~((size_t)kBlockMoveAlign - 1)), keepBefore + (size_t)GET_AVAIL_BYTES(p)); } @@ -198,7 +211,7 @@ int MatchFinder_NeedMove(CMatchFinder *p) return 0; if (p->streamEndWasReached || p->result != SZ_OK) return 0; - return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); + return ((size_t)(p->bufBase + p->blockSize - p->buffer) <= p->keepSizeAfter); } void MatchFinder_ReadIfRequired(CMatchFinder *p) @@ -214,6 +227,8 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p) p->cutValue = 32; p->btMode = 1; p->numHashBytes = 4; + p->numHashBytes_Min = 2; + p->numHashOutBits = 0; p->bigHash = 0; } @@ -222,8 +237,10 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p) void MatchFinder_Construct(CMatchFinder *p) { unsigned i; - p->bufferBase = NULL; + p->buffer = NULL; + p->bufBase = NULL; p->directInput = 0; + p->stream = NULL; p->hash = NULL; p->expectedDataSize = (UInt64)(Int64)-1; MatchFinder_SetDefaultSettings(p); @@ -238,6 +255,8 @@ void MatchFinder_Construct(CMatchFinder *p) } } +#undef kCrcPoly + static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) { ISzAlloc_Free(alloc, p->hash); @@ -252,7 +271,7 @@ void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) { - size_t sizeInBytes = (size_t)num * sizeof(CLzRef); + const size_t sizeInBytes = (size_t)num * sizeof(CLzRef); if (sizeInBytes / sizeof(CLzRef) != num) return NULL; return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); @@ -298,6 +317,62 @@ static UInt32 GetBlockSize(CMatchFinder *p, UInt32 historySize) } +// input is historySize +static UInt32 MatchFinder_GetHashMask2(CMatchFinder *p, UInt32 hs) +{ + if (p->numHashBytes == 2) + return (1 << 16) - 1; + if (hs != 0) + hs--; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + // we propagated 16 bits in (hs). Low 16 bits must be set later + if (hs >= (1 << 24)) + { + if (p->numHashBytes == 3) + hs = (1 << 24) - 1; + /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ + } + // (hash_size >= (1 << 16)) : Required for (numHashBytes > 2) + hs |= (1 << 16) - 1; /* don't change it! */ + // bt5: we adjust the size with recommended minimum size + if (p->numHashBytes >= 5) + hs |= (256 << kLzHash_CrcShift_2) - 1; + return hs; +} + +// input is historySize +static UInt32 MatchFinder_GetHashMask(CMatchFinder *p, UInt32 hs) +{ + if (p->numHashBytes == 2) + return (1 << 16) - 1; + if (hs != 0) + hs--; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + // we propagated 16 bits in (hs). Low 16 bits must be set later + hs >>= 1; + if (hs >= (1 << 24)) + { + if (p->numHashBytes == 3) + hs = (1 << 24) - 1; + else + hs >>= 1; + /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ + } + // (hash_size >= (1 << 16)) : Required for (numHashBytes > 2) + hs |= (1 << 16) - 1; /* don't change it! */ + // bt5: we adjust the size with recommended minimum size + if (p->numHashBytes >= 5) + hs |= (256 << kLzHash_CrcShift_2) - 1; + return hs; +} + + int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc) @@ -318,78 +393,91 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, p->blockSize = 0; if (p->directInput || LzInWindow_Create2(p, GetBlockSize(p, historySize), alloc)) { - const UInt32 newCyclicBufferSize = historySize + 1; // do not change it - UInt32 hs; - p->matchMaxLen = matchMaxLen; + size_t hashSizeSum; { - // UInt32 hs4; - p->fixedHashSize = 0; - hs = (1 << 16) - 1; - if (p->numHashBytes != 2) + UInt32 hs; + UInt32 hsCur; + + if (p->numHashOutBits != 0) { - hs = historySize; - if (hs > p->expectedDataSize) - hs = (UInt32)p->expectedDataSize; - if (hs != 0) - hs--; - hs |= (hs >> 1); - hs |= (hs >> 2); - hs |= (hs >> 4); - hs |= (hs >> 8); - // we propagated 16 bits in (hs). Low 16 bits must be set later - hs >>= 1; - if (hs >= (1 << 24)) - { - if (p->numHashBytes == 3) - hs = (1 << 24) - 1; - else - hs >>= 1; - /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ - } - - // hs = ((UInt32)1 << 25) - 1; // for test - + unsigned numBits = p->numHashOutBits; + const unsigned nbMax = + (p->numHashBytes == 2 ? 16 : + (p->numHashBytes == 3 ? 24 : 32)); + if (numBits > nbMax) + numBits = nbMax; + if (numBits >= 32) + hs = (UInt32)0 - 1; + else + hs = ((UInt32)1 << numBits) - 1; // (hash_size >= (1 << 16)) : Required for (numHashBytes > 2) hs |= (1 << 16) - 1; /* don't change it! */ - - // bt5: we adjust the size with recommended minimum size if (p->numHashBytes >= 5) hs |= (256 << kLzHash_CrcShift_2) - 1; + { + const UInt32 hs2 = MatchFinder_GetHashMask2(p, historySize); + if (hs > hs2) + hs = hs2; + } + hsCur = hs; + if (p->expectedDataSize < historySize) + { + const UInt32 hs2 = MatchFinder_GetHashMask2(p, (UInt32)p->expectedDataSize); + if (hsCur > hs2) + hsCur = hs2; + } + } + else + { + hs = MatchFinder_GetHashMask(p, historySize); + hsCur = hs; + if (p->expectedDataSize < historySize) + { + hsCur = MatchFinder_GetHashMask(p, (UInt32)p->expectedDataSize); + if (hsCur > hs) // is it possible? + hsCur = hs; + } } - p->hashMask = hs; - hs++; - - /* - hs4 = (1 << 20); - if (hs4 > hs) - hs4 = hs; - // hs4 = (1 << 16); // for test - p->hash4Mask = hs4 - 1; - */ - if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; - if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; - // if (p->numHashBytes > 4) p->fixedHashSize += hs4; // kHash4Size; - hs += p->fixedHashSize; + p->hashMask = hsCur; + + hashSizeSum = hs; + hashSizeSum++; + if (hashSizeSum < hs) + return 0; + { + UInt32 fixedHashSize = 0; + if (p->numHashBytes > 2 && p->numHashBytes_Min <= 2) fixedHashSize += kHash2Size; + if (p->numHashBytes > 3 && p->numHashBytes_Min <= 3) fixedHashSize += kHash3Size; + // if (p->numHashBytes > 4) p->fixedHashSize += hs4; // kHash4Size; + hashSizeSum += fixedHashSize; + p->fixedHashSize = fixedHashSize; + } } + p->matchMaxLen = matchMaxLen; + { size_t newSize; size_t numSons; + const UInt32 newCyclicBufferSize = historySize + 1; // do not change it p->historySize = historySize; - p->hashSizeSum = hs; p->cyclicBufferSize = newCyclicBufferSize; // it must be = (historySize + 1) numSons = newCyclicBufferSize; if (p->btMode) numSons <<= 1; - newSize = hs + numSons; + newSize = hashSizeSum + numSons; + + if (numSons < newCyclicBufferSize || newSize < numSons) + return 0; // aligned size is not required here, but it can be better for some loops #define NUM_REFS_ALIGN_MASK 0xF newSize = (newSize + NUM_REFS_ALIGN_MASK) & ~(size_t)NUM_REFS_ALIGN_MASK; - if (p->hash && p->numRefs == newSize) + // 22.02: we don't reallocate buffer, if old size is enough + if (p->hash && p->numRefs >= newSize) return 1; MatchFinder_FreeThisClassMemory(p, alloc); @@ -398,7 +486,7 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, if (p->hash) { - p->son = p->hash + p->hashSizeSum; + p->son = p->hash + hashSizeSum; return 1; } } @@ -470,7 +558,8 @@ void MatchFinder_Init_HighHash(CMatchFinder *p) void MatchFinder_Init_4(CMatchFinder *p) { - p->buffer = p->bufferBase; + if (!p->directInput) + p->buffer = p->bufBase; { /* kEmptyHashValue = 0 (Zero) is used in hash tables as NO-VALUE marker. the code in CMatchFinderMt expects (pos = 1) */ @@ -488,8 +577,9 @@ void MatchFinder_Init_4(CMatchFinder *p) #define CYC_TO_POS_OFFSET 0 // #define CYC_TO_POS_OFFSET 1 // for debug -void MatchFinder_Init(CMatchFinder *p) +void MatchFinder_Init(void *_p) { + CMatchFinder *p = (CMatchFinder *)_p; MatchFinder_Init_HighHash(p); MatchFinder_Init_LowHash(p); MatchFinder_Init_4(p); @@ -507,42 +597,42 @@ void MatchFinder_Init(CMatchFinder *p) #ifdef MY_CPU_X86_OR_AMD64 - #if defined(__clang__) && (__clang_major__ >= 8) \ - || defined(__GNUC__) && (__GNUC__ >= 8) \ - || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900) - #define USE_SATUR_SUB_128 - #define USE_AVX2 - #define ATTRIB_SSE41 __attribute__((__target__("sse4.1"))) - #define ATTRIB_AVX2 __attribute__((__target__("avx2"))) + #if defined(__clang__) && (__clang_major__ >= 4) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40701) + // || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900) + + #define USE_LZFIND_SATUR_SUB_128 + #define USE_LZFIND_SATUR_SUB_256 + #define LZFIND_ATTRIB_SSE41 __attribute__((__target__("sse4.1"))) + #define LZFIND_ATTRIB_AVX2 __attribute__((__target__("avx2"))) #elif defined(_MSC_VER) #if (_MSC_VER >= 1600) - #define USE_SATUR_SUB_128 - #if (_MSC_VER >= 1900) - #define USE_AVX2 - #include // avx - #endif + #define USE_LZFIND_SATUR_SUB_128 + #endif + #if (_MSC_VER >= 1900) + #define USE_LZFIND_SATUR_SUB_256 #endif #endif -// #elif defined(MY_CPU_ARM_OR_ARM64) -#elif defined(MY_CPU_ARM64) +#elif defined(MY_CPU_ARM64) \ + /* || (defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) */ - #if defined(__clang__) && (__clang_major__ >= 8) \ - || defined(__GNUC__) && (__GNUC__ >= 8) - #define USE_SATUR_SUB_128 + #if defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) + #define USE_LZFIND_SATUR_SUB_128 #ifdef MY_CPU_ARM64 - // #define ATTRIB_SSE41 __attribute__((__target__(""))) + // #define LZFIND_ATTRIB_SSE41 __attribute__((__target__(""))) #else - // #define ATTRIB_SSE41 __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #define LZFIND_ATTRIB_SSE41 __attribute__((__target__("fpu=neon"))) #endif #elif defined(_MSC_VER) #if (_MSC_VER >= 1910) - #define USE_SATUR_SUB_128 + #define USE_LZFIND_SATUR_SUB_128 #endif #endif - #if defined(_MSC_VER) && defined(MY_CPU_ARM64) + #if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else #include @@ -550,121 +640,130 @@ void MatchFinder_Init(CMatchFinder *p) #endif -/* -#ifndef ATTRIB_SSE41 - #define ATTRIB_SSE41 -#endif -#ifndef ATTRIB_AVX2 - #define ATTRIB_AVX2 -#endif -*/ -#ifdef USE_SATUR_SUB_128 +#ifdef USE_LZFIND_SATUR_SUB_128 -// #define _SHOW_HW_STATUS +// #define Z7_SHOW_HW_STATUS -#ifdef _SHOW_HW_STATUS +#ifdef Z7_SHOW_HW_STATUS #include -#define _PRF(x) x -_PRF(;) +#define PRF(x) x +PRF(;) #else -#define _PRF(x) +#define PRF(x) #endif + #ifdef MY_CPU_ARM_OR_ARM64 #ifdef MY_CPU_ARM64 -// #define FORCE_SATUR_SUB_128 +// #define FORCE_LZFIND_SATUR_SUB_128 #endif +typedef uint32x4_t LzFind_v128; +#define SASUB_128_V(v, s) \ + vsubq_u32(vmaxq_u32(v, s), s) -typedef uint32x4_t v128; -#define SASUB_128(i) \ - *(v128 *)(void *)(items + (i) * 4) = \ - vsubq_u32(vmaxq_u32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); - -#else +#else // MY_CPU_ARM_OR_ARM64 #include // sse4.1 -typedef __m128i v128; -#define SASUB_128(i) \ - *(v128 *)(void *)(items + (i) * 4) = \ - _mm_sub_epi32(_mm_max_epu32(*(const v128 *)(const void *)(items + (i) * 4), sub2), sub2); // SSE 4.1 +typedef __m128i LzFind_v128; +// SSE 4.1 +#define SASUB_128_V(v, s) \ + _mm_sub_epi32(_mm_max_epu32(v, s), s) -#endif +#endif // MY_CPU_ARM_OR_ARM64 +#define SASUB_128(i) \ + *( LzFind_v128 *)( void *)(items + (i) * 4) = SASUB_128_V( \ + *(const LzFind_v128 *)(const void *)(items + (i) * 4), sub2); + -MY_NO_INLINE +Z7_NO_INLINE static -#ifdef ATTRIB_SSE41 -ATTRIB_SSE41 +#ifdef LZFIND_ATTRIB_SSE41 +LZFIND_ATTRIB_SSE41 #endif void -MY_FAST_CALL +Z7_FASTCALL LzFind_SaturSub_128(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - v128 sub2 = + const LzFind_v128 sub2 = #ifdef MY_CPU_ARM_OR_ARM64 vdupq_n_u32(subValue); #else _mm_set_epi32((Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); #endif + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE do { - SASUB_128(0) - SASUB_128(1) - SASUB_128(2) - SASUB_128(3) - items += 4 * 4; + SASUB_128(0) SASUB_128(1) items += 2 * 4; + SASUB_128(0) SASUB_128(1) items += 2 * 4; } while (items != lim); } -#ifdef USE_AVX2 +#ifdef USE_LZFIND_SATUR_SUB_256 #include // avx +/* +clang :immintrin.h uses +#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ + defined(__AVX2__) +#include +#endif +so we need for clang-cl */ + +#if defined(__clang__) +#include +#include +#endif -#define SASUB_256(i) *(__m256i *)(void *)(items + (i) * 8) = _mm256_sub_epi32(_mm256_max_epu32(*(const __m256i *)(const void *)(items + (i) * 8), sub2), sub2); // AVX2 +// AVX2: +#define SASUB_256(i) \ + *( __m256i *)( void *)(items + (i) * 8) = \ + _mm256_sub_epi32(_mm256_max_epu32( \ + *(const __m256i *)(const void *)(items + (i) * 8), sub2), sub2); -MY_NO_INLINE +Z7_NO_INLINE static -#ifdef ATTRIB_AVX2 -ATTRIB_AVX2 +#ifdef LZFIND_ATTRIB_AVX2 +LZFIND_ATTRIB_AVX2 #endif void -MY_FAST_CALL +Z7_FASTCALL LzFind_SaturSub_256(UInt32 subValue, CLzRef *items, const CLzRef *lim) { - __m256i sub2 = _mm256_set_epi32( + const __m256i sub2 = _mm256_set_epi32( (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue, (Int32)subValue); + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE do { - SASUB_256(0) - SASUB_256(1) - items += 2 * 8; + SASUB_256(0) SASUB_256(1) items += 2 * 8; + SASUB_256(0) SASUB_256(1) items += 2 * 8; } while (items != lim); } -#endif // USE_AVX2 +#endif // USE_LZFIND_SATUR_SUB_256 -#ifndef FORCE_SATUR_SUB_128 -typedef void (MY_FAST_CALL *LZFIND_SATUR_SUB_CODE_FUNC)( +#ifndef FORCE_LZFIND_SATUR_SUB_128 +typedef void (Z7_FASTCALL *LZFIND_SATUR_SUB_CODE_FUNC)( UInt32 subValue, CLzRef *items, const CLzRef *lim); static LZFIND_SATUR_SUB_CODE_FUNC g_LzFind_SaturSub; -#endif // FORCE_SATUR_SUB_128 +#endif // FORCE_LZFIND_SATUR_SUB_128 -#endif // USE_SATUR_SUB_128 +#endif // USE_LZFIND_SATUR_SUB_128 // kEmptyHashValue must be zero -// #define SASUB_32(i) v = items[i]; m = v - subValue; if (v < subValue) m = kEmptyHashValue; items[i] = m; -#define SASUB_32(i) v = items[i]; if (v < subValue) v = subValue; items[i] = v - subValue; +// #define SASUB_32(i) { UInt32 v = items[i]; UInt32 m = v - subValue; if (v < subValue) m = kEmptyHashValue; items[i] = m; } +#define SASUB_32(i) { UInt32 v = items[i]; if (v < subValue) v = subValue; items[i] = v - subValue; } -#ifdef FORCE_SATUR_SUB_128 +#ifdef FORCE_LZFIND_SATUR_SUB_128 #define DEFAULT_SaturSub LzFind_SaturSub_128 @@ -672,24 +771,19 @@ static LZFIND_SATUR_SUB_CODE_FUNC g_LzFind_SaturSub; #define DEFAULT_SaturSub LzFind_SaturSub_32 -MY_NO_INLINE +Z7_NO_INLINE static void -MY_FAST_CALL +Z7_FASTCALL LzFind_SaturSub_32(UInt32 subValue, CLzRef *items, const CLzRef *lim) { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE do { - UInt32 v; - SASUB_32(0) - SASUB_32(1) - SASUB_32(2) - SASUB_32(3) - SASUB_32(4) - SASUB_32(5) - SASUB_32(6) - SASUB_32(7) - items += 8; + SASUB_32(0) SASUB_32(1) items += 2; + SASUB_32(0) SASUB_32(1) items += 2; + SASUB_32(0) SASUB_32(1) items += 2; + SASUB_32(0) SASUB_32(1) items += 2; } while (items != lim); } @@ -697,27 +791,23 @@ LzFind_SaturSub_32(UInt32 subValue, CLzRef *items, const CLzRef *lim) #endif -MY_NO_INLINE +Z7_NO_INLINE void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) { - #define K_NORM_ALIGN_BLOCK_SIZE (1 << 6) - - CLzRef *lim; - - for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (K_NORM_ALIGN_BLOCK_SIZE - 1)) != 0; numItems--) + #define LZFIND_NORM_ALIGN_BLOCK_SIZE (1 << 7) + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (LZFIND_NORM_ALIGN_BLOCK_SIZE - 1)) != 0; numItems--) { - UInt32 v; - SASUB_32(0); + SASUB_32(0) items++; } - { - #define K_NORM_ALIGN_MASK (K_NORM_ALIGN_BLOCK_SIZE / 4 - 1) - lim = items + (numItems & ~(size_t)K_NORM_ALIGN_MASK); - numItems &= K_NORM_ALIGN_MASK; + const size_t k_Align_Mask = (LZFIND_NORM_ALIGN_BLOCK_SIZE / 4 - 1); + CLzRef *lim = items + (numItems & ~(size_t)k_Align_Mask); + numItems &= k_Align_Mask; if (items != lim) { - #if defined(USE_SATUR_SUB_128) && !defined(FORCE_SATUR_SUB_128) + #if defined(USE_LZFIND_SATUR_SUB_128) && !defined(FORCE_LZFIND_SATUR_SUB_128) if (g_LzFind_SaturSub) g_LzFind_SaturSub(subValue, items, lim); else @@ -726,12 +816,10 @@ void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) } items = lim; } - - + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE for (; numItems != 0; numItems--) { - UInt32 v; - SASUB_32(0); + SASUB_32(0) items++; } } @@ -740,7 +828,7 @@ void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) // call MatchFinder_CheckLimits() only after (p->pos++) update -MY_NO_INLINE +Z7_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { if (// !p->streamEndWasReached && p->result == SZ_OK && @@ -768,11 +856,14 @@ static void MatchFinder_CheckLimits(CMatchFinder *p) const UInt32 subValue = (p->pos - p->historySize - 1) /* & ~(UInt32)(kNormalizeAlign - 1) */; // const UInt32 subValue = (1 << 15); // for debug // printf("\nMatchFinder_Normalize() subValue == 0x%x\n", subValue); - size_t numSonRefs = p->cyclicBufferSize; - if (p->btMode) - numSonRefs <<= 1; - Inline_MatchFinder_ReduceOffsets(p, subValue); - MatchFinder_Normalize3(subValue, p->hash, (size_t)p->hashSizeSum + numSonRefs); + MatchFinder_REDUCE_OFFSETS(p, subValue) + MatchFinder_Normalize3(subValue, p->hash, (size_t)p->hashMask + 1 + p->fixedHashSize); + { + size_t numSonRefs = p->cyclicBufferSize; + if (p->btMode) + numSonRefs <<= 1; + MatchFinder_Normalize3(subValue, p->son, numSonRefs); + } } if (p->cyclicBufferPos == p->cyclicBufferSize) @@ -785,7 +876,7 @@ static void MatchFinder_CheckLimits(CMatchFinder *p) /* (lenLimit > maxLen) */ -MY_FORCE_INLINE +Z7_FORCE_INLINE static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, UInt32 *d, unsigned maxLen) @@ -867,7 +958,7 @@ static UInt32 * Hc_GetMatchesSpec(size_t lenLimit, UInt32 curMatch, UInt32 pos, } -MY_FORCE_INLINE +Z7_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, UInt32 *d, UInt32 maxLen) @@ -998,13 +1089,15 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const #define MOVE_POS \ - ++p->cyclicBufferPos; \ + p->cyclicBufferPos++; \ p->buffer++; \ - { const UInt32 pos1 = p->pos + 1; p->pos = pos1; if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } + { const UInt32 pos1 = p->pos + 1; \ + p->pos = pos1; \ + if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } #define MOVE_POS_RET MOVE_POS return distances; -MY_NO_INLINE +Z7_NO_INLINE static void MatchFinder_MovePos(CMatchFinder *p) { /* we go here at the end of stream data, when (avail < num_hash_bytes) @@ -1015,24 +1108,30 @@ static void MatchFinder_MovePos(CMatchFinder *p) if (p->btMode) p->sons[(p->cyclicBufferPos << p->btMode) + 1] = 0; // kEmptyHashValue */ - MOVE_POS; + MOVE_POS } #define GET_MATCHES_HEADER2(minLen, ret_op) \ - unsigned lenLimit; UInt32 hv; Byte *cur; UInt32 curMatch; \ - lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + UInt32 hv; const Byte *cur; UInt32 curMatch; \ + UInt32 lenLimit = p->lenLimit; \ + if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; } \ cur = p->buffer; #define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return distances) -#define SKIP_HEADER(minLen) do { GET_MATCHES_HEADER2(minLen, continue) +#define SKIP_HEADER(minLen) \ + do { GET_MATCHES_HEADER2(minLen, continue) -#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue +#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, \ + p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue -#define SKIP_FOOTER SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS; } while (--num); +#define SKIP_FOOTER \ + SkipMatchesSpec(MF_PARAMS(p)); \ + MOVE_POS \ + } while (--num); #define GET_MATCHES_FOOTER_BASE(_maxLen_, func) \ - distances = func(MF_PARAMS(p), \ - distances, (UInt32)_maxLen_); MOVE_POS_RET; + distances = func(MF_PARAMS(p), distances, (UInt32)_maxLen_); \ + MOVE_POS_RET #define GET_MATCHES_FOOTER_BT(_maxLen_) \ GET_MATCHES_FOOTER_BASE(_maxLen_, GetMatchesSpec1) @@ -1049,10 +1148,11 @@ static void MatchFinder_MovePos(CMatchFinder *p) for (; c != lim; c++) if (*(c + diff) != *c) break; \ maxLen = (unsigned)(c - cur); } -static UInt32* Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt2_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; GET_MATCHES_HEADER(2) - HASH2_CALC; + HASH2_CALC curMatch = p->hash[hv]; p->hash[hv] = p->pos; GET_MATCHES_FOOTER_BT(1) @@ -1061,7 +1161,7 @@ static UInt32* Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; + HASH_ZIP_CALC curMatch = p->hash[hv]; p->hash[hv] = p->pos; GET_MATCHES_FOOTER_BT(2) @@ -1074,15 +1174,16 @@ UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) mmm = pos; -static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt3_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; UInt32 h2, d2, pos; unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(3) - HASH3_CALC; + HASH3_CALC hash = p->hash; pos = p->pos; @@ -1107,7 +1208,7 @@ static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (maxLen == lenLimit) { SkipMatchesSpec(MF_PARAMS(p)); - MOVE_POS_RET; + MOVE_POS_RET } } @@ -1115,15 +1216,16 @@ static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt4_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; UInt32 h2, h3, d2, d3, pos; unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(4) - HASH4_CALC; + HASH4_CALC hash = p->hash; pos = p->pos; @@ -1183,14 +1285,16 @@ static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt5_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; - UInt32 h2, h3, d2, d3, maxLen, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(5) - HASH5_CALC; + HASH5_CALC hash = p->hash; pos = p->pos; @@ -1246,7 +1350,7 @@ static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (maxLen == lenLimit) { SkipMatchesSpec(MF_PARAMS(p)); - MOVE_POS_RET; + MOVE_POS_RET } break; } @@ -1255,15 +1359,16 @@ static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Hc4_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; UInt32 h2, h3, d2, d3, pos; unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(4) - HASH4_CALC; + HASH4_CALC hash = p->hash; pos = p->pos; @@ -1314,23 +1419,25 @@ static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS_RET; + MOVE_POS_RET } break; } - GET_MATCHES_FOOTER_HC(maxLen); + GET_MATCHES_FOOTER_HC(maxLen) } -static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32 * Hc5_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; - UInt32 h2, h3, d2, d3, maxLen, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(5) - HASH5_CALC; + HASH5_CALC hash = p->hash; pos = p->pos; @@ -1382,34 +1489,35 @@ static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (*(cur - d2 + 3) != cur[3]) break; UPDATE_maxLen - distances[-2] = maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS_RET; + MOVE_POS_RET } break; } - GET_MATCHES_FOOTER_HC(maxLen); + GET_MATCHES_FOOTER_HC(maxLen) } UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; + HASH_ZIP_CALC curMatch = p->hash[hv]; p->hash[hv] = p->pos; GET_MATCHES_FOOTER_HC(2) } -static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt2_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(2) { - HASH2_CALC; + HASH2_CALC curMatch = p->hash[hv]; p->hash[hv] = p->pos; } @@ -1420,20 +1528,21 @@ void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { SKIP_HEADER(3) { - HASH_ZIP_CALC; + HASH_ZIP_CALC curMatch = p->hash[hv]; p->hash[hv] = p->pos; } SKIP_FOOTER } -static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt3_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(3) { UInt32 h2; UInt32 *hash; - HASH3_CALC; + HASH3_CALC hash = p->hash; curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = @@ -1442,13 +1551,14 @@ static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_FOOTER } -static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt4_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(4) { UInt32 h2, h3; UInt32 *hash; - HASH4_CALC; + HASH4_CALC hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; hash [h2] = @@ -1458,13 +1568,14 @@ static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_FOOTER } -static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt5_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(5) { UInt32 h2, h3; UInt32 *hash; - HASH5_CALC; + HASH5_CALC hash = p->hash; curMatch = (hash + kFix5HashSize)[hv]; hash [h2] = @@ -1478,7 +1589,7 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) #define HC_SKIP_HEADER(minLen) \ do { if (p->lenLimit < minLen) { MatchFinder_MovePos(p); num--; continue; } { \ - Byte *cur; \ + const Byte *cur; \ UInt32 *hash; \ UInt32 *son; \ UInt32 pos = p->pos; \ @@ -1505,12 +1616,13 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) }} while(num); \ -static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Hc4_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; HC_SKIP_HEADER(4) UInt32 h2, h3; - HASH4_CALC; + HASH4_CALC curMatch = (hash + kFix4HashSize)[hv]; hash [h2] = (hash + kFix3HashSize)[h3] = @@ -1520,8 +1632,9 @@ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) } -static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Hc5_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; HC_SKIP_HEADER(5) UInt32 h2, h3; @@ -1540,7 +1653,7 @@ void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) { HC_SKIP_HEADER(3) - HASH_ZIP_CALC; + HASH_ZIP_CALC curMatch = hash[hv]; hash[hv] = pos; @@ -1550,57 +1663,57 @@ void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable) { - vTable->Init = (Mf_Init_Func)MatchFinder_Init; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; + vTable->Init = MatchFinder_Init; + vTable->GetNumAvailableBytes = MatchFinder_GetNumAvailableBytes; + vTable->GetPointerToCurrentPos = MatchFinder_GetPointerToCurrentPos; if (!p->btMode) { if (p->numHashBytes <= 4) { - vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; + vTable->GetMatches = Hc4_MatchFinder_GetMatches; + vTable->Skip = Hc4_MatchFinder_Skip; } else { - vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; + vTable->GetMatches = Hc5_MatchFinder_GetMatches; + vTable->Skip = Hc5_MatchFinder_Skip; } } else if (p->numHashBytes == 2) { - vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; + vTable->GetMatches = Bt2_MatchFinder_GetMatches; + vTable->Skip = Bt2_MatchFinder_Skip; } else if (p->numHashBytes == 3) { - vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; + vTable->GetMatches = Bt3_MatchFinder_GetMatches; + vTable->Skip = Bt3_MatchFinder_Skip; } else if (p->numHashBytes == 4) { - vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; + vTable->GetMatches = Bt4_MatchFinder_GetMatches; + vTable->Skip = Bt4_MatchFinder_Skip; } else { - vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; + vTable->GetMatches = Bt5_MatchFinder_GetMatches; + vTable->Skip = Bt5_MatchFinder_Skip; } } -void LzFindPrepare() +void LzFindPrepare(void) { - #ifndef FORCE_SATUR_SUB_128 - #ifdef USE_SATUR_SUB_128 + #ifndef FORCE_LZFIND_SATUR_SUB_128 + #ifdef USE_LZFIND_SATUR_SUB_128 LZFIND_SATUR_SUB_CODE_FUNC f = NULL; #ifdef MY_CPU_ARM_OR_ARM64 { if (CPU_IsSupported_NEON()) { // #pragma message ("=== LzFind NEON") - _PRF(printf("\n=== LzFind NEON\n")); + PRF(printf("\n=== LzFind NEON\n")); f = LzFind_SaturSub_128; } // f = 0; // for debug @@ -1609,20 +1722,25 @@ void LzFindPrepare() if (CPU_IsSupported_SSE41()) { // #pragma message ("=== LzFind SSE41") - _PRF(printf("\n=== LzFind SSE41\n")); + PRF(printf("\n=== LzFind SSE41\n")); f = LzFind_SaturSub_128; - #ifdef USE_AVX2 + #ifdef USE_LZFIND_SATUR_SUB_256 if (CPU_IsSupported_AVX2()) { // #pragma message ("=== LzFind AVX2") - _PRF(printf("\n=== LzFind AVX2\n")); + PRF(printf("\n=== LzFind AVX2\n")); f = LzFind_SaturSub_256; } #endif } #endif // MY_CPU_ARM_OR_ARM64 g_LzFind_SaturSub = f; - #endif // USE_SATUR_SUB_128 - #endif // FORCE_SATUR_SUB_128 + #endif // USE_LZFIND_SATUR_SUB_128 + #endif // FORCE_LZFIND_SATUR_SUB_128 } + + +#undef MOVE_POS +#undef MOVE_POS_RET +#undef PRF diff --git a/multiarc/src/formats/7z/C/LzFind.h b/multiarc/src/formats/7z/C/LzFind.h old mode 100755 new mode 100644 index eea873ff6..67e8a6e02 --- a/multiarc/src/formats/7z/C/LzFind.h +++ b/multiarc/src/formats/7z/C/LzFind.h @@ -1,8 +1,8 @@ /* LzFind.h -- Match finder for LZ algorithms -2021-07-13 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ -#ifndef __LZ_FIND_H -#define __LZ_FIND_H +#ifndef ZIP7_INC_LZ_FIND_H +#define ZIP7_INC_LZ_FIND_H #include "7zTypes.h" @@ -10,9 +10,9 @@ EXTERN_C_BEGIN typedef UInt32 CLzRef; -typedef struct _CMatchFinder +typedef struct { - Byte *buffer; + const Byte *buffer; UInt32 pos; UInt32 posLimit; UInt32 streamPos; /* wrap over Zero is allowed (streamPos < pos). Use (UInt32)(streamPos - pos) */ @@ -32,8 +32,8 @@ typedef struct _CMatchFinder UInt32 hashMask; UInt32 cutValue; - Byte *bufferBase; - ISeqInStream *stream; + Byte *bufBase; + ISeqInStreamPtr stream; UInt32 blockSize; UInt32 keepSizeBefore; @@ -43,7 +43,9 @@ typedef struct _CMatchFinder size_t directInputRem; UInt32 historySize; UInt32 fixedHashSize; - UInt32 hashSizeSum; + Byte numHashBytes_Min; + Byte numHashOutBits; + Byte _pad2_[2]; SRes result; UInt32 crc[256]; size_t numRefs; @@ -69,24 +71,45 @@ void MatchFinder_ReadIfRequired(CMatchFinder *p); void MatchFinder_Construct(CMatchFinder *p); -/* Conditions: - historySize <= 3 GB - keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB +/* (directInput = 0) is default value. + It's required to provide correct (directInput) value + before calling MatchFinder_Create(). + You can set (directInput) by any of the following calls: + - MatchFinder_SET_DIRECT_INPUT_BUF() + - MatchFinder_SET_STREAM() + - MatchFinder_SET_STREAM_MODE() */ + +#define MatchFinder_SET_DIRECT_INPUT_BUF(p, _src_, _srcLen_) { \ + (p)->stream = NULL; \ + (p)->directInput = 1; \ + (p)->buffer = (_src_); \ + (p)->directInputRem = (_srcLen_); } + +/* +#define MatchFinder_SET_STREAM_MODE(p) { \ + (p)->directInput = 0; } +*/ + +#define MatchFinder_SET_STREAM(p, _stream_) { \ + (p)->stream = _stream_; \ + (p)->directInput = 0; } + + int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc); void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); -// void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); /* -#define Inline_MatchFinder_InitPos(p, val) \ +#define MatchFinder_INIT_POS(p, val) \ (p)->pos = (val); \ (p)->streamPos = (val); */ -#define Inline_MatchFinder_ReduceOffsets(p, subValue) \ +// void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); +#define MatchFinder_REDUCE_OFFSETS(p, subValue) \ (p)->pos -= (subValue); \ (p)->streamPos -= (subValue); @@ -107,7 +130,7 @@ typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); typedef UInt32 * (*Mf_GetMatches_Func)(void *object, UInt32 *distances); typedef void (*Mf_Skip_Func)(void *object, UInt32); -typedef struct _IMatchFinder +typedef struct { Mf_Init_Func Init; Mf_GetNumAvailableBytes_Func GetNumAvailableBytes; @@ -121,7 +144,8 @@ void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable); void MatchFinder_Init_LowHash(CMatchFinder *p); void MatchFinder_Init_HighHash(CMatchFinder *p); void MatchFinder_Init_4(CMatchFinder *p); -void MatchFinder_Init(CMatchFinder *p); +// void MatchFinder_Init(CMatchFinder *p); +void MatchFinder_Init(void *p); UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); diff --git a/multiarc/src/formats/7z/C/LzFindMt.c b/multiarc/src/formats/7z/C/LzFindMt.c old mode 100755 new mode 100644 index 791c6ff1b..dbabe9ae7 --- a/multiarc/src/formats/7z/C/LzFindMt.c +++ b/multiarc/src/formats/7z/C/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2021-12-21 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -69,7 +69,7 @@ extern UInt64 g_NumIters_Bytes; UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } -#define __MT_HASH4_CALC { \ +#define MT_HASH4_CALC { \ UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ h2 = temp & (kHash2Size - 1); \ temp ^= ((UInt32)cur[2] << 8); \ @@ -79,14 +79,14 @@ extern UInt64 g_NumIters_Bytes; */ -MY_NO_INLINE +Z7_NO_INLINE static void MtSync_Construct(CMtSync *p) { p->affinity = 0; p->wasCreated = False; p->csWasInitialized = False; p->csWasEntered = False; - Thread_Construct(&p->thread); + Thread_CONSTRUCT(&p->thread) Event_Construct(&p->canStart); Event_Construct(&p->wasStopped); Semaphore_Construct(&p->freeSemaphore); @@ -94,7 +94,7 @@ static void MtSync_Construct(CMtSync *p) } -#define DEBUG_BUFFER_LOCK // define it to debug lock state +// #define DEBUG_BUFFER_LOCK // define it to debug lock state #ifdef DEBUG_BUFFER_LOCK #include @@ -116,7 +116,7 @@ static void MtSync_Construct(CMtSync *p) (p)->csWasEntered = False; } -MY_NO_INLINE +Z7_NO_INLINE static UInt32 MtSync_GetNextBlock(CMtSync *p) { UInt32 numBlocks = 0; @@ -140,14 +140,14 @@ static UInt32 MtSync_GetNextBlock(CMtSync *p) // buffer is UNLOCKED here Semaphore_Wait(&p->filledSemaphore); - LOCK_BUFFER(p); + LOCK_BUFFER(p) return numBlocks; } /* if Writing (Processing) thread was started, we must call MtSync_StopWriting() */ -MY_NO_INLINE +Z7_NO_INLINE static void MtSync_StopWriting(CMtSync *p) { if (!Thread_WasCreated(&p->thread) || p->needStart) @@ -185,7 +185,7 @@ static void MtSync_StopWriting(CMtSync *p) } -MY_NO_INLINE +Z7_NO_INLINE static void MtSync_Destruct(CMtSync *p) { PRF(printf("\nMtSync_Destruct %p\n", p)); @@ -220,11 +220,11 @@ static void MtSync_Destruct(CMtSync *p) // #define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } // we want to get real system error codes here instead of SZ_ERROR_THREAD -#define RINOK_THREAD(x) RINOK(x) +#define RINOK_THREAD(x) RINOK_WRes(x) // call it before each new file (when new starting is required): -MY_NO_INLINE +Z7_NO_INLINE static SRes MtSync_Init(CMtSync *p, UInt32 numBlocks) { WRes wres; @@ -245,12 +245,12 @@ static WRes MtSync_Create_WRes(CMtSync *p, THREAD_FUNC_TYPE startAddress, void * if (p->wasCreated) return SZ_OK; - RINOK_THREAD(CriticalSection_Init(&p->cs)); + RINOK_THREAD(CriticalSection_Init(&p->cs)) p->csWasInitialized = True; p->csWasEntered = False; - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped)); + RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart)) + RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped)) p->needStart = True; p->exit = True; /* p->exit is unused before (canStart) Event. @@ -264,13 +264,13 @@ static WRes MtSync_Create_WRes(CMtSync *p, THREAD_FUNC_TYPE startAddress, void * else wres = Thread_Create(&p->thread, startAddress, obj); - RINOK_THREAD(wres); + RINOK_THREAD(wres) p->wasCreated = True; return SZ_OK; } -MY_NO_INLINE +Z7_NO_INLINE static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj) { const WRes wres = MtSync_Create_WRes(p, startAddress, obj); @@ -519,7 +519,7 @@ static void HashThreadFunc(CMatchFinderMt *mt) if (mf->pos > (UInt32)kMtMaxValForNormalize - num) { const UInt32 subValue = (mf->pos - mf->historySize - 1); // & ~(UInt32)(kNormalizeAlign - 1); - Inline_MatchFinder_ReduceOffsets(mf, subValue); + MatchFinder_REDUCE_OFFSETS(mf, subValue) MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, (size_t)mf->hashMask + 1); } @@ -560,7 +560,7 @@ static void HashThreadFunc(CMatchFinderMt *mt) */ -UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, +UInt32 * Z7_FASTCALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 *posRes); @@ -749,7 +749,7 @@ static void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) } -MY_NO_INLINE +Z7_NO_INLINE static void BtThreadFunc(CMatchFinderMt *mt) { CMtSync *p = &mt->btSync; @@ -864,21 +864,22 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB if (!MatchFinder_Create(mf, historySize, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter, alloc)) return SZ_ERROR_MEM; - RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p)); - RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p)); + RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p)) + RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p)) return SZ_OK; } SRes MatchFinderMt_InitMt(CMatchFinderMt *p) { - RINOK(MtSync_Init(&p->hashSync, kMtHashNumBlocks)); + RINOK(MtSync_Init(&p->hashSync, kMtHashNumBlocks)) return MtSync_Init(&p->btSync, kMtBtNumBlocks); } -static void MatchFinderMt_Init(CMatchFinderMt *p) +static void MatchFinderMt_Init(void *_p) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; CMatchFinder *mf = MF(p); p->btBufPos = @@ -941,7 +942,7 @@ void MatchFinderMt_ReleaseStream(CMatchFinderMt *p) } -MY_NO_INLINE +Z7_NO_INLINE static UInt32 MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) { if (p->failure_LZ_BT) @@ -981,8 +982,9 @@ static UInt32 MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) -static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) +static const Byte * MatchFinderMt_GetPointerToCurrentPos(void *_p) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; return p->pointerToCurPos; } @@ -990,8 +992,9 @@ static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) #define GET_NEXT_BLOCK_IF_REQUIRED if (p->btBufPos == p->btBufPosLimit) MatchFinderMt_GetNextBlock_Bt(p); -static UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p) +static UInt32 MatchFinderMt_GetNumAvailableBytes(void *_p) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; if (p->btBufPos != p->btBufPosLimit) return p->btNumAvailBytes; return MatchFinderMt_GetNextBlock_Bt(p); @@ -1163,7 +1166,7 @@ UInt32* MatchFinderMt_GetMatches_Bt4(CMatchFinderMt *p, UInt32 *d) */ -static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) +static UInt32 * MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) { UInt32 h2, h3, /* h4, */ c2, c3 /* , c4 */; UInt32 *hash = p->hash; @@ -1179,9 +1182,8 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) (hash + kFix3HashSize)[h3] = m; // (hash + kFix4HashSize)[h4] = m; - #define _USE_H2 - - #ifdef _USE_H2 + // #define BT5_USE_H2 + // #ifdef BT5_USE_H2 if (c2 >= matchMinPos && cur[(ptrdiff_t)c2 - (ptrdiff_t)m] == cur[0]) { d[1] = m - c2 - 1; @@ -1197,8 +1199,8 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) } d[0] = 3; d += 2; - - #ifdef _USE_H4 + + #ifdef BT5_USE_H4 if (c4 >= matchMinPos) if ( cur[(ptrdiff_t)c4 - (ptrdiff_t)m] == cur[0] && @@ -1214,7 +1216,7 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) d[0] = 2; d += 2; } - #endif + // #endif if (c3 >= matchMinPos && cur[(ptrdiff_t)c3 - (ptrdiff_t)m] == cur[0]) { @@ -1228,7 +1230,7 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) d += 2; } - #ifdef _USE_H4 + #ifdef BT5_USE_H4 if (c4 >= matchMinPos) if ( cur[(ptrdiff_t)c4 - (ptrdiff_t)m] == cur[0] && @@ -1244,8 +1246,9 @@ static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) } -static UInt32* MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) +static UInt32 * MatchFinderMt2_GetMatches(void *_p, UInt32 *d) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; const UInt32 *bt = p->btBufPos; const UInt32 len = *bt++; const UInt32 *btLim = bt + len; @@ -1268,8 +1271,9 @@ static UInt32* MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) -static UInt32* MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) +static UInt32 * MatchFinderMt_GetMatches(void *_p, UInt32 *d) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; const UInt32 *bt = p->btBufPos; UInt32 len = *bt++; const UInt32 avail = p->btNumAvailBytes - 1; @@ -1316,14 +1320,16 @@ static UInt32* MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) #define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; #define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += (size_t)*p->btBufPos + 1; } while (--num != 0); -static void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) +static void MatchFinderMt0_Skip(void *_p, UInt32 num) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; SKIP_HEADER2_MT { p->btNumAvailBytes--; SKIP_FOOTER_MT } -static void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) +static void MatchFinderMt2_Skip(void *_p, UInt32 num) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; SKIP_HEADER_MT(2) UInt32 h2; MT_HASH2_CALC @@ -1331,8 +1337,9 @@ static void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) SKIP_FOOTER_MT } -static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) +static void MatchFinderMt3_Skip(void *_p, UInt32 num) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; SKIP_HEADER_MT(3) UInt32 h2, h3; MT_HASH3_CALC @@ -1362,39 +1369,46 @@ static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder2 *vTable) { - vTable->Init = (Mf_Init_Func)MatchFinderMt_Init; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinderMt_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinderMt_GetPointerToCurrentPos; - vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches; + vTable->Init = MatchFinderMt_Init; + vTable->GetNumAvailableBytes = MatchFinderMt_GetNumAvailableBytes; + vTable->GetPointerToCurrentPos = MatchFinderMt_GetPointerToCurrentPos; + vTable->GetMatches = MatchFinderMt_GetMatches; switch (MF(p)->numHashBytes) { case 2: p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)NULL; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; - vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; + p->MixMatchesFunc = NULL; + vTable->Skip = MatchFinderMt0_Skip; + vTable->GetMatches = MatchFinderMt2_GetMatches; break; case 3: p->GetHeadsFunc = MF(p)->bigHash ? GetHeads3b : GetHeads3; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip; + p->MixMatchesFunc = MixMatches2; + vTable->Skip = MatchFinderMt2_Skip; break; case 4: p->GetHeadsFunc = MF(p)->bigHash ? GetHeads4b : GetHeads4; // it's fast inline version of GetMatches() - // vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches_Bt4; + // vTable->GetMatches = MatchFinderMt_GetMatches_Bt4; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches3; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip; + p->MixMatchesFunc = MixMatches3; + vTable->Skip = MatchFinderMt3_Skip; break; default: p->GetHeadsFunc = MF(p)->bigHash ? GetHeads5b : GetHeads5; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4; + p->MixMatchesFunc = MixMatches4; vTable->Skip = - (Mf_Skip_Func)MatchFinderMt3_Skip; - // (Mf_Skip_Func)MatchFinderMt4_Skip; + MatchFinderMt3_Skip; + // MatchFinderMt4_Skip; break; } } + +#undef RINOK_THREAD +#undef PRF +#undef MF +#undef GetUi24hi_from32 +#undef LOCK_BUFFER +#undef UNLOCK_BUFFER diff --git a/multiarc/src/formats/7z/C/LzFindMt.h b/multiarc/src/formats/7z/C/LzFindMt.h old mode 100755 new mode 100644 index 660b7244d..fcb479da9 --- a/multiarc/src/formats/7z/C/LzFindMt.h +++ b/multiarc/src/formats/7z/C/LzFindMt.h @@ -1,15 +1,15 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2021-07-12 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ -#ifndef __LZ_FIND_MT_H -#define __LZ_FIND_MT_H +#ifndef ZIP7_INC_LZ_FIND_MT_H +#define ZIP7_INC_LZ_FIND_MT_H #include "LzFind.h" #include "Threads.h" EXTERN_C_BEGIN -typedef struct _CMtSync +typedef struct { UInt32 numProcessedBlocks; CThread thread; @@ -31,7 +31,10 @@ typedef struct _CMtSync // UInt32 numBlocks_Sent; } CMtSync; -typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances); + +struct CMatchFinderMt_; + +typedef UInt32 * (*Mf_Mix_Matches)(struct CMatchFinderMt_ *p, UInt32 matchMinPos, UInt32 *distances); /* kMtCacheLineDummy must be >= size_of_CPU_cache_line */ #define kMtCacheLineDummy 128 @@ -39,7 +42,7 @@ typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distance typedef void (*Mf_GetHeads)(const Byte *buffer, UInt32 pos, UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc); -typedef struct _CMatchFinderMt +typedef struct CMatchFinderMt_ { /* LZ */ const Byte *pointerToCurPos; diff --git a/multiarc/src/formats/7z/C/LzFindOpt.c b/multiarc/src/formats/7z/C/LzFindOpt.c old mode 100755 new mode 100644 index 8ff006e07..85bdc136d --- a/multiarc/src/formats/7z/C/LzFindOpt.c +++ b/multiarc/src/formats/7z/C/LzFindOpt.c @@ -1,5 +1,5 @@ /* LzFindOpt.c -- multithreaded Match finder for LZ algorithms -2021-07-13 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -41,8 +41,8 @@ UInt64 g_NumIters_Bytes; // #define CYC_TO_POS_OFFSET 1 // for debug /* -MY_NO_INLINE -UInt32 * MY_FAST_CALL GetMatchesSpecN_1(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, +Z7_NO_INLINE +UInt32 * Z7_FASTCALL GetMatchesSpecN_1(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, UInt32 *posRes) { do @@ -214,13 +214,13 @@ else to eliminate "movsx" BUG in old MSVC x64 compiler. */ -UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, +UInt32 * Z7_FASTCALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 *posRes); -MY_NO_INLINE -UInt32 * MY_FAST_CALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, +Z7_NO_INLINE +UInt32 * Z7_FASTCALL GetMatchesSpecN_2(const Byte *lenLimit, size_t pos, const Byte *cur, CLzRef *son, UInt32 _cutValue, UInt32 *d, size_t _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 *posRes) @@ -404,7 +404,7 @@ else /* typedef UInt32 uint32plus; // size_t -UInt32 * MY_FAST_CALL GetMatchesSpecN_3(uint32plus lenLimit, size_t pos, const Byte *cur, CLzRef *son, +UInt32 * Z7_FASTCALL GetMatchesSpecN_3(uint32plus lenLimit, size_t pos, const Byte *cur, CLzRef *son, UInt32 _cutValue, UInt32 *d, uint32plus _maxLen, const UInt32 *hash, const UInt32 *limit, const UInt32 *size, size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 *posRes) diff --git a/multiarc/src/formats/7z/C/LzHash.h b/multiarc/src/formats/7z/C/LzHash.h old mode 100755 new mode 100644 index 77b898cfa..2b6290b64 --- a/multiarc/src/formats/7z/C/LzHash.h +++ b/multiarc/src/formats/7z/C/LzHash.h @@ -1,8 +1,8 @@ -/* LzHash.h -- HASH functions for LZ algorithms -2019-10-30 : Igor Pavlov : Public domain */ +/* LzHash.h -- HASH constants for LZ algorithms +2023-03-05 : Igor Pavlov : Public domain */ -#ifndef __LZ_HASH_H -#define __LZ_HASH_H +#ifndef ZIP7_INC_LZ_HASH_H +#define ZIP7_INC_LZ_HASH_H /* (kHash2Size >= (1 << 8)) : Required diff --git a/multiarc/src/formats/7z/C/Lzma2Dec.c b/multiarc/src/formats/7z/C/Lzma2Dec.c old mode 100755 new mode 100644 index ac970a843..8bf54e499 --- a/multiarc/src/formats/7z/C/Lzma2Dec.c +++ b/multiarc/src/formats/7z/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2021-02-09 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -71,14 +71,14 @@ static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props) SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; - RINOK(Lzma2Dec_GetOldProps(prop, props)); + RINOK(Lzma2Dec_GetOldProps(prop, props)) return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc); } SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; - RINOK(Lzma2Dec_GetOldProps(prop, props)); + RINOK(Lzma2Dec_GetOldProps(prop, props)) return LzmaDec_Allocate(&p->decoder, props, LZMA_PROPS_SIZE, alloc); } @@ -157,8 +157,10 @@ static unsigned Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } + + default: + return LZMA2_STATE_ERROR; } - return LZMA2_STATE_ERROR; } static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT size) @@ -474,8 +476,8 @@ SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, SizeT outSize = *destLen, inSize = *srcLen; *destLen = *srcLen = 0; *status = LZMA_STATUS_NOT_SPECIFIED; - Lzma2Dec_Construct(&p); - RINOK(Lzma2Dec_AllocateProbs(&p, prop, alloc)); + Lzma2Dec_CONSTRUCT(&p) + RINOK(Lzma2Dec_AllocateProbs(&p, prop, alloc)) p.decoder.dic = dest; p.decoder.dicBufSize = outSize; Lzma2Dec_Init(&p); @@ -487,3 +489,5 @@ SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, Lzma2Dec_FreeProbs(&p, alloc); return res; } + +#undef PRF diff --git a/multiarc/src/formats/7z/C/Lzma2Dec.h b/multiarc/src/formats/7z/C/Lzma2Dec.h old mode 100755 new mode 100644 index b8ddeac89..1f5233a72 --- a/multiarc/src/formats/7z/C/Lzma2Dec.h +++ b/multiarc/src/formats/7z/C/Lzma2Dec.h @@ -1,8 +1,8 @@ /* Lzma2Dec.h -- LZMA2 Decoder -2018-02-19 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ -#ifndef __LZMA2_DEC_H -#define __LZMA2_DEC_H +#ifndef ZIP7_INC_LZMA2_DEC_H +#define ZIP7_INC_LZMA2_DEC_H #include "LzmaDec.h" @@ -22,9 +22,10 @@ typedef struct CLzmaDec decoder; } CLzma2Dec; -#define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) +#define Lzma2Dec_CONSTRUCT(p) LzmaDec_CONSTRUCT(&(p)->decoder) +#define Lzma2Dec_Construct(p) Lzma2Dec_CONSTRUCT(p) +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); @@ -90,7 +91,7 @@ Lzma2Dec_GetUnpackExtra() returns the value that shows at current input positon. */ -#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0) /* ---------- One Call Interface ---------- */ diff --git a/multiarc/src/formats/7z/C/Lzma2DecMt.c b/multiarc/src/formats/7z/C/Lzma2DecMt.c old mode 100755 new mode 100644 index 9f1dc52ba..4bc4ddeb5 --- a/multiarc/src/formats/7z/C/Lzma2DecMt.c +++ b/multiarc/src/formats/7z/C/Lzma2DecMt.c @@ -1,44 +1,44 @@ /* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread -2021-04-01 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ #include "Precomp.h" // #define SHOW_DEBUG_INFO - -// #define _7ZIP_ST +// #define Z7_ST #ifdef SHOW_DEBUG_INFO #include #endif -#ifndef _7ZIP_ST -#ifdef SHOW_DEBUG_INFO -#define PRF(x) x -#else -#define PRF(x) -#endif -#define PRF_STR(s) PRF(printf("\n" s "\n")) -#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) -#endif - #include "Alloc.h" #include "Lzma2Dec.h" #include "Lzma2DecMt.h" -#ifndef _7ZIP_ST +#ifndef Z7_ST #include "MtDec.h" #define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) #endif +#ifndef Z7_ST +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif +#define PRF_STR(s) PRF(printf("\n" s "\n");) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2);) +#endif + + void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) { p->inBufSize_ST = 1 << 20; p->outStep_ST = 1 << 20; - #ifndef _7ZIP_ST + #ifndef Z7_ST p->numThreads = 1; p->inBufSize_MT = 1 << 18; p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; @@ -48,7 +48,7 @@ void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) -#ifndef _7ZIP_ST +#ifndef Z7_ST /* ---------- CLzma2DecMtThread ---------- */ @@ -81,7 +81,7 @@ typedef struct /* ---------- CLzma2DecMt ---------- */ -typedef struct +struct CLzma2DecMt { // ISzAllocPtr alloc; ISzAllocPtr allocMid; @@ -90,9 +90,9 @@ typedef struct CLzma2DecMtProps props; Byte prop; - ISeqInStream *inStream; - ISeqOutStream *outStream; - ICompressProgress *progress; + ISeqInStreamPtr inStream; + ISeqOutStreamPtr outStream; + ICompressProgressPtr progress; BoolInt finishMode; BoolInt outSize_Defined; @@ -111,14 +111,13 @@ typedef struct size_t inPos; size_t inLim; - #ifndef _7ZIP_ST + #ifndef Z7_ST UInt64 outProcessed_Parse; BoolInt mtc_WasConstructed; CMtDec mtc; - CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + CLzma2DecMtThread coders[MTDEC_THREADS_MAX]; #endif - -} CLzma2DecMt; +}; @@ -142,11 +141,11 @@ CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) // Lzma2DecMtProps_Init(&p->props); - #ifndef _7ZIP_ST + #ifndef Z7_ST p->mtc_WasConstructed = False; { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CLzma2DecMtThread *t = &p->coders[i]; t->dec_created = False; @@ -156,16 +155,16 @@ CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) } #endif - return p; + return (CLzma2DecMtHandle)(void *)p; } -#ifndef _7ZIP_ST +#ifndef Z7_ST static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CLzma2DecMtThread *t = &p->coders[i]; if (t->outBuf) @@ -196,13 +195,15 @@ static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) } -void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +// #define GET_CLzma2DecMt_p CLzma2DecMt *p = (CLzma2DecMt *)(void *)pp; + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p) { - CLzma2DecMt *p = (CLzma2DecMt *)pp; + // GET_CLzma2DecMt_p Lzma2DecMt_FreeSt(p); - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->mtc_WasConstructed) { @@ -211,7 +212,7 @@ void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) } { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CLzma2DecMtThread *t = &p->coders[i]; if (t->dec_created) @@ -226,19 +227,19 @@ void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) #endif - ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, p); } -#ifndef _7ZIP_ST +#ifndef Z7_ST static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) { CLzma2DecMt *me = (CLzma2DecMt *)obj; CLzma2DecMtThread *t = &me->coders[coderIndex]; - PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize) cc->state = MTDEC_PARSE_CONTINUE; @@ -246,7 +247,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa { if (!t->dec_created) { - Lzma2Dec_Construct(&t->dec); + Lzma2Dec_CONSTRUCT(&t->dec) t->dec_created = True; AlignOffsetAlloc_CreateVTable(&t->alloc); { @@ -297,7 +298,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa // that must be finished at position <= outBlockMax. { - const SizeT srcOrig = cc->srcSize; + const size_t srcOrig = cc->srcSize; SizeT srcSize_Point = 0; SizeT dicPos_Point = 0; @@ -306,10 +307,10 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa for (;;) { - SizeT srcCur = srcOrig - cc->srcSize; + SizeT srcCur = (SizeT)(srcOrig - cc->srcSize); status = Lzma2Dec_Parse(&t->dec, - limit - t->dec.decoder.dicPos, + (SizeT)limit - t->dec.decoder.dicPos, cc->src + cc->srcSize, &srcCur, checkFinishBlock); @@ -333,7 +334,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa if (t->dec.decoder.dicPos >= (1 << 14)) break; dicPos_Point = t->dec.decoder.dicPos; - srcSize_Point = cc->srcSize; + srcSize_Point = (SizeT)cc->srcSize; continue; } @@ -391,7 +392,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa if (unpackRem != 0) { /* we also reserve space for max possible number of output bytes of current LZMA chunk */ - SizeT rem = limit - dicPos; + size_t rem = limit - dicPos; if (rem > unpackRem) rem = unpackRem; dicPos += rem; @@ -444,7 +445,7 @@ static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) } t->dec.decoder.dic = dest; - t->dec.decoder.dicBufSize = t->outPreSize; + t->dec.decoder.dicBufSize = (SizeT)t->outPreSize; t->needInit = True; @@ -462,7 +463,7 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, UNUSED_VAR(srcFinished) - PRF_STR_INT_2("Code", coderIndex, srcSize); + PRF_STR_INT_2("Code", coderIndex, srcSize) *inCodePos = t->inCodeSize; *outCodePos = 0; @@ -476,13 +477,13 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, { ELzmaStatus status; - size_t srcProcessed = srcSize; + SizeT srcProcessed = (SizeT)srcSize; BoolInt blockWasFinished = ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); SRes res = Lzma2Dec_DecodeToDic(&t->dec, - t->outPreSize, + (SizeT)t->outPreSize, src, &srcProcessed, blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, &status); @@ -540,7 +541,7 @@ static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, UNUSED_VAR(srcSize) UNUSED_VAR(isCross) - PRF_STR_INT_2("Write", coderIndex, srcSize); + PRF_STR_INT_2("Write", coderIndex, srcSize) *needContinue = False; *canRecode = True; @@ -588,7 +589,7 @@ static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, *needContinue = needContinue2; return SZ_OK; } - RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)) } } @@ -611,11 +612,11 @@ static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) { if (!p->dec_created) { - Lzma2Dec_Construct(&p->dec); + Lzma2Dec_CONSTRUCT(&p->dec) p->dec_created = True; } - RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)) if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) { @@ -634,7 +635,7 @@ static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p - #ifndef _7ZIP_ST + #ifndef Z7_ST , BoolInt tMode #endif ) @@ -646,7 +647,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p CLzma2Dec *dec; - #ifndef _7ZIP_ST + #ifndef Z7_ST if (tMode) { Lzma2DecMt_FreeOutBufs(p); @@ -654,7 +655,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p } #endif - RINOK(Lzma2Dec_Prepare_ST(p)); + RINOK(Lzma2Dec_Prepare_ST(p)) dec = &p->dec; @@ -681,7 +682,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p if (inPos == inLim) { - #ifndef _7ZIP_ST + #ifndef Z7_ST if (tMode) { inData = MtDec_Read(&p->mtc, &inLim); @@ -710,7 +711,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p { SizeT next = dec->decoder.dicBufSize; if (next - wrPos > p->props.outStep_ST) - next = wrPos + p->props.outStep_ST; + next = wrPos + (SizeT)p->props.outStep_ST; size = next - dicPos; } @@ -726,7 +727,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p } } - inProcessed = inLim - inPos; + inProcessed = (SizeT)(inLim - inPos); res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); @@ -755,7 +756,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p dec->decoder.dicPos = 0; wrPos = dec->decoder.dicPos; - RINOK(res2); + RINOK(res2) if (needStop) { @@ -788,7 +789,7 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p UInt64 outDelta = p->outProcessed - outPrev; if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) { - RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)) inPrev = p->inProcessed; outPrev = p->outProcessed; } @@ -798,20 +799,20 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p -SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, Byte prop, const CLzma2DecMtProps *props, - ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + ISeqOutStreamPtr outStream, const UInt64 *outDataSize, int finishMode, // Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, // const Byte *inData, size_t inDataSize, UInt64 *inProcessed, // UInt64 *outProcessed, int *isMT, - ICompressProgress *progress) + ICompressProgressPtr progress) { - CLzma2DecMt *p = (CLzma2DecMt *)pp; - #ifndef _7ZIP_ST + // GET_CLzma2DecMt_p + #ifndef Z7_ST BoolInt tMode; #endif @@ -845,7 +846,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, *isMT = False; - #ifndef _7ZIP_ST + #ifndef Z7_ST tMode = False; @@ -939,7 +940,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, p->readWasFinished = p->mtc.readWasFinished; p->inProcessed = p->mtc.inProcessed; - PRF_STR("----- decoding ST -----"); + PRF_STR("----- decoding ST -----") } } @@ -950,7 +951,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, { SRes res = Lzma2Dec_Decode_ST(p - #ifndef _7ZIP_ST + #ifndef Z7_ST , tMode #endif ); @@ -967,7 +968,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, res = p->readRes; /* - #ifndef _7ZIP_ST + #ifndef Z7_ST if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) res = p->mtc.parseRes; #endif @@ -980,13 +981,13 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, /* ---------- Read from CLzma2DecMtHandle Interface ---------- */ -SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, +SRes Lzma2DecMt_Init(CLzma2DecMtHandle p, Byte prop, const CLzma2DecMtProps *props, const UInt64 *outDataSize, int finishMode, - ISeqInStream *inStream) + ISeqInStreamPtr inStream) { - CLzma2DecMt *p = (CLzma2DecMt *)pp; + // GET_CLzma2DecMt_p if (prop > 40) return SZ_ERROR_UNSUPPORTED; @@ -1015,11 +1016,11 @@ SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, } -SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, +SRes Lzma2DecMt_Read(CLzma2DecMtHandle p, Byte *data, size_t *outSize, UInt64 *inStreamProcessed) { - CLzma2DecMt *p = (CLzma2DecMt *)pp; + // GET_CLzma2DecMt_p ELzmaFinishMode finishMode; SRes readRes; size_t size = *outSize; @@ -1055,8 +1056,8 @@ SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); } - inCur = p->inLim - p->inPos; - outCur = size; + inCur = (SizeT)(p->inLim - p->inPos); + outCur = (SizeT)size; res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, p->inBuf + p->inPos, &inCur, finishMode, &status); @@ -1088,3 +1089,7 @@ SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, return readRes; } } + +#undef PRF +#undef PRF_STR +#undef PRF_STR_INT_2 diff --git a/multiarc/src/formats/7z/C/Lzma2DecMt.h b/multiarc/src/formats/7z/C/Lzma2DecMt.h old mode 100755 new mode 100644 index 7791c310b..93a5cd59e --- a/multiarc/src/formats/7z/C/Lzma2DecMt.h +++ b/multiarc/src/formats/7z/C/Lzma2DecMt.h @@ -1,8 +1,8 @@ /* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread -2018-02-17 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ -#ifndef __LZMA2_DEC_MT_H -#define __LZMA2_DEC_MT_H +#ifndef ZIP7_INC_LZMA2_DEC_MT_H +#define ZIP7_INC_LZMA2_DEC_MT_H #include "7zTypes.h" @@ -13,7 +13,7 @@ typedef struct size_t inBufSize_ST; size_t outStep_ST; - #ifndef _7ZIP_ST + #ifndef Z7_ST unsigned numThreads; size_t inBufSize_MT; size_t outBlockMax; @@ -38,7 +38,9 @@ void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ -typedef void * CLzma2DecMtHandle; +typedef struct CLzma2DecMt CLzma2DecMt; +typedef CLzma2DecMt * CLzma2DecMtHandle; +// Z7_DECLARE_HANDLE(CLzma2DecMtHandle) CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); @@ -46,11 +48,11 @@ void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, Byte prop, const CLzma2DecMtProps *props, - ISeqOutStream *outStream, + ISeqOutStreamPtr outStream, const UInt64 *outDataSize, // NULL means undefined int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished // Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, // const Byte *inData, size_t inDataSize, // out variables: @@ -58,7 +60,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ // UInt64 *outProcessed, - ICompressProgress *progress); + ICompressProgressPtr progress); /* ---------- Read from CLzma2DecMtHandle Interface ---------- */ @@ -67,7 +69,7 @@ SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, Byte prop, const CLzma2DecMtProps *props, const UInt64 *outDataSize, int finishMode, - ISeqInStream *inStream); + ISeqInStreamPtr inStream); SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, Byte *data, size_t *outSize, diff --git a/multiarc/src/formats/7z/C/Lzma2Enc.c b/multiarc/src/formats/7z/C/Lzma2Enc.c old mode 100755 new mode 100644 index e61a5dfe0..703e146b5 --- a/multiarc/src/formats/7z/C/Lzma2Enc.c +++ b/multiarc/src/formats/7z/C/Lzma2Enc.c @@ -1,18 +1,18 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2021-02-09 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ #include "Precomp.h" #include -/* #define _7ZIP_ST */ +/* #define Z7_ST */ #include "Lzma2Enc.h" -#ifndef _7ZIP_ST +#ifndef Z7_ST #include "MtCoder.h" #else -#define MTCODER__THREADS_MAX 1 +#define MTCODER_THREADS_MAX 1 #endif #define LZMA2_CONTROL_LZMA (1 << 7) @@ -40,7 +40,7 @@ typedef struct { ISeqInStream vt; - ISeqInStream *realStream; + ISeqInStreamPtr realStream; UInt64 limit; UInt64 processed; int finished; @@ -53,15 +53,15 @@ static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) p->finished = 0; } -static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +static SRes LimitedSeqInStream_Read(ISeqInStreamPtr pp, void *data, size_t *size) { - CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CLimitedSeqInStream) size_t size2 = *size; SRes res = SZ_OK; if (p->limit != (UInt64)(Int64)-1) { - UInt64 rem = p->limit - p->processed; + const UInt64 rem = p->limit - p->processed; if (size2 > rem) size2 = (size_t)rem; } @@ -95,8 +95,8 @@ static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) { SizeT propsSize = LZMA_PROPS_SIZE; Byte propsEncoded[LZMA_PROPS_SIZE]; - RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); - RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); + RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)) + RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)) p->propsByte = propsEncoded[0]; p->propsAreSet = True; } @@ -111,23 +111,23 @@ static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) } -SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle p, ISeqInStreamPtr inStream, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, +SRes LzmaEnc_MemPrepare(CLzmaEncHandle p, const Byte *src, SizeT srcLen, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle p, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); -const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); -void LzmaEnc_Finish(CLzmaEncHandle pp); -void LzmaEnc_SaveState(CLzmaEncHandle pp); -void LzmaEnc_RestoreState(CLzmaEncHandle pp); +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle p); +void LzmaEnc_Finish(CLzmaEncHandle p); +void LzmaEnc_SaveState(CLzmaEncHandle p); +void LzmaEnc_RestoreState(CLzmaEncHandle p); /* -UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle p); */ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, - size_t *packSizeRes, ISeqOutStream *outStream) + size_t *packSizeRes, ISeqOutStreamPtr outStream) { size_t packSizeLimit = *packSizeRes; size_t packSize = packSizeLimit; @@ -167,7 +167,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, while (unpackSize > 0) { - UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; + const UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; if (packSizeLimit - destPos < u + 3) return SZ_ERROR_OUTPUT_EOF; outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET); @@ -196,9 +196,9 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, { size_t destPos = 0; - UInt32 u = unpackSize - 1; - UInt32 pm = (UInt32)(packSize - 1); - unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); + const UInt32 u = unpackSize - 1; + const UInt32 pm = (UInt32)(packSize - 1); + const unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); PRF(printf(" ")); @@ -231,7 +231,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, void Lzma2EncProps_Init(CLzma2EncProps *p) { LzmaEncProps_Init(&p->lzmaProps); - p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; + p->blockSize = LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO; p->numBlockThreads_Reduced = -1; p->numBlockThreads_Max = -1; p->numTotalThreads = -1; @@ -251,8 +251,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) t2 = p->numBlockThreads_Max; t3 = p->numTotalThreads; - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; + if (t2 > MTCODER_THREADS_MAX) + t2 = MTCODER_THREADS_MAX; if (t3 <= 0) { @@ -268,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) t1 = 1; t2 = t3; } - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; + if (t2 > MTCODER_THREADS_MAX) + t2 = MTCODER_THREADS_MAX; } else if (t1 <= 0) { @@ -286,8 +286,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) fileSize = p->lzmaProps.reduceSize; - if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO + if ( p->blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID + && p->blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) p->lzmaProps.reduceSize = p->blockSize; @@ -297,19 +297,19 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) t1 = p->lzmaProps.numThreads; - if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + if (p->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) { t2r = t2 = 1; t3 = t1; } - else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) + else if (p->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO && t2 <= 1) { /* if there is no block multi-threading, we use SOLID block */ - p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + p->blockSize = LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID; } else { - if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + if (p->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) { const UInt32 kMinSize = (UInt32)1 << 20; const UInt32 kMaxSize = (UInt32)1 << 28; @@ -344,7 +344,7 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p) } -static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) +static SRes Progress(ICompressProgressPtr p, UInt64 inSize, UInt64 outSize) { return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; } @@ -352,7 +352,7 @@ static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) /* ---------- Lzma2 ---------- */ -typedef struct +struct CLzma2Enc { Byte propEncoded; CLzma2EncProps props; @@ -363,23 +363,22 @@ typedef struct ISzAllocPtr alloc; ISzAllocPtr allocBig; - CLzma2EncInt coders[MTCODER__THREADS_MAX]; + CLzma2EncInt coders[MTCODER_THREADS_MAX]; - #ifndef _7ZIP_ST + #ifndef Z7_ST - ISeqOutStream *outStream; + ISeqOutStreamPtr outStream; Byte *outBuf; size_t outBuf_Rem; /* remainder in outBuf */ size_t outBufSize; /* size of allocated outBufs[i] */ - size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; + size_t outBufsDataSizes[MTCODER_BLOCKS_MAX]; BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; - Byte *outBufs[MTCODER__BLOCKS_MAX]; + Byte *outBufs[MTCODER_BLOCKS_MAX]; #endif - -} CLzma2Enc; +}; @@ -396,30 +395,30 @@ CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) p->allocBig = allocBig; { unsigned i; - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) p->coders[i].enc = NULL; } - #ifndef _7ZIP_ST + #ifndef Z7_ST p->mtCoder_WasConstructed = False; { unsigned i; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) p->outBufs[i] = NULL; p->outBufSize = 0; } #endif - return p; + return (CLzma2EncHandle)p; } -#ifndef _7ZIP_ST +#ifndef Z7_ST static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) { unsigned i; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) if (p->outBufs[i]) { ISzAlloc_Free(p->alloc, p->outBufs[i]); @@ -430,12 +429,13 @@ static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) #endif +// #define GET_CLzma2Enc_p CLzma2Enc *p = (CLzma2Enc *)(void *)p; -void Lzma2Enc_Destroy(CLzma2EncHandle pp) +void Lzma2Enc_Destroy(CLzma2EncHandle p) { - CLzma2Enc *p = (CLzma2Enc *)pp; + // GET_CLzma2Enc_p unsigned i; - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) { CLzma2EncInt *t = &p->coders[i]; if (t->enc) @@ -446,7 +446,7 @@ void Lzma2Enc_Destroy(CLzma2EncHandle pp) } - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->mtCoder_WasConstructed) { MtCoder_Destruct(&p->mtCoder); @@ -458,13 +458,13 @@ void Lzma2Enc_Destroy(CLzma2EncHandle pp) ISzAlloc_Free(p->alloc, p->tempBufLzma); p->tempBufLzma = NULL; - ISzAlloc_Free(p->alloc, pp); + ISzAlloc_Free(p->alloc, p); } -SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) +SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props) { - CLzma2Enc *p = (CLzma2Enc *)pp; + // GET_CLzma2Enc_p CLzmaEncProps lzmaProps = props->lzmaProps; LzmaEncProps_Normalize(&lzmaProps); if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX) @@ -475,16 +475,16 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) } -void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize) { - CLzma2Enc *p = (CLzma2Enc *)pp; + // GET_CLzma2Enc_p p->expectedDataSize = expectedDataSiize; } -Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) +Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p) { - CLzma2Enc *p = (CLzma2Enc *)pp; + // GET_CLzma2Enc_p unsigned i; UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps); for (i = 0; i < 40; i++) @@ -497,12 +497,12 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) static SRes Lzma2Enc_EncodeMt1( CLzma2Enc *me, CLzma2EncInt *p, - ISeqOutStream *outStream, + ISeqOutStreamPtr outStream, Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, const Byte *inData, size_t inDataSize, int finished, - ICompressProgress *progress) + ICompressProgressPtr progress) { UInt64 unpackTotal = 0; UInt64 packTotal = 0; @@ -540,12 +540,12 @@ static SRes Lzma2Enc_EncodeMt1( } } - RINOK(Lzma2EncInt_InitStream(p, &me->props)); + RINOK(Lzma2EncInt_InitStream(p, &me->props)) for (;;) { SRes res = SZ_OK; - size_t inSizeCur = 0; + SizeT inSizeCur = 0; Lzma2EncInt_InitBlock(p); @@ -559,7 +559,7 @@ static SRes Lzma2Enc_EncodeMt1( if (me->expectedDataSize != (UInt64)(Int64)-1 && me->expectedDataSize >= unpackTotal) expected = me->expectedDataSize - unpackTotal; - if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + if (me->props.blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID && expected > me->props.blockSize) expected = (size_t)me->props.blockSize; @@ -569,14 +569,14 @@ static SRes Lzma2Enc_EncodeMt1( &limitedInStream.vt, LZMA2_KEEP_WINDOW_SIZE, me->alloc, - me->allocBig)); + me->allocBig)) } else { - inSizeCur = inDataSize - (size_t)unpackTotal; - if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + inSizeCur = (SizeT)(inDataSize - (size_t)unpackTotal); + if (me->props.blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID && inSizeCur > me->props.blockSize) - inSizeCur = (size_t)me->props.blockSize; + inSizeCur = (SizeT)(size_t)me->props.blockSize; // LzmaEnc_SetDataSize(p->enc, inSizeCur); @@ -584,7 +584,7 @@ static SRes Lzma2Enc_EncodeMt1( inData + (size_t)unpackTotal, inSizeCur, LZMA2_KEEP_WINDOW_SIZE, me->alloc, - me->allocBig)); + me->allocBig)) } for (;;) @@ -621,7 +621,7 @@ static SRes Lzma2Enc_EncodeMt1( unpackTotal += p->srcPos; - RINOK(res); + RINOK(res) if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) return SZ_ERROR_FAIL; @@ -652,12 +652,12 @@ static SRes Lzma2Enc_EncodeMt1( -#ifndef _7ZIP_ST +#ifndef Z7_ST -static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, +static SRes Lzma2Enc_MtCallback_Code(void *p, unsigned coderIndex, unsigned outBufIndex, const Byte *src, size_t srcSize, int finished) { - CLzma2Enc *me = (CLzma2Enc *)pp; + CLzma2Enc *me = (CLzma2Enc *)p; size_t destSize = me->outBufSize; SRes res; CMtProgressThunk progressThunk; @@ -692,9 +692,9 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out } -static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) +static SRes Lzma2Enc_MtCallback_Write(void *p, unsigned outBufIndex) { - CLzma2Enc *me = (CLzma2Enc *)pp; + CLzma2Enc *me = (CLzma2Enc *)p; size_t size = me->outBufsDataSizes[outBufIndex]; const Byte *data = me->outBufs[outBufIndex]; @@ -713,14 +713,14 @@ static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) -SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, - ISeqOutStream *outStream, +SRes Lzma2Enc_Encode2(CLzma2EncHandle p, + ISeqOutStreamPtr outStream, Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, const Byte *inData, size_t inDataSize, - ICompressProgress *progress) + ICompressProgressPtr progress) { - CLzma2Enc *p = (CLzma2Enc *)pp; + // GET_CLzma2Enc_p if (inStream && inData) return SZ_ERROR_PARAM; @@ -730,11 +730,11 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, { unsigned i; - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) p->coders[i].propsAreSet = False; } - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->props.numBlockThreads_Reduced > 1) { @@ -772,7 +772,7 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ { - size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; + const size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; if (destBlockSize < p->mtCoder.blockSize) return SZ_ERROR_PARAM; if (p->outBufSize != destBlockSize) @@ -784,7 +784,7 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, p->mtCoder.expectedDataSize = p->expectedDataSize; { - SRes res = MtCoder_Code(&p->mtCoder); + const SRes res = MtCoder_Code(&p->mtCoder); if (!outStream) *outBufSize = (size_t)(p->outBuf - outBuf); return res; @@ -801,3 +801,5 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, True, /* finished */ progress); } + +#undef PRF diff --git a/multiarc/src/formats/7z/C/Lzma2Enc.h b/multiarc/src/formats/7z/C/Lzma2Enc.h old mode 100755 new mode 100644 index 6a6110ff7..cb25275c6 --- a/multiarc/src/formats/7z/C/Lzma2Enc.h +++ b/multiarc/src/formats/7z/C/Lzma2Enc.h @@ -1,15 +1,15 @@ /* Lzma2Enc.h -- LZMA2 Encoder -2017-07-27 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ -#ifndef __LZMA2_ENC_H -#define __LZMA2_ENC_H +#ifndef ZIP7_INC_LZMA2_ENC_H +#define ZIP7_INC_LZMA2_ENC_H #include "LzmaEnc.h" EXTERN_C_BEGIN -#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 -#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) +#define LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO 0 +#define LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID ((UInt64)(Int64)-1) typedef struct { @@ -36,7 +36,9 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p); SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ -typedef void * CLzma2EncHandle; +typedef struct CLzma2Enc CLzma2Enc; +typedef CLzma2Enc * CLzma2EncHandle; +// Z7_DECLARE_HANDLE(CLzma2EncHandle) CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); void Lzma2Enc_Destroy(CLzma2EncHandle p); @@ -44,11 +46,11 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); SRes Lzma2Enc_Encode2(CLzma2EncHandle p, - ISeqOutStream *outStream, + ISeqOutStreamPtr outStream, Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, const Byte *inData, size_t inDataSize, - ICompressProgress *progress); + ICompressProgressPtr progress); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/Lzma86.h b/multiarc/src/formats/7z/C/Lzma86.h old mode 100755 new mode 100644 index bebed5cb7..e7707e2c4 --- a/multiarc/src/formats/7z/C/Lzma86.h +++ b/multiarc/src/formats/7z/C/Lzma86.h @@ -1,8 +1,8 @@ /* Lzma86.h -- LZMA + x86 (BCJ) Filter -2013-01-18 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ -#ifndef __LZMA86_H -#define __LZMA86_H +#ifndef ZIP7_INC_LZMA86_H +#define ZIP7_INC_LZMA86_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/Lzma86Dec.c b/multiarc/src/formats/7z/C/Lzma86Dec.c old mode 100755 new mode 100644 index 21031745c..f094d4c35 --- a/multiarc/src/formats/7z/C/Lzma86Dec.c +++ b/multiarc/src/formats/7z/C/Lzma86Dec.c @@ -1,5 +1,5 @@ /* Lzma86Dec.c -- LZMA + x86 (BCJ) Filter Decoder -2016-05-16 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -46,9 +46,8 @@ SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen) return res; if (useFilter == 1) { - UInt32 x86State; - x86_Convert_Init(x86State); - x86_Convert(dest, *destLen, 0, &x86State, 0); + UInt32 x86State = Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL; + z7_BranchConvSt_X86_Dec(dest, *destLen, 0, &x86State); } return SZ_OK; } diff --git a/multiarc/src/formats/7z/C/Lzma86Enc.c b/multiarc/src/formats/7z/C/Lzma86Enc.c old mode 100755 new mode 100644 index 14fcd65c6..0cdde1c9a --- a/multiarc/src/formats/7z/C/Lzma86Enc.c +++ b/multiarc/src/formats/7z/C/Lzma86Enc.c @@ -1,5 +1,5 @@ /* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2018-07-04 : Igor Pavlov : Public domain */ +2023-03-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -46,9 +46,8 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, memcpy(filteredStream, src, srcLen); } { - UInt32 x86State; - x86_Convert_Init(x86State); - x86_Convert(filteredStream, srcLen, 0, &x86State, 1); + UInt32 x86State = Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL; + z7_BranchConvSt_X86_Enc(filteredStream, srcLen, 0, &x86State); } } diff --git a/multiarc/src/formats/7z/C/LzmaDec.c b/multiarc/src/formats/7z/C/LzmaDec.c old mode 100755 new mode 100644 index d6742e5af..69bb8bba9 --- a/multiarc/src/formats/7z/C/LzmaDec.c +++ b/multiarc/src/formats/7z/C/LzmaDec.c @@ -1,5 +1,5 @@ /* LzmaDec.c -- LZMA Decoder -2021-04-01 : Igor Pavlov : Public domain */ +2023-04-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,15 +8,15 @@ /* #include "CpuArch.h" */ #include "LzmaDec.h" -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) +// #define kNumTopBits 24 +#define kTopValue ((UInt32)1 << 24) #define kNumBitModelTotalBits 11 #define kBitModelTotal (1 << kNumBitModelTotalBits) #define RC_INIT_SIZE 5 -#ifndef _LZMA_DEC_OPT +#ifndef Z7_LZMA_DEC_OPT #define kNumMoveBits 5 #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } @@ -25,14 +25,14 @@ #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ - { UPDATE_0(p); i = (i + i); A0; } else \ - { UPDATE_1(p); i = (i + i) + 1; A1; } + { UPDATE_0(p) i = (i + i); A0; } else \ + { UPDATE_1(p) i = (i + i) + 1; A1; } #define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } #define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ - { UPDATE_0(p + i); A0; } else \ - { UPDATE_1(p + i); A1; } + { UPDATE_0(p + i) A0; } else \ + { UPDATE_1(p + i) A1; } #define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) #define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) #define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) @@ -40,19 +40,19 @@ #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } -/* #define _LZMA_SIZE_OPT */ +/* #define Z7_LZMA_SIZE_OPT */ -#ifdef _LZMA_SIZE_OPT +#ifdef Z7_LZMA_SIZE_OPT #define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) #else #define TREE_6_DECODE(probs, i) \ { i = 1; \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i) \ + TREE_GET_BIT(probs, i) \ + TREE_GET_BIT(probs, i) \ + TREE_GET_BIT(probs, i) \ + TREE_GET_BIT(probs, i) \ + TREE_GET_BIT(probs, i) \ i -= 0x40; } #endif @@ -64,25 +64,25 @@ probLit = prob + (offs + bit + symbol); \ GET_BIT2(probLit, symbol, offs ^= bit; , ;) -#endif // _LZMA_DEC_OPT +#endif // Z7_LZMA_DEC_OPT #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_INPUT_EOF; range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ - { UPDATE_0_CHECK; i = (i + i); A0; } else \ - { UPDATE_1_CHECK; i = (i + i) + 1; A1; } + { UPDATE_0_CHECK i = (i + i); A0; } else \ + { UPDATE_1_CHECK i = (i + i) + 1; A1; } #define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) #define TREE_DECODE_CHECK(probs, limit, i) \ { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } #define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ - { UPDATE_0_CHECK; i += m; m += m; } else \ - { UPDATE_1_CHECK; m += m; i += m; } + { UPDATE_0_CHECK i += m; m += m; } else \ + { UPDATE_1_CHECK m += m; i += m; } #define kNumPosBitsMax 4 @@ -224,14 +224,14 @@ LZMA_DECODE_REAL() */ -#ifdef _LZMA_DEC_OPT +#ifdef Z7_LZMA_DEC_OPT -int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); +int Z7_FASTCALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); #else static -int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +int Z7_FASTCALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { CLzmaProb *probs = GET_PROBS; unsigned state = (unsigned)p->state; @@ -263,7 +263,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit IF_BIT_0(prob) { unsigned symbol; - UPDATE_0(prob); + UPDATE_0(prob) prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); @@ -273,7 +273,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit { state -= (state < 4) ? state : 3; symbol = 1; - #ifdef _LZMA_SIZE_OPT + #ifdef Z7_LZMA_SIZE_OPT do { NORMAL_LITER_DEC } while (symbol < 0x100); #else NORMAL_LITER_DEC @@ -292,7 +292,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit unsigned offs = 0x100; state -= (state < 10) ? 3 : 6; symbol = 1; - #ifdef _LZMA_SIZE_OPT + #ifdef Z7_LZMA_SIZE_OPT do { unsigned bit; @@ -321,25 +321,25 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit } { - UPDATE_1(prob); + UPDATE_1(prob) prob = probs + IsRep + state; IF_BIT_0(prob) { - UPDATE_0(prob); + UPDATE_0(prob) state += kNumStates; prob = probs + LenCoder; } else { - UPDATE_1(prob); + UPDATE_1(prob) prob = probs + IsRepG0 + state; IF_BIT_0(prob) { - UPDATE_0(prob); + UPDATE_0(prob) prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { - UPDATE_0(prob); + UPDATE_0(prob) // that case was checked before with kBadRepCode // if (checkDicSize == 0 && processedPos == 0) { len = kMatchSpecLen_Error_Data + 1; break; } @@ -353,30 +353,30 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit state = state < kNumLitStates ? 9 : 11; continue; } - UPDATE_1(prob); + UPDATE_1(prob) } else { UInt32 distance; - UPDATE_1(prob); + UPDATE_1(prob) prob = probs + IsRepG1 + state; IF_BIT_0(prob) { - UPDATE_0(prob); + UPDATE_0(prob) distance = rep1; } else { - UPDATE_1(prob); + UPDATE_1(prob) prob = probs + IsRepG2 + state; IF_BIT_0(prob) { - UPDATE_0(prob); + UPDATE_0(prob) distance = rep2; } else { - UPDATE_1(prob); + UPDATE_1(prob) distance = rep3; rep3 = rep2; } @@ -389,37 +389,37 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit prob = probs + RepLenCoder; } - #ifdef _LZMA_SIZE_OPT + #ifdef Z7_LZMA_SIZE_OPT { unsigned lim, offset; CLzmaProb *probLen = prob + LenChoice; IF_BIT_0(probLen) { - UPDATE_0(probLen); + UPDATE_0(probLen) probLen = prob + LenLow + GET_LEN_STATE; offset = 0; lim = (1 << kLenNumLowBits); } else { - UPDATE_1(probLen); + UPDATE_1(probLen) probLen = prob + LenChoice2; IF_BIT_0(probLen) { - UPDATE_0(probLen); + UPDATE_0(probLen) probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; lim = (1 << kLenNumLowBits); } else { - UPDATE_1(probLen); + UPDATE_1(probLen) probLen = prob + LenHigh; offset = kLenNumLowSymbols * 2; lim = (1 << kLenNumHighBits); } } - TREE_DECODE(probLen, lim, len); + TREE_DECODE(probLen, lim, len) len += offset; } #else @@ -427,32 +427,32 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit CLzmaProb *probLen = prob + LenChoice; IF_BIT_0(probLen) { - UPDATE_0(probLen); + UPDATE_0(probLen) probLen = prob + LenLow + GET_LEN_STATE; len = 1; - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len) + TREE_GET_BIT(probLen, len) + TREE_GET_BIT(probLen, len) len -= 8; } else { - UPDATE_1(probLen); + UPDATE_1(probLen) probLen = prob + LenChoice2; IF_BIT_0(probLen) { - UPDATE_0(probLen); + UPDATE_0(probLen) probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len) + TREE_GET_BIT(probLen, len) + TREE_GET_BIT(probLen, len) } else { - UPDATE_1(probLen); + UPDATE_1(probLen) probLen = prob + LenHigh; - TREE_DECODE(probLen, (1 << kLenNumHighBits), len); + TREE_DECODE(probLen, (1 << kLenNumHighBits), len) len += kLenNumLowSymbols * 2; } } @@ -464,7 +464,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit UInt32 distance; prob = probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); - TREE_6_DECODE(prob, distance); + TREE_6_DECODE(prob, distance) if (distance >= kStartPosModelIndex) { unsigned posSlot = (unsigned)distance; @@ -479,7 +479,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit distance++; do { - REV_BIT_VAR(prob, distance, m); + REV_BIT_VAR(prob, distance, m) } while (--numDirectBits); distance -= m; @@ -514,10 +514,10 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit distance <<= kNumAlignBits; { unsigned i = 1; - REV_BIT_CONST(prob, i, 1); - REV_BIT_CONST(prob, i, 2); - REV_BIT_CONST(prob, i, 4); - REV_BIT_LAST (prob, i, 8); + REV_BIT_CONST(prob, i, 1) + REV_BIT_CONST(prob, i, 2) + REV_BIT_CONST(prob, i, 4) + REV_BIT_LAST (prob, i, 8) distance |= i; } if (distance == (UInt32)0xFFFFFFFF) @@ -592,7 +592,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit } while (dicPos < limit && buf < bufLimit); - NORMALIZE; + NORMALIZE p->buf = buf; p->range = range; @@ -613,7 +613,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit -static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) +static void Z7_FASTCALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { unsigned len = (unsigned)p->remainLen; if (len == 0 /* || len >= kMatchSpecLenStart */) @@ -683,7 +683,7 @@ and we support the following state of (p->checkDicSize): (p->checkDicSize == p->prop.dicSize) */ -static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +static int Z7_FASTCALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { if (p->checkDicSize == 0) { @@ -767,54 +767,54 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byt else { unsigned len; - UPDATE_1_CHECK; + UPDATE_1_CHECK prob = probs + IsRep + state; IF_BIT_0_CHECK(prob) { - UPDATE_0_CHECK; + UPDATE_0_CHECK state = 0; prob = probs + LenCoder; res = DUMMY_MATCH; } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK res = DUMMY_REP; prob = probs + IsRepG0 + state; IF_BIT_0_CHECK(prob) { - UPDATE_0_CHECK; + UPDATE_0_CHECK prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { - UPDATE_0_CHECK; + UPDATE_0_CHECK break; } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK } } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK prob = probs + IsRepG1 + state; IF_BIT_0_CHECK(prob) { - UPDATE_0_CHECK; + UPDATE_0_CHECK } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK prob = probs + IsRepG2 + state; IF_BIT_0_CHECK(prob) { - UPDATE_0_CHECK; + UPDATE_0_CHECK } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK } } } @@ -826,31 +826,31 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byt const CLzmaProb *probLen = prob + LenChoice; IF_BIT_0_CHECK(probLen) { - UPDATE_0_CHECK; + UPDATE_0_CHECK probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK probLen = prob + LenChoice2; IF_BIT_0_CHECK(probLen) { - UPDATE_0_CHECK; + UPDATE_0_CHECK probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; limit = 1 << kLenNumLowBits; } else { - UPDATE_1_CHECK; + UPDATE_1_CHECK probLen = prob + LenHigh; offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } } - TREE_DECODE_CHECK(probLen, limit, len); + TREE_DECODE_CHECK(probLen, limit, len) len += offset; } @@ -860,7 +860,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byt prob = probs + PosSlot + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); - TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); + TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot) if (posSlot >= kStartPosModelIndex) { unsigned numDirectBits = ((posSlot >> 1) - 1); @@ -888,7 +888,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byt unsigned m = 1; do { - REV_BIT_CHECK(prob, i, m); + REV_BIT_CHECK(prob, i, m) } while (--numDirectBits); } @@ -897,7 +897,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, const Byt } break; } - NORMALIZE_CHECK; + NORMALIZE_CHECK *bufOut = buf; return res; @@ -943,7 +943,7 @@ When the decoder lookahead, and the lookahead symbol is not end_marker, we have */ -#define RETURN__NOT_FINISHED__FOR_FINISH \ +#define RETURN_NOT_FINISHED_FOR_FINISH \ *status = LZMA_STATUS_NOT_FINISHED; \ return SZ_ERROR_DATA; // for strict mode // return SZ_OK; // for relaxed mode @@ -1029,7 +1029,7 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr } if (p->remainLen != 0) { - RETURN__NOT_FINISHED__FOR_FINISH; + RETURN_NOT_FINISHED_FOR_FINISH } checkEndMarkNow = 1; } @@ -1072,7 +1072,7 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr for (i = 0; i < (unsigned)dummyProcessed; i++) p->tempBuf[i] = src[i]; // p->remainLen = kMatchSpecLen_Error_Data; - RETURN__NOT_FINISHED__FOR_FINISH; + RETURN_NOT_FINISHED_FOR_FINISH } bufLimit = src; @@ -1150,7 +1150,7 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr (*srcLen) += (unsigned)dummyProcessed - p->tempBufSize; p->tempBufSize = (unsigned)dummyProcessed; // p->remainLen = kMatchSpecLen_Error_Data; - RETURN__NOT_FINISHED__FOR_FINISH; + RETURN_NOT_FINISHED_FOR_FINISH } } @@ -1299,8 +1299,8 @@ static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAl SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + RINOK(LzmaProps_Decode(&propNew, props, propsSize)) + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)) p->prop = propNew; return SZ_OK; } @@ -1309,14 +1309,14 @@ SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAll { CLzmaProps propNew; SizeT dicBufSize; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + RINOK(LzmaProps_Decode(&propNew, props, propsSize)) + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)) { UInt32 dictSize = propNew.dicSize; SizeT mask = ((UInt32)1 << 12) - 1; if (dictSize >= ((UInt32)1 << 30)) mask = ((UInt32)1 << 22) - 1; - else if (dictSize >= ((UInt32)1 << 22)) mask = ((UInt32)1 << 20) - 1;; + else if (dictSize >= ((UInt32)1 << 22)) mask = ((UInt32)1 << 20) - 1; dicBufSize = ((SizeT)dictSize + mask) & ~mask; if (dicBufSize < dictSize) dicBufSize = dictSize; @@ -1348,8 +1348,8 @@ SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, *status = LZMA_STATUS_NOT_SPECIFIED; if (inSize < RC_INIT_SIZE) return SZ_ERROR_INPUT_EOF; - LzmaDec_Construct(&p); - RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)); + LzmaDec_CONSTRUCT(&p) + RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)) p.dic = dest; p.dicBufSize = outSize; LzmaDec_Init(&p); diff --git a/multiarc/src/formats/7z/C/LzmaDec.h b/multiarc/src/formats/7z/C/LzmaDec.h old mode 100755 new mode 100644 index 6f1296250..b0ce28fa0 --- a/multiarc/src/formats/7z/C/LzmaDec.h +++ b/multiarc/src/formats/7z/C/LzmaDec.h @@ -1,19 +1,19 @@ /* LzmaDec.h -- LZMA Decoder -2020-03-19 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __LZMA_DEC_H -#define __LZMA_DEC_H +#ifndef ZIP7_INC_LZMA_DEC_H +#define ZIP7_INC_LZMA_DEC_H #include "7zTypes.h" EXTERN_C_BEGIN -/* #define _LZMA_PROB32 */ -/* _LZMA_PROB32 can increase the speed on some CPUs, +/* #define Z7_LZMA_PROB32 */ +/* Z7_LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ typedef -#ifdef _LZMA_PROB32 +#ifdef Z7_LZMA_PROB32 UInt32 #else UInt16 @@ -25,7 +25,7 @@ typedef #define LZMA_PROPS_SIZE 5 -typedef struct _CLzmaProps +typedef struct { Byte lc; Byte lp; @@ -73,7 +73,8 @@ typedef struct Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } +#define LzmaDec_CONSTRUCT(p) { (p)->dic = NULL; (p)->probs = NULL; } +#define LzmaDec_Construct(p) LzmaDec_CONSTRUCT(p) void LzmaDec_Init(CLzmaDec *p); diff --git a/multiarc/src/formats/7z/C/LzmaEnc.c b/multiarc/src/formats/7z/C/LzmaEnc.c old mode 100755 new mode 100644 index c8b31a19e..37b2787db --- a/multiarc/src/formats/7z/C/LzmaEnc.c +++ b/multiarc/src/formats/7z/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2022-07-15: Igor Pavlov : Public domain */ +2024-01-24: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -16,22 +16,22 @@ #include "LzmaEnc.h" #include "LzFind.h" -#ifndef _7ZIP_ST +#ifndef Z7_ST #include "LzFindMt.h" #endif /* the following LzmaEnc_* declarations is internal LZMA interface for LZMA2 encoder */ -SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle p, ISeqInStreamPtr inStream, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, +SRes LzmaEnc_MemPrepare(CLzmaEncHandle p, const Byte *src, SizeT srcLen, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle p, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); -const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); -void LzmaEnc_Finish(CLzmaEncHandle pp); -void LzmaEnc_SaveState(CLzmaEncHandle pp); -void LzmaEnc_RestoreState(CLzmaEncHandle pp); +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle p); +void LzmaEnc_Finish(CLzmaEncHandle p); +void LzmaEnc_SaveState(CLzmaEncHandle p); +void LzmaEnc_RestoreState(CLzmaEncHandle p); #ifdef SHOW_STAT static unsigned g_STAT_OFFSET = 0; @@ -40,8 +40,8 @@ static unsigned g_STAT_OFFSET = 0; /* for good normalization speed we still reserve 256 MB before 4 GB range */ #define kLzmaMaxHistorySize ((UInt32)15 << 28) -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) +// #define kNumTopBits 24 +#define kTopValue ((UInt32)1 << 24) #define kNumBitModelTotalBits 11 #define kBitModelTotal (1 << kNumBitModelTotalBits) @@ -60,6 +60,7 @@ void LzmaEncProps_Init(CLzmaEncProps *p) p->dictSize = p->mc = 0; p->reduceSize = (UInt64)(Int64)-1; p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; + p->numHashOutBits = 0; p->writeEndMark = 0; p->affinity = 0; } @@ -99,7 +100,7 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p) if (p->numThreads < 0) p->numThreads = - #ifndef _7ZIP_ST + #ifndef Z7_ST ((p->btMode && p->algo) ? 2 : 1); #else 1; @@ -194,11 +195,11 @@ unsigned GetPosSlot1(UInt32 pos); unsigned GetPosSlot1(UInt32 pos) { unsigned res; - BSR2_RET(pos, res); + BSR2_RET(pos, res) return res; } -#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } +#define GetPosSlot2(pos, res) { BSR2_RET(pos, res) } +#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res) } #else // ! LZMA_LOG_BSR @@ -293,7 +294,7 @@ typedef struct #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) typedef -#ifdef _LZMA_PROB32 +#ifdef Z7_LZMA_PROB32 UInt32 #else UInt16 @@ -350,7 +351,7 @@ typedef struct Byte *buf; Byte *bufLim; Byte *bufBase; - ISeqOutStream *outStream; + ISeqOutStreamPtr outStream; UInt64 processed; SRes res; } CRangeEnc; @@ -383,7 +384,7 @@ typedef struct typedef UInt32 CProbPrice; -typedef struct +struct CLzmaEnc { void *matchFinderObj; IMatchFinder2 matchFinder; @@ -426,7 +427,7 @@ typedef struct UInt32 dictSize; SRes result; - #ifndef _7ZIP_ST + #ifndef Z7_ST BoolInt mtMode; // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; @@ -439,7 +440,7 @@ typedef struct // we suppose that we have 8-bytes alignment after CMatchFinder - #ifndef _7ZIP_ST + #ifndef Z7_ST Byte pad[128]; #endif @@ -479,77 +480,59 @@ typedef struct CSaveState saveState; // BoolInt mf_Failure; - #ifndef _7ZIP_ST + #ifndef Z7_ST Byte pad2[128]; #endif -} CLzmaEnc; +}; #define MFB (p->matchFinderBase) /* -#ifndef _7ZIP_ST +#ifndef Z7_ST #define MFB (p->matchFinderMt.MatchFinder) #endif */ -#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); - -void LzmaEnc_SaveState(CLzmaEncHandle pp) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - CSaveState *dest = &p->saveState; - - dest->state = p->state; - - dest->lenProbs = p->lenProbs; - dest->repLenProbs = p->repLenProbs; - - COPY_ARR(dest, p, reps); - - COPY_ARR(dest, p, posAlignEncoder); - COPY_ARR(dest, p, isRep); - COPY_ARR(dest, p, isRepG0); - COPY_ARR(dest, p, isRepG1); - COPY_ARR(dest, p, isRepG2); - COPY_ARR(dest, p, isMatch); - COPY_ARR(dest, p, isRep0Long); - COPY_ARR(dest, p, posSlotEncoder); - COPY_ARR(dest, p, posEncoders); - - memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); +// #define GET_CLzmaEnc_p CLzmaEnc *p = (CLzmaEnc*)(void *)p; +// #define GET_const_CLzmaEnc_p const CLzmaEnc *p = (const CLzmaEnc*)(const void *)p; + +#define COPY_ARR(dest, src, arr) memcpy((dest)->arr, (src)->arr, sizeof((src)->arr)); + +#define COPY_LZMA_ENC_STATE(d, s, p) \ + (d)->state = (s)->state; \ + COPY_ARR(d, s, reps) \ + COPY_ARR(d, s, posAlignEncoder) \ + COPY_ARR(d, s, isRep) \ + COPY_ARR(d, s, isRepG0) \ + COPY_ARR(d, s, isRepG1) \ + COPY_ARR(d, s, isRepG2) \ + COPY_ARR(d, s, isMatch) \ + COPY_ARR(d, s, isRep0Long) \ + COPY_ARR(d, s, posSlotEncoder) \ + COPY_ARR(d, s, posEncoders) \ + (d)->lenProbs = (s)->lenProbs; \ + (d)->repLenProbs = (s)->repLenProbs; \ + memcpy((d)->litProbs, (s)->litProbs, ((size_t)0x300 * sizeof(CLzmaProb)) << (p)->lclp); + +void LzmaEnc_SaveState(CLzmaEncHandle p) +{ + // GET_CLzmaEnc_p + CSaveState *v = &p->saveState; + COPY_LZMA_ENC_STATE(v, p, p) } - -void LzmaEnc_RestoreState(CLzmaEncHandle pp) +void LzmaEnc_RestoreState(CLzmaEncHandle p) { - CLzmaEnc *dest = (CLzmaEnc *)pp; - const CSaveState *p = &dest->saveState; - - dest->state = p->state; - - dest->lenProbs = p->lenProbs; - dest->repLenProbs = p->repLenProbs; - - COPY_ARR(dest, p, reps); - - COPY_ARR(dest, p, posAlignEncoder); - COPY_ARR(dest, p, isRep); - COPY_ARR(dest, p, isRepG0); - COPY_ARR(dest, p, isRepG1); - COPY_ARR(dest, p, isRepG2); - COPY_ARR(dest, p, isMatch); - COPY_ARR(dest, p, isRep0Long); - COPY_ARR(dest, p, posSlotEncoder); - COPY_ARR(dest, p, posEncoders); - - memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); + // GET_CLzmaEnc_p + const CSaveState *v = &p->saveState; + COPY_LZMA_ENC_STATE(p, v, p) } - -SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) +Z7_NO_INLINE +SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props2) { - CLzmaEnc *p = (CLzmaEnc *)pp; + // GET_CLzmaEnc_p CLzmaEncProps props = *props2; LzmaEncProps_Normalize(&props); @@ -585,6 +568,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->fastMode = (props.algo == 0); // p->_maxMode = True; MFB.btMode = (Byte)(props.btMode ? 1 : 0); + // MFB.btMode = (Byte)(props.btMode); { unsigned numHashBytes = 4; if (props.btMode) @@ -595,13 +579,15 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) if (props.numHashBytes >= 5) numHashBytes = 5; MFB.numHashBytes = numHashBytes; + // MFB.numHashBytes_Min = 2; + MFB.numHashOutBits = (Byte)props.numHashOutBits; } MFB.cutValue = props.mc; p->writeEndMark = (BoolInt)props.writeEndMark; - #ifndef _7ZIP_ST + #ifndef Z7_ST /* if (newMultiThread != _multiThread) { @@ -618,9 +604,9 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) } -void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize) { - CLzmaEnc *p = (CLzmaEnc *)pp; + // GET_CLzmaEnc_p MFB.expectedDataSize = expectedDataSiize; } @@ -684,7 +670,7 @@ static void RangeEnc_Init(CRangeEnc *p) p->res = SZ_OK; } -MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) +Z7_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { const size_t num = (size_t)(p->buf - p->bufBase); if (p->res == SZ_OK) @@ -696,7 +682,7 @@ MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) p->buf = p->bufBase; } -MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +Z7_NO_INLINE static void Z7_FASTCALL RangeEnc_ShiftLow(CRangeEnc *p) { UInt32 low = (UInt32)p->low; unsigned high = (unsigned)(p->low >> 32); @@ -741,9 +727,9 @@ static void RangeEnc_FlushData(CRangeEnc *p) ttt = *(prob); \ newBound = (range >> kNumBitModelTotalBits) * ttt; -// #define _LZMA_ENC_USE_BRANCH +// #define Z7_LZMA_ENC_USE_BRANCH -#ifdef _LZMA_ENC_USE_BRANCH +#ifdef Z7_LZMA_ENC_USE_BRANCH #define RC_BIT(p, prob, bit) { \ RC_BIT_PRE(p, prob) \ @@ -811,7 +797,7 @@ static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) CLzmaProb *prob = probs + (sym >> 8); UInt32 bit = (sym >> 7) & 1; sym <<= 1; - RC_BIT(p, prob, bit); + RC_BIT(p, prob, bit) } while (sym < 0x10000); p->range = range; @@ -833,7 +819,7 @@ static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UIn bit = (sym >> 7) & 1; sym <<= 1; offs &= ~(matchByte ^ sym); - RC_BIT(p, prob, bit); + RC_BIT(p, prob, bit) } while (sym < 0x10000); p->range = range; @@ -867,10 +853,10 @@ static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) #define GET_PRICE(prob, bit) \ - p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits] #define GET_PRICEa(prob, bit) \ - ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits] #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] @@ -921,7 +907,7 @@ static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBi unsigned bit = sym & 1; // RangeEnc_EncodeBit(rc, probs + m, bit); sym >>= 1; - RC_BIT(rc, probs + m, bit); + RC_BIT(rc, probs + m, bit) m = (m << 1) | bit; } while (--numBits); @@ -944,15 +930,15 @@ static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posS UInt32 range, ttt, newBound; CLzmaProb *probs = p->low; range = rc->range; - RC_BIT_PRE(rc, probs); + RC_BIT_PRE(rc, probs) if (sym >= kLenNumLowSymbols) { - RC_BIT_1(rc, probs); + RC_BIT_1(rc, probs) probs += kLenNumLowSymbols; - RC_BIT_PRE(rc, probs); + RC_BIT_PRE(rc, probs) if (sym >= kLenNumLowSymbols * 2) { - RC_BIT_1(rc, probs); + RC_BIT_1(rc, probs) rc->range = range; // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); @@ -965,11 +951,11 @@ static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posS { unsigned m; unsigned bit; - RC_BIT_0(rc, probs); + RC_BIT_0(rc, probs) probs += (posState << (1 + kLenNumLowBits)); - bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; - bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; - bit = sym & 1; RC_BIT(rc, probs + m, bit); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit) m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit) m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit) rc->range = range; } } @@ -990,7 +976,7 @@ static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *price } -MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( +Z7_NO_INLINE static void Z7_FASTCALL LenPriceEnc_UpdateTables( CLenPriceEnc *p, unsigned numPosStates, const CLenEnc *enc, @@ -1054,14 +1040,14 @@ MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( UInt32 price = b; do { - unsigned bit = sym & 1; + const unsigned bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); } while (sym >= 2); { - unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + const unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); } @@ -1070,7 +1056,7 @@ MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( { unsigned posState; - size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + const size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); for (posState = 1; posState < numPosStates; posState++) memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); } @@ -1152,7 +1138,7 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) + GET_PRICE_1(p->isRep[state]) \ + GET_PRICE_0(p->isRepG0[state]) -MY_FORCE_INLINE +Z7_FORCE_INLINE static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; @@ -1331,7 +1317,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) LitEnc_GetPrice(probs, curByte, p->ProbPrices)); } - MakeAs_Lit(&p->opt[1]); + MakeAs_Lit(&p->opt[1]) matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); @@ -1343,7 +1329,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (shortRepPrice < p->opt[1].price) { p->opt[1].price = shortRepPrice; - MakeAs_ShortRep(&p->opt[1]); + MakeAs_ShortRep(&p->opt[1]) } if (last < 2) { @@ -1410,7 +1396,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) else { unsigned slot; - GetPosSlot2(dist, slot); + GetPosSlot2(dist, slot) price += p->alignPrices[dist & kAlignMask]; price += p->posSlotPrices[lenToPosState][slot]; } @@ -1486,7 +1472,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) unsigned delta = best - cur; if (delta != 0) { - MOVE_POS(p, delta); + MOVE_POS(p, delta) } } cur = best; @@ -1633,7 +1619,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { nextOpt->price = litPrice; nextOpt->len = 1; - MakeAs_Lit(nextOpt); + MakeAs_Lit(nextOpt) nextIsLit = True; } } @@ -1667,7 +1653,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { nextOpt->price = shortRepPrice; nextOpt->len = 1; - MakeAs_ShortRep(nextOpt); + MakeAs_ShortRep(nextOpt) nextIsLit = False; } } @@ -1871,7 +1857,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) dist = MATCHES[(size_t)offs + 1]; // if (dist >= kNumFullDistances) - GetPosSlot2(dist, posSlot); + GetPosSlot2(dist, posSlot) for (len = /*2*/ startLen; ; len++) { @@ -1962,7 +1948,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) break; dist = MATCHES[(size_t)offs + 1]; // if (dist >= kNumFullDistances) - GetPosSlot2(dist, posSlot); + GetPosSlot2(dist, posSlot) } } } @@ -2138,7 +2124,7 @@ static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { UInt32 ttt, newBound; RC_BIT_PRE(p, probs + m) - RC_BIT_1(&p->rc, probs + m); + RC_BIT_1(&p->rc, probs + m) m = (m << 1) + 1; } while (m < (1 << kNumPosSlotBits)); @@ -2163,7 +2149,7 @@ static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { UInt32 ttt, newBound; RC_BIT_PRE(p, probs + m) - RC_BIT_1(&p->rc, probs + m); + RC_BIT_1(&p->rc, probs + m) m = (m << 1) + 1; } while (m < kAlignTableSize); @@ -2179,7 +2165,7 @@ static SRes CheckErrors(CLzmaEnc *p) if (p->rc.res != SZ_OK) p->result = SZ_ERROR_WRITE; - #ifndef _7ZIP_ST + #ifndef Z7_ST if ( // p->mf_Failure || (p->mtMode && @@ -2187,7 +2173,7 @@ static SRes CheckErrors(CLzmaEnc *p) p->matchFinderMt.failure_LZ_BT)) ) { - p->result = MY_HRES_ERROR__INTERNAL_ERROR; + p->result = MY_HRES_ERROR_INTERNAL_ERROR; // printf("\nCheckErrors p->matchFinderMt.failureLZ\n"); } #endif @@ -2201,7 +2187,7 @@ static SRes CheckErrors(CLzmaEnc *p) } -MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) +Z7_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { /* ReleaseMFStream(); */ p->finished = True; @@ -2213,7 +2199,7 @@ MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) } -MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) +Z7_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { unsigned i; const CProbPrice *ProbPrices = p->ProbPrices; @@ -2237,7 +2223,7 @@ MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) } -MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) +Z7_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) { // int y; for (y = 0; y < 100; y++) { @@ -2337,7 +2323,7 @@ static void LzmaEnc_Construct(CLzmaEnc *p) RangeEnc_Construct(&p->rc); MatchFinder_Construct(&MFB); - #ifndef _7ZIP_ST + #ifndef Z7_ST p->matchFinderMt.MatchFinder = &MFB; MatchFinderMt_Construct(&p->matchFinderMt); #endif @@ -2345,7 +2331,7 @@ static void LzmaEnc_Construct(CLzmaEnc *p) { CLzmaEncProps props; LzmaEncProps_Init(&props); - LzmaEnc_SetProps(p, &props); + LzmaEnc_SetProps((CLzmaEncHandle)(void *)p, &props); } #ifndef LZMA_LOG_BSR @@ -2376,7 +2362,7 @@ static void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) static void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - #ifndef _7ZIP_ST + #ifndef Z7_ST MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); #endif @@ -2387,21 +2373,22 @@ static void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBi void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); + // GET_CLzmaEnc_p + LzmaEnc_Destruct(p, alloc, allocBig); ISzAlloc_Free(alloc, p); } -MY_NO_INLINE +Z7_NO_INLINE static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; if (p->needInit) { - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->mtMode) { - RINOK(MatchFinderMt_InitMt(&p->matchFinderMt)); + RINOK(MatchFinderMt_InitMt(&p->matchFinderMt)) } #endif p->matchFinder.Init(p->matchFinderObj); @@ -2410,7 +2397,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa if (p->finished) return p->result; - RINOK(CheckErrors(p)); + RINOK(CheckErrors(p)) nowPos32 = (UInt32)p->nowPos64; startPos32 = nowPos32; @@ -2473,7 +2460,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa const Byte *data; unsigned state; - RC_BIT_0(&p->rc, probs); + RC_BIT_0(&p->rc, probs) p->rc.range = range; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; probs = LIT_PROBS(nowPos32, *(data - 1)); @@ -2487,53 +2474,53 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa } else { - RC_BIT_1(&p->rc, probs); + RC_BIT_1(&p->rc, probs) probs = &p->isRep[p->state]; RC_BIT_PRE(&p->rc, probs) if (dist < LZMA_NUM_REPS) { - RC_BIT_1(&p->rc, probs); + RC_BIT_1(&p->rc, probs) probs = &p->isRepG0[p->state]; RC_BIT_PRE(&p->rc, probs) if (dist == 0) { - RC_BIT_0(&p->rc, probs); + RC_BIT_0(&p->rc, probs) probs = &p->isRep0Long[p->state][posState]; RC_BIT_PRE(&p->rc, probs) if (len != 1) { - RC_BIT_1_BASE(&p->rc, probs); + RC_BIT_1_BASE(&p->rc, probs) } else { - RC_BIT_0_BASE(&p->rc, probs); + RC_BIT_0_BASE(&p->rc, probs) p->state = kShortRepNextStates[p->state]; } } else { - RC_BIT_1(&p->rc, probs); + RC_BIT_1(&p->rc, probs) probs = &p->isRepG1[p->state]; RC_BIT_PRE(&p->rc, probs) if (dist == 1) { - RC_BIT_0_BASE(&p->rc, probs); + RC_BIT_0_BASE(&p->rc, probs) dist = p->reps[1]; } else { - RC_BIT_1(&p->rc, probs); + RC_BIT_1(&p->rc, probs) probs = &p->isRepG2[p->state]; RC_BIT_PRE(&p->rc, probs) if (dist == 2) { - RC_BIT_0_BASE(&p->rc, probs); + RC_BIT_0_BASE(&p->rc, probs) dist = p->reps[2]; } else { - RC_BIT_1_BASE(&p->rc, probs); + RC_BIT_1_BASE(&p->rc, probs) dist = p->reps[3]; p->reps[3] = p->reps[2]; } @@ -2557,7 +2544,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa else { unsigned posSlot; - RC_BIT_0(&p->rc, probs); + RC_BIT_0(&p->rc, probs) p->rc.range = range; p->state = kMatchNextStates[p->state]; @@ -2571,7 +2558,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa p->reps[0] = dist + 1; p->matchPriceCount++; - GetPosSlot(dist, posSlot); + GetPosSlot(dist, posSlot) // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); { UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); @@ -2582,7 +2569,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; sym <<= 1; - RC_BIT(&p->rc, prob, bit); + RC_BIT(&p->rc, prob, bit) } while (sym < (1 << kNumPosSlotBits * 2)); p->rc.range = range; @@ -2626,10 +2613,10 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa { unsigned m = 1; unsigned bit; - bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; - bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; - bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; - bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit) m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit) m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit) m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit) p->rc.range = range; // p->alignPriceCount++; } @@ -2704,17 +2691,17 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, if (!RangeEnc_Alloc(&p->rc, alloc)) return SZ_ERROR_MEM; - #ifndef _7ZIP_ST + #ifndef Z7_ST p->mtMode = (p->multiThread && !p->fastMode && (MFB.btMode != 0)); #endif { - unsigned lclp = p->lc + p->lp; + const unsigned lclp = p->lc + p->lp; if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) { LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((size_t)0x300 * sizeof(CLzmaProb)) << lclp); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((size_t)0x300 * sizeof(CLzmaProb)) << lclp); if (!p->litProbs || !p->saveState.litProbs) { LzmaEnc_FreeLits(p, alloc); @@ -2748,15 +2735,14 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, (numFastBytes + LZMA_MATCH_LEN_MAX + 1) */ - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->mtMode) { RINOK(MatchFinderMt_Create(&p->matchFinderMt, dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX + 1 /* 18.04 */ - , allocBig)); + , allocBig)) p->matchFinderObj = &p->matchFinderMt; - MFB.bigHash = (Byte)( - (p->dictSize > kBigHashDicLimit && MFB.hashMask >= 0xFFFFFF) ? 1 : 0); + MFB.bigHash = (Byte)(MFB.hashMask >= 0xFFFFFF ? 1 : 0); MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); } else @@ -2816,8 +2802,8 @@ static void LzmaEnc_Init(CLzmaEnc *p) } { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - UInt32 k; + const size_t num = (size_t)0x300 << (p->lp + p->lc); + size_t k; CLzmaProb *probs = p->litProbs; for (k = 0; k < num; k++) probs[k] = kProbInitValue; @@ -2872,59 +2858,53 @@ static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr p->finished = False; p->result = SZ_OK; - RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); + p->nowPos64 = 0; + p->needInit = 1; + RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)) LzmaEnc_Init(p); LzmaEnc_InitPrices(p); - p->nowPos64 = 0; return SZ_OK; } -static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, +static SRes LzmaEnc_Prepare(CLzmaEncHandle p, + ISeqOutStreamPtr outStream, + ISeqInStreamPtr inStream, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)pp; - MFB.stream = inStream; - p->needInit = 1; + // GET_CLzmaEnc_p + MatchFinder_SET_STREAM(&MFB, inStream) p->rc.outStream = outStream; return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); } -SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, - ISeqInStream *inStream, UInt32 keepWindowSize, +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle p, + ISeqInStreamPtr inStream, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)pp; - MFB.stream = inStream; - p->needInit = 1; + // GET_CLzmaEnc_p + MatchFinder_SET_STREAM(&MFB, inStream) return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } -static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) -{ - MFB.directInput = 1; - MFB.bufferBase = (Byte *)src; - MFB.directInputRem = srcLen; -} - -SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +SRes LzmaEnc_MemPrepare(CLzmaEncHandle p, + const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)pp; - LzmaEnc_SetInputBuf(p, src, srcLen); - p->needInit = 1; - - LzmaEnc_SetDataSize(pp, srcLen); + // GET_CLzmaEnc_p + MatchFinder_SET_DIRECT_INPUT_BUF(&MFB, src, srcLen) + LzmaEnc_SetDataSize(p, srcLen); return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); } -void LzmaEnc_Finish(CLzmaEncHandle pp) +void LzmaEnc_Finish(CLzmaEncHandle p) { - #ifndef _7ZIP_ST - CLzmaEnc *p = (CLzmaEnc *)pp; + #ifndef Z7_ST + // GET_CLzmaEnc_p if (p->mtMode) MatchFinderMt_ReleaseStream(&p->matchFinderMt); #else - UNUSED_VAR(pp); + UNUSED_VAR(p) #endif } @@ -2933,13 +2913,13 @@ typedef struct { ISeqOutStream vt; Byte *data; - SizeT rem; + size_t rem; BoolInt overflow; } CLzmaEnc_SeqOutStreamBuf; -static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) +static size_t SeqOutStreamBuf_Write(ISeqOutStreamPtr pp, const void *data, size_t size) { - CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CLzmaEnc_SeqOutStreamBuf) if (p->rem < size) { size = p->rem; @@ -2956,25 +2936,25 @@ static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, s /* -UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle p) { - const CLzmaEnc *p = (CLzmaEnc *)pp; + GET_const_CLzmaEnc_p return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); } */ -const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle p) { - const CLzmaEnc *p = (CLzmaEnc *)pp; + // GET_const_CLzmaEnc_p return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; } // (desiredPackSize == 0) is not allowed -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle p, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { - CLzmaEnc *p = (CLzmaEnc *)pp; + // GET_CLzmaEnc_p UInt64 nowPos64; SRes res; CLzmaEnc_SeqOutStreamBuf outStream; @@ -3006,12 +2986,12 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, } -MY_NO_INLINE -static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) +Z7_NO_INLINE +static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgressPtr progress) { SRes res = SZ_OK; - #ifndef _7ZIP_ST + #ifndef Z7_ST Byte allocaDummy[0x300]; allocaDummy[0] = 0; allocaDummy[1] = allocaDummy[0]; @@ -3033,7 +3013,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) } } - LzmaEnc_Finish(p); + LzmaEnc_Finish((CLzmaEncHandle)(void *)p); /* if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&MFB)) @@ -3045,21 +3025,22 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) } -SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, +SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, ICompressProgressPtr progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); - return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); + // GET_CLzmaEnc_p + RINOK(LzmaEnc_Prepare(p, outStream, inStream, alloc, allocBig)) + return LzmaEnc_Encode2(p, progress); } -SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) +SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *props, SizeT *size) { if (*size < LZMA_PROPS_SIZE) return SZ_ERROR_PARAM; *size = LZMA_PROPS_SIZE; { - const CLzmaEnc *p = (const CLzmaEnc *)pp; + // GET_CLzmaEnc_p const UInt32 dictSize = p->dictSize; UInt32 v; props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); @@ -3083,23 +3064,24 @@ SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) while (v < dictSize); } - SetUi32(props + 1, v); + SetUi32(props + 1, v) return SZ_OK; } } -unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p) { - return (unsigned)((CLzmaEnc *)pp)->writeEndMark; + // GET_CLzmaEnc_p + return (unsigned)p->writeEndMark; } -SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) +SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + int writeEndMark, ICompressProgressPtr progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { SRes res; - CLzmaEnc *p = (CLzmaEnc *)pp; + // GET_CLzmaEnc_p CLzmaEnc_SeqOutStreamBuf outStream; @@ -3111,7 +3093,7 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte p->writeEndMark = writeEndMark; p->rc.outStream = &outStream.vt; - res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); + res = LzmaEnc_MemPrepare(p, src, srcLen, 0, alloc, allocBig); if (res == SZ_OK) { @@ -3120,7 +3102,7 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte res = SZ_ERROR_FAIL; } - *destLen -= outStream.rem; + *destLen -= (SizeT)outStream.rem; if (outStream.overflow) return SZ_ERROR_OUTPUT_EOF; return res; @@ -3129,9 +3111,9 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) + ICompressProgressPtr progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); + CLzmaEncHandle p = LzmaEnc_Create(alloc); SRes res; if (!p) return SZ_ERROR_MEM; @@ -3151,10 +3133,10 @@ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, /* -#ifndef _7ZIP_ST -void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]) +#ifndef Z7_ST +void LzmaEnc_GetLzThreads(CLzmaEncHandle p, HANDLE lz_threads[2]) { - const CLzmaEnc *p = (CLzmaEnc *)pp; + GET_const_CLzmaEnc_p lz_threads[0] = p->matchFinderMt.hashSync.thread; lz_threads[1] = p->matchFinderMt.btSync.thread; } diff --git a/multiarc/src/formats/7z/C/LzmaEnc.h b/multiarc/src/formats/7z/C/LzmaEnc.h old mode 100755 new mode 100644 index bc2ed5042..9f8039a10 --- a/multiarc/src/formats/7z/C/LzmaEnc.h +++ b/multiarc/src/formats/7z/C/LzmaEnc.h @@ -1,8 +1,8 @@ /* LzmaEnc.h -- LZMA Encoder -2019-10-30 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ -#ifndef __LZMA_ENC_H -#define __LZMA_ENC_H +#ifndef ZIP7_INC_LZMA_ENC_H +#define ZIP7_INC_LZMA_ENC_H #include "7zTypes.h" @@ -10,7 +10,7 @@ EXTERN_C_BEGIN #define LZMA_PROPS_SIZE 5 -typedef struct _CLzmaEncProps +typedef struct { int level; /* 0 <= level <= 9 */ UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version @@ -23,10 +23,13 @@ typedef struct _CLzmaEncProps int fb; /* 5 <= fb <= 273, default = 32 */ int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ int numHashBytes; /* 2, 3 or 4, default = 4 */ + unsigned numHashOutBits; /* default = ? */ UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ int numThreads; /* 1 or 2, default = 2 */ + // int _pad; + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. Encoder uses this value to reduce dictionary size */ @@ -51,7 +54,9 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ -typedef void * CLzmaEncHandle; +typedef struct CLzmaEnc CLzmaEnc; +typedef CLzmaEnc * CLzmaEncHandle; +// Z7_DECLARE_HANDLE(CLzmaEncHandle) CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); @@ -61,17 +66,17 @@ void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); -SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, + ICompressProgressPtr progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + int writeEndMark, ICompressProgressPtr progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); /* ---------- One Call Interface ---------- */ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + ICompressProgressPtr progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/LzmaLib.c b/multiarc/src/formats/7z/C/LzmaLib.c old mode 100755 new mode 100644 index 706e9e58c..785e88487 --- a/multiarc/src/formats/7z/C/LzmaLib.c +++ b/multiarc/src/formats/7z/C/LzmaLib.c @@ -1,12 +1,14 @@ /* LzmaLib.c -- LZMA library wrapper -2015-06-13 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" #include "Alloc.h" #include "LzmaDec.h" #include "LzmaEnc.h" #include "LzmaLib.h" -MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, +Z7_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, unsigned char *outProps, size_t *outPropsSize, int level, /* 0 <= level <= 9, default = 5 */ unsigned dictSize, /* use (1 << N) or (3 << N). 4 KB < dictSize <= 128 MB */ @@ -32,7 +34,7 @@ MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char } -MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t *srcLen, +Z7_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t *srcLen, const unsigned char *props, size_t propsSize) { ELzmaStatus status; diff --git a/multiarc/src/formats/7z/C/LzmaLib.h b/multiarc/src/formats/7z/C/LzmaLib.h old mode 100755 new mode 100644 index c343a8596..d7c0724de --- a/multiarc/src/formats/7z/C/LzmaLib.h +++ b/multiarc/src/formats/7z/C/LzmaLib.h @@ -1,14 +1,14 @@ /* LzmaLib.h -- LZMA library interface -2021-04-03 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __LZMA_LIB_H -#define __LZMA_LIB_H +#ifndef ZIP7_INC_LZMA_LIB_H +#define ZIP7_INC_LZMA_LIB_H #include "7zTypes.h" EXTERN_C_BEGIN -#define MY_STDAPI int MY_STD_CALL +#define Z7_STDAPI int Z7_STDCALL #define LZMA_PROPS_SIZE 5 @@ -100,7 +100,7 @@ numThreads - The number of thereads. 1 or 2. The default value is 2. SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) */ -MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, +Z7_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, unsigned char *outProps, size_t *outPropsSize, /* *outPropsSize must be = 5 */ int level, /* 0 <= level <= 9, default = 5 */ unsigned dictSize, /* default = (1 << 24) */ @@ -130,7 +130,7 @@ LzmaUncompress SZ_ERROR_INPUT_EOF - it needs more bytes in input buffer (src) */ -MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, SizeT *srcLen, +Z7_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, SizeT *srcLen, const unsigned char *props, size_t propsSize); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/MtCoder.c b/multiarc/src/formats/7z/C/MtCoder.c old mode 100755 new mode 100644 index 99dc9090a..03959b6ca --- a/multiarc/src/formats/7z/C/MtCoder.c +++ b/multiarc/src/formats/7z/C/MtCoder.c @@ -1,28 +1,28 @@ /* MtCoder.c -- Multi-thread Coder -2021-12-21 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -#ifndef _7ZIP_ST +#ifndef Z7_ST -static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +static SRes MtProgressThunk_Progress(ICompressProgressPtr pp, UInt64 inSize, UInt64 outSize) { - CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CMtProgressThunk) UInt64 inSize2 = 0; UInt64 outSize2 = 0; if (inSize != (UInt64)(Int64)-1) { - inSize2 = inSize - thunk->inSize; - thunk->inSize = inSize; + inSize2 = inSize - p->inSize; + p->inSize = inSize; } if (outSize != (UInt64)(Int64)-1) { - outSize2 = outSize - thunk->outSize; - thunk->outSize = outSize; + outSize2 = outSize - p->outSize; + p->outSize = outSize; } - return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); + return MtProgress_ProgressAdd(p->mtProgress, inSize2, outSize2); } @@ -36,20 +36,12 @@ void MtProgressThunk_CreateVTable(CMtProgressThunk *p) #define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static WRes ArEvent_OptCreate_And_Reset(CEvent *p) -{ - if (Event_IsCreated(p)) - return Event_Reset(p); - return AutoResetEvent_CreateNotSignaled(p); -} - - static THREAD_FUNC_DECL ThreadFunc(void *pp); static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t) { - WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent); + WRes wres = AutoResetEvent_OptCreate_And_Reset(&t->startEvent); if (wres == 0) { t->stop = False; @@ -84,24 +76,6 @@ static void MtCoderThread_Destruct(CMtCoderThread *t) -static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) -{ - size_t size = *processedSize; - *processedSize = 0; - while (size != 0) - { - size_t cur = size; - SRes res = ISeqInStream_Read(stream, data, &cur); - *processedSize += cur; - data += cur; - size -= cur; - RINOK(res); - if (cur == 0) - return SZ_OK; - } - return SZ_OK; -} - /* ThreadFunc2() returns: @@ -152,7 +126,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) } if (res == SZ_OK) { - res = FullRead(mtc->inStream, t->inBuf, &size); + res = SeqInStream_ReadMax(mtc->inStream, t->inBuf, &size); readProcessed = mtc->readProcessed + size; mtc->readProcessed = readProcessed; } @@ -253,7 +227,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) block->finished = finished; } - #ifdef MTCODER__USE_WRITE_THREAD + #ifdef MTCODER_USE_WRITE_THREAD RINOK_THREAD(Event_Set(&mtc->writeEvents[bi])) #else { @@ -352,7 +326,7 @@ static THREAD_FUNC_DECL ThreadFunc(void *pp) MtProgress_SetError(&mtc->mtProgress, res); } - #ifndef MTCODER__USE_WRITE_THREAD + #ifndef MTCODER_USE_WRITE_THREAD { unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads); if (numFinished == mtc->numStartedThreads) @@ -389,7 +363,7 @@ void MtCoder_Construct(CMtCoder *p) Event_Construct(&p->readEvent); Semaphore_Construct(&p->blocksSemaphore); - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) { CMtCoderThread *t = &p->threads[i]; t->mtCoder = p; @@ -397,11 +371,11 @@ void MtCoder_Construct(CMtCoder *p) t->inBuf = NULL; t->stop = False; Event_Construct(&t->startEvent); - Thread_Construct(&t->thread); + Thread_CONSTRUCT(&t->thread) } - #ifdef MTCODER__USE_WRITE_THREAD - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + #ifdef MTCODER_USE_WRITE_THREAD + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) Event_Construct(&p->writeEvents[i]); #else Event_Construct(&p->finishedEvent); @@ -424,14 +398,14 @@ static void MtCoder_Free(CMtCoder *p) Event_Set(&p->readEvent); */ - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) MtCoderThread_Destruct(&p->threads[i]); Event_Close(&p->readEvent); Semaphore_Close(&p->blocksSemaphore); - #ifdef MTCODER__USE_WRITE_THREAD - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + #ifdef MTCODER_USE_WRITE_THREAD + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) Event_Close(&p->writeEvents[i]); #else Event_Close(&p->finishedEvent); @@ -455,20 +429,20 @@ SRes MtCoder_Code(CMtCoder *p) unsigned i; SRes res = SZ_OK; - if (numThreads > MTCODER__THREADS_MAX) - numThreads = MTCODER__THREADS_MAX; - numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads); + if (numThreads > MTCODER_THREADS_MAX) + numThreads = MTCODER_THREADS_MAX; + numBlocksMax = MTCODER_GET_NUM_BLOCKS_FROM_THREADS(numThreads); if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++; if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++; if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++; - if (numBlocksMax > MTCODER__BLOCKS_MAX) - numBlocksMax = MTCODER__BLOCKS_MAX; + if (numBlocksMax > MTCODER_BLOCKS_MAX) + numBlocksMax = MTCODER_BLOCKS_MAX; if (p->blockSize != p->allocatedBufsSize) { - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) { CMtCoderThread *t = &p->threads[i]; if (t->inBuf) @@ -484,23 +458,23 @@ SRes MtCoder_Code(CMtCoder *p) MtProgress_Init(&p->mtProgress, p->progress); - #ifdef MTCODER__USE_WRITE_THREAD + #ifdef MTCODER_USE_WRITE_THREAD for (i = 0; i < numBlocksMax; i++) { - RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i])); + RINOK_THREAD(AutoResetEvent_OptCreate_And_Reset(&p->writeEvents[i])) } #else - RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + RINOK_THREAD(AutoResetEvent_OptCreate_And_Reset(&p->finishedEvent)) #endif { - RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent)); - RINOK_THREAD(Semaphore_OptCreateInit(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); + RINOK_THREAD(AutoResetEvent_OptCreate_And_Reset(&p->readEvent)) + RINOK_THREAD(Semaphore_OptCreateInit(&p->blocksSemaphore, (UInt32)numBlocksMax, (UInt32)numBlocksMax)) } - for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++) + for (i = 0; i < MTCODER_BLOCKS_MAX - 1; i++) p->freeBlockList[i] = i + 1; - p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1; + p->freeBlockList[MTCODER_BLOCKS_MAX - 1] = (unsigned)(int)-1; p->freeBlockHead = 0; p->readProcessed = 0; @@ -508,10 +482,10 @@ SRes MtCoder_Code(CMtCoder *p) p->numBlocksMax = numBlocksMax; p->stopReading = False; - #ifndef MTCODER__USE_WRITE_THREAD + #ifndef MTCODER_USE_WRITE_THREAD p->writeIndex = 0; p->writeRes = SZ_OK; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) p->ReadyBlocks[i] = False; p->numFinishedThreads = 0; #endif @@ -522,12 +496,12 @@ SRes MtCoder_Code(CMtCoder *p) // for (i = 0; i < numThreads; i++) { CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++]; - RINOK(MtCoderThread_CreateAndStart(nextThread)); + RINOK(MtCoderThread_CreateAndStart(nextThread)) } RINOK_THREAD(Event_Set(&p->readEvent)) - #ifdef MTCODER__USE_WRITE_THREAD + #ifdef MTCODER_USE_WRITE_THREAD { unsigned bi = 0; @@ -582,7 +556,7 @@ SRes MtCoder_Code(CMtCoder *p) if (res == SZ_OK) res = p->mtProgress.res; - #ifndef MTCODER__USE_WRITE_THREAD + #ifndef MTCODER_USE_WRITE_THREAD if (res == SZ_OK) res = p->writeRes; #endif @@ -593,3 +567,5 @@ SRes MtCoder_Code(CMtCoder *p) } #endif + +#undef RINOK_THREAD diff --git a/multiarc/src/formats/7z/C/MtCoder.h b/multiarc/src/formats/7z/C/MtCoder.h old mode 100755 new mode 100644 index 5a5f4d11b..1231d3c2a --- a/multiarc/src/formats/7z/C/MtCoder.h +++ b/multiarc/src/formats/7z/C/MtCoder.h @@ -1,30 +1,30 @@ /* MtCoder.h -- Multi-thread Coder -2018-07-04 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ -#ifndef __MT_CODER_H -#define __MT_CODER_H +#ifndef ZIP7_INC_MT_CODER_H +#define ZIP7_INC_MT_CODER_H #include "MtDec.h" EXTERN_C_BEGIN /* - if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream - if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream + if ( defined MTCODER_USE_WRITE_THREAD) : main thread writes all data blocks to output stream + if (not defined MTCODER_USE_WRITE_THREAD) : any coder thread can write data blocks to output stream */ -/* #define MTCODER__USE_WRITE_THREAD */ +/* #define MTCODER_USE_WRITE_THREAD */ -#ifndef _7ZIP_ST - #define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1) - #define MTCODER__THREADS_MAX 64 - #define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3) +#ifndef Z7_ST + #define MTCODER_GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1) + #define MTCODER_THREADS_MAX 64 + #define MTCODER_BLOCKS_MAX (MTCODER_GET_NUM_BLOCKS_FROM_THREADS(MTCODER_THREADS_MAX) + 3) #else - #define MTCODER__THREADS_MAX 1 - #define MTCODER__BLOCKS_MAX 1 + #define MTCODER_THREADS_MAX 1 + #define MTCODER_BLOCKS_MAX 1 #endif -#ifndef _7ZIP_ST +#ifndef Z7_ST typedef struct @@ -37,15 +37,15 @@ typedef struct void MtProgressThunk_CreateVTable(CMtProgressThunk *p); -#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } +#define MtProgressThunk_INIT(p) { (p)->inSize = 0; (p)->outSize = 0; } -struct _CMtCoder; +struct CMtCoder_; typedef struct { - struct _CMtCoder *mtCoder; + struct CMtCoder_ *mtCoder; unsigned index; int stop; Byte *inBuf; @@ -71,7 +71,7 @@ typedef struct } CMtCoderBlock; -typedef struct _CMtCoder +typedef struct CMtCoder_ { /* input variables */ @@ -79,11 +79,11 @@ typedef struct _CMtCoder unsigned numThreadsMax; UInt64 expectedDataSize; - ISeqInStream *inStream; + ISeqInStreamPtr inStream; const Byte *inData; size_t inDataSize; - ICompressProgress *progress; + ICompressProgressPtr progress; ISzAllocPtr allocBig; IMtCoderCallback2 *mtCallback; @@ -100,13 +100,13 @@ typedef struct _CMtCoder BoolInt stopReading; SRes readRes; - #ifdef MTCODER__USE_WRITE_THREAD - CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX]; + #ifdef MTCODER_USE_WRITE_THREAD + CAutoResetEvent writeEvents[MTCODER_BLOCKS_MAX]; #else CAutoResetEvent finishedEvent; SRes writeRes; unsigned writeIndex; - Byte ReadyBlocks[MTCODER__BLOCKS_MAX]; + Byte ReadyBlocks[MTCODER_BLOCKS_MAX]; LONG numFinishedThreads; #endif @@ -120,11 +120,11 @@ typedef struct _CMtCoder CCriticalSection cs; unsigned freeBlockHead; - unsigned freeBlockList[MTCODER__BLOCKS_MAX]; + unsigned freeBlockList[MTCODER_BLOCKS_MAX]; CMtProgress mtProgress; - CMtCoderBlock blocks[MTCODER__BLOCKS_MAX]; - CMtCoderThread threads[MTCODER__THREADS_MAX]; + CMtCoderBlock blocks[MTCODER_BLOCKS_MAX]; + CMtCoderThread threads[MTCODER_THREADS_MAX]; } CMtCoder; diff --git a/multiarc/src/formats/7z/C/MtDec.c b/multiarc/src/formats/7z/C/MtDec.c old mode 100755 new mode 100644 index 45a671396..96274b6f3 --- a/multiarc/src/formats/7z/C/MtDec.c +++ b/multiarc/src/formats/7z/C/MtDec.c @@ -1,5 +1,5 @@ /* MtDec.c -- Multi-thread Decoder -2021-12-21 : Igor Pavlov : Public domain */ +2024-02-20 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -14,7 +14,7 @@ #include "MtDec.h" -#ifndef _7ZIP_ST +#ifndef Z7_ST #ifdef SHOW_DEBUG_INFO #define PRF(x) x @@ -24,7 +24,7 @@ #define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) -void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +void MtProgress_Init(CMtProgress *p, ICompressProgressPtr progress) { p->progress = progress; p->res = SZ_OK; @@ -81,36 +81,28 @@ void MtProgress_SetError(CMtProgress *p, SRes res) #define RINOK_THREAD(x) RINOK_WRes(x) -static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +struct CMtDecBufLink_ { - if (Event_IsCreated(p)) - return Event_Reset(p); - return AutoResetEvent_CreateNotSignaled(p); -} - - -struct __CMtDecBufLink -{ - struct __CMtDecBufLink *next; + struct CMtDecBufLink_ *next; void *pad[3]; }; -typedef struct __CMtDecBufLink CMtDecBufLink; +typedef struct CMtDecBufLink_ CMtDecBufLink; #define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) #define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) -static THREAD_FUNC_DECL ThreadFunc(void *pp); +static THREAD_FUNC_DECL MtDec_ThreadFunc(void *pp); static WRes MtDecThread_CreateEvents(CMtDecThread *t) { - WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + WRes wres = AutoResetEvent_OptCreate_And_Reset(&t->canWrite); if (wres == 0) { - wres = ArEvent_OptCreate_And_Reset(&t->canRead); + wres = AutoResetEvent_OptCreate_And_Reset(&t->canRead); if (wres == 0) return SZ_OK; } @@ -126,7 +118,7 @@ static SRes MtDecThread_CreateAndStart(CMtDecThread *t) { if (Thread_WasCreated(&t->thread)) return SZ_OK; - wres = Thread_Create(&t->thread, ThreadFunc, t); + wres = Thread_Create(&t->thread, MtDec_ThreadFunc, t); if (wres == 0) return SZ_OK; } @@ -167,7 +159,7 @@ static void MtDecThread_CloseThread(CMtDecThread *t) static void MtDec_CloseThreads(CMtDec *p) { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) MtDecThread_CloseThread(&p->threads[i]); } @@ -179,25 +171,6 @@ static void MtDecThread_Destruct(CMtDecThread *t) -static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) -{ - size_t size = *processedSize; - *processedSize = 0; - while (size != 0) - { - size_t cur = size; - SRes res = ISeqInStream_Read(stream, data, &cur); - *processedSize += cur; - data += cur; - size -= cur; - RINOK(res); - if (cur == 0) - return SZ_OK; - } - return SZ_OK; -} - - static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted) { SRes res; @@ -253,7 +226,7 @@ Byte *MtDec_GetCrossBuff(CMtDec *p) /* - ThreadFunc2() returns: + MtDec_ThreadFunc2() returns: 0 - in all normal cases (even for stream error or memory allocation error) (!= 0) - WRes error return by system threading function */ @@ -261,11 +234,11 @@ Byte *MtDec_GetCrossBuff(CMtDec *p) // #define MTDEC_ProgessStep (1 << 22) #define MTDEC_ProgessStep (1 << 0) -static WRes ThreadFunc2(CMtDecThread *t) +static WRes MtDec_ThreadFunc2(CMtDecThread *t) { CMtDec *p = t->mtDec; - PRF_STR_INT("ThreadFunc2", t->index); + PRF_STR_INT("MtDec_ThreadFunc2", t->index) // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); @@ -295,13 +268,13 @@ static WRes ThreadFunc2(CMtDecThread *t) // CMtDecCallbackInfo parse; CMtDecThread *nextThread; - PRF_STR_INT("=============== Event_Wait(&t->canRead)", t->index); + PRF_STR_INT("=============== Event_Wait(&t->canRead)", t->index) - RINOK_THREAD(Event_Wait(&t->canRead)); + RINOK_THREAD(Event_Wait(&t->canRead)) if (p->exitThread) return 0; - PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index) // if (t->index == 3) return 19; // for test @@ -373,7 +346,7 @@ static WRes ThreadFunc2(CMtDecThread *t) { size = p->inBufSize; - res = FullRead(p->inStream, data, &size); + res = SeqInStream_ReadMax(p->inStream, data, &size); // size = 10; // test @@ -615,7 +588,7 @@ static WRes ThreadFunc2(CMtDecThread *t) // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration - // - otherwise we stop decoding and exit from ThreadFunc2() + // - otherwise we stop decoding and exit from MtDec_ThreadFunc2() // Don't change (finish) variable in the further code @@ -688,7 +661,7 @@ static WRes ThreadFunc2(CMtDecThread *t) // ---------- WRITE ---------- - RINOK_THREAD(Event_Wait(&t->canWrite)); + RINOK_THREAD(Event_Wait(&t->canWrite)) { BoolInt isErrorMode = False; @@ -801,14 +774,14 @@ static WRes ThreadFunc2(CMtDecThread *t) if (!finish) { - RINOK_THREAD(Event_Set(&nextThread->canWrite)); + RINOK_THREAD(Event_Set(&nextThread->canWrite)) } else { if (needContinue) { // we restore decoding with new iteration - RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)) } else { @@ -817,7 +790,7 @@ static WRes ThreadFunc2(CMtDecThread *t) return SZ_OK; p->exitThread = True; } - RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + RINOK_THREAD(Event_Set(&p->threads[0].canRead)) } } } @@ -836,7 +809,17 @@ static WRes ThreadFunc2(CMtDecThread *t) #endif -static THREAD_FUNC_DECL ThreadFunc1(void *pp) +typedef + #ifdef _WIN32 + UINT_PTR + #elif 1 + uintptr_t + #else + ptrdiff_t + #endif + MY_uintptr_t; + +static THREAD_FUNC_DECL MtDec_ThreadFunc1(void *pp) { WRes res; @@ -845,10 +828,10 @@ static THREAD_FUNC_DECL ThreadFunc1(void *pp) // fprintf(stdout, "\n%d = %p\n", t->index, &t); - res = ThreadFunc2(t); + res = MtDec_ThreadFunc2(t); p = t->mtDec; if (res == 0) - return (THREAD_FUNC_RET_TYPE)(UINT_PTR)p->exitThreadWRes; + return (THREAD_FUNC_RET_TYPE)(MY_uintptr_t)p->exitThreadWRes; { // it's unexpected situation for some threading function error if (p->exitThreadWRes == 0) @@ -859,17 +842,17 @@ static THREAD_FUNC_DECL ThreadFunc1(void *pp) Event_Set(&p->threads[0].canWrite); MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); } - return (THREAD_FUNC_RET_TYPE)(UINT_PTR)res; + return (THREAD_FUNC_RET_TYPE)(MY_uintptr_t)res; } -static MY_NO_INLINE THREAD_FUNC_DECL ThreadFunc(void *pp) +static Z7_NO_INLINE THREAD_FUNC_DECL MtDec_ThreadFunc(void *pp) { #ifdef USE_ALLOCA CMtDecThread *t = (CMtDecThread *)pp; // fprintf(stderr, "\n%d = %p - before", t->index, &t); t->allocaPtr = alloca(t->index * 128); #endif - return ThreadFunc1(pp); + return MtDec_ThreadFunc1(pp); } @@ -883,7 +866,7 @@ int MtDec_PrepareRead(CMtDec *p) { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) if (i > p->numStartedThreads || p->numFilledThreads <= (i >= p->filledThreadStart ? @@ -987,7 +970,7 @@ void MtDec_Construct(CMtDec *p) p->allocatedBufsSize = 0; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CMtDecThread *t = &p->threads[i]; t->mtDec = p; @@ -995,7 +978,7 @@ void MtDec_Construct(CMtDec *p) t->inBuf = NULL; Event_Construct(&t->canRead); Event_Construct(&t->canWrite); - Thread_Construct(&t->thread); + Thread_CONSTRUCT(&t->thread) } // Event_Construct(&p->finishedEvent); @@ -1010,7 +993,7 @@ static void MtDec_Free(CMtDec *p) p->exitThread = True; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) MtDecThread_Destruct(&p->threads[i]); // Event_Close(&p->finishedEvent); @@ -1061,15 +1044,15 @@ SRes MtDec_Code(CMtDec *p) { unsigned numThreads = p->numThreadsMax; - if (numThreads > MTDEC__THREADS_MAX) - numThreads = MTDEC__THREADS_MAX; + if (numThreads > MTDEC_THREADS_MAX) + numThreads = MTDEC_THREADS_MAX; p->numStartedThreads_Limit = numThreads; p->numStartedThreads = 0; } if (p->inBufSize != p->allocatedBufsSize) { - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CMtDecThread *t = &p->threads[i]; if (t->inBuf) @@ -1086,7 +1069,7 @@ SRes MtDec_Code(CMtDec *p) MtProgress_Init(&p->mtProgress, p->progress); - // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + // RINOK_THREAD(AutoResetEvent_OptCreate_And_Reset(&p->finishedEvent)) p->exitThread = False; p->exitThreadWRes = 0; @@ -1098,8 +1081,8 @@ SRes MtDec_Code(CMtDec *p) wres = MtDecThread_CreateEvents(nextThread); if (wres == 0) { wres = Event_Set(&nextThread->canWrite); if (wres == 0) { wres = Event_Set(&nextThread->canRead); - if (wres == 0) { THREAD_FUNC_RET_TYPE res = ThreadFunc(nextThread); - wres = (WRes)(UINT_PTR)res; + if (wres == 0) { THREAD_FUNC_RET_TYPE res = MtDec_ThreadFunc(nextThread); + wres = (WRes)(MY_uintptr_t)res; if (wres != 0) { p->needContinue = False; @@ -1137,3 +1120,5 @@ SRes MtDec_Code(CMtDec *p) } #endif + +#undef PRF diff --git a/multiarc/src/formats/7z/C/MtDec.h b/multiarc/src/formats/7z/C/MtDec.h old mode 100755 new mode 100644 index c2da46ae2..c28e8d9ac --- a/multiarc/src/formats/7z/C/MtDec.h +++ b/multiarc/src/formats/7z/C/MtDec.h @@ -1,46 +1,46 @@ /* MtDec.h -- Multi-thread Decoder -2020-03-05 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __MT_DEC_H -#define __MT_DEC_H +#ifndef ZIP7_INC_MT_DEC_H +#define ZIP7_INC_MT_DEC_H #include "7zTypes.h" -#ifndef _7ZIP_ST +#ifndef Z7_ST #include "Threads.h" #endif EXTERN_C_BEGIN -#ifndef _7ZIP_ST +#ifndef Z7_ST -#ifndef _7ZIP_ST - #define MTDEC__THREADS_MAX 32 +#ifndef Z7_ST + #define MTDEC_THREADS_MAX 32 #else - #define MTDEC__THREADS_MAX 1 + #define MTDEC_THREADS_MAX 1 #endif typedef struct { - ICompressProgress *progress; + ICompressProgressPtr progress; SRes res; UInt64 totalInSize; UInt64 totalOutSize; CCriticalSection cs; } CMtProgress; -void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +void MtProgress_Init(CMtProgress *p, ICompressProgressPtr progress); SRes MtProgress_Progress_ST(CMtProgress *p); SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); SRes MtProgress_GetError(CMtProgress *p); void MtProgress_SetError(CMtProgress *p, SRes res); -struct _CMtDec; +struct CMtDec; typedef struct { - struct _CMtDec *mtDec; + struct CMtDec_ *mtDec; unsigned index; void *inBuf; @@ -117,7 +117,7 @@ typedef struct -typedef struct _CMtDec +typedef struct CMtDec_ { /* input variables */ @@ -126,11 +126,11 @@ typedef struct _CMtDec // size_t inBlockMax; unsigned numThreadsMax_2; - ISeqInStream *inStream; + ISeqInStreamPtr inStream; // const Byte *inData; // size_t inDataSize; - ICompressProgress *progress; + ICompressProgressPtr progress; ISzAllocPtr alloc; IMtDecCallback2 *mtCallback; @@ -171,11 +171,11 @@ typedef struct _CMtDec unsigned filledThreadStart; unsigned numFilledThreads; - #ifndef _7ZIP_ST + #ifndef Z7_ST BoolInt needInterrupt; UInt64 interruptIndex; CMtProgress mtProgress; - CMtDecThread threads[MTDEC__THREADS_MAX]; + CMtDecThread threads[MTDEC_THREADS_MAX]; #endif } CMtDec; diff --git a/multiarc/src/formats/7z/C/Ppmd.h b/multiarc/src/formats/7z/C/Ppmd.h old mode 100755 new mode 100644 index b19879208..66b26266f --- a/multiarc/src/formats/7z/C/Ppmd.h +++ b/multiarc/src/formats/7z/C/Ppmd.h @@ -1,9 +1,9 @@ /* Ppmd.h -- PPMD codec common code -2021-04-13 : Igor Pavlov : Public domain +2023-03-05 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ -#ifndef __PPMD_H -#define __PPMD_H +#ifndef ZIP7_INC_PPMD_H +#define ZIP7_INC_PPMD_H #include "CpuArch.h" @@ -48,8 +48,10 @@ typedef struct Byte Count; /* Count to next change of Shift */ } CPpmd_See; -#define Ppmd_See_Update(p) if ((p)->Shift < PPMD_PERIOD_BITS && --(p)->Count == 0) \ - { (p)->Summ = (UInt16)((p)->Summ << 1); (p)->Count = (Byte)(3 << (p)->Shift++); } +#define Ppmd_See_UPDATE(p) \ + { if ((p)->Shift < PPMD_PERIOD_BITS && --(p)->Count == 0) \ + { (p)->Summ = (UInt16)((p)->Summ << 1); \ + (p)->Count = (Byte)(3 << (p)->Shift++); }} typedef struct diff --git a/multiarc/src/formats/7z/C/Ppmd7.c b/multiarc/src/formats/7z/C/Ppmd7.c old mode 100755 new mode 100644 index cf401cb37..efcc5d86e --- a/multiarc/src/formats/7z/C/Ppmd7.c +++ b/multiarc/src/formats/7z/C/Ppmd7.c @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -14,7 +14,7 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ MY_ALIGN(16) static const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; MY_ALIGN(16) -static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; +static const UInt16 PPMD7_kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; #define MAX_FREQ 124 #define UNIT_SIZE 12 @@ -33,7 +33,7 @@ static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x #define ONE_STATE(ctx) Ppmd7Context_OneState(ctx) #define SUFFIX(ctx) CTX((ctx)->Suffix) -typedef CPpmd7_Context * CTX_PTR; +typedef CPpmd7_Context * PPMD7_CTX_PTR; struct CPpmd7_Node_; @@ -107,14 +107,14 @@ BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) // ---------- Internal Memory Allocator ---------- /* We can use CPpmd7_Node in list of free units (as in Ppmd8) - But we still need one additional list walk pass in GlueFreeBlocks(). - So we use simple CPpmd_Void_Ref instead of CPpmd7_Node in InsertNode() / RemoveNode() + But we still need one additional list walk pass in Ppmd7_GlueFreeBlocks(). + So we use simple CPpmd_Void_Ref instead of CPpmd7_Node in Ppmd7_InsertNode() / Ppmd7_RemoveNode() */ #define EMPTY_NODE 0 -static void InsertNode(CPpmd7 *p, void *node, unsigned indx) +static void Ppmd7_InsertNode(CPpmd7 *p, void *node, unsigned indx) { *((CPpmd_Void_Ref *)node) = p->FreeList[indx]; // ((CPpmd7_Node *)node)->Next = (CPpmd7_Node_Ref)p->FreeList[indx]; @@ -124,7 +124,7 @@ static void InsertNode(CPpmd7 *p, void *node, unsigned indx) } -static void *RemoveNode(CPpmd7 *p, unsigned indx) +static void *Ppmd7_RemoveNode(CPpmd7 *p, unsigned indx) { CPpmd_Void_Ref *node = (CPpmd_Void_Ref *)Ppmd7_GetPtr(p, p->FreeList[indx]); p->FreeList[indx] = *node; @@ -134,32 +134,32 @@ static void *RemoveNode(CPpmd7 *p, unsigned indx) } -static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) +static void Ppmd7_SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) { unsigned i, nu = I2U(oldIndx) - I2U(newIndx); ptr = (Byte *)ptr + U2B(I2U(newIndx)); if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); + Ppmd7_InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); } - InsertNode(p, ptr, i); + Ppmd7_InsertNode(p, ptr, i); } /* we use CPpmd7_Node_Union union to solve XLC -O2 strict pointer aliasing problem */ -typedef union _CPpmd7_Node_Union +typedef union { CPpmd7_Node Node; CPpmd7_Node_Ref NextRef; } CPpmd7_Node_Union; -/* Original PPmdH (Ppmd7) code uses doubly linked list in GlueFreeBlocks() +/* Original PPmdH (Ppmd7) code uses doubly linked list in Ppmd7_GlueFreeBlocks() we use single linked list similar to Ppmd8 code */ -static void GlueFreeBlocks(CPpmd7 *p) +static void Ppmd7_GlueFreeBlocks(CPpmd7 *p) { /* we use first UInt16 field of 12-bytes UNITs as record type stamp @@ -239,27 +239,27 @@ static void GlueFreeBlocks(CPpmd7 *p) if (nu == 0) continue; for (; nu > 128; nu -= 128, node += 128) - InsertNode(p, node, PPMD_NUM_INDEXES - 1); + Ppmd7_InsertNode(p, node, PPMD_NUM_INDEXES - 1); if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, node + k, (unsigned)nu - k - 1); + Ppmd7_InsertNode(p, node + k, (unsigned)nu - k - 1); } - InsertNode(p, node, i); + Ppmd7_InsertNode(p, node, i); } } -MY_NO_INLINE -static void *AllocUnitsRare(CPpmd7 *p, unsigned indx) +Z7_NO_INLINE +static void *Ppmd7_AllocUnitsRare(CPpmd7 *p, unsigned indx) { unsigned i; if (p->GlueCount == 0) { - GlueFreeBlocks(p); + Ppmd7_GlueFreeBlocks(p); if (p->FreeList[indx] != 0) - return RemoveNode(p, indx); + return Ppmd7_RemoveNode(p, indx); } i = indx; @@ -277,17 +277,17 @@ static void *AllocUnitsRare(CPpmd7 *p, unsigned indx) while (p->FreeList[i] == 0); { - void *block = RemoveNode(p, i); - SplitBlock(p, block, i, indx); + void *block = Ppmd7_RemoveNode(p, i); + Ppmd7_SplitBlock(p, block, i, indx); return block; } } -static void *AllocUnits(CPpmd7 *p, unsigned indx) +static void *Ppmd7_AllocUnits(CPpmd7 *p, unsigned indx) { if (p->FreeList[indx] != 0) - return RemoveNode(p, indx); + return Ppmd7_RemoveNode(p, indx); { UInt32 numBytes = U2B(I2U(indx)); Byte *lo = p->LoUnit; @@ -297,13 +297,22 @@ static void *AllocUnits(CPpmd7 *p, unsigned indx) return lo; } } - return AllocUnitsRare(p, indx); + return Ppmd7_AllocUnitsRare(p, indx); } -#define MyMem12Cpy(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } +#define MEM_12_CPY(dest, src, num) \ + { UInt32 *d = (UInt32 *)(dest); \ + const UInt32 *z = (const UInt32 *)(src); \ + unsigned n = (num); \ + do { \ + d[0] = z[0]; \ + d[1] = z[1]; \ + d[2] = z[2]; \ + z += 3; \ + d += 3; \ + } while (--n); \ + } /* @@ -315,12 +324,12 @@ static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU return oldPtr; if (p->FreeList[i1] != 0) { - void *ptr = RemoveNode(p, i1); - MyMem12Cpy(ptr, oldPtr, newNU); - InsertNode(p, oldPtr, i0); + void *ptr = Ppmd7_RemoveNode(p, i1); + MEM_12_CPY(ptr, oldPtr, newNU) + Ppmd7_InsertNode(p, oldPtr, i0); return ptr; } - SplitBlock(p, oldPtr, i0, i1); + Ppmd7_SplitBlock(p, oldPtr, i0, i1); return oldPtr; } */ @@ -329,14 +338,14 @@ static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) { - Ppmd_SET_SUCCESSOR(p, v); + Ppmd_SET_SUCCESSOR(p, v) } -MY_NO_INLINE +Z7_NO_INLINE static -void RestartModel(CPpmd7 *p) +void Ppmd7_RestartModel(CPpmd7 *p) { unsigned i, k; @@ -352,8 +361,8 @@ void RestartModel(CPpmd7 *p) p->PrevSuccess = 0; { - CPpmd7_Context *mc = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ - CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + CPpmd7_Context *mc = (PPMD7_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* Ppmd7_AllocUnits(p, PPMD_NUM_INDEXES - 1); */ p->LoUnit += U2B(256 / 2); p->MaxContext = p->MinContext = mc; @@ -391,7 +400,7 @@ void RestartModel(CPpmd7 *p) { unsigned m; UInt16 *dest = p->BinSumm[i] + k; - UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 2)); + const UInt16 val = (UInt16)(PPMD_BIN_SCALE - PPMD7_kInitBinEsc[k] / (i + 2)); for (m = 0; m < 64; m += 8) dest[m] = val; } @@ -423,13 +432,13 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) { p->MaxOrder = maxOrder; - RestartModel(p); + Ppmd7_RestartModel(p); } /* - CreateSuccessors() + Ppmd7_CreateSuccessors() It's called when (FoundState->Successor) is RAW-Successor, that is the link to position in Raw text. So we create Context records and write the links to @@ -445,10 +454,10 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) also it can return pointer to real context of same order, */ -MY_NO_INLINE -static CTX_PTR CreateSuccessors(CPpmd7 *p) +Z7_NO_INLINE +static PPMD7_CTX_PTR Ppmd7_CreateSuccessors(CPpmd7 *p) { - CTX_PTR c = p->MinContext; + PPMD7_CTX_PTR c = p->MinContext; CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); Byte newSym, newFreq; unsigned numPs = 0; @@ -522,15 +531,15 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p) do { - CTX_PTR c1; + PPMD7_CTX_PTR c1; /* = AllocContext(p); */ if (p->HiUnit != p->LoUnit) - c1 = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); + c1 = (PPMD7_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); else if (p->FreeList[0] != 0) - c1 = (CTX_PTR)RemoveNode(p, 0); + c1 = (PPMD7_CTX_PTR)Ppmd7_RemoveNode(p, 0); else { - c1 = (CTX_PTR)AllocUnitsRare(p, 0); + c1 = (PPMD7_CTX_PTR)Ppmd7_AllocUnitsRare(p, 0); if (!c1) return NULL; } @@ -550,16 +559,16 @@ static CTX_PTR CreateSuccessors(CPpmd7 *p) -#define SwapStates(s) \ +#define SWAP_STATES(s) \ { CPpmd_State tmp = s[0]; s[0] = s[-1]; s[-1] = tmp; } void Ppmd7_UpdateModel(CPpmd7 *p); -MY_NO_INLINE +Z7_NO_INLINE void Ppmd7_UpdateModel(CPpmd7 *p) { CPpmd_Void_Ref maxSuccessor, minSuccessor; - CTX_PTR c, mc; + PPMD7_CTX_PTR c, mc; unsigned s0, ns; @@ -592,7 +601,7 @@ void Ppmd7_UpdateModel(CPpmd7 *p) if (s[0].Freq >= s[-1].Freq) { - SwapStates(s); + SWAP_STATES(s) s--; } } @@ -610,10 +619,10 @@ void Ppmd7_UpdateModel(CPpmd7 *p) { /* MAX ORDER context */ /* (FoundState->Successor) is RAW-Successor. */ - p->MaxContext = p->MinContext = CreateSuccessors(p); + p->MaxContext = p->MinContext = Ppmd7_CreateSuccessors(p); if (!p->MinContext) { - RestartModel(p); + Ppmd7_RestartModel(p); return; } SetSuccessor(p->FoundState, REF(p->MinContext)); @@ -629,7 +638,7 @@ void Ppmd7_UpdateModel(CPpmd7 *p) p->Text = text; if (text >= p->UnitsStart) { - RestartModel(p); + Ppmd7_RestartModel(p); return; } maxSuccessor = REF(text); @@ -645,10 +654,10 @@ void Ppmd7_UpdateModel(CPpmd7 *p) if (minSuccessor <= maxSuccessor) { // minSuccessor is RAW-Successor. So we will create real contexts records: - CTX_PTR cs = CreateSuccessors(p); + PPMD7_CTX_PTR cs = Ppmd7_CreateSuccessors(p); if (!cs) { - RestartModel(p); + Ppmd7_RestartModel(p); return; } minSuccessor = REF(cs); @@ -711,27 +720,27 @@ void Ppmd7_UpdateModel(CPpmd7 *p) if ((ns1 & 1) == 0) { /* Expand for one UNIT */ - unsigned oldNU = ns1 >> 1; - unsigned i = U2I(oldNU); + const unsigned oldNU = ns1 >> 1; + const unsigned i = U2I(oldNU); if (i != U2I((size_t)oldNU + 1)) { - void *ptr = AllocUnits(p, i + 1); + void *ptr = Ppmd7_AllocUnits(p, i + 1); void *oldPtr; if (!ptr) { - RestartModel(p); + Ppmd7_RestartModel(p); return; } oldPtr = STATS(c); - MyMem12Cpy(ptr, oldPtr, oldNU); - InsertNode(p, oldPtr, i); + MEM_12_CPY(ptr, oldPtr, oldNU) + Ppmd7_InsertNode(p, oldPtr, i); c->Union4.Stats = STATS_REF(ptr); } } sum = c->Union2.SummFreq; /* max increase of Escape_Freq is 3 here. total increase of Union2.SummFreq for all symbols is less than 256 here */ - sum += (UInt32)(2 * ns1 < ns) + 2 * ((unsigned)(4 * ns1 <= ns) & (sum <= 8 * ns1)); + sum += (UInt32)(unsigned)((2 * ns1 < ns) + 2 * ((unsigned)(4 * ns1 <= ns) & (sum <= 8 * ns1))); /* original PPMdH uses 16-bit variable for (sum) here. But (sum < 0x9000). So we don't truncate (sum) to 16-bit */ // sum = (UInt16)sum; @@ -739,10 +748,10 @@ void Ppmd7_UpdateModel(CPpmd7 *p) else { // instead of One-symbol context we create 2-symbol context - CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); + CPpmd_State *s = (CPpmd_State*)Ppmd7_AllocUnits(p, 0); if (!s) { - RestartModel(p); + Ppmd7_RestartModel(p); return; } { @@ -761,7 +770,7 @@ void Ppmd7_UpdateModel(CPpmd7 *p) // (max(s->freq) == 120), when we convert from 1-symbol into 2-symbol context s->Freq = (Byte)freq; // max(InitEsc = PPMD7_kExpEscape[*]) is 25. So the max(escapeFreq) is 26 here - sum = freq + p->InitEsc + (ns > 3); + sum = (UInt32)(freq + p->InitEsc + (ns > 3)); } } @@ -795,8 +804,8 @@ void Ppmd7_UpdateModel(CPpmd7 *p) -MY_NO_INLINE -static void Rescale(CPpmd7 *p) +Z7_NO_INLINE +static void Ppmd7_Rescale(CPpmd7 *p) { unsigned i, adder, sumFreq, escFreq; CPpmd_State *stats = STATS(p->MinContext); @@ -885,7 +894,7 @@ static void Rescale(CPpmd7 *p) *s = *stats; s->Freq = (Byte)freq; // (freq <= 260 / 4) p->FoundState = s; - InsertNode(p, stats, U2I(n0)); + Ppmd7_InsertNode(p, stats, U2I(n0)); return; } @@ -899,13 +908,13 @@ static void Rescale(CPpmd7 *p) { if (p->FreeList[i1] != 0) { - void *ptr = RemoveNode(p, i1); + void *ptr = Ppmd7_RemoveNode(p, i1); p->MinContext->Union4.Stats = STATS_REF(ptr); - MyMem12Cpy(ptr, (const void *)stats, n1); - InsertNode(p, stats, i0); + MEM_12_CPY(ptr, (const void *)stats, n1) + Ppmd7_InsertNode(p, stats, i0); } else - SplitBlock(p, stats, i0, i1); + Ppmd7_SplitBlock(p, stats, i0, i1); } } } @@ -933,10 +942,10 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) p->HiBitsFlag; { // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ - unsigned summ = (UInt16)see->Summ; // & 0xFFFF - unsigned r = (summ >> see->Shift); + const unsigned summ = (UInt16)see->Summ; // & 0xFFFF + const unsigned r = (summ >> see->Shift); see->Summ = (UInt16)(summ - r); - *escFreq = r + (r == 0); + *escFreq = (UInt32)(r + (r == 0)); } } else @@ -948,9 +957,9 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) } -static void NextContext(CPpmd7 *p) +static void Ppmd7_NextContext(CPpmd7 *p) { - CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); + PPMD7_CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); if (p->OrderFall == 0 && (const Byte *)c > p->Text) p->MaxContext = p->MinContext = c; else @@ -967,12 +976,12 @@ void Ppmd7_Update1(CPpmd7 *p) s->Freq = (Byte)freq; if (freq > s[-1].Freq) { - SwapStates(s); + SWAP_STATES(s) p->FoundState = --s; if (freq > MAX_FREQ) - Rescale(p); + Ppmd7_Rescale(p); } - NextContext(p); + Ppmd7_NextContext(p); } @@ -981,15 +990,15 @@ void Ppmd7_Update1_0(CPpmd7 *p) CPpmd_State *s = p->FoundState; CPpmd7_Context *mc = p->MinContext; unsigned freq = s->Freq; - unsigned summFreq = mc->Union2.SummFreq; + const unsigned summFreq = mc->Union2.SummFreq; p->PrevSuccess = (2 * freq > summFreq); - p->RunLength += (int)p->PrevSuccess; + p->RunLength += (Int32)p->PrevSuccess; mc->Union2.SummFreq = (UInt16)(summFreq + 4); freq += 4; s->Freq = (Byte)freq; if (freq > MAX_FREQ) - Rescale(p); - NextContext(p); + Ppmd7_Rescale(p); + Ppmd7_NextContext(p); } @@ -1000,7 +1009,7 @@ void Ppmd7_UpdateBin(CPpmd7 *p) p->FoundState->Freq = (Byte)(freq + (freq < 128)); p->PrevSuccess = 1; p->RunLength++; - NextContext(p); + Ppmd7_NextContext(p); } */ @@ -1013,7 +1022,7 @@ void Ppmd7_Update2(CPpmd7 *p) p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); s->Freq = (Byte)freq; if (freq > MAX_FREQ) - Rescale(p); + Ppmd7_Rescale(p); Ppmd7_UpdateModel(p); } @@ -1042,8 +1051,8 @@ Last UNIT of array at offset (Size - 12) is root order-0 CPpmd7_Context record. The code can free UNITs memory blocks that were allocated to store CPpmd_State vectors. The code doesn't free UNITs allocated for CPpmd7_Context records. -The code calls RestartModel(), when there is no free memory for allocation. -And RestartModel() changes the state to orignal start state, with full free block. +The code calls Ppmd7_RestartModel(), when there is no free memory for allocation. +And Ppmd7_RestartModel() changes the state to orignal start state, with full free block. The code allocates UNITs with the following order: @@ -1051,14 +1060,14 @@ The code allocates UNITs with the following order: Allocation of 1 UNIT for Context record - from free space (HiUnit) down to (LoUnit) - from FreeList[0] - - AllocUnitsRare() + - Ppmd7_AllocUnitsRare() -AllocUnits() for CPpmd_State vectors: +Ppmd7_AllocUnits() for CPpmd_State vectors: - from FreeList[i] - from free space (LoUnit) up to (HiUnit) - - AllocUnitsRare() + - Ppmd7_AllocUnitsRare() -AllocUnitsRare() +Ppmd7_AllocUnitsRare() - if (GlueCount == 0) { Glue lists, GlueCount = 255, allocate from FreeList[i]] } - loop for all higher sized FreeList[...] lists @@ -1093,8 +1102,8 @@ The PPMd code tries to fulfill the condition: We have (Sum(Stats[].Freq) <= 256 * 124), because of (MAX_FREQ = 124) So (4 = 128 - 124) is average reserve for Escape_Freq for each symbol. If (CPpmd_State::Freq) is not aligned for 4, the reserve can be 5, 6 or 7. -SummFreq and Escape_Freq can be changed in Rescale() and *Update*() functions. -Rescale() can remove symbols only from max-order contexts. So Escape_Freq can increase after multiple calls of Rescale() for +SummFreq and Escape_Freq can be changed in Ppmd7_Rescale() and *Update*() functions. +Ppmd7_Rescale() can remove symbols only from max-order contexts. So Escape_Freq can increase after multiple calls of Ppmd7_Rescale() for max-order context. When the PPMd code still break (Total <= RC::Range) condition in range coder, @@ -1102,3 +1111,21 @@ we have two ways to resolve that problem: 1) we can report error, if we want to keep compatibility with original PPMd code that has no fix for such cases. 2) we can reduce (Total) value to (RC::Range) by reducing (Escape_Freq) part of (Total) value. */ + +#undef MAX_FREQ +#undef UNIT_SIZE +#undef U2B +#undef U2I +#undef I2U +#undef I2U_UInt16 +#undef REF +#undef STATS_REF +#undef CTX +#undef STATS +#undef ONE_STATE +#undef SUFFIX +#undef NODE +#undef EMPTY_NODE +#undef MEM_12_CPY +#undef SUCCESSOR +#undef SWAP_STATES diff --git a/multiarc/src/formats/7z/C/Ppmd7.h b/multiarc/src/formats/7z/C/Ppmd7.h old mode 100755 new mode 100644 index d31809aeb..d9eb326d6 --- a/multiarc/src/formats/7z/C/Ppmd7.h +++ b/multiarc/src/formats/7z/C/Ppmd7.h @@ -1,11 +1,11 @@ /* Ppmd7.h -- Ppmd7 (PPMdH) compression codec -2021-04-13 : Igor Pavlov : Public domain +2023-04-02 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain */ -#ifndef __PPMD7_H -#define __PPMD7_H +#ifndef ZIP7_INC_PPMD7_H +#define ZIP7_INC_PPMD7_H #include "Ppmd.h" @@ -55,7 +55,7 @@ typedef struct UInt32 Range; UInt32 Code; UInt32 Low; - IByteIn *Stream; + IByteInPtr Stream; } CPpmd7_RangeDec; @@ -66,7 +66,7 @@ typedef struct // Byte _dummy_[3]; UInt64 Low; UInt64 CacheSize; - IByteOut *Stream; + IByteOutPtr Stream; } CPpmd7z_RangeEnc; diff --git a/multiarc/src/formats/7z/C/Ppmd7Dec.c b/multiarc/src/formats/7z/C/Ppmd7Dec.c old mode 100755 new mode 100644 index 55d74ff9d..081ab8955 --- a/multiarc/src/formats/7z/C/Ppmd7Dec.c +++ b/multiarc/src/formats/7z/C/Ppmd7Dec.c @@ -1,5 +1,5 @@ /* Ppmd7Dec.c -- Ppmd7z (PPMdH with 7z Range Coder) Decoder -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain */ @@ -8,7 +8,7 @@ This code is based on: #include "Ppmd7.h" -#define kTopValue (1 << 24) +#define kTopValue ((UInt32)1 << 24) #define READ_BYTE(p) IByteIn_Read((p)->Stream) @@ -37,9 +37,9 @@ BoolInt Ppmd7z_RangeDec_Init(CPpmd7_RangeDec *p) #define R (&p->rc.dec) -MY_FORCE_INLINE -// MY_NO_INLINE -static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd7z_RD_Decode(CPpmd7 *p, UInt32 start, UInt32 size) { @@ -48,18 +48,18 @@ static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) RC_NORM_LOCAL(R) } -#define RC_Decode(start, size) RangeDec_Decode(p, start, size); -#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) -#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) +#define RC_Decode(start, size) Ppmd7z_RD_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) #define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) -typedef CPpmd7_Context * CTX_PTR; +// typedef CPpmd7_Context * CTX_PTR; #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd7_UpdateModel(CPpmd7 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] -// MY_FORCE_INLINE +#define MASK(sym) ((Byte *)charMask)[sym] +// Z7_FORCE_INLINE // static int Ppmd7z_DecodeSymbol(CPpmd7 *p) { @@ -70,7 +70,7 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); unsigned i; UInt32 count, hiCnt; - UInt32 summFreq = p->MinContext->Union2.SummFreq; + const UInt32 summFreq = p->MinContext->Union2.SummFreq; @@ -81,7 +81,7 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) if ((Int32)(count -= s->Freq) < 0) { Byte sym; - RC_DecodeFinal(0, s->Freq); + RC_DecodeFinal(0, s->Freq) p->FoundState = s; sym = s->Symbol; Ppmd7_Update1_0(p); @@ -96,7 +96,7 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) if ((Int32)(count -= (++s)->Freq) < 0) { Byte sym; - RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) p->FoundState = s; sym = s->Symbol; Ppmd7_Update1(p); @@ -109,10 +109,10 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) return PPMD7_SYM_ERROR; hiCnt -= count; - RC_Decode(hiCnt, summFreq - hiCnt); + RC_Decode(hiCnt, summFreq - hiCnt) p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) // i = p->MinContext->NumStats - 1; // do { MASK((--s)->Symbol) = 0; } while (--i); { @@ -120,8 +120,8 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -152,7 +152,7 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) // Ppmd7_UpdateBin(p); { unsigned freq = s->Freq; - CTX_PTR c = CTX(SUCCESSOR(s)); + CPpmd7_Context *c = CTX(SUCCESSOR(s)); sym = s->Symbol; p->FoundState = s; p->PrevSuccess = 1; @@ -176,7 +176,7 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) R->Range -= size0; RC_NORM_LOCAL(R) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; p->PrevSuccess = 0; } @@ -209,17 +209,17 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) unsigned num2 = num / 2; num &= 1; - hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); s += num; p->MinContext = mc; do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; - hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); - hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); } while (--num2); } @@ -238,20 +238,20 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) s = Ppmd7_GetStats(p, p->MinContext); hiCnt = count; - // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // count -= s->Freq & (UInt32)(MASK(s->Symbol)); // if ((Int32)count >= 0) { for (;;) { - count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - }; + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + } } s--; - RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) // new (see->Summ) value can overflow over 16-bits in some rare cases - Ppmd_See_Update(see); + Ppmd_See_UPDATE(see) p->FoundState = s; sym = s->Symbol; Ppmd7_Update2(p); @@ -261,7 +261,7 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) if (count >= freqSum) return PPMD7_SYM_ERROR; - RC_Decode(hiCnt, freqSum - hiCnt); + RC_Decode(hiCnt, freqSum - hiCnt) // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. // new (see->Summ) value can overflow over 16-bits in some rare cases @@ -295,3 +295,18 @@ Byte *Ppmd7z_DecodeSymbols(CPpmd7 *p, Byte *buf, const Byte *lim) return buf; } */ + +#undef kTopValue +#undef READ_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Decode +#undef RC_DecodeFinal +#undef RC_GetThreshold +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/multiarc/src/formats/7z/C/Ppmd7Enc.c b/multiarc/src/formats/7z/C/Ppmd7Enc.c old mode 100755 new mode 100644 index 62139c5b6..49cbbe649 --- a/multiarc/src/formats/7z/C/Ppmd7Enc.c +++ b/multiarc/src/formats/7z/C/Ppmd7Enc.c @@ -1,5 +1,5 @@ /* Ppmd7Enc.c -- Ppmd7z (PPMdH with 7z Range Coder) Encoder -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain */ @@ -8,7 +8,7 @@ This code is based on: #include "Ppmd7.h" -#define kTopValue (1 << 24) +#define kTopValue ((UInt32)1 << 24) #define R (&p->rc.enc) @@ -20,8 +20,8 @@ void Ppmd7z_Init_RangeEnc(CPpmd7 *p) R->CacheSize = 1; } -MY_NO_INLINE -static void RangeEnc_ShiftLow(CPpmd7 *p) +Z7_NO_INLINE +static void Ppmd7z_RangeEnc_ShiftLow(CPpmd7 *p) { if ((UInt32)R->Low < (UInt32)0xFF000000 || (unsigned)(R->Low >> 32) != 0) { @@ -38,53 +38,53 @@ static void RangeEnc_ShiftLow(CPpmd7 *p) R->Low = (UInt32)((UInt32)R->Low << 8); } -#define RC_NORM_BASE(p) if (R->Range < kTopValue) { R->Range <<= 8; RangeEnc_ShiftLow(p); -#define RC_NORM_1(p) RC_NORM_BASE(p) } -#define RC_NORM(p) RC_NORM_BASE(p) RC_NORM_BASE(p) }} +#define RC_NORM_BASE(p) if (R->Range < kTopValue) { R->Range <<= 8; Ppmd7z_RangeEnc_ShiftLow(p); +#define RC_NORM_1(p) RC_NORM_BASE(p) } +#define RC_NORM(p) RC_NORM_BASE(p) RC_NORM_BASE(p) }} // we must use only one type of Normalization from two: LOCAL or REMOTE #define RC_NORM_LOCAL(p) // RC_NORM(p) #define RC_NORM_REMOTE(p) RC_NORM(p) /* -#define RangeEnc_Encode(p, start, _size_) \ +#define Ppmd7z_RangeEnc_Encode(p, start, _size_) \ { UInt32 size = _size_; \ R->Low += start * R->Range; \ R->Range *= size; \ RC_NORM_LOCAL(p); } */ -MY_FORCE_INLINE -// MY_NO_INLINE -static void RangeEnc_Encode(CPpmd7 *p, UInt32 start, UInt32 size) +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd7z_RangeEnc_Encode(CPpmd7 *p, UInt32 start, UInt32 size) { R->Low += start * R->Range; R->Range *= size; - RC_NORM_LOCAL(p); + RC_NORM_LOCAL(p) } void Ppmd7z_Flush_RangeEnc(CPpmd7 *p) { unsigned i; for (i = 0; i < 5; i++) - RangeEnc_ShiftLow(p); + Ppmd7z_RangeEnc_ShiftLow(p); } -#define RC_Encode(start, size) RangeEnc_Encode(p, start, size); -#define RC_EncodeFinal(start, size) RC_Encode(start, size); RC_NORM_REMOTE(p); +#define RC_Encode(start, size) Ppmd7z_RangeEnc_Encode(p, start, size); +#define RC_EncodeFinal(start, size) RC_Encode(start, size) RC_NORM_REMOTE(p) #define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) #define SUFFIX(ctx) CTX((ctx)->Suffix) -typedef CPpmd7_Context * CTX_PTR; +// typedef CPpmd7_Context * CTX_PTR; #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd7_UpdateModel(CPpmd7 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] -MY_FORCE_INLINE +Z7_FORCE_INLINE static void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) { @@ -104,7 +104,7 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) if (s->Symbol == symbol) { // R->Range /= p->MinContext->Union2.SummFreq; - RC_EncodeFinal(0, s->Freq); + RC_EncodeFinal(0, s->Freq) p->FoundState = s; Ppmd7_Update1_0(p); return; @@ -117,7 +117,7 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) if ((++s)->Symbol == symbol) { // R->Range /= p->MinContext->Union2.SummFreq; - RC_EncodeFinal(sum, s->Freq); + RC_EncodeFinal(sum, s->Freq) p->FoundState = s; Ppmd7_Update1(p); return; @@ -127,10 +127,10 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) while (--i); // R->Range /= p->MinContext->Union2.SummFreq; - RC_Encode(sum, p->MinContext->Union2.SummFreq - sum); + RC_Encode(sum, p->MinContext->Union2.SummFreq - sum) p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) // MASK(s->Symbol) = 0; // i = p->MinContext->NumStats - 1; // do { MASK((--s)->Symbol) = 0; } while (--i); @@ -139,8 +139,8 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -153,20 +153,20 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) UInt16 *prob = Ppmd7_GetBinSumm(p); CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); UInt32 pr = *prob; - UInt32 bound = (R->Range >> 14) * pr; + const UInt32 bound = (R->Range >> 14) * pr; pr = PPMD_UPDATE_PROB_1(pr); if (s->Symbol == symbol) { *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); // RangeEnc_EncodeBit_0(p, bound); R->Range = bound; - RC_NORM_1(p); + RC_NORM_1(p) // p->FoundState = s; // Ppmd7_UpdateBin(p); { - unsigned freq = s->Freq; - CTX_PTR c = CTX(SUCCESSOR(s)); + const unsigned freq = s->Freq; + CPpmd7_Context *c = CTX(SUCCESSOR(s)); p->FoundState = s; p->PrevSuccess = 1; p->RunLength++; @@ -187,7 +187,7 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) R->Range -= bound; RC_NORM_LOCAL(p) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) MASK(s->Symbol) = 0; p->PrevSuccess = 0; } @@ -248,14 +248,14 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) do { - unsigned cur = s->Symbol; + const unsigned cur = s->Symbol; if ((int)cur == symbol) { - UInt32 low = sum; - UInt32 freq = s->Freq; + const UInt32 low = sum; + const UInt32 freq = s->Freq; unsigned num2; - Ppmd_See_Update(see); + Ppmd_See_UPDATE(see) p->FoundState = s; sum += escFreq; @@ -265,21 +265,20 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) if (num2 != 0) { s += i; - for (;;) + do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; sum += (s[-2].Freq & (unsigned)(MASK(sym0))); sum += (s[-1].Freq & (unsigned)(MASK(sym1))); - if (--num2 == 0) - break; } + while (--num2); } R->Range /= sum; - RC_EncodeFinal(low, freq); + RC_EncodeFinal(low, freq) Ppmd7_Update2(p); return; } @@ -289,21 +288,21 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) while (--i); { - UInt32 total = sum + escFreq; + const UInt32 total = sum + escFreq; see->Summ = (UInt16)(see->Summ + total); R->Range /= total; - RC_Encode(sum, escFreq); + RC_Encode(sum, escFreq) } { - CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); + const CPpmd_State *s2 = Ppmd7_GetStats(p, p->MinContext); s--; MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -321,3 +320,18 @@ void Ppmd7z_EncodeSymbols(CPpmd7 *p, const Byte *buf, const Byte *lim) Ppmd7z_EncodeSymbol(p, *buf); } } + +#undef kTopValue +#undef WRITE_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Encode +#undef RC_EncodeFinal +#undef SUFFIX +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/multiarc/src/formats/7z/C/Ppmd7aDec.c b/multiarc/src/formats/7z/C/Ppmd7aDec.c old mode 100755 new mode 100644 index c42457849..ef86dde79 --- a/multiarc/src/formats/7z/C/Ppmd7aDec.c +++ b/multiarc/src/formats/7z/C/Ppmd7aDec.c @@ -1,5 +1,5 @@ /* Ppmd7aDec.c -- PPMd7a (PPMdH) Decoder -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -8,8 +8,8 @@ This code is based on: #include "Ppmd7.h" -#define kTop (1 << 24) -#define kBot (1 << 15) +#define kTop ((UInt32)1 << 24) +#define kBot ((UInt32)1 << 15) #define READ_BYTE(p) IByteIn_Read((p)->Stream) @@ -37,9 +37,9 @@ BoolInt Ppmd7a_RangeDec_Init(CPpmd7_RangeDec *p) #define R (&p->rc.dec) -MY_FORCE_INLINE -// MY_NO_INLINE -static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd7a_RD_Decode(CPpmd7 *p, UInt32 start, UInt32 size) { start *= R->Range; R->Low += start; @@ -48,9 +48,9 @@ static void RangeDec_Decode(CPpmd7 *p, UInt32 start, UInt32 size) RC_NORM_LOCAL(R) } -#define RC_Decode(start, size) RangeDec_Decode(p, start, size); -#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) -#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) +#define RC_Decode(start, size) Ppmd7a_RD_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) #define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) @@ -58,7 +58,7 @@ typedef CPpmd7_Context * CTX_PTR; #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd7_UpdateModel(CPpmd7 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] int Ppmd7a_DecodeSymbol(CPpmd7 *p) @@ -70,7 +70,7 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); unsigned i; UInt32 count, hiCnt; - UInt32 summFreq = p->MinContext->Union2.SummFreq; + const UInt32 summFreq = p->MinContext->Union2.SummFreq; if (summFreq > R->Range) return PPMD7_SYM_ERROR; @@ -81,7 +81,7 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) if ((Int32)(count -= s->Freq) < 0) { Byte sym; - RC_DecodeFinal(0, s->Freq); + RC_DecodeFinal(0, s->Freq) p->FoundState = s; sym = s->Symbol; Ppmd7_Update1_0(p); @@ -96,7 +96,7 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) if ((Int32)(count -= (++s)->Freq) < 0) { Byte sym; - RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) p->FoundState = s; sym = s->Symbol; Ppmd7_Update1(p); @@ -109,10 +109,10 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) return PPMD7_SYM_ERROR; hiCnt -= count; - RC_Decode(hiCnt, summFreq - hiCnt); + RC_Decode(hiCnt, summFreq - hiCnt) p->HiBitsFlag = PPMD7_HiBitsFlag_3(p->FoundState->Symbol); - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) // i = p->MinContext->NumStats - 1; // do { MASK((--s)->Symbol) = 0; } while (--i); { @@ -120,8 +120,8 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -176,7 +176,7 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - size0; RC_NORM_LOCAL(R) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; p->PrevSuccess = 0; } @@ -209,17 +209,17 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) unsigned num2 = num / 2; num &= 1; - hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); s += num; p->MinContext = mc; do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; - hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); - hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); } while (--num2); } @@ -238,20 +238,20 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) s = Ppmd7_GetStats(p, p->MinContext); hiCnt = count; - // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // count -= s->Freq & (UInt32)(MASK(s->Symbol)); // if ((Int32)count >= 0) { for (;;) { - count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - }; + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + } } s--; - RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) // new (see->Summ) value can overflow over 16-bits in some rare cases - Ppmd_See_Update(see); + Ppmd_See_UPDATE(see) p->FoundState = s; sym = s->Symbol; Ppmd7_Update2(p); @@ -261,7 +261,7 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) if (count >= freqSum) return PPMD7_SYM_ERROR; - RC_Decode(hiCnt, freqSum - hiCnt); + RC_Decode(hiCnt, freqSum - hiCnt) // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. // new (see->Summ) value can overflow over 16-bits in some rare cases @@ -277,3 +277,19 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) while (s != s2); } } + +#undef kTop +#undef kBot +#undef READ_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Decode +#undef RC_DecodeFinal +#undef RC_GetThreshold +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/multiarc/src/formats/7z/C/Ppmd8.c b/multiarc/src/formats/7z/C/Ppmd8.c old mode 100755 new mode 100644 index fda8b88a7..c6bdd8673 --- a/multiarc/src/formats/7z/C/Ppmd8.c +++ b/multiarc/src/formats/7z/C/Ppmd8.c @@ -1,5 +1,5 @@ /* Ppmd8.c -- PPMdI codec -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -14,7 +14,7 @@ This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ MY_ALIGN(16) static const Byte PPMD8_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; MY_ALIGN(16) -static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; +static const UInt16 PPMD8_kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; #define MAX_FREQ 124 #define UNIT_SIZE 12 @@ -33,7 +33,7 @@ static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x #define ONE_STATE(ctx) Ppmd8Context_OneState(ctx) #define SUFFIX(ctx) CTX((ctx)->Suffix) -typedef CPpmd8_Context * CTX_PTR; +typedef CPpmd8_Context * PPMD8_CTX_PTR; struct CPpmd8_Node_; @@ -114,7 +114,7 @@ BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) #define EMPTY_NODE 0xFFFFFFFF -static void InsertNode(CPpmd8 *p, void *node, unsigned indx) +static void Ppmd8_InsertNode(CPpmd8 *p, void *node, unsigned indx) { ((CPpmd8_Node *)node)->Stamp = EMPTY_NODE; ((CPpmd8_Node *)node)->Next = (CPpmd8_Node_Ref)p->FreeList[indx]; @@ -124,7 +124,7 @@ static void InsertNode(CPpmd8 *p, void *node, unsigned indx) } -static void *RemoveNode(CPpmd8 *p, unsigned indx) +static void *Ppmd8_RemoveNode(CPpmd8 *p, unsigned indx) { CPpmd8_Node *node = NODE((CPpmd8_Node_Ref)p->FreeList[indx]); p->FreeList[indx] = node->Next; @@ -134,16 +134,16 @@ static void *RemoveNode(CPpmd8 *p, unsigned indx) } -static void SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) +static void Ppmd8_SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) { unsigned i, nu = I2U(oldIndx) - I2U(newIndx); ptr = (Byte *)ptr + U2B(I2U(newIndx)); if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); + Ppmd8_InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); } - InsertNode(p, ptr, i); + Ppmd8_InsertNode(p, ptr, i); } @@ -159,7 +159,7 @@ static void SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx) -static void GlueFreeBlocks(CPpmd8 *p) +static void Ppmd8_GlueFreeBlocks(CPpmd8 *p) { /* we use first UInt32 field of 12-bytes UNITs as record type stamp @@ -239,27 +239,27 @@ static void GlueFreeBlocks(CPpmd8 *p) if (nu == 0) continue; for (; nu > 128; nu -= 128, node += 128) - InsertNode(p, node, PPMD_NUM_INDEXES - 1); + Ppmd8_InsertNode(p, node, PPMD_NUM_INDEXES - 1); if (I2U(i = U2I(nu)) != nu) { unsigned k = I2U(--i); - InsertNode(p, node + k, (unsigned)nu - k - 1); + Ppmd8_InsertNode(p, node + k, (unsigned)nu - k - 1); } - InsertNode(p, node, i); + Ppmd8_InsertNode(p, node, i); } } -MY_NO_INLINE -static void *AllocUnitsRare(CPpmd8 *p, unsigned indx) +Z7_NO_INLINE +static void *Ppmd8_AllocUnitsRare(CPpmd8 *p, unsigned indx) { unsigned i; if (p->GlueCount == 0) { - GlueFreeBlocks(p); + Ppmd8_GlueFreeBlocks(p); if (p->FreeList[indx] != 0) - return RemoveNode(p, indx); + return Ppmd8_RemoveNode(p, indx); } i = indx; @@ -277,17 +277,17 @@ static void *AllocUnitsRare(CPpmd8 *p, unsigned indx) while (p->FreeList[i] == 0); { - void *block = RemoveNode(p, i); - SplitBlock(p, block, i, indx); + void *block = Ppmd8_RemoveNode(p, i); + Ppmd8_SplitBlock(p, block, i, indx); return block; } } -static void *AllocUnits(CPpmd8 *p, unsigned indx) +static void *Ppmd8_AllocUnits(CPpmd8 *p, unsigned indx) { if (p->FreeList[indx] != 0) - return RemoveNode(p, indx); + return Ppmd8_RemoveNode(p, indx); { UInt32 numBytes = U2B(I2U(indx)); Byte *lo = p->LoUnit; @@ -297,13 +297,22 @@ static void *AllocUnits(CPpmd8 *p, unsigned indx) return lo; } } - return AllocUnitsRare(p, indx); + return Ppmd8_AllocUnitsRare(p, indx); } -#define MyMem12Cpy(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } +#define MEM_12_CPY(dest, src, num) \ + { UInt32 *d = (UInt32 *)(dest); \ + const UInt32 *z = (const UInt32 *)(src); \ + unsigned n = (num); \ + do { \ + d[0] = z[0]; \ + d[1] = z[1]; \ + d[2] = z[2]; \ + z += 3; \ + d += 3; \ + } while (--n); \ + } @@ -315,26 +324,26 @@ static void *ShrinkUnits(CPpmd8 *p, void *oldPtr, unsigned oldNU, unsigned newNU return oldPtr; if (p->FreeList[i1] != 0) { - void *ptr = RemoveNode(p, i1); - MyMem12Cpy(ptr, oldPtr, newNU); - InsertNode(p, oldPtr, i0); + void *ptr = Ppmd8_RemoveNode(p, i1); + MEM_12_CPY(ptr, oldPtr, newNU) + Ppmd8_InsertNode(p, oldPtr, i0); return ptr; } - SplitBlock(p, oldPtr, i0, i1); + Ppmd8_SplitBlock(p, oldPtr, i0, i1); return oldPtr; } static void FreeUnits(CPpmd8 *p, void *ptr, unsigned nu) { - InsertNode(p, ptr, U2I(nu)); + Ppmd8_InsertNode(p, ptr, U2I(nu)); } static void SpecialFreeUnit(CPpmd8 *p, void *ptr) { if ((Byte *)ptr != p->UnitsStart) - InsertNode(p, ptr, 0); + Ppmd8_InsertNode(p, ptr, 0); else { #ifdef PPMD8_FREEZE_SUPPORT @@ -352,10 +361,10 @@ static void *MoveUnitsUp(CPpmd8 *p, void *oldPtr, unsigned nu) void *ptr; if ((Byte *)oldPtr > p->UnitsStart + (1 << 14) || REF(oldPtr) > p->FreeList[indx]) return oldPtr; - ptr = RemoveNode(p, indx); - MyMem12Cpy(ptr, oldPtr, nu); + ptr = Ppmd8_RemoveNode(p, indx); + MEM_12_CPY(ptr, oldPtr, nu) if ((Byte *)oldPtr != p->UnitsStart) - InsertNode(p, oldPtr, indx); + Ppmd8_InsertNode(p, oldPtr, indx); else p->UnitsStart += U2B(I2U(indx)); return ptr; @@ -411,22 +420,22 @@ static void ExpandTextArea(CPpmd8 *p) #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) -static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) +static void Ppmd8State_SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) { - Ppmd_SET_SUCCESSOR(p, v); + Ppmd_SET_SUCCESSOR(p, v) } #define RESET_TEXT(offs) { p->Text = p->Base + p->AlignOffset + (offs); } -MY_NO_INLINE +Z7_NO_INLINE static -void RestartModel(CPpmd8 *p) +void Ppmd8_RestartModel(CPpmd8 *p) { unsigned i, k, m; memset(p->FreeList, 0, sizeof(p->FreeList)); memset(p->Stamps, 0, sizeof(p->Stamps)); - RESET_TEXT(0); + RESET_TEXT(0) p->HiUnit = p->Text + p->Size; p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; p->GlueCount = 0; @@ -436,8 +445,8 @@ void RestartModel(CPpmd8 *p) p->PrevSuccess = 0; { - CPpmd8_Context *mc = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ - CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + CPpmd8_Context *mc = (PPMD8_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* Ppmd8_AllocUnits(p, PPMD_NUM_INDEXES - 1); */ p->LoUnit += U2B(256 / 2); p->MaxContext = p->MinContext = mc; @@ -452,7 +461,7 @@ void RestartModel(CPpmd8 *p) { s->Symbol = (Byte)i; s->Freq = 1; - SetSuccessor(s, 0); + Ppmd8State_SetSuccessor(s, 0); } } @@ -475,7 +484,7 @@ void RestartModel(CPpmd8 *p) { unsigned r; UInt16 *dest = p->BinSumm[m] + k; - UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 1)); + const UInt16 val = (UInt16)(PPMD_BIN_SCALE - PPMD8_kInitBinEsc[k] / (i + 1)); for (r = 0; r < 64; r += 8) dest[r] = val; } @@ -507,7 +516,7 @@ void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod) { p->MaxOrder = maxOrder; p->RestoreMethod = restoreMethod; - RestartModel(p); + Ppmd8_RestartModel(p); } @@ -531,7 +540,7 @@ Refresh() is called when we remove some symbols (successors) in context. It increases Escape_Freq for sum of all removed symbols. */ -static void Refresh(CPpmd8 *p, CTX_PTR ctx, unsigned oldNU, unsigned scale) +static void Refresh(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned oldNU, unsigned scale) { unsigned i = ctx->NumStats, escFreq, sumFreq, flags; CPpmd_State *s = (CPpmd_State *)ShrinkUnits(p, STATS(ctx), oldNU, (i + 2) >> 1); @@ -581,7 +590,7 @@ static void Refresh(CPpmd8 *p, CTX_PTR ctx, unsigned oldNU, unsigned scale) } -static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) +static void SWAP_STATES(CPpmd_State *t1, CPpmd_State *t2) { CPpmd_State tmp = *t1; *t1 = *t2; @@ -597,7 +606,7 @@ CutOff() reduces contexts: if the (Union4.Stats) is close to (UnitsStart), it moves it up. */ -static CPpmd_Void_Ref CutOff(CPpmd8 *p, CTX_PTR ctx, unsigned order) +static CPpmd_Void_Ref CutOff(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned order) { int ns = ctx->NumStats; unsigned nu; @@ -613,7 +622,7 @@ static CPpmd_Void_Ref CutOff(CPpmd8 *p, CTX_PTR ctx, unsigned order) successor = CutOff(p, CTX(successor), order + 1); else successor = 0; - SetSuccessor(s, successor); + Ppmd8State_SetSuccessor(s, successor); if (successor || order <= 9) /* O_BOUND */ return REF(ctx); } @@ -630,11 +639,11 @@ static CPpmd_Void_Ref CutOff(CPpmd8 *p, CTX_PTR ctx, unsigned order) if ((UInt32)((Byte *)stats - p->UnitsStart) <= (1 << 14) && (CPpmd_Void_Ref)ctx->Union4.Stats <= p->FreeList[indx]) { - void *ptr = RemoveNode(p, indx); + void *ptr = Ppmd8_RemoveNode(p, indx); ctx->Union4.Stats = STATS_REF(ptr); - MyMem12Cpy(ptr, (const void *)stats, nu); + MEM_12_CPY(ptr, (const void *)stats, nu) if ((Byte *)stats != p->UnitsStart) - InsertNode(p, stats, indx); + Ppmd8_InsertNode(p, stats, indx); else p->UnitsStart += U2B(I2U(indx)); stats = ptr; @@ -656,16 +665,16 @@ static CPpmd_Void_Ref CutOff(CPpmd8 *p, CTX_PTR ctx, unsigned order) } else { - SwapStates(s, s2); - SetSuccessor(s2, 0); + SWAP_STATES(s, s2); + Ppmd8State_SetSuccessor(s2, 0); } } else { if (order < p->MaxOrder) - SetSuccessor(s, CutOff(p, CTX(successor), order + 1)); + Ppmd8State_SetSuccessor(s, CutOff(p, CTX(successor), order + 1)); else - SetSuccessor(s, 0); + Ppmd8State_SetSuccessor(s, 0); } } while (--s >= stats); @@ -711,7 +720,7 @@ RemoveBinContexts() removes Bin Context without Successor, if suffix of that context is also binary. */ -static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, CTX_PTR ctx, unsigned order) +static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned order) { if (!ctx->NumStats) { @@ -721,7 +730,7 @@ static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, CTX_PTR ctx, unsigned order) successor = RemoveBinContexts(p, CTX(successor), order + 1); else successor = 0; - SetSuccessor(s, successor); + Ppmd8State_SetSuccessor(s, successor); /* Suffix context can be removed already, since different (high-order) Successors may refer to same context. So we check Flags == 0xFF (Stamp == EMPTY_NODE) */ if (!successor && (!SUFFIX(ctx)->NumStats || SUFFIX(ctx)->Flags == 0xFF)) @@ -737,9 +746,9 @@ static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, CTX_PTR ctx, unsigned order) { CPpmd_Void_Ref successor = SUCCESSOR(s); if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder) - SetSuccessor(s, RemoveBinContexts(p, CTX(successor), order + 1)); + Ppmd8State_SetSuccessor(s, RemoveBinContexts(p, CTX(successor), order + 1)); else - SetSuccessor(s, 0); + Ppmd8State_SetSuccessor(s, 0); } while (--s >= STATS(ctx)); } @@ -767,15 +776,15 @@ static UInt32 GetUsedMemory(const CPpmd8 *p) #endif -static void RestoreModel(CPpmd8 *p, CTX_PTR ctxError +static void RestoreModel(CPpmd8 *p, PPMD8_CTX_PTR ctxError #ifdef PPMD8_FREEZE_SUPPORT - , CTX_PTR fSuccessor + , PPMD8_CTX_PTR fSuccessor #endif ) { - CTX_PTR c; + PPMD8_CTX_PTR c; CPpmd_State *s; - RESET_TEXT(0); + RESET_TEXT(0) // we go here in cases of error of allocation for context (c1) // Order(MinContext) < Order(ctxError) <= Order(MaxContext) @@ -831,7 +840,7 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR ctxError else #endif if (p->RestoreMethod == PPMD8_RESTORE_METHOD_RESTART || GetUsedMemory(p) < (p->Size >> 1)) - RestartModel(p); + Ppmd8_RestartModel(p); else { while (p->MaxContext->Suffix) @@ -850,8 +859,8 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR ctxError -MY_NO_INLINE -static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PTR c) +Z7_NO_INLINE +static PPMD8_CTX_PTR Ppmd8_CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, PPMD8_CTX_PTR c) { CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); @@ -927,15 +936,15 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT do { - CTX_PTR c1; + PPMD8_CTX_PTR c1; /* = AllocContext(p); */ if (p->HiUnit != p->LoUnit) - c1 = (CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); + c1 = (PPMD8_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); else if (p->FreeList[0] != 0) - c1 = (CTX_PTR)RemoveNode(p, 0); + c1 = (PPMD8_CTX_PTR)Ppmd8_RemoveNode(p, 0); else { - c1 = (CTX_PTR)AllocUnitsRare(p, 0); + c1 = (PPMD8_CTX_PTR)Ppmd8_AllocUnitsRare(p, 0); if (!c1) return NULL; } @@ -943,9 +952,9 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT c1->NumStats = 0; c1->Union2.State2.Symbol = newSym; c1->Union2.State2.Freq = newFreq; - SetSuccessor(ONE_STATE(c1), upBranch); + Ppmd8State_SetSuccessor(ONE_STATE(c1), upBranch); c1->Suffix = REF(c); - SetSuccessor(ps[--numPs], REF(c1)); + Ppmd8State_SetSuccessor(ps[--numPs], REF(c1)); c = c1; } while (numPs != 0); @@ -954,10 +963,10 @@ static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PT } -static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) +static PPMD8_CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, PPMD8_CTX_PTR c) { CPpmd_State *s = NULL; - CTX_PTR c1 = c; + PPMD8_CTX_PTR c1 = c; CPpmd_Void_Ref upBranch = REF(p->Text); #ifdef PPMD8_FREEZE_SUPPORT @@ -967,7 +976,7 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) ps[numPs++] = p->FoundState; #endif - SetSuccessor(p->FoundState, upBranch); + Ppmd8State_SetSuccessor(p->FoundState, upBranch); p->OrderFall++; for (;;) @@ -985,8 +994,8 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) #ifdef PPMD8_FREEZE_SUPPORT if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) { - do { SetSuccessor(ps[--numPs], REF(c)); } while (numPs); - RESET_TEXT(1); + do { Ppmd8State_SetSuccessor(ps[--numPs], REF(c)); } while (numPs); + RESET_TEXT(1) p->OrderFall = 1; } #endif @@ -1014,7 +1023,7 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) #ifdef PPMD8_FREEZE_SUPPORT ps[numPs++] = s; #endif - SetSuccessor(s, upBranch); + Ppmd8State_SetSuccessor(s, upBranch); p->OrderFall++; } @@ -1022,8 +1031,8 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) { c = CTX(SUCCESSOR(s)); - do { SetSuccessor(ps[--numPs], REF(c)); } while (numPs); - RESET_TEXT(1); + do { Ppmd8State_SetSuccessor(ps[--numPs], REF(c)); } while (numPs); + RESET_TEXT(1) p->OrderFall = 1; return c; } @@ -1031,15 +1040,15 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) #endif if (SUCCESSOR(s) <= upBranch) { - CTX_PTR successor; + PPMD8_CTX_PTR successor; CPpmd_State *s2 = p->FoundState; p->FoundState = s; - successor = CreateSuccessors(p, False, NULL, c); + successor = Ppmd8_CreateSuccessors(p, False, NULL, c); if (!successor) - SetSuccessor(s, 0); + Ppmd8State_SetSuccessor(s, 0); else - SetSuccessor(s, REF(successor)); + Ppmd8State_SetSuccessor(s, REF(successor)); p->FoundState = s2; } @@ -1047,7 +1056,7 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) CPpmd_Void_Ref successor = SUCCESSOR(s); if (p->OrderFall == 1 && c1 == p->MaxContext) { - SetSuccessor(p->FoundState, successor); + Ppmd8State_SetSuccessor(p->FoundState, successor); p->Text--; } if (successor == 0) @@ -1059,11 +1068,11 @@ static CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, CTX_PTR c) void Ppmd8_UpdateModel(CPpmd8 *p); -MY_NO_INLINE +Z7_NO_INLINE void Ppmd8_UpdateModel(CPpmd8 *p) { CPpmd_Void_Ref maxSuccessor, minSuccessor = SUCCESSOR(p->FoundState); - CTX_PTR c; + PPMD8_CTX_PTR c; unsigned s0, ns, fFreq = p->FoundState->Freq; Byte flag, fSymbol = p->FoundState->Symbol; { @@ -1096,7 +1105,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) if (s[0].Freq >= s[-1].Freq) { - SwapStates(&s[0], &s[-1]); + SWAP_STATES(&s[0], &s[-1]); s--; } } @@ -1112,14 +1121,14 @@ void Ppmd8_UpdateModel(CPpmd8 *p) c = p->MaxContext; if (p->OrderFall == 0 && minSuccessor) { - CTX_PTR cs = CreateSuccessors(p, True, s, p->MinContext); + PPMD8_CTX_PTR cs = Ppmd8_CreateSuccessors(p, True, s, p->MinContext); if (!cs) { - SetSuccessor(p->FoundState, 0); + Ppmd8State_SetSuccessor(p->FoundState, 0); RESTORE_MODEL(c, CTX(minSuccessor)); return; } - SetSuccessor(p->FoundState, REF(cs)); + Ppmd8State_SetSuccessor(p->FoundState, REF(cs)); p->MinContext = p->MaxContext = cs; return; } @@ -1141,7 +1150,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) if (!minSuccessor) { - CTX_PTR cs = ReduceOrder(p, s, p->MinContext); + PPMD8_CTX_PTR cs = ReduceOrder(p, s, p->MinContext); if (!cs) { RESTORE_MODEL(c, NULL); @@ -1151,7 +1160,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) } else if ((Byte *)Ppmd8_GetPtr(p, minSuccessor) < p->UnitsStart) { - CTX_PTR cs = CreateSuccessors(p, False, s, p->MinContext); + PPMD8_CTX_PTR cs = Ppmd8_CreateSuccessors(p, False, s, p->MinContext); if (!cs) { RESTORE_MODEL(c, NULL); @@ -1169,7 +1178,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) else if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE) { maxSuccessor = minSuccessor; - RESET_TEXT(0); + RESET_TEXT(0) p->OrderFall = 0; } #endif @@ -1215,11 +1224,11 @@ void Ppmd8_UpdateModel(CPpmd8 *p) if ((ns1 & 1) != 0) { /* Expand for one UNIT */ - unsigned oldNU = (ns1 + 1) >> 1; - unsigned i = U2I(oldNU); + const unsigned oldNU = (ns1 + 1) >> 1; + const unsigned i = U2I(oldNU); if (i != U2I((size_t)oldNU + 1)) { - void *ptr = AllocUnits(p, i + 1); + void *ptr = Ppmd8_AllocUnits(p, i + 1); void *oldPtr; if (!ptr) { @@ -1227,15 +1236,15 @@ void Ppmd8_UpdateModel(CPpmd8 *p) return; } oldPtr = STATS(c); - MyMem12Cpy(ptr, oldPtr, oldNU); - InsertNode(p, oldPtr, i); + MEM_12_CPY(ptr, oldPtr, oldNU) + Ppmd8_InsertNode(p, oldPtr, i); c->Union4.Stats = STATS_REF(ptr); } } sum = c->Union2.SummFreq; /* max increase of Escape_Freq is 1 here. an average increase is 1/3 per symbol */ - sum += (3 * ns1 + 1 < ns); + sum += (UInt32)(unsigned)(3 * ns1 + 1 < ns); /* original PPMdH uses 16-bit variable for (sum) here. But (sum < ???). Do we need to truncate (sum) to 16-bit */ // sum = (UInt16)sum; @@ -1243,7 +1252,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) else { - CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); + CPpmd_State *s = (CPpmd_State*)Ppmd8_AllocUnits(p, 0); if (!s) { RESTORE_MODEL(c, CTX(minSuccessor)); @@ -1255,7 +1264,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) s->Symbol = c->Union2.State2.Symbol; s->Successor_0 = c->Union4.State4.Successor_0; s->Successor_1 = c->Union4.State4.Successor_1; - // SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of + // Ppmd8State_SetSuccessor(s, c->Union4.Stats); // call it only for debug purposes to check the order of // (Successor_0 and Successor_1) in LE/BE. c->Union4.Stats = REF(s); if (freq < MAX_FREQ / 4 - 1) @@ -1265,7 +1274,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) s->Freq = (Byte)freq; - sum = freq + p->InitEsc + (ns > 2); // Ppmd8 (> 2) + sum = (UInt32)(freq + p->InitEsc + (ns > 2)); // Ppmd8 (> 2) } } @@ -1275,7 +1284,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) UInt32 sf = (UInt32)s0 + sum; s->Symbol = fSymbol; c->NumStats = (Byte)(ns1 + 1); - SetSuccessor(s, maxSuccessor); + Ppmd8State_SetSuccessor(s, maxSuccessor); c->Flags |= flag; if (cf < 6 * sf) { @@ -1299,8 +1308,8 @@ void Ppmd8_UpdateModel(CPpmd8 *p) -MY_NO_INLINE -static void Rescale(CPpmd8 *p) +Z7_NO_INLINE +static void Ppmd8_Rescale(CPpmd8 *p) { unsigned i, adder, sumFreq, escFreq; CPpmd_State *stats = STATS(p->MinContext); @@ -1389,7 +1398,7 @@ static void Rescale(CPpmd8 *p) *s = *stats; s->Freq = (Byte)freq; p->FoundState = s; - InsertNode(p, stats, U2I(n0)); + Ppmd8_InsertNode(p, stats, U2I(n0)); return; } @@ -1437,10 +1446,10 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) { // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ - unsigned summ = (UInt16)see->Summ; // & 0xFFFF - unsigned r = (summ >> see->Shift); + const unsigned summ = (UInt16)see->Summ; // & 0xFFFF + const unsigned r = (summ >> see->Shift); see->Summ = (UInt16)(summ - r); - *escFreq = r + (r == 0); + *escFreq = (UInt32)(r + (r == 0)); } } else @@ -1452,9 +1461,9 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) } -static void NextContext(CPpmd8 *p) +static void Ppmd8_NextContext(CPpmd8 *p) { - CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); + PPMD8_CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart) p->MaxContext = p->MinContext = c; else @@ -1471,12 +1480,12 @@ void Ppmd8_Update1(CPpmd8 *p) s->Freq = (Byte)freq; if (freq > s[-1].Freq) { - SwapStates(s, &s[-1]); + SWAP_STATES(s, &s[-1]); p->FoundState = --s; if (freq > MAX_FREQ) - Rescale(p); + Ppmd8_Rescale(p); } - NextContext(p); + Ppmd8_NextContext(p); } @@ -1485,15 +1494,15 @@ void Ppmd8_Update1_0(CPpmd8 *p) CPpmd_State *s = p->FoundState; CPpmd8_Context *mc = p->MinContext; unsigned freq = s->Freq; - unsigned summFreq = mc->Union2.SummFreq; + const unsigned summFreq = mc->Union2.SummFreq; p->PrevSuccess = (2 * freq >= summFreq); // Ppmd8 (>=) - p->RunLength += (int)p->PrevSuccess; + p->RunLength += (Int32)p->PrevSuccess; mc->Union2.SummFreq = (UInt16)(summFreq + 4); freq += 4; s->Freq = (Byte)freq; if (freq > MAX_FREQ) - Rescale(p); - NextContext(p); + Ppmd8_Rescale(p); + Ppmd8_NextContext(p); } @@ -1504,7 +1513,7 @@ void Ppmd8_UpdateBin(CPpmd8 *p) p->FoundState->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196) p->PrevSuccess = 1; p->RunLength++; - NextContext(p); + Ppmd8_NextContext(p); } */ @@ -1517,7 +1526,7 @@ void Ppmd8_Update2(CPpmd8 *p) p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4); s->Freq = (Byte)freq; if (freq > MAX_FREQ) - Rescale(p); + Ppmd8_Rescale(p); Ppmd8_UpdateModel(p); } @@ -1526,7 +1535,7 @@ void Ppmd8_Update2(CPpmd8 *p) GlueCount, and Glue method BinSum See / EscFreq - CreateSuccessors updates more suffix contexts + Ppmd8_CreateSuccessors updates more suffix contexts Ppmd8_UpdateModel consts. PrevSuccess Update @@ -1535,3 +1544,31 @@ void Ppmd8_Update2(CPpmd8 *p) (1 << 3) - there is symbol in Stats with (sym >= 0x40) in (1 << 4) - main symbol of context is (sym >= 0x40) */ + +#undef RESET_TEXT +#undef FLAG_RESCALED +#undef FLAG_PREV_HIGH +#undef HiBits_Prepare +#undef HiBits_Convert_3 +#undef HiBits_Convert_4 +#undef PPMD8_HiBitsFlag_3 +#undef PPMD8_HiBitsFlag_4 +#undef RESTORE_MODEL + +#undef MAX_FREQ +#undef UNIT_SIZE +#undef U2B +#undef U2I +#undef I2U + +#undef REF +#undef STATS_REF +#undef CTX +#undef STATS +#undef ONE_STATE +#undef SUFFIX +#undef NODE +#undef EMPTY_NODE +#undef MEM_12_CPY +#undef SUCCESSOR +#undef SWAP_STATES diff --git a/multiarc/src/formats/7z/C/Ppmd8.h b/multiarc/src/formats/7z/C/Ppmd8.h old mode 100755 new mode 100644 index fe93fe7cd..d5bb57e15 --- a/multiarc/src/formats/7z/C/Ppmd8.h +++ b/multiarc/src/formats/7z/C/Ppmd8.h @@ -1,11 +1,11 @@ /* Ppmd8.h -- Ppmd8 (PPMdI) compression codec -2021-04-13 : Igor Pavlov : Public domain +2023-04-02 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ -#ifndef __PPMD8_H -#define __PPMD8_H +#ifndef ZIP7_INC_PPMD8_H +#define ZIP7_INC_PPMD8_H #include "Ppmd.h" @@ -87,8 +87,8 @@ typedef struct UInt32 Low; union { - IByteIn *In; - IByteOut *Out; + IByteInPtr In; + IByteOutPtr Out; } Stream; Byte Indx2Units[PPMD_NUM_INDEXES + 2]; // +2 for alignment diff --git a/multiarc/src/formats/7z/C/Ppmd8Dec.c b/multiarc/src/formats/7z/C/Ppmd8Dec.c old mode 100755 new mode 100644 index d205de283..ff911674a --- a/multiarc/src/formats/7z/C/Ppmd8Dec.c +++ b/multiarc/src/formats/7z/C/Ppmd8Dec.c @@ -1,5 +1,5 @@ /* Ppmd8Dec.c -- Ppmd8 (PPMdI) Decoder -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -8,8 +8,8 @@ This code is based on: #include "Ppmd8.h" -#define kTop (1 << 24) -#define kBot (1 << 15) +#define kTop ((UInt32)1 << 24) +#define kBot ((UInt32)1 << 15) #define READ_BYTE(p) IByteIn_Read((p)->Stream.In) @@ -37,9 +37,9 @@ BoolInt Ppmd8_Init_RangeDec(CPpmd8 *p) #define R p -MY_FORCE_INLINE -// MY_NO_INLINE -static void RangeDec_Decode(CPpmd8 *p, UInt32 start, UInt32 size) +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd8_RD_Decode(CPpmd8 *p, UInt32 start, UInt32 size) { start *= R->Range; R->Low += start; @@ -48,17 +48,17 @@ static void RangeDec_Decode(CPpmd8 *p, UInt32 start, UInt32 size) RC_NORM_LOCAL(R) } -#define RC_Decode(start, size) RangeDec_Decode(p, start, size); -#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) -#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) +#define RC_Decode(start, size) Ppmd8_RD_Decode(p, start, size); +#define RC_DecodeFinal(start, size) RC_Decode(start, size) RC_NORM_REMOTE(R) +#define RC_GetThreshold(total) (R->Code / (R->Range /= (total))) #define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) -typedef CPpmd8_Context * CTX_PTR; +// typedef CPpmd8_Context * CTX_PTR; #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd8_UpdateModel(CPpmd8 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] int Ppmd8_DecodeSymbol(CPpmd8 *p) @@ -81,7 +81,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) if ((Int32)(count -= s->Freq) < 0) { Byte sym; - RC_DecodeFinal(0, s->Freq); + RC_DecodeFinal(0, s->Freq) p->FoundState = s; sym = s->Symbol; Ppmd8_Update1_0(p); @@ -96,7 +96,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) if ((Int32)(count -= (++s)->Freq) < 0) { Byte sym; - RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) p->FoundState = s; sym = s->Symbol; Ppmd8_Update1(p); @@ -109,10 +109,10 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) return PPMD8_SYM_ERROR; hiCnt -= count; - RC_Decode(hiCnt, summFreq - hiCnt); + RC_Decode(hiCnt, summFreq - hiCnt) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) // i = p->MinContext->NumStats - 1; // do { MASK((--s)->Symbol) = 0; } while (--i); { @@ -120,8 +120,8 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -152,7 +152,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) // Ppmd8_UpdateBin(p); { unsigned freq = s->Freq; - CTX_PTR c = CTX(SUCCESSOR(s)); + CPpmd8_Context *c = CTX(SUCCESSOR(s)); sym = s->Symbol; p->FoundState = s; p->PrevSuccess = 1; @@ -176,7 +176,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - size0; RC_NORM_LOCAL(R) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) MASK(Ppmd8Context_OneState(p->MinContext)->Symbol) = 0; p->PrevSuccess = 0; } @@ -209,17 +209,17 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) unsigned num2 = num / 2; num &= 1; - hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); s += num; p->MinContext = mc; do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; - hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); - hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); } while (--num2); } @@ -227,7 +227,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) see = Ppmd8_MakeEscFreq(p, numMasked, &freqSum); freqSum += hiCnt; freqSum2 = freqSum; - PPMD8_CORRECT_SUM_RANGE(R, freqSum2); + PPMD8_CORRECT_SUM_RANGE(R, freqSum2) count = RC_GetThreshold(freqSum2); @@ -235,7 +235,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) if (count < hiCnt) { Byte sym; - // Ppmd_See_Update(see); // new (see->Summ) value can overflow over 16-bits in some rare cases + // Ppmd_See_UPDATE(see) // new (see->Summ) value can overflow over 16-bits in some rare cases s = Ppmd8_GetStats(p, p->MinContext); hiCnt = count; @@ -243,15 +243,15 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) { for (;;) { - count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; } } s--; - RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq); + RC_DecodeFinal((hiCnt - count) - s->Freq, s->Freq) // new (see->Summ) value can overflow over 16-bits in some rare cases - Ppmd_See_Update(see); + Ppmd_See_UPDATE(see) p->FoundState = s; sym = s->Symbol; Ppmd8_Update2(p); @@ -261,7 +261,7 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) if (count >= freqSum2) return PPMD8_SYM_ERROR; - RC_Decode(hiCnt, freqSum2 - hiCnt); + RC_Decode(hiCnt, freqSum2 - hiCnt) // We increase (see->Summ) for sum of Freqs of all non_Masked symbols. // new (see->Summ) value can overflow over 16-bits in some rare cases @@ -277,3 +277,19 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) while (s != s2); } } + +#undef kTop +#undef kBot +#undef READ_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Decode +#undef RC_DecodeFinal +#undef RC_GetThreshold +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/multiarc/src/formats/7z/C/Ppmd8Enc.c b/multiarc/src/formats/7z/C/Ppmd8Enc.c old mode 100755 new mode 100644 index 32ff8052e..b0e34c45e --- a/multiarc/src/formats/7z/C/Ppmd8Enc.c +++ b/multiarc/src/formats/7z/C/Ppmd8Enc.c @@ -1,5 +1,5 @@ /* Ppmd8Enc.c -- Ppmd8 (PPMdI) Encoder -2021-04-13 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -8,8 +8,8 @@ This code is based on: #include "Ppmd8.h" -#define kTop (1 << 24) -#define kBot (1 << 15) +#define kTop ((UInt32)1 << 24) +#define kBot ((UInt32)1 << 15) #define WRITE_BYTE(p) IByteOut_Write(p->Stream.Out, (Byte)(p->Low >> 24)) @@ -54,13 +54,13 @@ void Ppmd8_Flush_RangeEnc(CPpmd8 *p) -MY_FORCE_INLINE -// MY_NO_INLINE -static void RangeEnc_Encode(CPpmd8 *p, UInt32 start, UInt32 size, UInt32 total) +Z7_FORCE_INLINE +// Z7_NO_INLINE +static void Ppmd8_RangeEnc_Encode(CPpmd8 *p, UInt32 start, UInt32 size, UInt32 total) { R->Low += start * (R->Range /= total); R->Range *= size; - RC_NORM_LOCAL(R); + RC_NORM_LOCAL(R) } @@ -72,19 +72,19 @@ static void RangeEnc_Encode(CPpmd8 *p, UInt32 start, UInt32 size, UInt32 total) -#define RC_Encode(start, size, total) RangeEnc_Encode(p, start, size, total); -#define RC_EncodeFinal(start, size, total) RC_Encode(start, size, total); RC_NORM_REMOTE(p); +#define RC_Encode(start, size, total) Ppmd8_RangeEnc_Encode(p, start, size, total); +#define RC_EncodeFinal(start, size, total) RC_Encode(start, size, total) RC_NORM_REMOTE(p) #define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref)) -typedef CPpmd8_Context * CTX_PTR; +// typedef CPpmd8_Context * CTX_PTR; #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd8_UpdateModel(CPpmd8 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] -// MY_FORCE_INLINE +// Z7_FORCE_INLINE // static void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) { @@ -104,7 +104,7 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) if (s->Symbol == symbol) { - RC_EncodeFinal(0, s->Freq, summFreq); + RC_EncodeFinal(0, s->Freq, summFreq) p->FoundState = s; Ppmd8_Update1_0(p); return; @@ -117,7 +117,7 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) if ((++s)->Symbol == symbol) { - RC_EncodeFinal(sum, s->Freq, summFreq); + RC_EncodeFinal(sum, s->Freq, summFreq) p->FoundState = s; Ppmd8_Update1(p); return; @@ -127,10 +127,10 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) while (--i); - RC_Encode(sum, summFreq - sum, summFreq); + RC_Encode(sum, summFreq - sum, summFreq) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) // MASK(s->Symbol) = 0; // i = p->MinContext->NumStats; // do { MASK((--s)->Symbol) = 0; } while (--i); @@ -139,8 +139,8 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -153,20 +153,20 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) UInt16 *prob = Ppmd8_GetBinSumm(p); CPpmd_State *s = Ppmd8Context_OneState(p->MinContext); UInt32 pr = *prob; - UInt32 bound = (R->Range >> 14) * pr; + const UInt32 bound = (R->Range >> 14) * pr; pr = PPMD_UPDATE_PROB_1(pr); if (s->Symbol == symbol) { *prob = (UInt16)(pr + (1 << PPMD_INT_BITS)); // RangeEnc_EncodeBit_0(p, bound); R->Range = bound; - RC_NORM(R); + RC_NORM(R) // p->FoundState = s; // Ppmd8_UpdateBin(p); { - unsigned freq = s->Freq; - CTX_PTR c = CTX(SUCCESSOR(s)); + const unsigned freq = s->Freq; + CPpmd8_Context *c = CTX(SUCCESSOR(s)); p->FoundState = s; p->PrevSuccess = 1; p->RunLength++; @@ -187,7 +187,7 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) R->Range = (R->Range & ~((UInt32)PPMD_BIN_SCALE - 1)) - bound; RC_NORM_LOCAL(R) - PPMD_SetAllBitsIn256Bytes(charMask); + PPMD_SetAllBitsIn256Bytes(charMask) MASK(s->Symbol) = 0; p->PrevSuccess = 0; } @@ -248,14 +248,14 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) do { - unsigned cur = s->Symbol; + const unsigned cur = s->Symbol; if ((int)cur == symbol) { - UInt32 low = sum; - UInt32 freq = s->Freq; + const UInt32 low = sum; + const UInt32 freq = s->Freq; unsigned num2; - Ppmd_See_Update(see); + Ppmd_See_UPDATE(see) p->FoundState = s; sum += escFreq; @@ -265,21 +265,20 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) if (num2 != 0) { s += i; - for (;;) + do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; sum += (s[-2].Freq & (unsigned)(MASK(sym0))); sum += (s[-1].Freq & (unsigned)(MASK(sym1))); - if (--num2 == 0) - break; } + while (--num2); } - PPMD8_CORRECT_SUM_RANGE(p, sum); + PPMD8_CORRECT_SUM_RANGE(p, sum) - RC_EncodeFinal(low, freq, sum); + RC_EncodeFinal(low, freq, sum) Ppmd8_Update2(p); return; } @@ -291,19 +290,19 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) { UInt32 total = sum + escFreq; see->Summ = (UInt16)(see->Summ + total); - PPMD8_CORRECT_SUM_RANGE(p, total); + PPMD8_CORRECT_SUM_RANGE(p, total) - RC_Encode(sum, total - sum, total); + RC_Encode(sum, total - sum, total) } { - CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); + const CPpmd_State *s2 = Ppmd8_GetStats(p, p->MinContext); s--; MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -312,3 +311,27 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) } } } + + + + + + + + + +#undef kTop +#undef kBot +#undef WRITE_BYTE +#undef RC_NORM_BASE +#undef RC_NORM_1 +#undef RC_NORM +#undef RC_NORM_LOCAL +#undef RC_NORM_REMOTE +#undef R +#undef RC_Encode +#undef RC_EncodeFinal + +#undef CTX +#undef SUCCESSOR +#undef MASK diff --git a/multiarc/src/formats/7z/C/Precomp.h b/multiarc/src/formats/7z/C/Precomp.h old mode 100755 new mode 100644 index e8ff8b40e..7747fdd74 --- a/multiarc/src/formats/7z/C/Precomp.h +++ b/multiarc/src/formats/7z/C/Precomp.h @@ -1,10 +1,127 @@ -/* Precomp.h -- StdAfx -2013-11-12 : Igor Pavlov : Public domain */ +/* Precomp.h -- precompilation file +2024-01-25 : Igor Pavlov : Public domain */ -#ifndef __7Z_PRECOMP_H -#define __7Z_PRECOMP_H +#ifndef ZIP7_INC_PRECOMP_H +#define ZIP7_INC_PRECOMP_H + +/* + this file must be included before another *.h files and before . + this file is included from the following files: + C\*.c + C\Util\*\Precomp.h <- C\Util\*\*.c + CPP\Common\Common.h <- *\StdAfx.h <- *\*.cpp + + this file can set the following macros: + Z7_LARGE_PAGES 1 + Z7_LONG_PATH 1 + Z7_WIN32_WINNT_MIN 0x0500 (or higher) : we require at least win2000+ for 7-Zip + _WIN32_WINNT 0x0500 (or higher) + WINVER _WIN32_WINNT + UNICODE 1 + _UNICODE 1 +*/ #include "Compiler.h" -/* #include "7zTypes.h" */ + +#ifdef _MSC_VER +// #pragma warning(disable : 4206) // nonstandard extension used : translation unit is empty +#if _MSC_VER >= 1912 +// #pragma warning(disable : 5039) // pointer or reference to potentially throwing function passed to 'extern "C"' function under - EHc.Undefined behavior may occur if this function throws an exception. +#endif +#endif + +/* +// for debug: +#define UNICODE 1 +#define _UNICODE 1 +#define _WIN32_WINNT 0x0500 // win2000 +#ifndef WINVER + #define WINVER _WIN32_WINNT +#endif +*/ + +#ifdef _WIN32 +/* + this "Precomp.h" file must be included before , + if we want to define _WIN32_WINNT before . +*/ + +#ifndef Z7_LARGE_PAGES +#ifndef Z7_NO_LARGE_PAGES +#define Z7_LARGE_PAGES 1 +#endif +#endif + +#ifndef Z7_LONG_PATH +#ifndef Z7_NO_LONG_PATH +#define Z7_LONG_PATH 1 +#endif +#endif + +#ifndef Z7_DEVICE_FILE +#ifndef Z7_NO_DEVICE_FILE +// #define Z7_DEVICE_FILE 1 +#endif +#endif + +// we don't change macros if included after +#ifndef _WINDOWS_ + +#ifndef Z7_WIN32_WINNT_MIN + #if defined(_M_ARM64) || defined(__aarch64__) + // #define Z7_WIN32_WINNT_MIN 0x0a00 // win10 + #define Z7_WIN32_WINNT_MIN 0x0600 // vista + #elif defined(_M_ARM) && defined(_M_ARMT) && defined(_M_ARM_NT) + // #define Z7_WIN32_WINNT_MIN 0x0602 // win8 + #define Z7_WIN32_WINNT_MIN 0x0600 // vista + #elif defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(_M_IA64) + #define Z7_WIN32_WINNT_MIN 0x0503 // win2003 + // #elif defined(_M_IX86) || defined(__i386__) + // #define Z7_WIN32_WINNT_MIN 0x0500 // win2000 + #else // x86 and another(old) systems + #define Z7_WIN32_WINNT_MIN 0x0500 // win2000 + // #define Z7_WIN32_WINNT_MIN 0x0502 // win2003 // for debug + #endif +#endif // Z7_WIN32_WINNT_MIN + + +#ifndef Z7_DO_NOT_DEFINE_WIN32_WINNT +#ifdef _WIN32_WINNT + // #error Stop_Compiling_Bad_WIN32_WINNT +#else + #ifndef Z7_NO_DEFINE_WIN32_WINNT +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define _WIN32_WINNT Z7_WIN32_WINNT_MIN +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER + #endif +#endif // _WIN32_WINNT + +#ifndef WINVER + #define WINVER _WIN32_WINNT +#endif +#endif // Z7_DO_NOT_DEFINE_WIN32_WINNT + + +#ifndef _MBCS +#ifndef Z7_NO_UNICODE +// UNICODE and _UNICODE are used by and by 7-zip code. + +#ifndef UNICODE +#define UNICODE 1 +#endif + +#ifndef _UNICODE +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#define _UNICODE 1 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // Z7_NO_UNICODE +#endif // _MBCS +#endif // _WINDOWS_ + +// #include "7zWindows.h" + +#endif // _WIN32 #endif diff --git a/multiarc/src/formats/7z/C/RotateDefs.h b/multiarc/src/formats/7z/C/RotateDefs.h old mode 100755 new mode 100644 index 8f01d1a6c..c16b4f8e6 --- a/multiarc/src/formats/7z/C/RotateDefs.h +++ b/multiarc/src/formats/7z/C/RotateDefs.h @@ -1,14 +1,14 @@ /* RotateDefs.h -- Rotate functions -2015-03-25 : Igor Pavlov : Public domain */ +2023-06-18 : Igor Pavlov : Public domain */ -#ifndef __ROTATE_DEFS_H -#define __ROTATE_DEFS_H +#ifndef ZIP7_INC_ROTATE_DEFS_H +#define ZIP7_INC_ROTATE_DEFS_H #ifdef _MSC_VER #include -/* don't use _rotl with MINGW. It can insert slow call to function. */ +/* don't use _rotl with old MINGW. It can insert slow call to function. */ /* #if (_MSC_VER >= 1200) */ #pragma intrinsic(_rotl) @@ -18,12 +18,32 @@ #define rotlFixed(x, n) _rotl((x), (n)) #define rotrFixed(x, n) _rotr((x), (n)) +#if (_MSC_VER >= 1300) +#define Z7_ROTL64(x, n) _rotl64((x), (n)) +#define Z7_ROTR64(x, n) _rotr64((x), (n)) +#else +#define Z7_ROTL64(x, n) (((x) << (n)) | ((x) >> (64 - (n)))) +#define Z7_ROTR64(x, n) (((x) >> (n)) | ((x) << (64 - (n)))) +#endif + #else /* new compilers can translate these macros to fast commands. */ +#if defined(__clang__) && (__clang_major__ >= 4) \ + || defined(__GNUC__) && (__GNUC__ >= 5) +/* GCC 4.9.0 and clang 3.5 can recognize more correct version: */ +#define rotlFixed(x, n) (((x) << (n)) | ((x) >> (-(n) & 31))) +#define rotrFixed(x, n) (((x) >> (n)) | ((x) << (-(n) & 31))) +#define Z7_ROTL64(x, n) (((x) << (n)) | ((x) >> (-(n) & 63))) +#define Z7_ROTR64(x, n) (((x) >> (n)) | ((x) << (-(n) & 63))) +#else +/* for old GCC / clang: */ #define rotlFixed(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) #define rotrFixed(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) +#define Z7_ROTL64(x, n) (((x) << (n)) | ((x) >> (64 - (n)))) +#define Z7_ROTR64(x, n) (((x) >> (n)) | ((x) << (64 - (n)))) +#endif #endif diff --git a/multiarc/src/formats/7z/C/Sha1.c b/multiarc/src/formats/7z/C/Sha1.c old mode 100755 new mode 100644 index 9665b5b55..4c9289289 --- a/multiarc/src/formats/7z/C/Sha1.c +++ b/multiarc/src/formats/7z/C/Sha1.c @@ -1,5 +1,5 @@ /* Sha1.c -- SHA-1 Hash -2021-07-13 : Igor Pavlov : Public domain +2024-03-01 : Igor Pavlov : Public domain This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -15,50 +15,50 @@ This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ l #endif #ifdef MY_CPU_X86_OR_AMD64 - #ifdef _MSC_VER - #if _MSC_VER >= 1200 - #define _SHA_SUPPORTED - #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define _SHA_SUPPORTED - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 8) // fix that check - #define _SHA_SUPPORTED - #endif - #elif defined(__INTEL_COMPILER) - #if (__INTEL_COMPILER >= 1800) // fix that check - #define _SHA_SUPPORTED - #endif + #if defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) \ + || defined(_MSC_VER) && (_MSC_VER >= 1200) + #define Z7_COMPILER_SHA1_SUPPORTED #endif -#elif defined(MY_CPU_ARM_OR_ARM64) - #ifdef _MSC_VER - #if _MSC_VER >= 1910 && _MSC_VER >= 1929 && _MSC_FULL_VER >= 192930037 - #define _SHA_SUPPORTED +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) \ + && (!defined(Z7_MSC_VER_ORIGINAL) || (_MSC_VER >= 1929) && (_MSC_FULL_VER >= 192930037)) + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define Z7_COMPILER_SHA1_SUPPORTED + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) + #define Z7_COMPILER_SHA1_SUPPORTED #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define _SHA_SUPPORTED #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define _SHA_SUPPORTED #endif #endif #endif -void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); -#ifdef _SHA_SUPPORTED - void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +#ifdef Z7_COMPILER_SHA1_SUPPORTED + void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); - static SHA1_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS = Sha1_UpdateBlocks; - static SHA1_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS_HW; + static SHA1_FUNC_UPDATE_BLOCKS g_SHA1_FUNC_UPDATE_BLOCKS = Sha1_UpdateBlocks; + static SHA1_FUNC_UPDATE_BLOCKS g_SHA1_FUNC_UPDATE_BLOCKS_HW; - #define UPDATE_BLOCKS(p) p->func_UpdateBlocks + #define SHA1_UPDATE_BLOCKS(p) p->func_UpdateBlocks #else - #define UPDATE_BLOCKS(p) Sha1_UpdateBlocks + #define SHA1_UPDATE_BLOCKS(p) Sha1_UpdateBlocks #endif @@ -66,16 +66,16 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) { SHA1_FUNC_UPDATE_BLOCKS func = Sha1_UpdateBlocks; - #ifdef _SHA_SUPPORTED + #ifdef Z7_COMPILER_SHA1_SUPPORTED if (algo != SHA1_ALGO_SW) { if (algo == SHA1_ALGO_DEFAULT) - func = g_FUNC_UPDATE_BLOCKS; + func = g_SHA1_FUNC_UPDATE_BLOCKS; else { if (algo != SHA1_ALGO_HW) return False; - func = g_FUNC_UPDATE_BLOCKS_HW; + func = g_SHA1_FUNC_UPDATE_BLOCKS_HW; if (!func) return False; } @@ -91,21 +91,22 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) /* define it for speed optimization */ -// #define _SHA1_UNROLL +// #define Z7_SHA1_UNROLL // allowed unroll steps: (1, 2, 4, 5, 20) -#ifdef _SHA1_UNROLL +#undef Z7_SHA1_BIG_W +#ifdef Z7_SHA1_UNROLL #define STEP_PRE 20 #define STEP_MAIN 20 #else - #define _SHA1_BIG_W + #define Z7_SHA1_BIG_W #define STEP_PRE 5 #define STEP_MAIN 5 #endif -#ifdef _SHA1_BIG_W +#ifdef Z7_SHA1_BIG_W #define kNumW 80 #define w(i) W[i] #else @@ -150,11 +151,11 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) */ #define M5(i, fx, wx0, wx1) \ - T5 ( a,b,c,d,e, fx, wx0((i) ) ); \ - T5 ( e,a,b,c,d, fx, wx1((i)+1) ); \ - T5 ( d,e,a,b,c, fx, wx1((i)+2) ); \ - T5 ( c,d,e,a,b, fx, wx1((i)+3) ); \ - T5 ( b,c,d,e,a, fx, wx1((i)+4) ); \ + T5 ( a,b,c,d,e, fx, wx0((i) ) ) \ + T5 ( e,a,b,c,d, fx, wx1((i)+1) ) \ + T5 ( d,e,a,b,c, fx, wx1((i)+2) ) \ + T5 ( c,d,e,a,b, fx, wx1((i)+3) ) \ + T5 ( b,c,d,e,a, fx, wx1((i)+4) ) \ #define R5(i, fx, wx) \ M5 ( i, fx, wx, wx) \ @@ -163,17 +164,17 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) #if STEP_PRE > 5 #define R20_START \ - R5 ( 0, f0, w0); \ - R5 ( 5, f0, w0); \ - R5 ( 10, f0, w0); \ - M5 ( 15, f0, w0, w1); \ + R5 ( 0, f0, w0) \ + R5 ( 5, f0, w0) \ + R5 ( 10, f0, w0) \ + M5 ( 15, f0, w0, w1) \ #elif STEP_PRE == 5 #define R20_START \ { size_t i; for (i = 0; i < 15; i += STEP_PRE) \ - { R5(i, f0, w0); } } \ - M5 ( 15, f0, w0, w1); \ + { R5(i, f0, w0) } } \ + M5 ( 15, f0, w0, w1) \ #else @@ -187,8 +188,8 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) #define R20_START \ { size_t i; for (i = 0; i < 16; i += STEP_PRE) \ - { R_PRE(i, f0, w0); } } \ - R4 ( 16, f0, w1); \ + { R_PRE(i, f0, w0) } } \ + R4 ( 16, f0, w1) \ #endif @@ -197,10 +198,10 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) #if STEP_MAIN > 5 #define R20(ii, fx) \ - R5 ( (ii) , fx, w1); \ - R5 ( (ii) + 5 , fx, w1); \ - R5 ( (ii) + 10, fx, w1); \ - R5 ( (ii) + 15, fx, w1); \ + R5 ( (ii) , fx, w1) \ + R5 ( (ii) + 5 , fx, w1) \ + R5 ( (ii) + 10, fx, w1) \ + R5 ( (ii) + 15, fx, w1) \ #else @@ -216,7 +217,7 @@ BoolInt Sha1_SetFunction(CSha1 *p, unsigned algo) #define R20(ii, fx) \ { size_t i; for (i = (ii); i < (ii) + 20; i += STEP_MAIN) \ - { R_MAIN(i, fx, w1); } } \ + { R_MAIN(i, fx, w1) } } \ #endif @@ -235,8 +236,8 @@ void Sha1_InitState(CSha1 *p) void Sha1_Init(CSha1 *p) { p->func_UpdateBlocks = - #ifdef _SHA_SUPPORTED - g_FUNC_UPDATE_BLOCKS; + #ifdef Z7_COMPILER_SHA1_SUPPORTED + g_SHA1_FUNC_UPDATE_BLOCKS; #else NULL; #endif @@ -244,8 +245,8 @@ void Sha1_Init(CSha1 *p) } -MY_NO_INLINE -void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks) +Z7_NO_INLINE +void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks) { UInt32 a, b, c, d, e; UInt32 W[kNumW]; @@ -266,9 +267,9 @@ void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t nu #endif R20_START - R20(20, f1); - R20(40, f2); - R20(60, f3); + R20(20, f1) + R20(40, f2) + R20(60, f3) a += state[0]; b += state[1]; @@ -288,7 +289,7 @@ void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t nu } -#define Sha1_UpdateBlock(p) UPDATE_BLOCKS(p)(p->state, p->buffer, 1) +#define Sha1_UpdateBlock(p) SHA1_UPDATE_BLOCKS(p)(p->state, p->buffer, 1) void Sha1_Update(CSha1 *p, const Byte *data, size_t size) { @@ -318,7 +319,7 @@ void Sha1_Update(CSha1 *p, const Byte *data, size_t size) } { size_t numBlocks = size >> 6; - UPDATE_BLOCKS(p)(p->state, data, numBlocks); + SHA1_UPDATE_BLOCKS(p)(p->state, data, numBlocks); size &= 0x3F; if (size == 0) return; @@ -361,18 +362,18 @@ void Sha1_Final(CSha1 *p, Byte *digest) memset(&p->buffer[pos], 0, (64 - 8) - pos); { - UInt64 numBits = (p->count << 3); - SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)); - SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); + const UInt64 numBits = (p->count << 3); + SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)) + SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)) } Sha1_UpdateBlock(p); - SetBe32(digest, p->state[0]); - SetBe32(digest + 4, p->state[1]); - SetBe32(digest + 8, p->state[2]); - SetBe32(digest + 12, p->state[3]); - SetBe32(digest + 16, p->state[4]); + SetBe32(digest, p->state[0]) + SetBe32(digest + 4, p->state[1]) + SetBe32(digest + 8, p->state[2]) + SetBe32(digest + 12, p->state[3]) + SetBe32(digest + 16, p->state[4]) @@ -384,10 +385,10 @@ void Sha1_Final(CSha1 *p, Byte *digest) void Sha1_PrepareBlock(const CSha1 *p, Byte *block, unsigned size) { const UInt64 numBits = (p->count + size) << 3; - SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 2], (UInt32)(numBits >> 32)); - SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 1], (UInt32)(numBits)); + SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 2], (UInt32)(numBits >> 32)) + SetBe32(&((UInt32 *)(void *)block)[SHA1_NUM_BLOCK_WORDS - 1], (UInt32)(numBits)) // SetBe32((UInt32 *)(block + size), 0x80000000); - SetUi32((UInt32 *)(void *)(block + size), 0x80); + SetUi32((UInt32 *)(void *)(block + size), 0x80) size += 4; while (size != (SHA1_NUM_BLOCK_WORDS - 2) * 4) { @@ -407,19 +408,19 @@ void Sha1_GetBlockDigest(const CSha1 *p, const Byte *data, Byte *destDigest) st[3] = p->state[3]; st[4] = p->state[4]; - UPDATE_BLOCKS(p)(st, data, 1); + SHA1_UPDATE_BLOCKS(p)(st, data, 1); - SetBe32(destDigest + 0 , st[0]); - SetBe32(destDigest + 1 * 4, st[1]); - SetBe32(destDigest + 2 * 4, st[2]); - SetBe32(destDigest + 3 * 4, st[3]); - SetBe32(destDigest + 4 * 4, st[4]); + SetBe32(destDigest + 0 , st[0]) + SetBe32(destDigest + 1 * 4, st[1]) + SetBe32(destDigest + 2 * 4, st[2]) + SetBe32(destDigest + 3 * 4, st[3]) + SetBe32(destDigest + 4 * 4, st[4]) } -void Sha1Prepare() +void Sha1Prepare(void) { - #ifdef _SHA_SUPPORTED + #ifdef Z7_COMPILER_SHA1_SUPPORTED SHA1_FUNC_UPDATE_BLOCKS f, f_hw; f = Sha1_UpdateBlocks; f_hw = NULL; @@ -435,7 +436,7 @@ void Sha1Prepare() #endif { // printf("\n========== HW SHA1 ======== \n"); - #if defined(MY_CPU_ARM_OR_ARM64) && defined(_MSC_VER) + #if 0 && defined(MY_CPU_ARM_OR_ARM64) && defined(_MSC_VER) /* there was bug in MSVC compiler for ARM64 -O2 before version VS2019 16.10 (19.29.30037). It generated incorrect SHA-1 code. 21.03 : we test sha1-hardware code at runtime initialization */ @@ -467,7 +468,31 @@ void Sha1Prepare() f = f_hw = Sha1_UpdateBlocks_HW; } } - g_FUNC_UPDATE_BLOCKS = f; - g_FUNC_UPDATE_BLOCKS_HW = f_hw; + g_SHA1_FUNC_UPDATE_BLOCKS = f; + g_SHA1_FUNC_UPDATE_BLOCKS_HW = f_hw; #endif } + +#undef kNumW +#undef w +#undef w0 +#undef w1 +#undef f0 +#undef f1 +#undef f2 +#undef f3 +#undef T1 +#undef T5 +#undef M5 +#undef R1 +#undef R2 +#undef R4 +#undef R5 +#undef R20_START +#undef R_PRE +#undef R_MAIN +#undef STEP_PRE +#undef STEP_MAIN +#undef Z7_SHA1_BIG_W +#undef Z7_SHA1_UNROLL +#undef Z7_COMPILER_SHA1_SUPPORTED diff --git a/multiarc/src/formats/7z/C/Sha1.h b/multiarc/src/formats/7z/C/Sha1.h old mode 100755 new mode 100644 index 345a816ac..fecd9d310 --- a/multiarc/src/formats/7z/C/Sha1.h +++ b/multiarc/src/formats/7z/C/Sha1.h @@ -1,8 +1,8 @@ /* Sha1.h -- SHA-1 Hash -2021-02-08 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __7Z_SHA1_H -#define __7Z_SHA1_H +#ifndef ZIP7_INC_SHA1_H +#define ZIP7_INC_SHA1_H #include "7zTypes.h" @@ -14,7 +14,7 @@ EXTERN_C_BEGIN #define SHA1_BLOCK_SIZE (SHA1_NUM_BLOCK_WORDS * 4) #define SHA1_DIGEST_SIZE (SHA1_NUM_DIGEST_WORDS * 4) -typedef void (MY_FAST_CALL *SHA1_FUNC_UPDATE_BLOCKS)(UInt32 state[5], const Byte *data, size_t numBlocks); +typedef void (Z7_FASTCALL *SHA1_FUNC_UPDATE_BLOCKS)(UInt32 state[5], const Byte *data, size_t numBlocks); /* if (the system supports different SHA1 code implementations) @@ -34,9 +34,9 @@ typedef struct { SHA1_FUNC_UPDATE_BLOCKS func_UpdateBlocks; UInt64 count; - UInt64 __pad_2[2]; + UInt64 _pad_2[2]; UInt32 state[SHA1_NUM_DIGEST_WORDS]; - UInt32 __pad_3[3]; + UInt32 _pad_3[3]; Byte buffer[SHA1_BLOCK_SIZE]; } CSha1; @@ -62,7 +62,7 @@ void Sha1_Final(CSha1 *p, Byte *digest); void Sha1_PrepareBlock(const CSha1 *p, Byte *block, unsigned size); void Sha1_GetBlockDigest(const CSha1 *p, const Byte *data, Byte *destDigest); -// void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); +// void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); /* call Sha1Prepare() once at program start. diff --git a/multiarc/src/formats/7z/C/Sha1Opt.c b/multiarc/src/formats/7z/C/Sha1Opt.c old mode 100755 new mode 100644 index 61da0b114..4e835f1fa --- a/multiarc/src/formats/7z/C/Sha1Opt.c +++ b/multiarc/src/formats/7z/C/Sha1Opt.c @@ -1,7 +1,9 @@ /* Sha1Opt.c -- SHA-1 optimized code for SHA-1 hardware instructions -2021-04-01 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "Compiler.h" +#include "CpuArch.h" #if defined(_MSC_VER) #if (_MSC_VER < 1900) && (_MSC_VER >= 1200) @@ -9,59 +11,54 @@ #endif #endif -#include "CpuArch.h" +// #define Z7_USE_HW_SHA_STUB // for debug #ifdef MY_CPU_X86_OR_AMD64 - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check + #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) // fix that check #define USE_HW_SHA - #ifndef __SHA__ - #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) - #if defined(_MSC_VER) - // SSSE3: for clang-cl: - #include - #define __SHA__ - #endif - #endif - #pragma clang diagnostic ignored "-Wvector-conversion" - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 8) // fix that check + #elif defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) #define USE_HW_SHA - #ifndef __SHA__ + #if !defined(_INTEL_COMPILER) + // icc defines __GNUC__, but icc doesn't support __attribute__(__target__) + #if !defined(__SHA__) || !defined(__SSSE3__) #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) - // #pragma GCC target("sha,ssse3") #endif - #endif - #elif defined(__INTEL_COMPILER) - #if (__INTEL_COMPILER >= 1800) // fix that check - #define USE_HW_SHA - #endif + #endif #elif defined(_MSC_VER) #ifdef USE_MY_MM #define USE_VER_MIN 1300 #else - #define USE_VER_MIN 1910 + #define USE_VER_MIN 1900 #endif - #if _MSC_VER >= USE_VER_MIN + #if (_MSC_VER >= USE_VER_MIN) #define USE_HW_SHA + #else + #define Z7_USE_HW_SHA_STUB #endif #endif // #endif // MY_CPU_X86_OR_AMD64 +#ifndef USE_HW_SHA + // #define Z7_USE_HW_SHA_STUB // for debug +#endif #ifdef USE_HW_SHA // #pragma message("Sha1 HW") -// #include -#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +// sse/sse2/ssse3: +#include +// sha*: #include -#else -#include -#if defined(_MSC_VER) && (_MSC_VER >= 1600) -// #include -#endif +#if defined (__clang__) && defined(_MSC_VER) + // #if !defined(__SSSE3__) + // #endif + #if !defined(__SHA__) + #include + #endif +#else #ifdef USE_MY_MM #include "My_mm.h" @@ -87,37 +84,37 @@ SHA1 uses: _mm_sha1* */ -#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); + #define XOR_SI128(dest, src) dest = _mm_xor_si128(dest, src); #define SHUFFLE_EPI8(dest, mask) dest = _mm_shuffle_epi8(dest, mask); #define SHUFFLE_EPI32(dest, mask) dest = _mm_shuffle_epi32(dest, mask); - -#define SHA1_RND4(abcd, e0, f) abcd = _mm_sha1rnds4_epu32(abcd, e0, f); -#define SHA1_NEXTE(e, m) e = _mm_sha1nexte_epu32(e, m); - - - - - -#define SHA1_MSG1(dest, src) dest = _mm_sha1msg1_epu32(dest, src); -#define SHA1_MSG2(dest, src) dest = _mm_sha1msg2_epu32(dest, src); +#ifdef __clang__ +#define SHA1_RNDS4_RET_TYPE_CAST (__m128i) +#else +#define SHA1_RNDS4_RET_TYPE_CAST +#endif +#define SHA1_RND4(abcd, e0, f) abcd = SHA1_RNDS4_RET_TYPE_CAST _mm_sha1rnds4_epu32(abcd, e0, f); +#define SHA1_NEXTE(e, m) e = _mm_sha1nexte_epu32(e, m); +#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); +#define SHA1_MSG1(dest, src) dest = _mm_sha1msg1_epu32(dest, src); +#define SHA1_MSG2(dest, src) dest = _mm_sha1msg2_epu32(dest, src); #define LOAD_SHUFFLE(m, k) \ m = _mm_loadu_si128((const __m128i *)(const void *)(data + (k) * 16)); \ - SHUFFLE_EPI8(m, mask); \ + SHUFFLE_EPI8(m, mask) \ #define SM1(m0, m1, m2, m3) \ - SHA1_MSG1(m0, m1); \ + SHA1_MSG1(m0, m1) \ #define SM2(m0, m1, m2, m3) \ - XOR_SI128(m3, m1); \ - SHA1_MSG2(m3, m2); \ + XOR_SI128(m3, m1) \ + SHA1_MSG2(m3, m2) \ #define SM3(m0, m1, m2, m3) \ - XOR_SI128(m3, m1); \ + XOR_SI128(m3, m1) \ SM1(m0, m1, m2, m3) \ - SHA1_MSG2(m3, m2); \ + SHA1_MSG2(m3, m2) \ #define NNN(m0, m1, m2, m3) @@ -139,9 +136,9 @@ SHA1 uses: #define R4(k, e0, e1, m0, m1, m2, m3, OP) \ e1 = abcd; \ - SHA1_RND4(abcd, e0, (k) / 5); \ - SHA1_NEXTE(e1, m1); \ - OP(m0, m1, m2, m3); \ + SHA1_RND4(abcd, e0, (k) / 5) \ + SHA1_NEXTE(e1, m1) \ + OP(m0, m1, m2, m3) \ #define R16(k, mx, OP0, OP1, OP2, OP3) \ R4 ( (k)*4+0, e0,e1, m0,m1,m2,m3, OP0 ) \ @@ -150,18 +147,18 @@ SHA1 uses: R4 ( (k)*4+3, e1,e0, m3,mx,m1,m2, OP3 ) \ #define PREPARE_STATE \ - SHUFFLE_EPI32 (abcd, 0x1B); \ - SHUFFLE_EPI32 (e0, 0x1B); \ + SHUFFLE_EPI32 (abcd, 0x1B) \ + SHUFFLE_EPI32 (e0, 0x1B) \ -void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); #ifdef ATTRIB_SHA ATTRIB_SHA #endif -void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) { const __m128i mask = _mm_set_epi32(0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f); @@ -190,15 +187,15 @@ void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t LOAD_SHUFFLE (m2, 2) LOAD_SHUFFLE (m3, 3) - ADD_EPI32(e0, m0); + ADD_EPI32(e0, m0) - R16 ( 0, m0, SM1, SM3, SM3, SM3 ); - R16 ( 1, m0, SM3, SM3, SM3, SM3 ); - R16 ( 2, m0, SM3, SM3, SM3, SM3 ); - R16 ( 3, m0, SM3, SM3, SM3, SM3 ); - R16 ( 4, e2, SM2, NNN, NNN, NNN ); + R16 ( 0, m0, SM1, SM3, SM3, SM3 ) + R16 ( 1, m0, SM3, SM3, SM3, SM3 ) + R16 ( 2, m0, SM3, SM3, SM3, SM3 ) + R16 ( 3, m0, SM3, SM3, SM3, SM3 ) + R16 ( 4, e2, SM2, NNN, NNN, NNN ) - ADD_EPI32(abcd, abcd_save); + ADD_EPI32(abcd, abcd_save) data += 64; } @@ -212,46 +209,124 @@ void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t #endif // USE_HW_SHA -#elif defined(MY_CPU_ARM_OR_ARM64) - #if defined(__ARM_NEON) || defined(__ARM_NEON__) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) \ + && (!defined(Z7_MSC_VER_ORIGINAL) || (_MSC_VER >= 1929) && (_MSC_FULL_VER >= 192930037)) + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_SHA + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_SHA #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_SHA #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_SHA #endif #endif - #endif + #ifdef USE_HW_SHA // #pragma message("=== Sha1 HW === ") +// __ARM_FEATURE_CRYPTO macro is deprecated in favor of the finer grained feature macro __ARM_FEATURE_SHA2 #if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_SHA __attribute__((__target__("crypto"))) +#else #define ATTRIB_SHA __attribute__((__target__("+crypto"))) +#endif #else +#if defined(__clang__) && (__clang_major__ >= 1) + #define ATTRIB_SHA __attribute__((__target__("armv8-a,sha2"))) +#else #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif #endif +#endif #else // _MSC_VER // for arm32 #define _ARM_USE_NEW_NEON_INTRINSICS #endif -#if defined(_MSC_VER) && defined(MY_CPU_ARM64) + + + + +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else + + + + + + + + + +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_SHA2 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + #include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_SHA2) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_SHA2 + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") #endif +#endif // Z7_MSC_VER_ORIGINAL + typedef uint32x4_t v128; // typedef __n128 v128; // MSVC +// the bug in clang 3.8.1: +// __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); +#if defined(__clang__) && (__clang_major__ <= 9) +#pragma GCC diagnostic ignored "-Wvector-conversion" +#endif #ifdef MY_CPU_BE #define MY_rev32_for_LE(x) @@ -266,19 +341,19 @@ typedef uint32x4_t v128; m = LOAD_128((data + (k) * 16)); \ MY_rev32_for_LE(m); \ -#define SU0(dest, src2, src3) dest = vsha1su0q_u32(dest, src2, src3); -#define SU1(dest, src) dest = vsha1su1q_u32(dest, src); -#define C(e) abcd = vsha1cq_u32(abcd, e, t); -#define P(e) abcd = vsha1pq_u32(abcd, e, t); -#define M(e) abcd = vsha1mq_u32(abcd, e, t); +#define SU0(dest, src2, src3) dest = vsha1su0q_u32(dest, src2, src3) +#define SU1(dest, src) dest = vsha1su1q_u32(dest, src) +#define C(e) abcd = vsha1cq_u32(abcd, e, t) +#define P(e) abcd = vsha1pq_u32(abcd, e, t) +#define M(e) abcd = vsha1mq_u32(abcd, e, t) #define H(e) e = vsha1h_u32(vgetq_lane_u32(abcd, 0)) #define T(m, c) t = vaddq_u32(m, c) -void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); #ifdef ATTRIB_SHA ATTRIB_SHA #endif -void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) { v128 abcd; v128 c0, c1, c2, c3; @@ -347,18 +422,19 @@ void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t #endif // MY_CPU_ARM_OR_ARM64 -#ifndef USE_HW_SHA - +#if !defined(USE_HW_SHA) && defined(Z7_USE_HW_SHA_STUB) // #error Stop_Compiling_UNSUPPORTED_SHA // #include -// #include "Sha1.h" -void MY_FAST_CALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); -#pragma message("Sha1 HW-SW stub was used") -void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); -void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) +// #include "Sha1.h" +// #if defined(_MSC_VER) +#pragma message("Sha1 HW-SW stub was used") +// #endif +void Z7_FASTCALL Sha1_UpdateBlocks (UInt32 state[5], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) { Sha1_UpdateBlocks(state, data, numBlocks); /* @@ -369,5 +445,28 @@ void MY_FAST_CALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t return; */ } - #endif + +#undef SU0 +#undef SU1 +#undef C +#undef P +#undef M +#undef H +#undef T +#undef MY_rev32_for_LE +#undef NNN +#undef LOAD_128 +#undef STORE_128 +#undef LOAD_SHUFFLE +#undef SM1 +#undef SM2 +#undef SM3 +#undef NNN +#undef R4 +#undef R16 +#undef PREPARE_STATE +#undef USE_HW_SHA +#undef ATTRIB_SHA +#undef USE_VER_MIN +#undef Z7_USE_HW_SHA_STUB diff --git a/multiarc/src/formats/7z/C/Sha256.c b/multiarc/src/formats/7z/C/Sha256.c old mode 100755 new mode 100644 index 8b3983ea7..14d3be9c6 --- a/multiarc/src/formats/7z/C/Sha256.c +++ b/multiarc/src/formats/7z/C/Sha256.c @@ -1,5 +1,5 @@ /* Sha256.c -- SHA-256 Hash -2021-04-01 : Igor Pavlov : Public domain +2024-03-01 : Igor Pavlov : Public domain This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -15,50 +15,50 @@ This code is based on public domain code from Wei Dai's Crypto++ library. */ #endif #ifdef MY_CPU_X86_OR_AMD64 - #ifdef _MSC_VER - #if _MSC_VER >= 1200 - #define _SHA_SUPPORTED - #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define _SHA_SUPPORTED - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 8) // fix that check - #define _SHA_SUPPORTED - #endif - #elif defined(__INTEL_COMPILER) - #if (__INTEL_COMPILER >= 1800) // fix that check - #define _SHA_SUPPORTED - #endif + #if defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) \ + || defined(_MSC_VER) && (_MSC_VER >= 1200) + #define Z7_COMPILER_SHA256_SUPPORTED #endif -#elif defined(MY_CPU_ARM_OR_ARM64) - #ifdef _MSC_VER - #if _MSC_VER >= 1910 - #define _SHA_SUPPORTED +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define Z7_COMPILER_SHA256_SUPPORTED + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) + #define Z7_COMPILER_SHA256_SUPPORTED #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define _SHA_SUPPORTED #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define _SHA_SUPPORTED #endif #endif #endif -void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); -#ifdef _SHA_SUPPORTED - void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +#ifdef Z7_COMPILER_SHA256_SUPPORTED + void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); - static SHA256_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS = Sha256_UpdateBlocks; - static SHA256_FUNC_UPDATE_BLOCKS g_FUNC_UPDATE_BLOCKS_HW; + static SHA256_FUNC_UPDATE_BLOCKS g_SHA256_FUNC_UPDATE_BLOCKS = Sha256_UpdateBlocks; + static SHA256_FUNC_UPDATE_BLOCKS g_SHA256_FUNC_UPDATE_BLOCKS_HW; - #define UPDATE_BLOCKS(p) p->func_UpdateBlocks + #define SHA256_UPDATE_BLOCKS(p) p->func_UpdateBlocks #else - #define UPDATE_BLOCKS(p) Sha256_UpdateBlocks + #define SHA256_UPDATE_BLOCKS(p) Sha256_UpdateBlocks #endif @@ -66,16 +66,16 @@ BoolInt Sha256_SetFunction(CSha256 *p, unsigned algo) { SHA256_FUNC_UPDATE_BLOCKS func = Sha256_UpdateBlocks; - #ifdef _SHA_SUPPORTED + #ifdef Z7_COMPILER_SHA256_SUPPORTED if (algo != SHA256_ALGO_SW) { if (algo == SHA256_ALGO_DEFAULT) - func = g_FUNC_UPDATE_BLOCKS; + func = g_SHA256_FUNC_UPDATE_BLOCKS; else { if (algo != SHA256_ALGO_HW) return False; - func = g_FUNC_UPDATE_BLOCKS_HW; + func = g_SHA256_FUNC_UPDATE_BLOCKS_HW; if (!func) return False; } @@ -92,17 +92,18 @@ BoolInt Sha256_SetFunction(CSha256 *p, unsigned algo) /* define it for speed optimization */ -#ifdef _SFX +#ifdef Z7_SFX #define STEP_PRE 1 #define STEP_MAIN 1 #else #define STEP_PRE 2 #define STEP_MAIN 4 - // #define _SHA256_UNROLL + // #define Z7_SHA256_UNROLL #endif +#undef Z7_SHA256_BIG_W #if STEP_MAIN != 16 - #define _SHA256_BIG_W + #define Z7_SHA256_BIG_W #endif @@ -124,8 +125,8 @@ void Sha256_InitState(CSha256 *p) void Sha256_Init(CSha256 *p) { p->func_UpdateBlocks = - #ifdef _SHA_SUPPORTED - g_FUNC_UPDATE_BLOCKS; + #ifdef Z7_COMPILER_SHA256_SUPPORTED + g_SHA256_FUNC_UPDATE_BLOCKS; #else NULL; #endif @@ -145,7 +146,7 @@ void Sha256_Init(CSha256 *p) #define blk2_main(j, i) s1(w(j, (i)-2)) + w(j, (i)-7) + s0(w(j, (i)-15)) -#ifdef _SHA256_BIG_W +#ifdef Z7_SHA256_BIG_W // we use +i instead of +(i) to change the order to solve CLANG compiler warning for signed/unsigned. #define w(j, i) W[(size_t)(j) + i] #define blk2(j, i) (w(j, i) = w(j, (i)-16) + blk2_main(j, i)) @@ -176,7 +177,7 @@ void Sha256_Init(CSha256 *p) #define R1_PRE(i) T1( W_PRE, i) #define R1_MAIN(i) T1( W_MAIN, i) -#if (!defined(_SHA256_UNROLL) || STEP_MAIN < 8) && (STEP_MAIN >= 4) +#if (!defined(Z7_SHA256_UNROLL) || STEP_MAIN < 8) && (STEP_MAIN >= 4) #define R2_MAIN(i) \ R1_MAIN(i) \ R1_MAIN(i + 1) \ @@ -185,7 +186,7 @@ void Sha256_Init(CSha256 *p) -#if defined(_SHA256_UNROLL) && STEP_MAIN >= 8 +#if defined(Z7_SHA256_UNROLL) && STEP_MAIN >= 8 #define T4( a,b,c,d,e,f,g,h, wx, i) \ h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + wx(i); \ @@ -223,8 +224,6 @@ void Sha256_Init(CSha256 *p) #endif -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); - // static extern MY_ALIGN(64) const UInt32 SHA256_K_ARRAY[64]; @@ -252,11 +251,11 @@ const UInt32 SHA256_K_ARRAY[64] = { #define K SHA256_K_ARRAY -MY_NO_INLINE -void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks) +Z7_NO_INLINE +void Z7_FASTCALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks) { UInt32 W - #ifdef _SHA256_BIG_W + #ifdef Z7_SHA256_BIG_W [64]; #else [16]; @@ -266,7 +265,7 @@ void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t UInt32 a,b,c,d,e,f,g,h; - #if !defined(_SHA256_UNROLL) || (STEP_MAIN <= 4) || (STEP_PRE <= 4) + #if !defined(Z7_SHA256_UNROLL) || (STEP_MAIN <= 4) || (STEP_PRE <= 4) UInt32 tmp; #endif @@ -297,12 +296,12 @@ void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t #else - R1_PRE(0); + R1_PRE(0) #if STEP_PRE >= 2 - R1_PRE(1); + R1_PRE(1) #if STEP_PRE >= 4 - R1_PRE(2); - R1_PRE(3); + R1_PRE(2) + R1_PRE(3) #endif #endif @@ -311,32 +310,32 @@ void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t for (j = 16; j < 64; j += STEP_MAIN) { - #if defined(_SHA256_UNROLL) && STEP_MAIN >= 8 + #if defined(Z7_SHA256_UNROLL) && STEP_MAIN >= 8 #if STEP_MAIN < 8 - R4_MAIN(0); + R4_MAIN(0) #else - R8_MAIN(0); + R8_MAIN(0) #if STEP_MAIN == 16 - R8_MAIN(8); + R8_MAIN(8) #endif #endif #else - R1_MAIN(0); + R1_MAIN(0) #if STEP_MAIN >= 2 - R1_MAIN(1); + R1_MAIN(1) #if STEP_MAIN >= 4 - R2_MAIN(2); + R2_MAIN(2) #if STEP_MAIN >= 8 - R2_MAIN(4); - R2_MAIN(6); + R2_MAIN(4) + R2_MAIN(6) #if STEP_MAIN >= 16 - R2_MAIN(8); - R2_MAIN(10); - R2_MAIN(12); - R2_MAIN(14); + R2_MAIN(8) + R2_MAIN(10) + R2_MAIN(12) + R2_MAIN(14) #endif #endif #endif @@ -367,7 +366,7 @@ void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t #undef s1 #undef K -#define Sha256_UpdateBlock(p) UPDATE_BLOCKS(p)(p->state, p->buffer, 1) +#define Sha256_UpdateBlock(p) SHA256_UPDATE_BLOCKS(p)(p->state, p->buffer, 1) void Sha256_Update(CSha256 *p, const Byte *data, size_t size) { @@ -397,7 +396,7 @@ void Sha256_Update(CSha256 *p, const Byte *data, size_t size) } { size_t numBlocks = size >> 6; - UPDATE_BLOCKS(p)(p->state, data, numBlocks); + SHA256_UPDATE_BLOCKS(p)(p->state, data, numBlocks); size &= 0x3F; if (size == 0) return; @@ -441,8 +440,8 @@ void Sha256_Final(CSha256 *p, Byte *digest) { UInt64 numBits = (p->count << 3); - SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)); - SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); + SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)) + SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)) } Sha256_UpdateBlock(p); @@ -451,8 +450,8 @@ void Sha256_Final(CSha256 *p, Byte *digest) { UInt32 v0 = p->state[i]; UInt32 v1 = p->state[(size_t)i + 1]; - SetBe32(digest , v0); - SetBe32(digest + 4, v1); + SetBe32(digest , v0) + SetBe32(digest + 4, v1) digest += 8; } @@ -460,9 +459,9 @@ void Sha256_Final(CSha256 *p, Byte *digest) } -void Sha256Prepare() +void Sha256Prepare(void) { - #ifdef _SHA_SUPPORTED + #ifdef Z7_COMPILER_SHA256_SUPPORTED SHA256_FUNC_UPDATE_BLOCKS f, f_hw; f = Sha256_UpdateBlocks; f_hw = NULL; @@ -480,7 +479,36 @@ void Sha256Prepare() // printf("\n========== HW SHA256 ======== \n"); f = f_hw = Sha256_UpdateBlocks_HW; } - g_FUNC_UPDATE_BLOCKS = f; - g_FUNC_UPDATE_BLOCKS_HW = f_hw; + g_SHA256_FUNC_UPDATE_BLOCKS = f; + g_SHA256_FUNC_UPDATE_BLOCKS_HW = f_hw; #endif } + +#undef S0 +#undef S1 +#undef s0 +#undef s1 +#undef Ch +#undef Maj +#undef W_MAIN +#undef W_PRE +#undef w +#undef blk2_main +#undef blk2 +#undef T1 +#undef T4 +#undef T8 +#undef R1_PRE +#undef R1_MAIN +#undef R2_MAIN +#undef R4 +#undef R4_PRE +#undef R4_MAIN +#undef R8 +#undef R8_PRE +#undef R8_MAIN +#undef STEP_PRE +#undef STEP_MAIN +#undef Z7_SHA256_BIG_W +#undef Z7_SHA256_UNROLL +#undef Z7_COMPILER_SHA256_SUPPORTED diff --git a/multiarc/src/formats/7z/C/Sha256.h b/multiarc/src/formats/7z/C/Sha256.h old mode 100755 new mode 100644 index aa38501e5..9e0422320 --- a/multiarc/src/formats/7z/C/Sha256.h +++ b/multiarc/src/formats/7z/C/Sha256.h @@ -1,8 +1,8 @@ /* Sha256.h -- SHA-256 Hash -2021-01-01 : Igor Pavlov : Public domain */ +2023-04-02 : Igor Pavlov : Public domain */ -#ifndef __7Z_SHA256_H -#define __7Z_SHA256_H +#ifndef ZIP7_INC_SHA256_H +#define ZIP7_INC_SHA256_H #include "7zTypes.h" @@ -14,7 +14,7 @@ EXTERN_C_BEGIN #define SHA256_BLOCK_SIZE (SHA256_NUM_BLOCK_WORDS * 4) #define SHA256_DIGEST_SIZE (SHA256_NUM_DIGEST_WORDS * 4) -typedef void (MY_FAST_CALL *SHA256_FUNC_UPDATE_BLOCKS)(UInt32 state[8], const Byte *data, size_t numBlocks); +typedef void (Z7_FASTCALL *SHA256_FUNC_UPDATE_BLOCKS)(UInt32 state[8], const Byte *data, size_t numBlocks); /* if (the system supports different SHA256 code implementations) @@ -34,7 +34,7 @@ typedef struct { SHA256_FUNC_UPDATE_BLOCKS func_UpdateBlocks; UInt64 count; - UInt64 __pad_2[2]; + UInt64 _pad_2[2]; UInt32 state[SHA256_NUM_DIGEST_WORDS]; Byte buffer[SHA256_BLOCK_SIZE]; @@ -62,7 +62,7 @@ void Sha256_Final(CSha256 *p, Byte *digest); -// void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); +// void Z7_FASTCALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); /* call Sha256Prepare() once at program start. diff --git a/multiarc/src/formats/7z/C/Sha256Opt.c b/multiarc/src/formats/7z/C/Sha256Opt.c old mode 100755 new mode 100644 index 2022abab1..eb3816664 --- a/multiarc/src/formats/7z/C/Sha256Opt.c +++ b/multiarc/src/formats/7z/C/Sha256Opt.c @@ -1,7 +1,9 @@ /* Sha256Opt.c -- SHA-256 optimized code for SHA-256 hardware instructions -2021-04-01 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "Compiler.h" +#include "CpuArch.h" #if defined(_MSC_VER) #if (_MSC_VER < 1900) && (_MSC_VER >= 1200) @@ -9,59 +11,54 @@ #endif #endif -#include "CpuArch.h" +// #define Z7_USE_HW_SHA_STUB // for debug #ifdef MY_CPU_X86_OR_AMD64 - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check + #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) // fix that check #define USE_HW_SHA - #ifndef __SHA__ - #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) - #if defined(_MSC_VER) - // SSSE3: for clang-cl: - #include - #define __SHA__ - #endif - #endif - - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 8) // fix that check + #elif defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) #define USE_HW_SHA - #ifndef __SHA__ + #if !defined(_INTEL_COMPILER) + // icc defines __GNUC__, but icc doesn't support __attribute__(__target__) + #if !defined(__SHA__) || !defined(__SSSE3__) #define ATTRIB_SHA __attribute__((__target__("sha,ssse3"))) - // #pragma GCC target("sha,ssse3") #endif - #endif - #elif defined(__INTEL_COMPILER) - #if (__INTEL_COMPILER >= 1800) // fix that check - #define USE_HW_SHA - #endif + #endif #elif defined(_MSC_VER) #ifdef USE_MY_MM #define USE_VER_MIN 1300 #else - #define USE_VER_MIN 1910 + #define USE_VER_MIN 1900 #endif - #if _MSC_VER >= USE_VER_MIN + #if (_MSC_VER >= USE_VER_MIN) #define USE_HW_SHA + #else + #define Z7_USE_HW_SHA_STUB #endif #endif // #endif // MY_CPU_X86_OR_AMD64 +#ifndef USE_HW_SHA + // #define Z7_USE_HW_SHA_STUB // for debug +#endif #ifdef USE_HW_SHA // #pragma message("Sha256 HW") -// #include -#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +// sse/sse2/ssse3: +#include +// sha*: #include -#else -#include -#if defined(_MSC_VER) && (_MSC_VER >= 1600) -// #include -#endif +#if defined (__clang__) && defined(_MSC_VER) + // #if !defined(__SSSE3__) + // #endif + #if !defined(__SHA__) + #include + #endif +#else #ifdef USE_MY_MM #include "My_mm.h" @@ -98,9 +95,9 @@ const UInt32 SHA256_K_ARRAY[64]; #define K SHA256_K_ARRAY -#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); -#define SHA256_MSG1(dest, src) dest = _mm_sha256msg1_epu32(dest, src); -#define SHA25G_MSG2(dest, src) dest = _mm_sha256msg2_epu32(dest, src); +#define ADD_EPI32(dest, src) dest = _mm_add_epi32(dest, src); +#define SHA256_MSG1(dest, src) dest = _mm_sha256msg1_epu32(dest, src); +#define SHA25G_MSG2(dest, src) dest = _mm_sha256msg2_epu32(dest, src); #define LOAD_SHUFFLE(m, k) \ @@ -112,7 +109,7 @@ const UInt32 SHA256_K_ARRAY[64]; #define SM2(g0, g1, g2, g3) \ tmp = _mm_alignr_epi8(g1, g0, 4); \ - ADD_EPI32(g2, tmp); \ + ADD_EPI32(g2, tmp) \ SHA25G_MSG2(g2, g1); \ // #define LS0(k, g0, g1, g2, g3) LOAD_SHUFFLE(g0, k) @@ -138,16 +135,16 @@ const UInt32 SHA256_K_ARRAY[64]; // We use scheme with 3 rounds ahead for SHA256_MSG1 / 2 rounds ahead for SHA256_MSG2 #define R4(k, g0, g1, g2, g3, OP0, OP1) \ - RND2_0(g0, k); \ - OP0(g0, g1, g2, g3); \ - RND2_1; \ - OP1(g0, g1, g2, g3); \ + RND2_0(g0, k) \ + OP0(g0, g1, g2, g3) \ + RND2_1 \ + OP1(g0, g1, g2, g3) \ #define R16(k, OP0, OP1, OP2, OP3, OP4, OP5, OP6, OP7) \ - R4 ( (k)*4+0, m0, m1, m2, m3, OP0, OP1 ) \ - R4 ( (k)*4+1, m1, m2, m3, m0, OP2, OP3 ) \ - R4 ( (k)*4+2, m2, m3, m0, m1, OP4, OP5 ) \ - R4 ( (k)*4+3, m3, m0, m1, m2, OP6, OP7 ) \ + R4 ( (k)*4+0, m0,m1,m2,m3, OP0, OP1 ) \ + R4 ( (k)*4+1, m1,m2,m3,m0, OP2, OP3 ) \ + R4 ( (k)*4+2, m2,m3,m0,m1, OP4, OP5 ) \ + R4 ( (k)*4+3, m3,m0,m1,m2, OP6, OP7 ) \ #define PREPARE_STATE \ tmp = _mm_shuffle_epi32(state0, 0x1B); /* abcd */ \ @@ -157,11 +154,11 @@ const UInt32 SHA256_K_ARRAY[64]; state1 = _mm_unpackhi_epi64(state1, tmp); /* abef */ \ -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); #ifdef ATTRIB_SHA ATTRIB_SHA #endif -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) { const __m128i mask = _mm_set_epi32(0x0c0d0e0f, 0x08090a0b, 0x04050607, 0x00010203); __m128i tmp; @@ -192,13 +189,13 @@ void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size - R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); - R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); - R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); - R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ) + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ) + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ) + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ) - ADD_EPI32(state0, state0_save); - ADD_EPI32(state1, state1_save); + ADD_EPI32(state0, state0_save) + ADD_EPI32(state1, state1_save) data += 64; } @@ -212,44 +209,117 @@ void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size #endif // USE_HW_SHA -#elif defined(MY_CPU_ARM_OR_ARM64) - #if defined(__ARM_NEON) || defined(__ARM_NEON__) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_SHA + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_SHA #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_SHA #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_SHA #endif #endif - #endif + #ifdef USE_HW_SHA // #pragma message("=== Sha256 HW === ") + #if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_SHA __attribute__((__target__("crypto"))) +#else #define ATTRIB_SHA __attribute__((__target__("+crypto"))) +#endif #else +#if defined(__clang__) && (__clang_major__ >= 1) + #define ATTRIB_SHA __attribute__((__target__("armv8-a,sha2"))) +#else #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif #endif +#endif #else // _MSC_VER // for arm32 #define _ARM_USE_NEW_NEON_INTRINSICS #endif -#if defined(_MSC_VER) && defined(MY_CPU_ARM64) + + + + +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else + + + + + + + + + +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_SHA2 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + #include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_SHA2) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_SHA2 + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") #endif +#endif // Z7_MSC_VER_ORIGINAL + typedef uint32x4_t v128; // typedef __n128 v128; // MSVC @@ -298,11 +368,11 @@ const UInt32 SHA256_K_ARRAY[64]; R4 ( (k)*4+3, m3, m0, m1, m2, OP6, OP7 ) \ -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); #ifdef ATTRIB_SHA ATTRIB_SHA #endif -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) { v128 state0, state1; @@ -326,10 +396,10 @@ void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size LOAD_SHUFFLE (m2, 2) LOAD_SHUFFLE (m3, 3) - R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); - R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); - R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); - R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ) + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ) + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ) + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ) state0 = vaddq_u32(state0, state0_save); state1 = vaddq_u32(state1, state1_save); @@ -347,18 +417,19 @@ void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size #endif // MY_CPU_ARM_OR_ARM64 -#ifndef USE_HW_SHA - +#if !defined(USE_HW_SHA) && defined(Z7_USE_HW_SHA_STUB) // #error Stop_Compiling_UNSUPPORTED_SHA // #include - +// We can compile this file with another C compiler, +// or we can compile asm version. +// So we can generate real code instead of this stub function. // #include "Sha256.h" -void MY_FAST_CALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); - +// #if defined(_MSC_VER) #pragma message("Sha256 HW-SW stub was used") - -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); -void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) +// #endif +void Z7_FASTCALL Sha256_UpdateBlocks (UInt32 state[8], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); +void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) { Sha256_UpdateBlocks(state, data, numBlocks); /* @@ -369,5 +440,28 @@ void MY_FAST_CALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size return; */ } - #endif + + + +#undef K +#undef RND2 +#undef RND2_0 +#undef RND2_1 + +#undef MY_rev32_for_LE +#undef NNN +#undef LOAD_128 +#undef STORE_128 +#undef LOAD_SHUFFLE +#undef SM1 +#undef SM2 + +#undef NNN +#undef R4 +#undef R16 +#undef PREPARE_STATE +#undef USE_HW_SHA +#undef ATTRIB_SHA +#undef USE_VER_MIN +#undef Z7_USE_HW_SHA_STUB diff --git a/multiarc/src/formats/7z/C/Sort.c b/multiarc/src/formats/7z/C/Sort.c old mode 100755 new mode 100644 diff --git a/multiarc/src/formats/7z/C/Sort.h b/multiarc/src/formats/7z/C/Sort.h old mode 100755 new mode 100644 index 2e2963a23..1817b652f --- a/multiarc/src/formats/7z/C/Sort.h +++ b/multiarc/src/formats/7z/C/Sort.h @@ -1,8 +1,8 @@ /* Sort.h -- Sort functions -2014-04-05 : Igor Pavlov : Public domain */ +2023-03-05 : Igor Pavlov : Public domain */ -#ifndef __7Z_SORT_H -#define __7Z_SORT_H +#ifndef ZIP7_INC_SORT_H +#define ZIP7_INC_SORT_H #include "7zTypes.h" diff --git a/multiarc/src/formats/7z/C/SwapBytes.c b/multiarc/src/formats/7z/C/SwapBytes.c new file mode 100644 index 000000000..9290592b7 --- /dev/null +++ b/multiarc/src/formats/7z/C/SwapBytes.c @@ -0,0 +1,835 @@ +/* SwapBytes.c -- Byte Swap conversion filter +2024-03-01 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Compiler.h" +#include "CpuArch.h" +#include "RotateDefs.h" +#include "SwapBytes.h" + +typedef UInt16 CSwapUInt16; +typedef UInt32 CSwapUInt32; + +// #define k_SwapBytes_Mode_BASE 0 + +#ifdef MY_CPU_X86_OR_AMD64 + +#define k_SwapBytes_Mode_SSE2 1 +#define k_SwapBytes_Mode_SSSE3 2 +#define k_SwapBytes_Mode_AVX2 3 + + // #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1900) + #if defined(__clang__) && (__clang_major__ >= 4) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40701) + #define k_SwapBytes_Mode_MAX k_SwapBytes_Mode_AVX2 + #define SWAP_ATTRIB_SSE2 __attribute__((__target__("sse2"))) + #define SWAP_ATTRIB_SSSE3 __attribute__((__target__("ssse3"))) + #define SWAP_ATTRIB_AVX2 __attribute__((__target__("avx2"))) + #elif defined(_MSC_VER) + #if (_MSC_VER == 1900) + #pragma warning(disable : 4752) // found Intel(R) Advanced Vector Extensions; consider using /arch:AVX + #endif + #if (_MSC_VER >= 1900) + #define k_SwapBytes_Mode_MAX k_SwapBytes_Mode_AVX2 + #elif (_MSC_VER >= 1500) // (VS2008) + #define k_SwapBytes_Mode_MAX k_SwapBytes_Mode_SSSE3 + #elif (_MSC_VER >= 1310) // (VS2003) + #define k_SwapBytes_Mode_MAX k_SwapBytes_Mode_SSE2 + #endif + #endif // _MSC_VER + +/* +// for debug +#ifdef k_SwapBytes_Mode_MAX +#undef k_SwapBytes_Mode_MAX +#endif +*/ + +#ifndef k_SwapBytes_Mode_MAX +#define k_SwapBytes_Mode_MAX 0 +#endif + +#if (k_SwapBytes_Mode_MAX != 0) && defined(MY_CPU_AMD64) + #define k_SwapBytes_Mode_MIN k_SwapBytes_Mode_SSE2 +#else + #define k_SwapBytes_Mode_MIN 0 +#endif + +#if (k_SwapBytes_Mode_MAX >= k_SwapBytes_Mode_AVX2) + #define USE_SWAP_AVX2 +#endif +#if (k_SwapBytes_Mode_MAX >= k_SwapBytes_Mode_SSSE3) + #define USE_SWAP_SSSE3 +#endif +#if (k_SwapBytes_Mode_MAX >= k_SwapBytes_Mode_SSE2) + #define USE_SWAP_128 +#endif + +#if k_SwapBytes_Mode_MAX <= k_SwapBytes_Mode_MIN || !defined(USE_SWAP_128) +#define FORCE_SWAP_MODE +#endif + + +#ifdef USE_SWAP_128 +/* + MMX + SSE + SSE2 + SSE3 + SSSE3 + SSE4.1 + SSE4.2 + SSE4A + AES + AVX, AVX2, FMA +*/ + +#include // sse2 +// typedef __m128i v128; + +#define SWAP2_128(i) { \ + const __m128i v = *(const __m128i *)(const void *)(items + (i) * 8); \ + *( __m128i *)( void *)(items + (i) * 8) = \ + _mm_or_si128( \ + _mm_slli_epi16(v, 8), \ + _mm_srli_epi16(v, 8)); } +// _mm_or_si128() has more ports to execute than _mm_add_epi16(). + +static +#ifdef SWAP_ATTRIB_SSE2 +SWAP_ATTRIB_SSE2 +#endif +void +Z7_FASTCALL +SwapBytes2_128(CSwapUInt16 *items, const CSwapUInt16 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP2_128(0) SWAP2_128(1) items += 2 * 8; + SWAP2_128(0) SWAP2_128(1) items += 2 * 8; + } + while (items != lim); +} + +/* +// sse2 +#define SWAP4_128_pack(i) { \ + __m128i v = *(const __m128i *)(const void *)(items + (i) * 4); \ + __m128i v0 = _mm_unpacklo_epi8(v, mask); \ + __m128i v1 = _mm_unpackhi_epi8(v, mask); \ + v0 = _mm_shufflelo_epi16(v0, 0x1b); \ + v1 = _mm_shufflelo_epi16(v1, 0x1b); \ + v0 = _mm_shufflehi_epi16(v0, 0x1b); \ + v1 = _mm_shufflehi_epi16(v1, 0x1b); \ + *(__m128i *)(void *)(items + (i) * 4) = _mm_packus_epi16(v0, v1); } + +static +#ifdef SWAP_ATTRIB_SSE2 +SWAP_ATTRIB_SSE2 +#endif +void +Z7_FASTCALL +SwapBytes4_128_pack(CSwapUInt32 *items, const CSwapUInt32 *lim) +{ + const __m128i mask = _mm_setzero_si128(); + // const __m128i mask = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, 0); + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP4_128_pack(0); items += 1 * 4; + // SWAP4_128_pack(0); SWAP4_128_pack(1); items += 2 * 4; + } + while (items != lim); +} + +// sse2 +#define SWAP4_128_shift(i) { \ + __m128i v = *(const __m128i *)(const void *)(items + (i) * 4); \ + __m128i v2; \ + v2 = _mm_or_si128( \ + _mm_slli_si128(_mm_and_si128(v, mask), 1), \ + _mm_and_si128(_mm_srli_si128(v, 1), mask)); \ + v = _mm_or_si128( \ + _mm_slli_epi32(v, 24), \ + _mm_srli_epi32(v, 24)); \ + *(__m128i *)(void *)(items + (i) * 4) = _mm_or_si128(v2, v); } + +static +#ifdef SWAP_ATTRIB_SSE2 +SWAP_ATTRIB_SSE2 +#endif +void +Z7_FASTCALL +SwapBytes4_128_shift(CSwapUInt32 *items, const CSwapUInt32 *lim) +{ + #define M1 0xff00 + const __m128i mask = _mm_set_epi32(M1, M1, M1, M1); + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + // SWAP4_128_shift(0) SWAP4_128_shift(1) items += 2 * 4; + // SWAP4_128_shift(0) SWAP4_128_shift(1) items += 2 * 4; + SWAP4_128_shift(0); items += 1 * 4; + } + while (items != lim); +} +*/ + + +#if defined(USE_SWAP_SSSE3) || defined(USE_SWAP_AVX2) + +#define SWAP_SHUF_REV_SEQ_2_VALS(v) (v)+1, (v) +#define SWAP_SHUF_REV_SEQ_4_VALS(v) (v)+3, (v)+2, (v)+1, (v) + +#define SWAP2_SHUF_MASK_16_BYTES \ + SWAP_SHUF_REV_SEQ_2_VALS (0 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (1 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (2 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (3 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (4 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (5 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (6 * 2), \ + SWAP_SHUF_REV_SEQ_2_VALS (7 * 2) + +#define SWAP4_SHUF_MASK_16_BYTES \ + SWAP_SHUF_REV_SEQ_4_VALS (0 * 4), \ + SWAP_SHUF_REV_SEQ_4_VALS (1 * 4), \ + SWAP_SHUF_REV_SEQ_4_VALS (2 * 4), \ + SWAP_SHUF_REV_SEQ_4_VALS (3 * 4) + +#if defined(USE_SWAP_AVX2) +/* if we use 256_BIT_INIT_MASK, each static array mask will be larger for 16 bytes */ +// #define SWAP_USE_256_BIT_INIT_MASK +#endif + +#if defined(SWAP_USE_256_BIT_INIT_MASK) && defined(USE_SWAP_AVX2) +#define SWAP_MASK_INIT_SIZE 32 +#else +#define SWAP_MASK_INIT_SIZE 16 +#endif + +MY_ALIGN(SWAP_MASK_INIT_SIZE) +static const Byte k_ShufMask_Swap2[] = +{ + SWAP2_SHUF_MASK_16_BYTES + #if SWAP_MASK_INIT_SIZE > 16 + , SWAP2_SHUF_MASK_16_BYTES + #endif +}; + +MY_ALIGN(SWAP_MASK_INIT_SIZE) +static const Byte k_ShufMask_Swap4[] = +{ + SWAP4_SHUF_MASK_16_BYTES + #if SWAP_MASK_INIT_SIZE > 16 + , SWAP4_SHUF_MASK_16_BYTES + #endif +}; + + +#ifdef USE_SWAP_SSSE3 + +#include // ssse3 + +#define SHUF_128(i) *(items + (i)) = \ + _mm_shuffle_epi8(*(items + (i)), mask); // SSSE3 + +// Z7_NO_INLINE +static +#ifdef SWAP_ATTRIB_SSSE3 +SWAP_ATTRIB_SSSE3 +#endif +Z7_ATTRIB_NO_VECTORIZE +void +Z7_FASTCALL +ShufBytes_128(void *items8, const void *lim8, const void *mask128_ptr) +{ + __m128i *items = (__m128i *)items8; + const __m128i *lim = (const __m128i *)lim8; + // const __m128i mask = _mm_set_epi8(SHUF_SWAP2_MASK_16_VALS); + // const __m128i mask = _mm_set_epi8(SHUF_SWAP4_MASK_16_VALS); + // const __m128i mask = _mm_load_si128((const __m128i *)(const void *)&(k_ShufMask_Swap4[0])); + // const __m128i mask = _mm_load_si128((const __m128i *)(const void *)&(k_ShufMask_Swap4[0])); + // const __m128i mask = *(const __m128i *)(const void *)&(k_ShufMask_Swap4[0]); + const __m128i mask = *(const __m128i *)mask128_ptr; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SHUF_128(0) SHUF_128(1) items += 2; + SHUF_128(0) SHUF_128(1) items += 2; + } + while (items != lim); +} + +#endif // USE_SWAP_SSSE3 + + + +#ifdef USE_SWAP_AVX2 + +#include // avx, avx2 +#if defined(__clang__) +#include +#include +#endif + +#define SHUF_256(i) *(items + (i)) = \ + _mm256_shuffle_epi8(*(items + (i)), mask); // AVX2 + +// Z7_NO_INLINE +static +#ifdef SWAP_ATTRIB_AVX2 +SWAP_ATTRIB_AVX2 +#endif +Z7_ATTRIB_NO_VECTORIZE +void +Z7_FASTCALL +ShufBytes_256(void *items8, const void *lim8, const void *mask128_ptr) +{ + __m256i *items = (__m256i *)items8; + const __m256i *lim = (const __m256i *)lim8; + /* + UNUSED_VAR(mask128_ptr) + __m256i mask = + for Swap4: _mm256_setr_epi8(SWAP4_SHUF_MASK_16_BYTES, SWAP4_SHUF_MASK_16_BYTES); + for Swap2: _mm256_setr_epi8(SWAP2_SHUF_MASK_16_BYTES, SWAP2_SHUF_MASK_16_BYTES); + */ + const __m256i mask = + #if SWAP_MASK_INIT_SIZE > 16 + *(const __m256i *)(const void *)mask128_ptr; + #else + /* msvc: broadcastsi128() version reserves the stack for no reason + msvc 19.29-: _mm256_insertf128_si256() / _mm256_set_m128i)) versions use non-avx movdqu xmm0,XMMWORD PTR [r8] + msvc 19.30+ (VS2022): replaces _mm256_set_m128i(m,m) to vbroadcastf128(m) as we want + */ + // _mm256_broadcastsi128_si256(*mask128_ptr); +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000) + #define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1) +#else + #define MY_mm256_set_m128i _mm256_set_m128i +#endif + MY_mm256_set_m128i( + *(const __m128i *)mask128_ptr, + *(const __m128i *)mask128_ptr); + #endif + + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SHUF_256(0) SHUF_256(1) items += 2; + SHUF_256(0) SHUF_256(1) items += 2; + } + while (items != lim); +} + +#endif // USE_SWAP_AVX2 +#endif // USE_SWAP_SSSE3 || USE_SWAP_AVX2 +#endif // USE_SWAP_128 + + + +// compile message "NEON intrinsics not available with the soft-float ABI" +#elif defined(MY_CPU_ARM_OR_ARM64) \ + && defined(MY_CPU_LE) \ + && !defined(Z7_DISABLE_ARM_NEON) + + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 6) + #if defined(__ARM_FP) + #if (defined(__ARM_ARCH) && (__ARM_ARCH >= 4)) \ + || defined(MY_CPU_ARM64) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) + #define USE_SWAP_128 + #ifdef MY_CPU_ARM64 + // #define SWAP_ATTRIB_NEON __attribute__((__target__(""))) + #else +#if defined(Z7_CLANG_VERSION) + // #define SWAP_ATTRIB_NEON __attribute__((__target__("neon"))) +#else + // #pragma message("SWAP_ATTRIB_NEON __attribute__((__target__(fpu=neon))") + #define SWAP_ATTRIB_NEON __attribute__((__target__("fpu=neon"))) +#endif + #endif // MY_CPU_ARM64 + #endif // __ARM_NEON + #endif // __ARM_ARCH + #endif // __ARM_FP + + #elif defined(_MSC_VER) + #if (_MSC_VER >= 1910) + #define USE_SWAP_128 + #endif + #endif + + #ifdef USE_SWAP_128 + #if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) + #include + #else + +/* +#if !defined(__ARM_NEON) +#if defined(Z7_GCC_VERSION) && (__GNUC__ < 5) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 5) && (Z7_GCC_VERSION < 90201) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 5) && (Z7_GCC_VERSION < 100100) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#pragma message("#define __ARM_NEON 1") +// #define __ARM_NEON 1 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif +*/ + #include + #endif + #endif + +#ifndef USE_SWAP_128 + #define FORCE_SWAP_MODE +#else + +#ifdef MY_CPU_ARM64 + // for debug : comment it + #define FORCE_SWAP_MODE +#else + #define k_SwapBytes_Mode_NEON 1 +#endif +// typedef uint8x16_t v128; +#define SWAP2_128(i) *(uint8x16_t *) (void *)(items + (i) * 8) = \ + vrev16q_u8(*(const uint8x16_t *)(const void *)(items + (i) * 8)); +#define SWAP4_128(i) *(uint8x16_t *) (void *)(items + (i) * 4) = \ + vrev32q_u8(*(const uint8x16_t *)(const void *)(items + (i) * 4)); + +// Z7_NO_INLINE +static +#ifdef SWAP_ATTRIB_NEON +SWAP_ATTRIB_NEON +#endif +Z7_ATTRIB_NO_VECTORIZE +void +Z7_FASTCALL +SwapBytes2_128(CSwapUInt16 *items, const CSwapUInt16 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP2_128(0) SWAP2_128(1) items += 2 * 8; + SWAP2_128(0) SWAP2_128(1) items += 2 * 8; + } + while (items != lim); +} + +// Z7_NO_INLINE +static +#ifdef SWAP_ATTRIB_NEON +SWAP_ATTRIB_NEON +#endif +Z7_ATTRIB_NO_VECTORIZE +void +Z7_FASTCALL +SwapBytes4_128(CSwapUInt32 *items, const CSwapUInt32 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP4_128(0) SWAP4_128(1) items += 2 * 4; + SWAP4_128(0) SWAP4_128(1) items += 2 * 4; + } + while (items != lim); +} + +#endif // USE_SWAP_128 + +#else // MY_CPU_ARM_OR_ARM64 +#define FORCE_SWAP_MODE +#endif // MY_CPU_ARM_OR_ARM64 + + + + + + +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_X86) + /* _byteswap_ushort() in MSVC x86 32-bit works via slow { mov dh, al; mov dl, ah } + So we use own versions of byteswap function */ + #if (_MSC_VER < 1400 ) // old MSVC-X86 without _rotr16() support + #define SWAP2_16(i) { UInt32 v = items[i]; v += (v << 16); v >>= 8; items[i] = (CSwapUInt16)v; } + #else // is new MSVC-X86 with fast _rotr16() + #include + #define SWAP2_16(i) { items[i] = _rotr16(items[i], 8); } + #endif +#else // is not MSVC-X86 + #define SWAP2_16(i) { CSwapUInt16 v = items[i]; items[i] = Z7_BSWAP16(v); } +#endif // MSVC-X86 + +#if defined(Z7_CPU_FAST_BSWAP_SUPPORTED) + #define SWAP4_32(i) { CSwapUInt32 v = items[i]; items[i] = Z7_BSWAP32(v); } +#else + #define SWAP4_32(i) \ + { UInt32 v = items[i]; \ + v = ((v & 0xff00ff) << 8) + ((v >> 8) & 0xff00ff); \ + v = rotlFixed(v, 16); \ + items[i] = v; } +#endif + + + + +#if defined(FORCE_SWAP_MODE) && defined(USE_SWAP_128) + #define DEFAULT_Swap2 SwapBytes2_128 + #if !defined(MY_CPU_X86_OR_AMD64) + #define DEFAULT_Swap4 SwapBytes4_128 + #endif +#endif + +#if !defined(DEFAULT_Swap2) || !defined(DEFAULT_Swap4) + +#define SWAP_BASE_FUNCS_PREFIXES \ +Z7_FORCE_INLINE \ +static \ +Z7_ATTRIB_NO_VECTOR \ +void Z7_FASTCALL + + +#if defined(MY_CPU_ARM_OR_ARM64) +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wlanguage-extension-token" +#endif +#endif + + +#ifdef MY_CPU_64BIT + +#if defined(MY_CPU_ARM64) \ + && defined(__ARM_ARCH) && (__ARM_ARCH >= 8) \ + && ( (defined(__GNUC__) && (__GNUC__ >= 4)) \ + || (defined(__clang__) && (__clang_major__ >= 4))) + + #define SWAP2_64_VAR(v) asm ("rev16 %x0,%x0" : "+r" (v)); + #define SWAP4_64_VAR(v) asm ("rev32 %x0,%x0" : "+r" (v)); + +#else // is not ARM64-GNU + +#if !defined(MY_CPU_X86_OR_AMD64) || (k_SwapBytes_Mode_MIN == 0) || !defined(USE_SWAP_128) + #define SWAP2_64_VAR(v) \ + v = ( 0x00ff00ff00ff00ff & (v >> 8)) \ + + ((0x00ff00ff00ff00ff & v) << 8); + /* plus gives faster code in MSVC */ +#endif + +#ifdef Z7_CPU_FAST_BSWAP_SUPPORTED + #define SWAP4_64_VAR(v) \ + v = Z7_BSWAP64(v); \ + v = Z7_ROTL64(v, 32); +#else + #define SWAP4_64_VAR(v) \ + v = ( 0x000000ff000000ff & (v >> 24)) \ + + ((0x000000ff000000ff & v) << 24 ) \ + + ( 0x0000ff000000ff00 & (v >> 8)) \ + + ((0x0000ff000000ff00 & v) << 8 ) \ + ; +#endif + +#endif // ARM64-GNU + + +#ifdef SWAP2_64_VAR + +#define SWAP2_64(i) { \ + UInt64 v = *(const UInt64 *)(const void *)(items + (i) * 4); \ + SWAP2_64_VAR(v) \ + *(UInt64 *)(void *)(items + (i) * 4) = v; } + +SWAP_BASE_FUNCS_PREFIXES +SwapBytes2_64(CSwapUInt16 *items, const CSwapUInt16 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP2_64(0) SWAP2_64(1) items += 2 * 4; + SWAP2_64(0) SWAP2_64(1) items += 2 * 4; + } + while (items != lim); +} + + #define DEFAULT_Swap2 SwapBytes2_64 + #if !defined(FORCE_SWAP_MODE) + #define SWAP2_DEFAULT_MODE 0 + #endif +#else // !defined(SWAP2_64_VAR) + #define DEFAULT_Swap2 SwapBytes2_128 + #if !defined(FORCE_SWAP_MODE) + #define SWAP2_DEFAULT_MODE 1 + #endif +#endif // SWAP2_64_VAR + + +#define SWAP4_64(i) { \ + UInt64 v = *(const UInt64 *)(const void *)(items + (i) * 2); \ + SWAP4_64_VAR(v) \ + *(UInt64 *)(void *)(items + (i) * 2) = v; } + +SWAP_BASE_FUNCS_PREFIXES +SwapBytes4_64(CSwapUInt32 *items, const CSwapUInt32 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP4_64(0) SWAP4_64(1) items += 2 * 2; + SWAP4_64(0) SWAP4_64(1) items += 2 * 2; + } + while (items != lim); +} + +#define DEFAULT_Swap4 SwapBytes4_64 + +#else // is not 64BIT + + +#if defined(MY_CPU_ARM_OR_ARM64) \ + && defined(__ARM_ARCH) && (__ARM_ARCH >= 6) \ + && ( (defined(__GNUC__) && (__GNUC__ >= 4)) \ + || (defined(__clang__) && (__clang_major__ >= 4))) + +#ifdef MY_CPU_64BIT + #define SWAP2_32_VAR(v) asm ("rev16 %w0,%w0" : "+r" (v)); +#else + #define SWAP2_32_VAR(v) asm ("rev16 %0,%0" : "+r" (v)); // for clang/gcc + // asm ("rev16 %r0,%r0" : "+r" (a)); // for gcc +#endif + +#elif defined(_MSC_VER) && (_MSC_VER < 1300) && defined(MY_CPU_X86) \ + || !defined(Z7_CPU_FAST_BSWAP_SUPPORTED) \ + || !defined(Z7_CPU_FAST_ROTATE_SUPPORTED) + // old msvc doesn't support _byteswap_ulong() + #define SWAP2_32_VAR(v) \ + v = ((v & 0xff00ff) << 8) + ((v >> 8) & 0xff00ff); + +#else // is not ARM and is not old-MSVC-X86 and fast BSWAP/ROTATE are supported + #define SWAP2_32_VAR(v) \ + v = Z7_BSWAP32(v); \ + v = rotlFixed(v, 16); + +#endif // GNU-ARM* + +#define SWAP2_32(i) { \ + UInt32 v = *(const UInt32 *)(const void *)(items + (i) * 2); \ + SWAP2_32_VAR(v); \ + *(UInt32 *)(void *)(items + (i) * 2) = v; } + + +SWAP_BASE_FUNCS_PREFIXES +SwapBytes2_32(CSwapUInt16 *items, const CSwapUInt16 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP2_32(0) SWAP2_32(1) items += 2 * 2; + SWAP2_32(0) SWAP2_32(1) items += 2 * 2; + } + while (items != lim); +} + + +SWAP_BASE_FUNCS_PREFIXES +SwapBytes4_32(CSwapUInt32 *items, const CSwapUInt32 *lim) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + SWAP4_32(0) SWAP4_32(1) items += 2; + SWAP4_32(0) SWAP4_32(1) items += 2; + } + while (items != lim); +} + +#define DEFAULT_Swap2 SwapBytes2_32 +#define DEFAULT_Swap4 SwapBytes4_32 +#if !defined(FORCE_SWAP_MODE) + #define SWAP2_DEFAULT_MODE 0 +#endif + +#endif // MY_CPU_64BIT +#endif // if !defined(DEFAULT_Swap2) || !defined(DEFAULT_Swap4) + + + +#if !defined(FORCE_SWAP_MODE) +static unsigned g_SwapBytes_Mode; +#endif + +/* size of largest unrolled loop iteration: 128 bytes = 4 * 32 bytes (AVX). */ +#define SWAP_ITERATION_BLOCK_SIZE_MAX (1 << 7) + +// 32 bytes for (AVX) or 2 * 16-bytes for NEON. +#define SWAP_VECTOR_ALIGN_SIZE (1 << 5) + +Z7_NO_INLINE +void z7_SwapBytes2(CSwapUInt16 *items, size_t numItems) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (SWAP_VECTOR_ALIGN_SIZE - 1)) != 0; numItems--) + { + SWAP2_16(0) + items++; + } + { + const size_t k_Align_Mask = SWAP_ITERATION_BLOCK_SIZE_MAX / sizeof(CSwapUInt16) - 1; + size_t numItems2 = numItems; + CSwapUInt16 *lim; + numItems &= k_Align_Mask; + numItems2 &= ~(size_t)k_Align_Mask; + lim = items + numItems2; + if (numItems2 != 0) + { + #if !defined(FORCE_SWAP_MODE) + #ifdef MY_CPU_X86_OR_AMD64 + #ifdef USE_SWAP_AVX2 + if (g_SwapBytes_Mode > k_SwapBytes_Mode_SSSE3) + ShufBytes_256((__m256i *)(void *)items, + (const __m256i *)(const void *)lim, + (const __m128i *)(const void *)&(k_ShufMask_Swap2[0])); + else + #endif + #ifdef USE_SWAP_SSSE3 + if (g_SwapBytes_Mode >= k_SwapBytes_Mode_SSSE3) + ShufBytes_128((__m128i *)(void *)items, + (const __m128i *)(const void *)lim, + (const __m128i *)(const void *)&(k_ShufMask_Swap2[0])); + else + #endif + #endif // MY_CPU_X86_OR_AMD64 + #if SWAP2_DEFAULT_MODE == 0 + if (g_SwapBytes_Mode != 0) + SwapBytes2_128(items, lim); + else + #endif + #endif // FORCE_SWAP_MODE + DEFAULT_Swap2(items, lim); + } + items = lim; + } + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + for (; numItems != 0; numItems--) + { + SWAP2_16(0) + items++; + } +} + + +Z7_NO_INLINE +void z7_SwapBytes4(CSwapUInt32 *items, size_t numItems) +{ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + for (; numItems != 0 && ((unsigned)(ptrdiff_t)items & (SWAP_VECTOR_ALIGN_SIZE - 1)) != 0; numItems--) + { + SWAP4_32(0) + items++; + } + { + const size_t k_Align_Mask = SWAP_ITERATION_BLOCK_SIZE_MAX / sizeof(CSwapUInt32) - 1; + size_t numItems2 = numItems; + CSwapUInt32 *lim; + numItems &= k_Align_Mask; + numItems2 &= ~(size_t)k_Align_Mask; + lim = items + numItems2; + if (numItems2 != 0) + { + #if !defined(FORCE_SWAP_MODE) + #ifdef MY_CPU_X86_OR_AMD64 + #ifdef USE_SWAP_AVX2 + if (g_SwapBytes_Mode > k_SwapBytes_Mode_SSSE3) + ShufBytes_256((__m256i *)(void *)items, + (const __m256i *)(const void *)lim, + (const __m128i *)(const void *)&(k_ShufMask_Swap4[0])); + else + #endif + #ifdef USE_SWAP_SSSE3 + if (g_SwapBytes_Mode >= k_SwapBytes_Mode_SSSE3) + ShufBytes_128((__m128i *)(void *)items, + (const __m128i *)(const void *)lim, + (const __m128i *)(const void *)&(k_ShufMask_Swap4[0])); + else + #endif + #else // MY_CPU_X86_OR_AMD64 + + if (g_SwapBytes_Mode != 0) + SwapBytes4_128(items, lim); + else + #endif // MY_CPU_X86_OR_AMD64 + #endif // FORCE_SWAP_MODE + DEFAULT_Swap4(items, lim); + } + items = lim; + } + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + for (; numItems != 0; numItems--) + { + SWAP4_32(0) + items++; + } +} + + +// #define SHOW_HW_STATUS + +#ifdef SHOW_HW_STATUS +#include +#define PRF(x) x +#else +#define PRF(x) +#endif + +void z7_SwapBytesPrepare(void) +{ +#ifndef FORCE_SWAP_MODE + unsigned mode = 0; // k_SwapBytes_Mode_BASE; + +#ifdef MY_CPU_ARM_OR_ARM64 + { + if (CPU_IsSupported_NEON()) + { + // #pragma message ("=== SwapBytes NEON") + PRF(printf("\n=== SwapBytes NEON\n");) + mode = k_SwapBytes_Mode_NEON; + } + } +#else // MY_CPU_ARM_OR_ARM64 + { + #ifdef USE_SWAP_AVX2 + if (CPU_IsSupported_AVX2()) + { + // #pragma message ("=== SwapBytes AVX2") + PRF(printf("\n=== SwapBytes AVX2\n");) + mode = k_SwapBytes_Mode_AVX2; + } + else + #endif + #ifdef USE_SWAP_SSSE3 + if (CPU_IsSupported_SSSE3()) + { + // #pragma message ("=== SwapBytes SSSE3") + PRF(printf("\n=== SwapBytes SSSE3\n");) + mode = k_SwapBytes_Mode_SSSE3; + } + else + #endif + #if !defined(MY_CPU_AMD64) + if (CPU_IsSupported_SSE2()) + #endif + { + // #pragma message ("=== SwapBytes SSE2") + PRF(printf("\n=== SwapBytes SSE2\n");) + mode = k_SwapBytes_Mode_SSE2; + } + } +#endif // MY_CPU_ARM_OR_ARM64 + g_SwapBytes_Mode = mode; + // g_SwapBytes_Mode = 0; // for debug +#endif // FORCE_SWAP_MODE + PRF(printf("\n=== SwapBytesPrepare\n");) +} + +#undef PRF diff --git a/multiarc/src/formats/7z/C/SwapBytes.h b/multiarc/src/formats/7z/C/SwapBytes.h new file mode 100644 index 000000000..d44246738 --- /dev/null +++ b/multiarc/src/formats/7z/C/SwapBytes.h @@ -0,0 +1,17 @@ +/* SwapBytes.h -- Byte Swap conversion filter +2023-04-02 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_SWAP_BYTES_H +#define ZIP7_INC_SWAP_BYTES_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +void z7_SwapBytes2(UInt16 *data, size_t numItems); +void z7_SwapBytes4(UInt32 *data, size_t numItems); +void z7_SwapBytesPrepare(void); + +EXTERN_C_END + +#endif diff --git a/multiarc/src/formats/7z/C/Threads.c b/multiarc/src/formats/7z/C/Threads.c old mode 100755 new mode 100644 index 58eb90ffa..464efeca4 --- a/multiarc/src/formats/7z/C/Threads.c +++ b/multiarc/src/formats/7z/C/Threads.c @@ -1,5 +1,5 @@ /* Threads.c -- multithreading library -2021-12-21 : Igor Pavlov : Public domain */ +2024-03-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,9 +11,9 @@ #include "Threads.h" -static WRes GetError() +static WRes GetError(void) { - DWORD res = GetLastError(); + const DWORD res = GetLastError(); return res ? (WRes)res : 1; } @@ -173,6 +173,9 @@ WRes CriticalSection_Init(CCriticalSection *p) Windows XP, 2003 : can raise a STATUS_NO_MEMORY exception Windows Vista+ : no exceptions */ #ifdef _MSC_VER + #ifdef __clang__ + #pragma GCC diagnostic ignored "-Wlanguage-extension-token" + #endif __try #endif { @@ -192,19 +195,26 @@ WRes CriticalSection_Init(CCriticalSection *p) // ---------- POSIX ---------- -#ifndef __APPLE__ -#ifndef _7ZIP_AFFINITY_DISABLE +#if defined(__linux__) && !defined(__APPLE__) && !defined(_AIX) && !defined(__ANDROID__) +#ifndef Z7_AFFINITY_DISABLE // _GNU_SOURCE can be required for pthread_setaffinity_np() / CPU_ZERO / CPU_SET -#define _GNU_SOURCE -#endif -#endif +// clang < 3.6 : unknown warning group '-Wreserved-id-macro' +// clang 3.6 - 12.01 : gives warning "macro name is a reserved identifier" +// clang >= 13 : do not give warning +#if !defined(_GNU_SOURCE) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #define _GNU_SOURCE +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif // !defined(_GNU_SOURCE) +#endif // Z7_AFFINITY_DISABLE +#endif // __linux__ #include "Threads.h" #include #include #include -#ifdef _7ZIP_AFFINITY_SUPPORTED +#ifdef Z7_AFFINITY_SUPPORTED // #include #endif @@ -212,15 +222,12 @@ WRes CriticalSection_Init(CCriticalSection *p) // #include // #define PRF(p) p #define PRF(p) - -#define Print(s) PRF(printf("\n%s\n", s)) - -// #include +#define Print(s) PRF(printf("\n%s\n", s);) WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, const CCpuSet *cpuSet) { // new thread in Posix probably inherits affinity from parrent thread - Print("Thread_Create_With_CpuSet"); + Print("Thread_Create_With_CpuSet") pthread_attr_t attr; int ret; @@ -228,7 +235,7 @@ WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, p->_created = 0; - RINOK(pthread_attr_init(&attr)); + RINOK(pthread_attr_init(&attr)) ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); @@ -236,8 +243,9 @@ WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, { if (cpuSet) { - #ifdef _7ZIP_AFFINITY_SUPPORTED - + // pthread_attr_setaffinity_np() is not supported for MUSL compile. + // so we check for __GLIBC__ here +#if defined(Z7_AFFINITY_SUPPORTED) && defined( __GLIBC__) /* printf("\n affinity :"); unsigned i; @@ -259,7 +267,7 @@ WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, // ret2 = pthread_attr_setaffinity_np(&attr, sizeof(*cpuSet), cpuSet); // if (ret2) ret = ret2; - #endif +#endif } ret = pthread_create(&p->_tid, &attr, func, param); @@ -292,7 +300,7 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param) WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, CAffinityMask affinity) { - Print("Thread_Create_WithAffinity"); + Print("Thread_Create_WithAffinity") CCpuSet cs; unsigned i; CpuSet_Zero(&cs); @@ -312,7 +320,7 @@ WRes Thread_Create_With_Affinity(CThread *p, THREAD_FUNC_TYPE func, LPVOID param WRes Thread_Close(CThread *p) { - // Print("Thread_Close"); + // Print("Thread_Close") int ret; if (!p->_created) return 0; @@ -326,7 +334,7 @@ WRes Thread_Close(CThread *p) WRes Thread_Wait_Close(CThread *p) { - // Print("Thread_Wait_Close"); + // Print("Thread_Wait_Close") void *thread_return; int ret; if (!p->_created) @@ -343,8 +351,8 @@ WRes Thread_Wait_Close(CThread *p) static WRes Event_Create(CEvent *p, int manualReset, int signaled) { - RINOK(pthread_mutex_init(&p->_mutex, NULL)); - RINOK(pthread_cond_init(&p->_cond, NULL)); + RINOK(pthread_mutex_init(&p->_mutex, NULL)) + RINOK(pthread_cond_init(&p->_cond, NULL)) p->_manual_reset = manualReset; p->_state = (signaled ? True : False); p->_created = 1; @@ -361,25 +369,32 @@ WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) { return AutoResetEvent_Create(p, 0); } +#if defined(Z7_LLVM_CLANG_VERSION) && (__clang_major__ == 13) +// freebsd: +#pragma GCC diagnostic ignored "-Wthread-safety-analysis" +#endif + WRes Event_Set(CEvent *p) { - RINOK(pthread_mutex_lock(&p->_mutex)); + RINOK(pthread_mutex_lock(&p->_mutex)) p->_state = True; - int res1 = pthread_cond_broadcast(&p->_cond); - int res2 = pthread_mutex_unlock(&p->_mutex); - return (res2 ? res2 : res1); + { + const int res1 = pthread_cond_broadcast(&p->_cond); + const int res2 = pthread_mutex_unlock(&p->_mutex); + return (res2 ? res2 : res1); + } } WRes Event_Reset(CEvent *p) { - RINOK(pthread_mutex_lock(&p->_mutex)); + RINOK(pthread_mutex_lock(&p->_mutex)) p->_state = False; return pthread_mutex_unlock(&p->_mutex); } WRes Event_Wait(CEvent *p) { - RINOK(pthread_mutex_lock(&p->_mutex)); + RINOK(pthread_mutex_lock(&p->_mutex)) while (p->_state == False) { // ETIMEDOUT @@ -400,8 +415,8 @@ WRes Event_Close(CEvent *p) return 0; p->_created = 0; { - int res1 = pthread_mutex_destroy(&p->_mutex); - int res2 = pthread_cond_destroy(&p->_cond); + const int res1 = pthread_mutex_destroy(&p->_mutex); + const int res2 = pthread_cond_destroy(&p->_cond); return (res1 ? res1 : res2); } } @@ -411,8 +426,8 @@ WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount) { if (initCount > maxCount || maxCount < 1) return EINVAL; - RINOK(pthread_mutex_init(&p->_mutex, NULL)); - RINOK(pthread_cond_init(&p->_cond, NULL)); + RINOK(pthread_mutex_init(&p->_mutex, NULL)) + RINOK(pthread_cond_init(&p->_cond, NULL)) p->_count = initCount; p->_maxCount = maxCount; p->_created = 1; @@ -448,7 +463,7 @@ WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 releaseCount) if (releaseCount < 1) return EINVAL; - RINOK(pthread_mutex_lock(&p->_mutex)); + RINOK(pthread_mutex_lock(&p->_mutex)) newCount = p->_count + releaseCount; if (newCount > p->_maxCount) @@ -458,13 +473,13 @@ WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 releaseCount) p->_count = newCount; ret = pthread_cond_broadcast(&p->_cond); } - RINOK(pthread_mutex_unlock(&p->_mutex)); + RINOK(pthread_mutex_unlock(&p->_mutex)) return ret; } WRes Semaphore_Wait(CSemaphore *p) { - RINOK(pthread_mutex_lock(&p->_mutex)); + RINOK(pthread_mutex_lock(&p->_mutex)) while (p->_count < 1) { pthread_cond_wait(&p->_cond, &p->_mutex); @@ -479,8 +494,8 @@ WRes Semaphore_Close(CSemaphore *p) return 0; p->_created = 0; { - int res1 = pthread_mutex_destroy(&p->_mutex); - int res2 = pthread_cond_destroy(&p->_cond); + const int res1 = pthread_mutex_destroy(&p->_mutex); + const int res2 = pthread_cond_destroy(&p->_cond); return (res1 ? res1 : res2); } } @@ -489,7 +504,7 @@ WRes Semaphore_Close(CSemaphore *p) WRes CriticalSection_Init(CCriticalSection *p) { - // Print("CriticalSection_Init"); + // Print("CriticalSection_Init") if (!p) return EINTR; return pthread_mutex_init(&p->_mutex, NULL); @@ -497,7 +512,7 @@ WRes CriticalSection_Init(CCriticalSection *p) void CriticalSection_Enter(CCriticalSection *p) { - // Print("CriticalSection_Enter"); + // Print("CriticalSection_Enter") if (p) { // int ret = @@ -507,7 +522,7 @@ void CriticalSection_Enter(CCriticalSection *p) void CriticalSection_Leave(CCriticalSection *p) { - // Print("CriticalSection_Leave"); + // Print("CriticalSection_Leave") if (p) { // int ret = @@ -517,7 +532,7 @@ void CriticalSection_Leave(CCriticalSection *p) void CriticalSection_Delete(CCriticalSection *p) { - // Print("CriticalSection_Delete"); + // Print("CriticalSection_Delete") if (p) { // int ret = @@ -527,14 +542,40 @@ void CriticalSection_Delete(CCriticalSection *p) LONG InterlockedIncrement(LONG volatile *addend) { - // Print("InterlockedIncrement"); + // Print("InterlockedIncrement") #ifdef USE_HACK_UNSAFE_ATOMIC LONG val = *addend + 1; *addend = val; return val; #else + + #if defined(__clang__) && (__clang_major__ >= 8) + #pragma GCC diagnostic ignored "-Watomic-implicit-seq-cst" + #endif return __sync_add_and_fetch(addend, 1); #endif } +LONG InterlockedDecrement(LONG volatile *addend) +{ + // Print("InterlockedDecrement") + #ifdef USE_HACK_UNSAFE_ATOMIC + LONG val = *addend - 1; + *addend = val; + return val; + #else + return __sync_sub_and_fetch(addend, 1); + #endif +} + #endif // _WIN32 + +WRes AutoResetEvent_OptCreate_And_Reset(CAutoResetEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + +#undef PRF +#undef Print diff --git a/multiarc/src/formats/7z/C/Threads.h b/multiarc/src/formats/7z/C/Threads.h old mode 100755 new mode 100644 index f8e1f6d8f..141dba7dd --- a/multiarc/src/formats/7z/C/Threads.h +++ b/multiarc/src/formats/7z/C/Threads.h @@ -1,19 +1,29 @@ /* Threads.h -- multithreading library -2021-12-21 : Igor Pavlov : Public domain */ +2024-03-28 : Igor Pavlov : Public domain */ -#ifndef __7Z_THREADS_H -#define __7Z_THREADS_H +#ifndef ZIP7_INC_THREADS_H +#define ZIP7_INC_THREADS_H #ifdef _WIN32 -#include +#include "7zWindows.h" + #else +#include "Compiler.h" + +// #define Z7_AFFINITY_DISABLE #if defined(__linux__) #if !defined(__APPLE__) && !defined(_AIX) && !defined(__ANDROID__) -#ifndef _7ZIP_AFFINITY_DISABLE -#define _7ZIP_AFFINITY_SUPPORTED -// #pragma message(" ==== _7ZIP_AFFINITY_SUPPORTED") -// #define _GNU_SOURCE +#ifndef Z7_AFFINITY_DISABLE +#define Z7_AFFINITY_SUPPORTED +// #pragma message(" ==== Z7_AFFINITY_SUPPORTED") +#if !defined(_GNU_SOURCE) +// #pragma message(" ==== _GNU_SOURCE set") +// we need _GNU_SOURCE for cpu_set_t, if we compile for MUSL +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#define _GNU_SOURCE +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif #endif #endif #endif @@ -35,7 +45,7 @@ WRes Handle_WaitObject(HANDLE h); typedef HANDLE CThread; -#define Thread_Construct(p) { *(p) = NULL; } +#define Thread_CONSTRUCT(p) { *(p) = NULL; } #define Thread_WasCreated(p) (*(p) != NULL) #define Thread_Close(p) HandlePtr_Close(p) // #define Thread_Wait(p) Handle_WaitObject(*(p)) @@ -54,42 +64,46 @@ typedef #endif THREAD_FUNC_RET_TYPE; +#define THREAD_FUNC_RET_ZERO 0 + typedef DWORD_PTR CAffinityMask; typedef DWORD_PTR CCpuSet; -#define CpuSet_Zero(p) { *(p) = 0; } -#define CpuSet_Set(p, cpu) { *(p) |= ((DWORD_PTR)1 << (cpu)); } +#define CpuSet_Zero(p) *(p) = (0) +#define CpuSet_Set(p, cpu) *(p) |= ((DWORD_PTR)1 << (cpu)) #else // _WIN32 -typedef struct _CThread +typedef struct { pthread_t _tid; int _created; } CThread; -#define Thread_Construct(p) { (p)->_tid = 0; (p)->_created = 0; } -#define Thread_WasCreated(p) ((p)->_created != 0) +#define Thread_CONSTRUCT(p) { (p)->_tid = 0; (p)->_created = 0; } +#define Thread_WasCreated(p) ((p)->_created != 0) WRes Thread_Close(CThread *p); // #define Thread_Wait Thread_Wait_Close typedef void * THREAD_FUNC_RET_TYPE; +#define THREAD_FUNC_RET_ZERO NULL + typedef UInt64 CAffinityMask; -#ifdef _7ZIP_AFFINITY_SUPPORTED +#ifdef Z7_AFFINITY_SUPPORTED typedef cpu_set_t CCpuSet; -#define CpuSet_Zero(p) CPU_ZERO(p) -#define CpuSet_Set(p, cpu) CPU_SET(cpu, p) -#define CpuSet_IsSet(p, cpu) CPU_ISSET(cpu, p) +#define CpuSet_Zero(p) CPU_ZERO(p) +#define CpuSet_Set(p, cpu) CPU_SET(cpu, p) +#define CpuSet_IsSet(p, cpu) CPU_ISSET(cpu, p) #else typedef UInt64 CCpuSet; -#define CpuSet_Zero(p) { *(p) = 0; } -#define CpuSet_Set(p, cpu) { *(p) |= ((UInt64)1 << (cpu)); } -#define CpuSet_IsSet(p, cpu) ((*(p) & ((UInt64)1 << (cpu))) != 0) +#define CpuSet_Zero(p) *(p) = (0) +#define CpuSet_Set(p, cpu) *(p) |= ((UInt64)1 << (cpu)) +#define CpuSet_IsSet(p, cpu) ((*(p) & ((UInt64)1 << (cpu))) != 0) #endif @@ -97,7 +111,7 @@ typedef UInt64 CCpuSet; #endif // _WIN32 -#define THREAD_FUNC_CALL_TYPE MY_STD_CALL +#define THREAD_FUNC_CALL_TYPE Z7_STDCALL #if defined(_WIN32) && defined(__GNUC__) /* GCC compiler for x86 32-bit uses the rule: @@ -170,7 +184,7 @@ WRes CriticalSection_Init(CCriticalSection *p); #else // _WIN32 -typedef struct _CEvent +typedef struct { int _created; int _manual_reset; @@ -189,13 +203,14 @@ WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled); WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p); WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled); WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); + WRes Event_Set(CEvent *p); WRes Event_Reset(CEvent *p); WRes Event_Wait(CEvent *p); WRes Event_Close(CEvent *p); -typedef struct _CSemaphore +typedef struct { int _created; UInt32 _count; @@ -215,7 +230,7 @@ WRes Semaphore_Wait(CSemaphore *p); WRes Semaphore_Close(CSemaphore *p); -typedef struct _CCriticalSection +typedef struct { pthread_mutex_t _mutex; } CCriticalSection; @@ -226,9 +241,12 @@ void CriticalSection_Enter(CCriticalSection *cs); void CriticalSection_Leave(CCriticalSection *cs); LONG InterlockedIncrement(LONG volatile *addend); +LONG InterlockedDecrement(LONG volatile *addend); #endif // _WIN32 +WRes AutoResetEvent_OptCreate_And_Reset(CAutoResetEvent *p); + EXTERN_C_END #endif diff --git a/multiarc/src/formats/7z/C/Xxh64.c b/multiarc/src/formats/7z/C/Xxh64.c new file mode 100644 index 000000000..dc02a0253 --- /dev/null +++ b/multiarc/src/formats/7z/C/Xxh64.c @@ -0,0 +1,327 @@ +/* Xxh64.c -- XXH64 hash calculation +original code: Copyright (c) Yann Collet. +2023-08-18 : modified by Igor Pavlov. +This source code is licensed under BSD 2-Clause License. +*/ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "RotateDefs.h" +#include "Xxh64.h" + +#define Z7_XXH_PRIME64_1 UINT64_CONST(0x9E3779B185EBCA87) +#define Z7_XXH_PRIME64_2 UINT64_CONST(0xC2B2AE3D27D4EB4F) +#define Z7_XXH_PRIME64_3 UINT64_CONST(0x165667B19E3779F9) +#define Z7_XXH_PRIME64_4 UINT64_CONST(0x85EBCA77C2B2AE63) +#define Z7_XXH_PRIME64_5 UINT64_CONST(0x27D4EB2F165667C5) + +void Xxh64State_Init(CXxh64State *p) +{ + const UInt64 seed = 0; + p->v[0] = seed + Z7_XXH_PRIME64_1 + Z7_XXH_PRIME64_2; + p->v[1] = seed + Z7_XXH_PRIME64_2; + p->v[2] = seed; + p->v[3] = seed - Z7_XXH_PRIME64_1; +} + +#if !defined(MY_CPU_64BIT) && defined(MY_CPU_X86) && defined(_MSC_VER) + #define Z7_XXH64_USE_ASM +#endif + +#if !defined(MY_CPU_64BIT) && defined(MY_CPU_X86) \ + && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1200 +/* we try to avoid __allmul calls in MSVC for 64-bit multiply. + But MSVC6 still uses __allmul for our code. + So for MSVC6 we use default 64-bit multiply without our optimization. +*/ +#define LOW32(b) ((UInt32)(b & 0xffffffff)) +/* MSVC compiler (MSVC > 1200) can use "mul" instruction + without __allmul for our MY_emulu MACRO. + MY_emulu is similar to __emulu(a, b) MACRO */ +#define MY_emulu(a, b) ((UInt64)(a) * (b)) +#define MY_SET_HIGH32(a) ((UInt64)(a) << 32) +#define MY_MUL32_SET_HIGH32(a, b) MY_SET_HIGH32((UInt32)(a) * (UInt32)(b)) +// /* +#define MY_MUL64(a, b) \ + ( MY_emulu((UInt32)(a), LOW32(b)) + \ + MY_SET_HIGH32( \ + (UInt32)((a) >> 32) * LOW32(b) + \ + (UInt32)(a) * (UInt32)((b) >> 32) \ + )) +// */ +/* +#define MY_MUL64(a, b) \ + ( MY_emulu((UInt32)(a), LOW32(b)) \ + + MY_MUL32_SET_HIGH32((a) >> 32, LOW32(b)) + \ + + MY_MUL32_SET_HIGH32(a, (b) >> 32) \ + ) +*/ + +#define MY_MUL_32_64(a32, b) \ + ( MY_emulu((UInt32)(a32), LOW32(b)) \ + + MY_MUL32_SET_HIGH32(a32, (b) >> 32) \ + ) + +#else +#define MY_MUL64(a, b) ((a) * (b)) +#define MY_MUL_32_64(a32, b) ((a32) * (UInt64)(b)) +#endif + + +static +Z7_FORCE_INLINE +UInt64 Xxh64_Round(UInt64 acc, UInt64 input) +{ + acc += MY_MUL64(input, Z7_XXH_PRIME64_2); + acc = Z7_ROTL64(acc, 31); + return MY_MUL64(acc, Z7_XXH_PRIME64_1); +} + +static UInt64 Xxh64_Merge(UInt64 acc, UInt64 val) +{ + acc ^= Xxh64_Round(0, val); + return MY_MUL64(acc, Z7_XXH_PRIME64_1) + Z7_XXH_PRIME64_4; +} + + +#ifdef Z7_XXH64_USE_ASM + +#define Z7_XXH_PRIME64_1_HIGH 0x9E3779B1 +#define Z7_XXH_PRIME64_1_LOW 0x85EBCA87 +#define Z7_XXH_PRIME64_2_HIGH 0xC2B2AE3D +#define Z7_XXH_PRIME64_2_LOW 0x27D4EB4F + +void +Z7_NO_INLINE +__declspec(naked) +Z7_FASTCALL +Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end) +{ + #if !defined(__clang__) + UNUSED_VAR(p) + UNUSED_VAR(data) + UNUSED_VAR(end) + #endif + __asm push ebx + __asm push ebp + __asm push esi + __asm push edi + + #define STACK_OFFSET 4 * 8 + __asm sub esp, STACK_OFFSET + +#define COPY_1(n) \ + __asm mov eax, [ecx + n * 4] \ + __asm mov [esp + n * 4], eax \ + +#define COPY_2(n) \ + __asm mov eax, [esp + n * 4] \ + __asm mov [ecx + n * 4], eax \ + + COPY_1(0) + __asm mov edi, [ecx + 1 * 4] \ + COPY_1(2) + COPY_1(3) + COPY_1(4) + COPY_1(5) + COPY_1(6) + COPY_1(7) + + __asm mov esi, edx \ + __asm mov [esp + 0 * 8 + 4], ecx + __asm mov ecx, Z7_XXH_PRIME64_2_LOW \ + __asm mov ebp, Z7_XXH_PRIME64_1_LOW \ + +#define R(n, state1, state1_reg) \ + __asm mov eax, [esi + n * 8] \ + __asm imul ebx, eax, Z7_XXH_PRIME64_2_HIGH \ + __asm add ebx, state1 \ + __asm mul ecx \ + __asm add edx, ebx \ + __asm mov ebx, [esi + n * 8 + 4] \ + __asm imul ebx, ecx \ + __asm add eax, [esp + n * 8] \ + __asm adc edx, ebx \ + __asm mov ebx, eax \ + __asm shld eax, edx, 31 \ + __asm shld edx, ebx, 31 \ + __asm imul state1_reg, eax, Z7_XXH_PRIME64_1_HIGH \ + __asm imul edx, ebp \ + __asm add state1_reg, edx \ + __asm mul ebp \ + __asm add state1_reg, edx \ + __asm mov [esp + n * 8], eax \ + +#define R2(n) \ + R(n, [esp + n * 8 + 4], ebx) \ + __asm mov [esp + n * 8 + 4], ebx \ + + __asm align 16 + __asm main_loop: + R(0, edi, edi) + R2(1) + R2(2) + R2(3) + __asm add esi, 32 + __asm cmp esi, [esp + STACK_OFFSET + 4 * 4 + 4] + __asm jne main_loop + + __asm mov ecx, [esp + 0 * 8 + 4] + + COPY_2(0) + __asm mov [ecx + 1 * 4], edi + COPY_2(2) + COPY_2(3) + COPY_2(4) + COPY_2(5) + COPY_2(6) + COPY_2(7) + + __asm add esp, STACK_OFFSET + __asm pop edi + __asm pop esi + __asm pop ebp + __asm pop ebx + __asm ret 4 +} + +#else + +void +Z7_NO_INLINE +Z7_FASTCALL +Xxh64State_UpdateBlocks(CXxh64State *p, const void *_data, const void *end) +{ + const Byte *data = (const Byte *)_data; + UInt64 v[4]; + v[0] = p->v[0]; + v[1] = p->v[1]; + v[2] = p->v[2]; + v[3] = p->v[3]; + do + { + v[0] = Xxh64_Round(v[0], GetUi64(data)); data += 8; + v[1] = Xxh64_Round(v[1], GetUi64(data)); data += 8; + v[2] = Xxh64_Round(v[2], GetUi64(data)); data += 8; + v[3] = Xxh64_Round(v[3], GetUi64(data)); data += 8; + } + while (data != end); + p->v[0] = v[0]; + p->v[1] = v[1]; + p->v[2] = v[2]; + p->v[3] = v[3]; +} + +#endif + +UInt64 Xxh64State_Digest(const CXxh64State *p, const void *_data, UInt64 count) +{ + UInt64 h = p->v[2]; + + if (count >= 32) + { + h = Z7_ROTL64(p->v[0], 1) + + Z7_ROTL64(p->v[1], 7) + + Z7_ROTL64(h, 12) + + Z7_ROTL64(p->v[3], 18); + h = Xxh64_Merge(h, p->v[0]); + h = Xxh64_Merge(h, p->v[1]); + h = Xxh64_Merge(h, p->v[2]); + h = Xxh64_Merge(h, p->v[3]); + } + else + h += Z7_XXH_PRIME64_5; + + h += count; + + // XXH64_finalize(): + { + unsigned cnt = (unsigned)count & 31; + const Byte *data = (const Byte *)_data; + while (cnt >= 8) + { + h ^= Xxh64_Round(0, GetUi64(data)); + data += 8; + h = Z7_ROTL64(h, 27); + h = MY_MUL64(h, Z7_XXH_PRIME64_1) + Z7_XXH_PRIME64_4; + cnt -= 8; + } + if (cnt >= 4) + { + const UInt32 v = GetUi32(data); + data += 4; + h ^= MY_MUL_32_64(v, Z7_XXH_PRIME64_1); + h = Z7_ROTL64(h, 23); + h = MY_MUL64(h, Z7_XXH_PRIME64_2) + Z7_XXH_PRIME64_3; + cnt -= 4; + } + while (cnt) + { + const UInt32 v = *data++; + h ^= MY_MUL_32_64(v, Z7_XXH_PRIME64_5); + h = Z7_ROTL64(h, 11); + h = MY_MUL64(h, Z7_XXH_PRIME64_1); + cnt--; + } + // XXH64_avalanche(h): + h ^= h >> 33; h = MY_MUL64(h, Z7_XXH_PRIME64_2); + h ^= h >> 29; h = MY_MUL64(h, Z7_XXH_PRIME64_3); + h ^= h >> 32; + return h; + } +} + + +void Xxh64_Init(CXxh64 *p) +{ + Xxh64State_Init(&p->state); + p->count = 0; + p->buf64[0] = 0; + p->buf64[1] = 0; + p->buf64[2] = 0; + p->buf64[3] = 0; +} + +void Xxh64_Update(CXxh64 *p, const void *_data, size_t size) +{ + const Byte *data = (const Byte *)_data; + unsigned cnt; + if (size == 0) + return; + cnt = (unsigned)p->count; + p->count += size; + + if (cnt &= 31) + { + unsigned rem = 32 - cnt; + Byte *dest = (Byte *)p->buf64 + cnt; + if (rem > size) + rem = (unsigned)size; + size -= rem; + cnt += rem; + // memcpy((Byte *)p->buf64 + cnt, data, rem); + do + *dest++ = *data++; + while (--rem); + if (cnt != 32) + return; + Xxh64State_UpdateBlocks(&p->state, p->buf64, &p->buf64[4]); + } + + if (size &= ~(size_t)31) + { + Xxh64State_UpdateBlocks(&p->state, data, data + size); + data += size; + } + + cnt = (unsigned)p->count & 31; + if (cnt) + { + // memcpy(p->buf64, data, cnt); + Byte *dest = (Byte *)p->buf64; + do + *dest++ = *data++; + while (--cnt); + } +} diff --git a/multiarc/src/formats/7z/C/Xxh64.h b/multiarc/src/formats/7z/C/Xxh64.h new file mode 100644 index 000000000..efef65eb3 --- /dev/null +++ b/multiarc/src/formats/7z/C/Xxh64.h @@ -0,0 +1,50 @@ +/* Xxh64.h -- XXH64 hash calculation interfaces +2023-08-18 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_XXH64_H +#define ZIP7_INC_XXH64_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define Z7_XXH64_BLOCK_SIZE (4 * 8) + +typedef struct +{ + UInt64 v[4]; +} CXxh64State; + +void Xxh64State_Init(CXxh64State *p); + +// end != data && end == data + Z7_XXH64_BLOCK_SIZE * numBlocks +void Z7_FASTCALL Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end); + +/* +Xxh64State_Digest(): +data: + the function processes only + (totalCount & (Z7_XXH64_BLOCK_SIZE - 1)) bytes in (data): (smaller than 32 bytes). +totalCount: total size of hashed stream: + it includes total size of data processed by previous Xxh64State_UpdateBlocks() calls, + and it also includes current processed size in (data). +*/ +UInt64 Xxh64State_Digest(const CXxh64State *p, const void *data, UInt64 totalCount); + + +typedef struct +{ + CXxh64State state; + UInt64 count; + UInt64 buf64[4]; +} CXxh64; + +void Xxh64_Init(CXxh64 *p); +void Xxh64_Update(CXxh64 *p, const void *data, size_t size); + +#define Xxh64_Digest(p) \ + Xxh64State_Digest(&(p)->state, (p)->buf64, (p)->count) + +EXTERN_C_END + +#endif diff --git a/multiarc/src/formats/7z/C/Xz.c b/multiarc/src/formats/7z/C/Xz.c old mode 100755 new mode 100644 index 7c53b6007..d07550d09 --- a/multiarc/src/formats/7z/C/Xz.c +++ b/multiarc/src/formats/7z/C/Xz.c @@ -1,5 +1,5 @@ /* Xz.c - Xz -2021-02-09 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -52,6 +52,7 @@ void XzCheck_Init(CXzCheck *p, unsigned mode) case XZ_CHECK_CRC32: p->crc = CRC_INIT_VAL; break; case XZ_CHECK_CRC64: p->crc64 = CRC64_INIT_VAL; break; case XZ_CHECK_SHA256: Sha256_Init(&p->sha); break; + default: break; } } @@ -62,6 +63,7 @@ void XzCheck_Update(CXzCheck *p, const void *data, size_t size) case XZ_CHECK_CRC32: p->crc = CrcUpdate(p->crc, data, size); break; case XZ_CHECK_CRC64: p->crc64 = Crc64Update(p->crc64, data, size); break; case XZ_CHECK_SHA256: Sha256_Update(&p->sha, (const Byte *)data, size); break; + default: break; } } @@ -70,7 +72,7 @@ int XzCheck_Final(CXzCheck *p, Byte *digest) switch (p->mode) { case XZ_CHECK_CRC32: - SetUi32(digest, CRC_GET_DIGEST(p->crc)); + SetUi32(digest, CRC_GET_DIGEST(p->crc)) break; case XZ_CHECK_CRC64: { diff --git a/multiarc/src/formats/7z/C/Xz.h b/multiarc/src/formats/7z/C/Xz.h old mode 100755 new mode 100644 index 849b944bf..42bc68534 --- a/multiarc/src/formats/7z/C/Xz.h +++ b/multiarc/src/formats/7z/C/Xz.h @@ -1,21 +1,24 @@ /* Xz.h - Xz interface -2021-04-01 : Igor Pavlov : Public domain */ +2024-01-26 : Igor Pavlov : Public domain */ -#ifndef __XZ_H -#define __XZ_H +#ifndef ZIP7_INC_XZ_H +#define ZIP7_INC_XZ_H #include "Sha256.h" +#include "Delta.h" EXTERN_C_BEGIN #define XZ_ID_Subblock 1 #define XZ_ID_Delta 3 -#define XZ_ID_X86 4 -#define XZ_ID_PPC 5 -#define XZ_ID_IA64 6 -#define XZ_ID_ARM 7 -#define XZ_ID_ARMT 8 +#define XZ_ID_X86 4 +#define XZ_ID_PPC 5 +#define XZ_ID_IA64 6 +#define XZ_ID_ARM 7 +#define XZ_ID_ARMT 8 #define XZ_ID_SPARC 9 +#define XZ_ID_ARM64 0xa +#define XZ_ID_RISCV 0xb #define XZ_ID_LZMA2 0x21 unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value); @@ -53,7 +56,7 @@ typedef struct #define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex, UInt32 *headerSizeRes); /* ---------- xz stream ---------- */ @@ -101,7 +104,7 @@ typedef UInt16 CXzStreamFlags; unsigned XzFlags_GetCheckSize(CXzStreamFlags f); SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf); -SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream); +SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStreamPtr inStream); typedef struct { @@ -112,6 +115,7 @@ typedef struct typedef struct { CXzStreamFlags flags; + // Byte _pad[6]; size_t numBlocks; CXzBlockSizes *blocks; UInt64 startOffset; @@ -134,7 +138,7 @@ typedef struct void Xzs_Construct(CXzs *p); void Xzs_Free(CXzs *p, ISzAllocPtr alloc); -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc); +SRes Xzs_ReadBackward(CXzs *p, ILookInStreamPtr inStream, Int64 *startOffset, ICompressProgressPtr progress, ISzAllocPtr alloc); UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); @@ -160,9 +164,9 @@ typedef enum } ECoderFinishMode; -typedef struct _IStateCoder +typedef struct { - void *p; + void *p; // state object; void (*Free)(void *p, ISzAllocPtr alloc); SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); @@ -174,6 +178,20 @@ typedef struct _IStateCoder } IStateCoder; +typedef struct +{ + UInt32 methodId; + UInt32 delta; + UInt32 ip; + UInt32 X86_State; + Byte delta_State[DELTA_STATE_SIZE]; +} CXzBcFilterStateBase; + +typedef SizeT (*Xz_Func_BcFilterStateBase_Filter)(CXzBcFilterStateBase *p, Byte *data, SizeT size); + +SRes Xz_StateCoder_Bc_SetFromMethod_Func(IStateCoder *p, UInt64 id, + Xz_Func_BcFilterStateBase_Filter func, ISzAllocPtr alloc); + #define MIXCODER_NUM_FILTERS_MAX 4 @@ -216,13 +234,13 @@ typedef enum typedef struct { EXzState state; - UInt32 pos; + unsigned pos; unsigned alignPos; unsigned indexPreSize; CXzStreamFlags streamFlags; - UInt32 blockHeaderSize; + unsigned blockHeaderSize; UInt64 packSize; UInt64 unpackSize; @@ -422,7 +440,7 @@ typedef struct size_t outStep_ST; // size of output buffer for Single-Thread decoding BoolInt ignoreErrors; // if set to 1, the decoder can ignore some errors and it skips broken parts of data. - #ifndef _7ZIP_ST + #ifndef Z7_ST unsigned numThreads; // the number of threads for Multi-Thread decoding. if (umThreads == 1) it will use Single-thread decoding size_t inBufSize_MT; // size of small input data buffers for Multi-Thread decoding. Big number of such small buffers can be created size_t memUseMax; // the limit of total memory usage for Multi-Thread decoding. @@ -432,8 +450,9 @@ typedef struct void XzDecMtProps_Init(CXzDecMtProps *p); - -typedef void * CXzDecMtHandle; +typedef struct CXzDecMt CXzDecMt; +typedef CXzDecMt * CXzDecMtHandle; +// Z7_DECLARE_HANDLE(CXzDecMtHandle) /* alloc : XzDecMt uses CAlignOffsetAlloc internally for addresses allocated by (alloc). @@ -503,14 +522,14 @@ SRes XzDecMt_Decode(CXzDecMtHandle p, const CXzDecMtProps *props, const UInt64 *outDataSize, // NULL means undefined int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished - ISeqOutStream *outStream, + ISeqOutStreamPtr outStream, // Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, // const Byte *inData, size_t inDataSize, CXzStatInfo *stat, // out: decoding results and statistics int *isMT, // out: 0 means that ST (Single-Thread) version was used // 1 means that MT (Multi-Thread) version was used - ICompressProgress *progress); + ICompressProgressPtr progress); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/XzCrc64.c b/multiarc/src/formats/7z/C/XzCrc64.c old mode 100755 new mode 100644 index b6d02cbeb..94fc1afb3 --- a/multiarc/src/formats/7z/C/XzCrc64.c +++ b/multiarc/src/formats/7z/C/XzCrc64.c @@ -1,5 +1,5 @@ /* XzCrc64.c -- CRC64 calculation -2017-05-23 : Igor Pavlov : Public domain */ +2023-12-08 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,45 +8,76 @@ #define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) -#ifdef MY_CPU_LE - #define CRC64_NUM_TABLES 4 +// for debug only : define Z7_CRC64_DEBUG_BE to test big-endian code in little-endian cpu +// #define Z7_CRC64_DEBUG_BE +#ifdef Z7_CRC64_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif + +#ifdef Z7_CRC64_NUM_TABLES + #define Z7_CRC64_NUM_TABLES_USE Z7_CRC64_NUM_TABLES #else - #define CRC64_NUM_TABLES 5 - #define CRC_UINT64_SWAP(v) \ - ((v >> 56) \ - | ((v >> 40) & ((UInt64)0xFF << 8)) \ - | ((v >> 24) & ((UInt64)0xFF << 16)) \ - | ((v >> 8) & ((UInt64)0xFF << 24)) \ - | ((v << 8) & ((UInt64)0xFF << 32)) \ - | ((v << 24) & ((UInt64)0xFF << 40)) \ - | ((v << 40) & ((UInt64)0xFF << 48)) \ - | ((v << 56))) - - UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); + #define Z7_CRC64_NUM_TABLES_USE 12 +#endif + +#if Z7_CRC64_NUM_TABLES_USE < 1 + #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES #endif + +#if Z7_CRC64_NUM_TABLES_USE != 1 + #ifndef MY_CPU_BE - UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); + #define FUNC_NAME_LE_2(s) XzCrc64UpdateT ## s + #define FUNC_NAME_LE_1(s) FUNC_NAME_LE_2(s) + #define FUNC_NAME_LE FUNC_NAME_LE_1(Z7_CRC64_NUM_TABLES_USE) + UInt64 Z7_FASTCALL FUNC_NAME_LE (UInt64 v, const void *data, size_t size, const UInt64 *table); +#endif +#ifndef MY_CPU_LE + #define FUNC_NAME_BE_2(s) XzCrc64UpdateBeT ## s + #define FUNC_NAME_BE_1(s) FUNC_NAME_BE_2(s) + #define FUNC_NAME_BE FUNC_NAME_BE_1(Z7_CRC64_NUM_TABLES_USE) + UInt64 Z7_FASTCALL FUNC_NAME_BE (UInt64 v, const void *data, size_t size, const UInt64 *table); #endif -typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); +#if defined(MY_CPU_LE) + #define FUNC_REF FUNC_NAME_LE +#elif defined(MY_CPU_BE) + #define FUNC_REF FUNC_NAME_BE +#else + #define FUNC_REF g_Crc64Update + static UInt64 (Z7_FASTCALL *FUNC_REF)(UInt64 v, const void *data, size_t size, const UInt64 *table); +#endif -static CRC64_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; +#endif + + +MY_ALIGN(64) +static UInt64 g_Crc64Table[256 * Z7_CRC64_NUM_TABLES_USE]; -UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) -{ - return g_Crc64Update(v, data, size, g_Crc64Table); -} -UInt64 MY_FAST_CALL Crc64Calc(const void *data, size_t size) +UInt64 Z7_FASTCALL Crc64Update(UInt64 v, const void *data, size_t size) { - return g_Crc64Update(CRC64_INIT_VAL, data, size, g_Crc64Table) ^ CRC64_INIT_VAL; +#if Z7_CRC64_NUM_TABLES_USE == 1 + #define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + const UInt64 *table = g_Crc64Table; + const Byte *p = (const Byte *)data; + const Byte *lim = p + size; + for (; p != lim; p++) + v = CRC64_UPDATE_BYTE_2(v, *p); + return v; + #undef CRC64_UPDATE_BYTE_2 +#else + return FUNC_REF (v, data, size, g_Crc64Table); +#endif } -void MY_FAST_CALL Crc64GenerateTable() + +Z7_NO_INLINE +void Z7_FASTCALL Crc64GenerateTable(void) { - UInt32 i; + unsigned i; for (i = 0; i < 256; i++) { UInt64 r = i; @@ -55,32 +86,55 @@ void MY_FAST_CALL Crc64GenerateTable() r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); g_Crc64Table[i] = r; } - for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) + +#if Z7_CRC64_NUM_TABLES_USE != 1 +#if 1 || 1 && defined(MY_CPU_X86) // low register count + for (i = 0; i < 256 * (Z7_CRC64_NUM_TABLES_USE - 1); i++) { - UInt64 r = g_Crc64Table[(size_t)i - 256]; - g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); + const UInt64 r0 = g_Crc64Table[(size_t)i]; + g_Crc64Table[(size_t)i + 256] = g_Crc64Table[(Byte)r0] ^ (r0 >> 8); } - - #ifdef MY_CPU_LE - - g_Crc64Update = XzCrc64UpdateT4; +#else + for (i = 0; i < 256 * (Z7_CRC64_NUM_TABLES_USE - 1); i += 2) + { + UInt64 r0 = g_Crc64Table[(size_t)(i) ]; + UInt64 r1 = g_Crc64Table[(size_t)(i) + 1]; + r0 = g_Crc64Table[(Byte)r0] ^ (r0 >> 8); + r1 = g_Crc64Table[(Byte)r1] ^ (r1 >> 8); + g_Crc64Table[(size_t)i + 256 ] = r0; + g_Crc64Table[(size_t)i + 256 + 1] = r1; + } +#endif - #else +#ifndef MY_CPU_LE { - #ifndef MY_CPU_BE +#ifndef MY_CPU_BE UInt32 k = 1; if (*(const Byte *)&k == 1) - g_Crc64Update = XzCrc64UpdateT4; + FUNC_REF = FUNC_NAME_LE; else - #endif +#endif { - for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) +#ifndef MY_CPU_BE + FUNC_REF = FUNC_NAME_BE; +#endif + for (i = 0; i < 256 * Z7_CRC64_NUM_TABLES_USE; i++) { - UInt64 x = g_Crc64Table[(size_t)i - 256]; - g_Crc64Table[i] = CRC_UINT64_SWAP(x); + const UInt64 x = g_Crc64Table[i]; + g_Crc64Table[i] = Z7_BSWAP64(x); } - g_Crc64Update = XzCrc64UpdateT1_BeT4; } } - #endif +#endif // ndef MY_CPU_LE +#endif // Z7_CRC64_NUM_TABLES_USE != 1 } + +#undef kCrc64Poly +#undef Z7_CRC64_NUM_TABLES_USE +#undef FUNC_REF +#undef FUNC_NAME_LE_2 +#undef FUNC_NAME_LE_1 +#undef FUNC_NAME_LE +#undef FUNC_NAME_BE_2 +#undef FUNC_NAME_BE_1 +#undef FUNC_NAME_BE diff --git a/multiarc/src/formats/7z/C/XzCrc64.h b/multiarc/src/formats/7z/C/XzCrc64.h old mode 100755 new mode 100644 index 08dbc330c..04f8153df --- a/multiarc/src/formats/7z/C/XzCrc64.h +++ b/multiarc/src/formats/7z/C/XzCrc64.h @@ -1,8 +1,8 @@ /* XzCrc64.h -- CRC64 calculation -2013-01-18 : Igor Pavlov : Public domain */ +2023-12-08 : Igor Pavlov : Public domain */ -#ifndef __XZ_CRC64_H -#define __XZ_CRC64_H +#ifndef ZIP7_INC_XZ_CRC64_H +#define ZIP7_INC_XZ_CRC64_H #include @@ -10,16 +10,16 @@ EXTERN_C_BEGIN -extern UInt64 g_Crc64Table[]; +// extern UInt64 g_Crc64Table[]; -void MY_FAST_CALL Crc64GenerateTable(void); +void Z7_FASTCALL Crc64GenerateTable(void); #define CRC64_INIT_VAL UINT64_CONST(0xFFFFFFFFFFFFFFFF) #define CRC64_GET_DIGEST(crc) ((crc) ^ CRC64_INIT_VAL) -#define CRC64_UPDATE_BYTE(crc, b) (g_Crc64Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +// #define CRC64_UPDATE_BYTE(crc, b) (g_Crc64Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) -UInt64 MY_FAST_CALL Crc64Update(UInt64 crc, const void *data, size_t size); -UInt64 MY_FAST_CALL Crc64Calc(const void *data, size_t size); +UInt64 Z7_FASTCALL Crc64Update(UInt64 crc, const void *data, size_t size); +// UInt64 Z7_FASTCALL Crc64Calc(const void *data, size_t size); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/XzCrc64Opt.c b/multiarc/src/formats/7z/C/XzCrc64Opt.c old mode 100755 new mode 100644 index 93a9ffff5..0c1fc2ffe --- a/multiarc/src/formats/7z/C/XzCrc64Opt.c +++ b/multiarc/src/formats/7z/C/XzCrc64Opt.c @@ -1,71 +1,261 @@ -/* XzCrc64Opt.c -- CRC64 calculation -2021-02-09 : Igor Pavlov : Public domain */ +/* XzCrc64Opt.c -- CRC64 calculation (optimized functions) +2023-12-08 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" +#if !defined(Z7_CRC64_NUM_TABLES) || Z7_CRC64_NUM_TABLES > 1 + +// for debug only : define Z7_CRC64_DEBUG_BE to test big-endian code in little-endian cpu +// #define Z7_CRC64_DEBUG_BE +#ifdef Z7_CRC64_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif + +#if defined(MY_CPU_64BIT) +#define Z7_CRC64_USE_64BIT +#endif + +// the value Z7_CRC64_NUM_TABLES_USE must be defined to same value as in XzCrc64.c +#ifdef Z7_CRC64_NUM_TABLES +#define Z7_CRC64_NUM_TABLES_USE Z7_CRC64_NUM_TABLES +#else +#define Z7_CRC64_NUM_TABLES_USE 12 +#endif + +#if Z7_CRC64_NUM_TABLES_USE % 4 || \ + Z7_CRC64_NUM_TABLES_USE < 4 || \ + Z7_CRC64_NUM_TABLES_USE > 4 * 4 + #error Stop_Compiling_Bad_CRC64_NUM_TABLES +#endif + + #ifndef MY_CPU_BE -#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + +#if defined(Z7_CRC64_USE_64BIT) && (Z7_CRC64_NUM_TABLES_USE % 8 == 0) + +#define Q64LE(n, d) \ + ( (table + ((n) * 8 + 7) * 0x100)[((d) ) & 0xFF] \ + ^ (table + ((n) * 8 + 6) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 5) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 4) * 0x100)[((d) >> 3 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 3) * 0x100)[((d) >> 4 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 2) * 0x100)[((d) >> 5 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 1) * 0x100)[((d) >> 6 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 0) * 0x100)[((d) >> 7 * 8)] ) + +#define R64(a) *((const UInt64 *)(const void *)p + (a)) + +#else + +#define Q32LE(n, d) \ + ( (table + ((n) * 4 + 3) * 0x100)[((d) ) & 0xFF] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 0) * 0x100)[((d) >> 3 * 8)] ) + +#define R32(a) *((const UInt32 *)(const void *)p + (a)) + +#endif + + +#define CRC64_FUNC_PRE_LE2(step) \ +UInt64 Z7_FASTCALL XzCrc64UpdateT ## step (UInt64 v, const void *data, size_t size, const UInt64 *table) + +#define CRC64_FUNC_PRE_LE(step) \ + CRC64_FUNC_PRE_LE2(step); \ + CRC64_FUNC_PRE_LE2(step) -UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); -UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) +CRC64_FUNC_PRE_LE(Z7_CRC64_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + const Byte *lim; + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC64_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC64_UPDATE_BYTE_2(v, *p); - for (; size >= 4; size -= 4, p += 4) + lim = p + size; + if (size >= Z7_CRC64_NUM_TABLES_USE) { - UInt32 d = (UInt32)v ^ *(const UInt32 *)(const void *)p; - v = (v >> 32) - ^ (table + 0x300)[((d ) & 0xFF)] - ^ (table + 0x200)[((d >> 8) & 0xFF)] - ^ (table + 0x100)[((d >> 16) & 0xFF)] - ^ (table + 0x000)[((d >> 24))]; + lim -= Z7_CRC64_NUM_TABLES_USE; + do + { +#if Z7_CRC64_NUM_TABLES_USE == 4 + const UInt32 d = (UInt32)v ^ R32(0); + v = (v >> 32) ^ Q32LE(0, d); +#elif Z7_CRC64_NUM_TABLES_USE == 8 +#ifdef Z7_CRC64_USE_64BIT + v ^= R64(0); + v = Q64LE(0, v); +#else + UInt32 v0, v1; + v0 = (UInt32)v ^ R32(0); + v1 = (UInt32)(v >> 32) ^ R32(1); + v = Q32LE(1, v0) ^ Q32LE(0, v1); +#endif +#elif Z7_CRC64_NUM_TABLES_USE == 12 + UInt32 w; + UInt32 v0, v1; + v0 = (UInt32)v ^ R32(0); + v1 = (UInt32)(v >> 32) ^ R32(1); + w = R32(2); + v = Q32LE(0, w); + v ^= Q32LE(2, v0) ^ Q32LE(1, v1); +#elif Z7_CRC64_NUM_TABLES_USE == 16 +#ifdef Z7_CRC64_USE_64BIT + UInt64 w; + UInt64 x; + w = R64(1); x = Q64LE(0, w); + v ^= R64(0); v = x ^ Q64LE(1, v); +#else + UInt32 v0, v1; + UInt32 r0, r1; + v0 = (UInt32)v ^ R32(0); + v1 = (UInt32)(v >> 32) ^ R32(1); + r0 = R32(2); + r1 = R32(3); + v = Q32LE(1, r0) ^ Q32LE(0, r1); + v ^= Q32LE(3, v0) ^ Q32LE(2, v1); +#endif +#else +#error Stop_Compiling_Bad_CRC64_NUM_TABLES +#endif + p += Z7_CRC64_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC64_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC64_UPDATE_BYTE_2(v, *p); return v; } +#undef CRC64_UPDATE_BYTE_2 +#undef R32 +#undef R64 +#undef Q32LE +#undef Q64LE +#undef CRC64_FUNC_PRE_LE +#undef CRC64_FUNC_PRE_LE2 + #endif + + #ifndef MY_CPU_LE -#define CRC_UINT64_SWAP(v) \ - ((v >> 56) \ - | ((v >> 40) & ((UInt64)0xFF << 8)) \ - | ((v >> 24) & ((UInt64)0xFF << 16)) \ - | ((v >> 8) & ((UInt64)0xFF << 24)) \ - | ((v << 8) & ((UInt64)0xFF << 32)) \ - | ((v << 24) & ((UInt64)0xFF << 40)) \ - | ((v << 40) & ((UInt64)0xFF << 48)) \ - | ((v << 56))) +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[((crc) >> 56) ^ (b)] ^ ((crc) << 8)) + +#if defined(Z7_CRC64_USE_64BIT) && (Z7_CRC64_NUM_TABLES_USE % 8 == 0) + +#define Q64BE(n, d) \ + ( (table + ((n) * 8 + 0) * 0x100)[(Byte)(d)] \ + ^ (table + ((n) * 8 + 1) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 2) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 3) * 0x100)[((d) >> 3 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 4) * 0x100)[((d) >> 4 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 5) * 0x100)[((d) >> 5 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 6) * 0x100)[((d) >> 6 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 7) * 0x100)[((d) >> 7 * 8)] ) + +#ifdef Z7_CRC64_DEBUG_BE + #define R64BE(a) GetBe64a((const UInt64 *)(const void *)p + (a)) +#else + #define R64BE(a) *((const UInt64 *)(const void *)p + (a)) +#endif + +#else -#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +#define Q32BE(n, d) \ + ( (table + ((n) * 4 + 0) * 0x100)[(Byte)(d)] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 3) * 0x100)[((d) >> 3 * 8)] ) -UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); -UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) +#ifdef Z7_CRC64_DEBUG_BE + #define R32BE(a) GetBe32a((const UInt32 *)(const void *)p + (a)) +#else + #define R32BE(a) *((const UInt32 *)(const void *)p + (a)) +#endif + +#endif + +#define CRC64_FUNC_PRE_BE2(step) \ +UInt64 Z7_FASTCALL XzCrc64UpdateBeT ## step (UInt64 v, const void *data, size_t size, const UInt64 *table) + +#define CRC64_FUNC_PRE_BE(step) \ + CRC64_FUNC_PRE_BE2(step); \ + CRC64_FUNC_PRE_BE2(step) + +CRC64_FUNC_PRE_BE(Z7_CRC64_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; - table += 0x100; - v = CRC_UINT64_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + const Byte *lim; + v = Z7_BSWAP64(v); + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC64_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC64_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 4; size -= 4, p += 4) + lim = p + size; + if (size >= Z7_CRC64_NUM_TABLES_USE) { - UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)(const void *)p; - v = (v << 32) - ^ (table + 0x000)[((d ) & 0xFF)] - ^ (table + 0x100)[((d >> 8) & 0xFF)] - ^ (table + 0x200)[((d >> 16) & 0xFF)] - ^ (table + 0x300)[((d >> 24))]; + lim -= Z7_CRC64_NUM_TABLES_USE; + do + { +#if Z7_CRC64_NUM_TABLES_USE == 4 + const UInt32 d = (UInt32)(v >> 32) ^ R32BE(0); + v = (v << 32) ^ Q32BE(0, d); +#elif Z7_CRC64_NUM_TABLES_USE == 12 + const UInt32 d1 = (UInt32)(v >> 32) ^ R32BE(0); + const UInt32 d0 = (UInt32)(v ) ^ R32BE(1); + const UInt32 w = R32BE(2); + v = Q32BE(0, w); + v ^= Q32BE(2, d1) ^ Q32BE(1, d0); + +#elif Z7_CRC64_NUM_TABLES_USE == 8 + #ifdef Z7_CRC64_USE_64BIT + v ^= R64BE(0); + v = Q64BE(0, v); + #else + const UInt32 d1 = (UInt32)(v >> 32) ^ R32BE(0); + const UInt32 d0 = (UInt32)(v ) ^ R32BE(1); + v = Q32BE(1, d1) ^ Q32BE(0, d0); + #endif +#elif Z7_CRC64_NUM_TABLES_USE == 16 + #ifdef Z7_CRC64_USE_64BIT + const UInt64 w = R64BE(1); + v ^= R64BE(0); + v = Q64BE(0, w) ^ Q64BE(1, v); + #else + const UInt32 d1 = (UInt32)(v >> 32) ^ R32BE(0); + const UInt32 d0 = (UInt32)(v ) ^ R32BE(1); + const UInt32 w1 = R32BE(2); + const UInt32 w0 = R32BE(3); + v = Q32BE(1, w1) ^ Q32BE(0, w0); + v ^= Q32BE(3, d1) ^ Q32BE(2, d0); + #endif +#elif +#error Stop_Compiling_Bad_CRC64_NUM_TABLES +#endif + p += Z7_CRC64_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC64_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC64_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT64_SWAP(v); + return Z7_BSWAP64(v); } +#undef CRC64_UPDATE_BYTE_2_BE +#undef R32BE +#undef R64BE +#undef Q32BE +#undef Q64BE +#undef CRC64_FUNC_PRE_BE +#undef CRC64_FUNC_PRE_BE2 + +#endif +#undef Z7_CRC64_NUM_TABLES_USE #endif diff --git a/multiarc/src/formats/7z/C/XzDec.c b/multiarc/src/formats/7z/C/XzDec.c old mode 100755 new mode 100644 index 3f96a37f9..3d1c98e63 --- a/multiarc/src/formats/7z/C/XzDec.c +++ b/multiarc/src/formats/7z/C/XzDec.c @@ -1,5 +1,5 @@ /* XzDec.c -- Xz Decode -2021-09-04 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -67,7 +67,8 @@ unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value) return 0; } -/* ---------- BraState ---------- */ + +/* ---------- XzBcFilterState ---------- */ #define BRA_BUF_SIZE (1 << 14) @@ -76,55 +77,60 @@ typedef struct size_t bufPos; size_t bufConv; size_t bufTotal; + Byte *buf; // must be aligned for 4 bytes + Xz_Func_BcFilterStateBase_Filter filter_func; + // int encodeMode; + CXzBcFilterStateBase base; + // Byte buf[BRA_BUF_SIZE]; +} CXzBcFilterState; - int encodeMode; - - UInt32 methodId; - UInt32 delta; - UInt32 ip; - UInt32 x86State; - Byte deltaState[DELTA_STATE_SIZE]; - - Byte buf[BRA_BUF_SIZE]; -} CBraState; -static void BraState_Free(void *pp, ISzAllocPtr alloc) +static void XzBcFilterState_Free(void *pp, ISzAllocPtr alloc) { - ISzAlloc_Free(alloc, pp); + if (pp) + { + CXzBcFilterState *p = ((CXzBcFilterState *)pp); + ISzAlloc_Free(alloc, p->buf); + ISzAlloc_Free(alloc, pp); + } } -static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) + +static SRes XzBcFilterState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { - CBraState *p = ((CBraState *)pp); - UNUSED_VAR(alloc); + CXzBcFilterStateBase *p = &((CXzBcFilterState *)pp)->base; + UNUSED_VAR(alloc) p->ip = 0; if (p->methodId == XZ_ID_Delta) { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - p->delta = (unsigned)props[0] + 1; + p->delta = (UInt32)props[0] + 1; } else { if (propSize == 4) { - UInt32 v = GetUi32(props); + const UInt32 v = GetUi32(props); switch (p->methodId) { case XZ_ID_PPC: case XZ_ID_ARM: case XZ_ID_SPARC: - if ((v & 3) != 0) + case XZ_ID_ARM64: + if (v & 3) return SZ_ERROR_UNSUPPORTED; break; case XZ_ID_ARMT: - if ((v & 1) != 0) + case XZ_ID_RISCV: + if (v & 1) return SZ_ERROR_UNSUPPORTED; break; case XZ_ID_IA64: - if ((v & 0xF) != 0) + if (v & 0xf) return SZ_ERROR_UNSUPPORTED; break; + default: break; } p->ip = v; } @@ -134,73 +140,91 @@ static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzA return SZ_OK; } -static void BraState_Init(void *pp) + +static void XzBcFilterState_Init(void *pp) { - CBraState *p = ((CBraState *)pp); + CXzBcFilterState *p = ((CXzBcFilterState *)pp); p->bufPos = p->bufConv = p->bufTotal = 0; - x86_Convert_Init(p->x86State); - if (p->methodId == XZ_ID_Delta) - Delta_Init(p->deltaState); + p->base.X86_State = Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL; + if (p->base.methodId == XZ_ID_Delta) + Delta_Init(p->base.delta_State); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; - -static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) +static const z7_Func_BranchConv g_Funcs_BranchConv_RISC_Dec[] = +{ + Z7_BRANCH_CONV_DEC_2 (BranchConv_PPC), + Z7_BRANCH_CONV_DEC_2 (BranchConv_IA64), + Z7_BRANCH_CONV_DEC_2 (BranchConv_ARM), + Z7_BRANCH_CONV_DEC_2 (BranchConv_ARMT), + Z7_BRANCH_CONV_DEC_2 (BranchConv_SPARC), + Z7_BRANCH_CONV_DEC_2 (BranchConv_ARM64), + Z7_BRANCH_CONV_DEC_2 (BranchConv_RISCV) +}; + +static SizeT XzBcFilterStateBase_Filter_Dec(CXzBcFilterStateBase *p, Byte *data, SizeT size) { - CBraState *p = ((CBraState *)pp); switch (p->methodId) { case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, data, size); - else - Delta_Decode(p->deltaState, p->delta, data, size); + Delta_Decode(p->delta_State, p->delta, data, size); break; case XZ_ID_X86: - size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + size = (SizeT)(z7_BranchConvSt_X86_Dec(data, size, p->ip, &p->X86_State) - data); + break; + default: + if (p->methodId >= XZ_ID_PPC) + { + const UInt32 i = p->methodId - XZ_ID_PPC; + if (i < Z7_ARRAY_SIZE(g_Funcs_BranchConv_RISC_Dec)) + size = (SizeT)(g_Funcs_BranchConv_RISC_Dec[i](data, size, p->ip) - data); + } break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) } p->ip += (UInt32)size; return size; } -static SRes BraState_Code2(void *pp, +static SizeT XzBcFilterState_Filter(void *pp, Byte *data, SizeT size) +{ + CXzBcFilterState *p = ((CXzBcFilterState *)pp); + return p->filter_func(&p->base, data, size); +} + + +static SRes XzBcFilterState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, int srcWasFinished, ECoderFinishMode finishMode, // int *wasFinished ECoderStatus *status) { - CBraState *p = ((CBraState *)pp); + CXzBcFilterState *p = ((CXzBcFilterState *)pp); SizeT destRem = *destLen; SizeT srcRem = *srcLen; - UNUSED_VAR(finishMode); + UNUSED_VAR(finishMode) *destLen = 0; *srcLen = 0; // *wasFinished = False; *status = CODER_STATUS_NOT_FINISHED; - while (destRem > 0) + while (destRem != 0) { - if (p->bufPos != p->bufConv) { size_t size = p->bufConv - p->bufPos; - if (size > destRem) - size = destRem; - memcpy(dest, p->buf + p->bufPos, size); - p->bufPos += size; - *destLen += size; - dest += size; - destRem -= size; - continue; + if (size) + { + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; + continue; + } } p->bufTotal -= p->bufPos; @@ -220,7 +244,7 @@ static SRes BraState_Code2(void *pp, if (p->bufTotal == 0) break; - p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); + p->bufConv = p->filter_func(&p->base, p->buf, p->bufTotal); if (p->bufConv == 0) { @@ -240,27 +264,37 @@ static SRes BraState_Code2(void *pp, } -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) +#define XZ_IS_SUPPORTED_FILTER_ID(id) \ + ((id) >= XZ_ID_Delta && (id) <= XZ_ID_RISCV) + +SRes Xz_StateCoder_Bc_SetFromMethod_Func(IStateCoder *p, UInt64 id, + Xz_Func_BcFilterStateBase_Filter func, ISzAllocPtr alloc) { - CBraState *decoder; - if (id < XZ_ID_Delta || id > XZ_ID_SPARC) + CXzBcFilterState *decoder; + if (!XZ_IS_SUPPORTED_FILTER_ID(id)) return SZ_ERROR_UNSUPPORTED; - decoder = (CBraState *)p->p; + decoder = (CXzBcFilterState *)p->p; if (!decoder) { - decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + decoder = (CXzBcFilterState *)ISzAlloc_Alloc(alloc, sizeof(CXzBcFilterState)); if (!decoder) return SZ_ERROR_MEM; + decoder->buf = ISzAlloc_Alloc(alloc, BRA_BUF_SIZE); + if (!decoder->buf) + { + ISzAlloc_Free(alloc, decoder); + return SZ_ERROR_MEM; + } p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code2 = BraState_Code2; - p->Filter = BraState_Filter; + p->Free = XzBcFilterState_Free; + p->SetProps = XzBcFilterState_SetProps; + p->Init = XzBcFilterState_Init; + p->Code2 = XzBcFilterState_Code2; + p->Filter = XzBcFilterState_Filter; + decoder->filter_func = func; } - decoder->methodId = (UInt32)id; - decoder->encodeMode = encodeMode; + decoder->base.methodId = (UInt32)id; + // decoder->encodeMode = encodeMode; return SZ_OK; } @@ -279,9 +313,9 @@ static void SbState_Free(void *pp, ISzAllocPtr alloc) static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { - UNUSED_VAR(pp); - UNUSED_VAR(props); - UNUSED_VAR(alloc); + UNUSED_VAR(pp) + UNUSED_VAR(props) + UNUSED_VAR(alloc) return (propSize == 0) ? SZ_OK : SZ_ERROR_UNSUPPORTED; } @@ -297,7 +331,7 @@ static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, { CSbDec *p = (CSbDec *)pp; SRes res; - UNUSED_VAR(srcWasFinished); + UNUSED_VAR(srcWasFinished) p->dest = dest; p->destLen = *destLen; p->src = src; @@ -389,7 +423,7 @@ static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *s ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ SRes res; - UNUSED_VAR(srcWasFinished); + UNUSED_VAR(srcWasFinished) if (spec->outBufMode) { SizeT dicPos = spec->decoder.decoder.dicPos; @@ -420,7 +454,7 @@ static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBuf p->Init = Lzma2State_Init; p->Code2 = Lzma2State_Code2; p->Filter = NULL; - Lzma2Dec_Construct(&spec->decoder); + Lzma2Dec_CONSTRUCT(&spec->decoder) } spec->outBufMode = False; if (outBuf) @@ -510,26 +544,24 @@ static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 met { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; - switch (methodId) - { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); - #ifdef USE_SUBBLOCK - case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); - #endif - } + if (methodId == XZ_ID_LZMA2) + return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); +#ifdef USE_SUBBLOCK + if (methodId == XZ_ID_Subblock) + return SbState_SetFromMethod(sc, p->alloc); +#endif if (coderIndex == 0) return SZ_ERROR_UNSUPPORTED; - return BraState_SetFromMethod(sc, methodId, 0, p->alloc); + return Xz_StateCoder_Bc_SetFromMethod_Func(sc, methodId, + XzBcFilterStateBase_Filter_Dec, p->alloc); } static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; - switch (methodId) - { - case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); - } + if (methodId == XZ_ID_LZMA2) + return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); return SZ_ERROR_UNSUPPORTED; } @@ -568,7 +600,7 @@ static SRes MixCoder_Code(CMixCoder *p, SizeT destLen2, srcLen2; int wasFinished; - PRF_STR("------- MixCoder Single ----------"); + PRF_STR("------- MixCoder Single ----------") srcLen2 = srcLenOrig; destLen2 = destLenOrig; @@ -615,14 +647,14 @@ static SRes MixCoder_Code(CMixCoder *p, processed = coder->Filter(coder->p, p->outBuf, processed); if (wasFinished || (destFinish && p->outWritten == destLenOrig)) processed = p->outWritten; - PRF_STR_INT("filter", i); + PRF_STR_INT("filter", i) } *destLen = processed; } return res; } - PRF_STR("standard mix"); + PRF_STR("standard mix") if (p->numCoders != 1) { @@ -772,14 +804,14 @@ static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ - { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ + { const unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; \ pos += s; } static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) { - unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + const unsigned numFilters = XzBlock_GetNumFilters(p) - 1; unsigned i; { const CXzFilter *f = &p->filters[numFilters]; @@ -795,8 +827,7 @@ static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) if (f->propsSize != 1) return False; } - else if (f->id < XZ_ID_Delta - || f->id > XZ_ID_SPARC + else if (!XZ_IS_SUPPORTED_FILTER_ID(f->id) || (f->propsSize != 0 && f->propsSize != 4)) return False; } @@ -821,22 +852,24 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) p->packSize = (UInt64)(Int64)-1; if (XzBlock_HasPackSize(p)) { - READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize); + READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize) if (p->packSize == 0 || p->packSize + headerSize >= (UInt64)1 << 63) return SZ_ERROR_ARCHIVE; } p->unpackSize = (UInt64)(Int64)-1; if (XzBlock_HasUnpackSize(p)) - READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize); + { + READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize) + } numFilters = XzBlock_GetNumFilters(p); for (i = 0; i < numFilters; i++) { CXzFilter *filter = p->filters + i; UInt64 size; - READ_VARINT_AND_CHECK(header, pos, headerSize, &filter->id); - READ_VARINT_AND_CHECK(header, pos, headerSize, &size); + READ_VARINT_AND_CHECK(header, pos, headerSize, &filter->id) + READ_VARINT_AND_CHECK(header, pos, headerSize, &size) if (size > headerSize - pos || size > XZ_FILTER_PROPS_SIZE_MAX) return SZ_ERROR_ARCHIVE; filter->propsSize = (UInt32)size; @@ -894,20 +927,20 @@ static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, siz MixCoder_Free(p); for (i = 0; i < numFilters; i++) { - RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)) } p->numCoders = numFilters; } else { - RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)) } for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i]; IStateCoder *sc = &p->coders[i]; - RINOK(sc->SetProps(sc->p, f->props, f->propsSize, p->alloc)); + RINOK(sc->SetProps(sc->p, f->props, f->propsSize, p->alloc)) } MixCoder_Init(p); @@ -1001,7 +1034,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, SRes res; ECoderFinishMode finishMode2 = finishMode; - BoolInt srcFinished2 = srcFinished; + BoolInt srcFinished2 = (BoolInt)srcFinished; BoolInt destFinish = False; if (p->block.packSize != (UInt64)(Int64)-1) @@ -1054,14 +1087,14 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, (*destLen) += destLen2; p->unpackSize += destLen2; - RINOK(res); + RINOK(res) if (*status != CODER_STATUS_FINISHED_WITH_MARK) { if (p->block.packSize == p->packSize && *status == CODER_STATUS_NEEDS_MORE_INPUT) { - PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT") *status = CODER_STATUS_NOT_SPECIFIED; return SZ_ERROR_DATA; } @@ -1078,7 +1111,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) { - PRF_STR("ERROR: block.size mismatch"); + PRF_STR("ERROR: block.size mismatch") return SZ_ERROR_DATA; } } @@ -1094,7 +1127,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, return SZ_OK; } - switch (p->state) + switch ((int)p->state) { case XZ_STATE_STREAM_HEADER: { @@ -1109,7 +1142,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } else { - RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); + RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)) p->numStartedStreams++; p->indexSize = 0; p->numBlocks = 0; @@ -1139,15 +1172,15 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, p->state = XZ_STATE_STREAM_INDEX; break; } - p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + p->blockHeaderSize = ((unsigned)p->buf[0] << 2) + 4; break; } if (p->pos != p->blockHeaderSize) { - UInt32 cur = p->blockHeaderSize - p->pos; + unsigned cur = p->blockHeaderSize - p->pos; if (cur > srcRem) - cur = (UInt32)srcRem; + cur = (unsigned)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; @@ -1155,7 +1188,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } else { - RINOK(XzBlock_Parse(&p->block, p->buf)); + RINOK(XzBlock_Parse(&p->block, p->buf)) if (!XzBlock_AreSupportedFilters(&p->block)) return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; @@ -1168,7 +1201,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, p->headerParsedOk = True; return SZ_OK; } - RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)) } break; } @@ -1189,8 +1222,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } else { - UInt32 checkSize = XzFlags_GetCheckSize(p->streamFlags); - UInt32 cur = checkSize - p->pos; + const unsigned checkSize = XzFlags_GetCheckSize(p->streamFlags); + unsigned cur = checkSize - p->pos; if (cur != 0) { if (srcRem == 0) @@ -1199,7 +1232,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, return SZ_OK; } if (cur > srcRem) - cur = (UInt32)srcRem; + cur = (unsigned)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; @@ -1288,9 +1321,9 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, case XZ_STATE_STREAM_FOOTER: { - UInt32 cur = XZ_STREAM_FOOTER_SIZE - p->pos; + unsigned cur = XZ_STREAM_FOOTER_SIZE - p->pos; if (cur > srcRem) - cur = (UInt32)srcRem; + cur = (unsigned)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; @@ -1325,6 +1358,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } case XZ_STATE_BLOCK: break; /* to disable GCC warning */ + + default: return SZ_ERROR_FAIL; } } /* @@ -1389,7 +1424,7 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) -#ifndef _7ZIP_ST +#ifndef Z7_ST #include "MtDec.h" #endif @@ -1400,7 +1435,7 @@ void XzDecMtProps_Init(CXzDecMtProps *p) p->outStep_ST = 1 << 20; p->ignoreErrors = False; - #ifndef _7ZIP_ST + #ifndef Z7_ST p->numThreads = 1; p->inBufSize_MT = 1 << 18; p->memUseMax = sizeof(size_t) << 28; @@ -1409,7 +1444,7 @@ void XzDecMtProps_Init(CXzDecMtProps *p) -#ifndef _7ZIP_ST +#ifndef Z7_ST /* ---------- CXzDecMtThread ---------- */ @@ -1448,7 +1483,7 @@ typedef struct /* ---------- CXzDecMt ---------- */ -typedef struct +struct CXzDecMt { CAlignOffsetAlloc alignOffsetAlloc; ISzAllocPtr allocMid; @@ -1456,9 +1491,9 @@ typedef struct CXzDecMtProps props; size_t unpackBlockMaxSize; - ISeqInStream *inStream; - ISeqOutStream *outStream; - ICompressProgress *progress; + ISeqInStreamPtr inStream; + ISeqOutStreamPtr outStream; + ICompressProgressPtr progress; BoolInt finishMode; BoolInt outSize_Defined; @@ -1481,7 +1516,7 @@ typedef struct ECoderStatus status; SRes codeRes; - #ifndef _7ZIP_ST + #ifndef Z7_ST BoolInt mainDecoderWasCalled; // int statErrorDefined; int finishedDecoderIndex; @@ -1504,10 +1539,9 @@ typedef struct BoolInt mtc_WasConstructed; CMtDec mtc; - CXzDecMtThread coders[MTDEC__THREADS_MAX]; + CXzDecMtThread coders[MTDEC_THREADS_MAX]; #endif - -} CXzDecMt; +}; @@ -1535,11 +1569,11 @@ CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) XzDecMtProps_Init(&p->props); - #ifndef _7ZIP_ST + #ifndef Z7_ST p->mtc_WasConstructed = False; { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CXzDecMtThread *coder = &p->coders[i]; coder->dec_created = False; @@ -1549,16 +1583,16 @@ CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) } #endif - return p; + return (CXzDecMtHandle)p; } -#ifndef _7ZIP_ST +#ifndef Z7_ST static void XzDecMt_FreeOutBufs(CXzDecMt *p) { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CXzDecMtThread *coder = &p->coders[i]; if (coder->outBuf) @@ -1595,13 +1629,15 @@ static void XzDecMt_FreeSt(CXzDecMt *p) } -void XzDecMt_Destroy(CXzDecMtHandle pp) +// #define GET_CXzDecMt_p CXzDecMt *p = pp; + +void XzDecMt_Destroy(CXzDecMtHandle p) { - CXzDecMt *p = (CXzDecMt *)pp; + // GET_CXzDecMt_p XzDecMt_FreeSt(p); - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->mtc_WasConstructed) { @@ -1610,7 +1646,7 @@ void XzDecMt_Destroy(CXzDecMtHandle pp) } { unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) + for (i = 0; i < MTDEC_THREADS_MAX; i++) { CXzDecMtThread *t = &p->coders[i]; if (t->dec_created) @@ -1625,12 +1661,12 @@ void XzDecMt_Destroy(CXzDecMtHandle pp) #endif - ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, p); } -#ifndef _7ZIP_ST +#ifndef Z7_ST static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) { @@ -1696,7 +1732,7 @@ static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbac coder->dec.parseMode = True; coder->dec.headerParsedOk = False; - PRF_STR_INT("Parse", srcSize2); + PRF_STR_INT("Parse", srcSize2) res = XzUnpacker_Code(&coder->dec, NULL, &destSize, @@ -1739,10 +1775,10 @@ static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbac } } { - UInt64 packSize = block->packSize; - UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); - UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); - UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + const UInt64 packSize = block->packSize; + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + const unsigned checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + const UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; // if (blockPackSum <= me->props.inBlockMax) // unpackBlockMaxSize { @@ -2071,7 +2107,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, } data += cur; size -= cur; - // PRF_STR_INT("Written size =", size); + // PRF_STR_INT("Written size =", size) if (size == 0) break; res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); @@ -2087,7 +2123,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, return res; } - RINOK(res); + RINOK(res) if (coder->inPreSize != coder->inCodeSize || coder->blockPackTotal != coder->inCodeSize) @@ -2106,13 +2142,13 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, // (coder->state == MTDEC_PARSE_END) means that there are no other working threads // so we can use mtc variables without lock - PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed) me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; { CXzUnpacker *dec = &me->dec; - PRF_STR_INT("PostSingle", srcSize); + PRF_STR_INT("PostSingle", srcSize) { size_t srcProcessed = srcSize; @@ -2186,7 +2222,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, me->mtc.crossEnd = srcSize; } - PRF_STR_INT("XZ_STATE_STREAM_HEADER crossEnd = ", (unsigned)me->mtc.crossEnd); + PRF_STR_INT("XZ_STATE_STREAM_HEADER crossEnd = ", (unsigned)me->mtc.crossEnd) return SZ_OK; } @@ -2277,7 +2313,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; if (inDelta >= (1 << 22)) { - RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)) inProgressPrev = me->mtc.inProcessed; } } @@ -2331,7 +2367,7 @@ void XzStatInfo_Clear(CXzStatInfo *p) */ static SRes XzDecMt_Decode_ST(CXzDecMt *p - #ifndef _7ZIP_ST + #ifndef Z7_ST , BoolInt tMode #endif , CXzStatInfo *stat) @@ -2343,11 +2379,11 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p CXzUnpacker *dec; - #ifndef _7ZIP_ST + #ifndef Z7_ST if (tMode) { XzDecMt_FreeOutBufs(p); - tMode = MtDec_PrepareRead(&p->mtc); + tMode = (BoolInt)MtDec_PrepareRead(&p->mtc); } #endif @@ -2400,7 +2436,7 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p if (inPos == inLim) { - #ifndef _7ZIP_ST + #ifndef Z7_ST if (tMode) { inData = MtDec_Read(&p->mtc, &inLim); @@ -2577,19 +2613,19 @@ static void XzStatInfo_SetStat(const CXzUnpacker *dec, -SRes XzDecMt_Decode(CXzDecMtHandle pp, +SRes XzDecMt_Decode(CXzDecMtHandle p, const CXzDecMtProps *props, const UInt64 *outDataSize, int finishMode, - ISeqOutStream *outStream, + ISeqOutStreamPtr outStream, // Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, // const Byte *inData, size_t inDataSize, CXzStatInfo *stat, int *isMT, - ICompressProgress *progress) + ICompressProgressPtr progress) { - CXzDecMt *p = (CXzDecMt *)pp; - #ifndef _7ZIP_ST + // GET_CXzDecMt_p + #ifndef Z7_ST BoolInt tMode; #endif @@ -2610,7 +2646,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, p->outSize = *outDataSize; } - p->finishMode = finishMode; + p->finishMode = (BoolInt)finishMode; // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test @@ -2640,7 +2676,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, */ - #ifndef _7ZIP_ST + #ifndef Z7_ST p->isBlockHeaderState_Parse = False; p->isBlockHeaderState_Write = False; @@ -2782,7 +2818,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, return res; } - PRF_STR("----- decoding ST -----"); + PRF_STR("----- decoding ST -----") } #endif @@ -2792,13 +2828,13 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, { SRes res = XzDecMt_Decode_ST(p - #ifndef _7ZIP_ST + #ifndef Z7_ST , tMode #endif , stat ); - #ifndef _7ZIP_ST + #ifndef Z7_ST // we must set error code from MT decoding at first if (p->mainErrorCode != SZ_OK) stat->DecodeRes = p->mainErrorCode; @@ -2835,3 +2871,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, return res; } } + +#undef PRF +#undef PRF_STR +#undef PRF_STR_INT_2 diff --git a/multiarc/src/formats/7z/C/XzEnc.c b/multiarc/src/formats/7z/C/XzEnc.c old mode 100755 new mode 100644 index be174cccf..c1affadfa --- a/multiarc/src/formats/7z/C/XzEnc.c +++ b/multiarc/src/formats/7z/C/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2021-04-01 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -18,42 +18,43 @@ #include "XzEnc.h" -// #define _7ZIP_ST +// #define Z7_ST -#ifndef _7ZIP_ST +#ifndef Z7_ST #include "MtCoder.h" #else -#define MTCODER__THREADS_MAX 1 -#define MTCODER__BLOCKS_MAX 1 +#define MTCODER_THREADS_MAX 1 +#define MTCODER_BLOCKS_MAX 1 #endif #define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) -/* max pack size for LZMA2 block + check-64bytrs: */ -#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) +#define XZ_CHECK_SIZE_MAX 64 +/* max pack size for LZMA2 block + pad4 + check_size: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + XZ_CHECK_SIZE_MAX) #define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) -#define XzBlock_ClearFlags(p) (p)->flags = 0; -#define XzBlock_SetNumFilters(p, n) (p)->flags = (Byte)((p)->flags | ((n) - 1)); +// #define XzBlock_ClearFlags(p) (p)->flags = 0; +#define XzBlock_ClearFlags_SetNumFilters(p, n) (p)->flags = (Byte)((n) - 1); #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; -static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) +static SRes WriteBytes(ISeqOutStreamPtr s, const void *buf, size_t size) { return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; } -static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) +static SRes WriteBytes_UpdateCrc(ISeqOutStreamPtr s, const void *buf, size_t size, UInt32 *crc) { *crc = CrcUpdate(*crc, buf, size); return WriteBytes(s, buf, size); } -static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) +static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStreamPtr s) { UInt32 crc; Byte header[XZ_STREAM_HEADER_SIZE]; @@ -61,12 +62,12 @@ static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) header[XZ_SIG_SIZE] = (Byte)(f >> 8); header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF); crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE); - SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc); + SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc) return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE); } -static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) +static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStreamPtr s) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; @@ -91,7 +92,7 @@ static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) header[pos++] = 0; header[0] = (Byte)(pos >> 2); - SetUi32(header + pos, CrcCalc(header, pos)); + SetUi32(header + pos, CrcCalc(header, pos)) return WriteBytes(s, header, pos + 4); } @@ -182,7 +183,7 @@ static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 size_t newSize = p->allocated * 2 + 16 * 2; if (newSize < p->size + pos) return SZ_ERROR_MEM; - RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); + RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)) } memcpy(p->blocks + p->size, buf, pos); p->size += pos; @@ -191,7 +192,7 @@ static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 } -static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) +static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStreamPtr s) { Byte buf[32]; UInt64 globalPos; @@ -200,8 +201,8 @@ static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, I globalPos = pos; buf[0] = 0; - RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); - RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); + RINOK(WriteBytes_UpdateCrc(s, buf, pos, &crc)) + RINOK(WriteBytes_UpdateCrc(s, p->blocks, p->size, &crc)) globalPos += p->size; pos = XZ_GET_PAD_SIZE(globalPos); @@ -211,12 +212,12 @@ static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, I globalPos += pos; crc = CrcUpdate(crc, buf + 4 - pos, pos); - SetUi32(buf + 4, CRC_GET_DIGEST(crc)); + SetUi32(buf + 4, CRC_GET_DIGEST(crc)) - SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); + SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)) buf[8 + 8] = (Byte)(flags >> 8); buf[8 + 9] = (Byte)(flags & 0xFF); - SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); + SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)) buf[8 + 10] = XZ_FOOTER_SIG_0; buf[8 + 11] = XZ_FOOTER_SIG_1; @@ -230,7 +231,7 @@ static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, I typedef struct { ISeqInStream vt; - ISeqInStream *realStream; + ISeqInStreamPtr realStream; const Byte *data; UInt64 limit; UInt64 processed; @@ -251,9 +252,9 @@ static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) XzCheck_Final(&p->check, digest); } -static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +static SRes SeqCheckInStream_Read(ISeqInStreamPtr pp, void *data, size_t *size) { - CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSeqCheckInStream) size_t size2 = *size; SRes res = SZ_OK; @@ -285,15 +286,15 @@ static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *si typedef struct { ISeqOutStream vt; - ISeqOutStream *realStream; + ISeqOutStreamPtr realStream; Byte *outBuf; size_t outBufLimit; UInt64 processed; } CSeqSizeOutStream; -static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) +static size_t SeqSizeOutStream_Write(ISeqOutStreamPtr pp, const void *data, size_t size) { - CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSeqSizeOutStream) if (p->realStream) size = ISeqOutStream_Write(p->realStream, data, size); else @@ -313,8 +314,8 @@ static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, typedef struct { - ISeqInStream p; - ISeqInStream *realStream; + ISeqInStream vt; + ISeqInStreamPtr realStream; IStateCoder StateCoder; Byte *buf; size_t curPos; @@ -323,7 +324,40 @@ typedef struct } CSeqInFilter; -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); +static const z7_Func_BranchConv g_Funcs_BranchConv_RISC_Enc[] = +{ + Z7_BRANCH_CONV_ENC_2 (BranchConv_PPC), + Z7_BRANCH_CONV_ENC_2 (BranchConv_IA64), + Z7_BRANCH_CONV_ENC_2 (BranchConv_ARM), + Z7_BRANCH_CONV_ENC_2 (BranchConv_ARMT), + Z7_BRANCH_CONV_ENC_2 (BranchConv_SPARC), + Z7_BRANCH_CONV_ENC_2 (BranchConv_ARM64), + Z7_BRANCH_CONV_ENC_2 (BranchConv_RISCV) +}; + +static SizeT XzBcFilterStateBase_Filter_Enc(CXzBcFilterStateBase *p, Byte *data, SizeT size) +{ + switch (p->methodId) + { + case XZ_ID_Delta: + Delta_Encode(p->delta_State, p->delta, data, size); + break; + case XZ_ID_X86: + size = (SizeT)(z7_BranchConvSt_X86_Enc(data, size, p->ip, &p->X86_State) - data); + break; + default: + if (p->methodId >= XZ_ID_PPC) + { + const UInt32 i = p->methodId - XZ_ID_PPC; + if (i < Z7_ARRAY_SIZE(g_Funcs_BranchConv_RISC_Enc)) + size = (SizeT)(g_Funcs_BranchConv_RISC_Enc[i](data, size, p->ip) - data); + } + break; + } + p->ip += (UInt32)size; + return size; +} + static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) { @@ -335,17 +369,17 @@ static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPt } p->curPos = p->endPos = 0; p->srcWasFinished = 0; - RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); - RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); + RINOK(Xz_StateCoder_Bc_SetFromMethod_Func(&p->StateCoder, props->id, XzBcFilterStateBase_Filter_Enc, alloc)) + RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)) p->StateCoder.Init(p->StateCoder.p); return SZ_OK; } -static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) +static SRes SeqInFilter_Read(ISeqInStreamPtr pp, void *data, size_t *size) { - CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); - size_t sizeOriginal = *size; + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSeqInFilter) + const size_t sizeOriginal = *size; if (sizeOriginal == 0) return SZ_OK; *size = 0; @@ -356,7 +390,7 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) { p->curPos = 0; p->endPos = FILTER_BUF_SIZE; - RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)); + RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)) if (p->endPos == 0) p->srcWasFinished = 1; } @@ -381,7 +415,7 @@ static void SeqInFilter_Construct(CSeqInFilter *p) { p->buf = NULL; p->StateCoder.p = NULL; - p->p.Read = SeqInFilter_Read; + p->vt.Read = SeqInFilter_Read; } static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) @@ -406,13 +440,13 @@ static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) typedef struct { ISeqInStream vt; - ISeqInStream *inStream; + ISeqInStreamPtr inStream; CSbEnc enc; } CSbEncInStream; -static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +static SRes SbEncInStream_Read(ISeqInStreamPtr pp, void *data, size_t *size) { - CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); + CSbEncInStream *p = Z7_CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); size_t sizeOriginal = *size; if (sizeOriginal == 0) return SZ_OK; @@ -422,7 +456,7 @@ static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) if (p->enc.needRead && !p->enc.readWasFinished) { size_t processed = p->enc.needReadSizeMax; - RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed)); + RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed)) p->enc.readPos += processed; if (processed == 0) { @@ -433,7 +467,7 @@ static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) } *size = sizeOriginal; - RINOK(SbEnc_Read(&p->enc, data, size)); + RINOK(SbEnc_Read(&p->enc, data, size)) if (*size != 0 || !p->enc.needRead) return SZ_OK; } @@ -473,7 +507,7 @@ void XzFilterProps_Init(CXzFilterProps *p) void XzProps_Init(CXzProps *p) { p->checkId = XZ_CHECK_CRC32; - p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; + p->blockSize = XZ_PROPS_BLOCK_SIZE_AUTO; p->numBlockThreads_Reduced = -1; p->numBlockThreads_Max = -1; p->numTotalThreads = -1; @@ -502,8 +536,8 @@ static void XzEncProps_Normalize_Fixed(CXzProps *p) t2 = p->numBlockThreads_Max; t3 = p->numTotalThreads; - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; + if (t2 > MTCODER_THREADS_MAX) + t2 = MTCODER_THREADS_MAX; if (t3 <= 0) { @@ -519,8 +553,8 @@ static void XzEncProps_Normalize_Fixed(CXzProps *p) t1 = 1; t2 = t3; } - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; + if (t2 > MTCODER_THREADS_MAX) + t2 = MTCODER_THREADS_MAX; } else if (t1 <= 0) { @@ -571,7 +605,7 @@ static void XzProps_Normalize(CXzProps *p) /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. Lzma2Enc_SetProps() will normalize lzma2Props later. */ - if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) + if (p->blockSize == XZ_PROPS_BLOCK_SIZE_SOLID) { p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; p->numBlockThreads_Reduced = 1; @@ -583,15 +617,15 @@ static void XzProps_Normalize(CXzProps *p) else { CLzma2EncProps *lzma2 = &p->lzma2Props; - if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + if (p->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) { // xz-auto p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; - if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + if (lzma2->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) { // if (xz-auto && lzma2-solid) - we use solid for both - p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; + p->blockSize = XZ_PROPS_BLOCK_SIZE_SOLID; p->numBlockThreads_Reduced = 1; p->numBlockThreads_Max = 1; if (p->lzma2Props.numTotalThreads <= 0) @@ -610,9 +644,9 @@ static void XzProps_Normalize(CXzProps *p) p->blockSize = tp.blockSize; // fixed or solid p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; p->numBlockThreads_Max = tp.numBlockThreads_Max; - if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) - lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + if (lzma2->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) + lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID + if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) lzma2->lzmaProps.reduceSize = tp.blockSize; lzma2->numBlockThreads_Reduced = 1; lzma2->numBlockThreads_Max = 1; @@ -631,9 +665,9 @@ static void XzProps_Normalize(CXzProps *p) r = p->blockSize; lzma2->lzmaProps.reduceSize = r; } - if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) - lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; - else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + if (lzma2->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) + lzma2->blockSize = LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID; + else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) lzma2->blockSize = p->blockSize; XzEncProps_Normalize_Fixed(p); @@ -704,17 +738,17 @@ typedef struct static SRes Xz_CompressBlock( CLzma2WithFilters *lzmaf, - ISeqOutStream *outStream, + ISeqOutStreamPtr outStream, Byte *outBufHeader, Byte *outBufData, size_t outBufDataLimit, - ISeqInStream *inStream, + ISeqInStreamPtr inStream, // UInt64 expectedSize, const Byte *inBuf, // used if (!inStream) size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored const CXzProps *props, - ICompressProgress *progress, + ICompressProgressPtr progress, int *inStreamFinished, /* only for inStream version */ CXzEncBlockInfo *blockSizes, ISzAllocPtr alloc, @@ -731,12 +765,12 @@ static SRes Xz_CompressBlock( *inStreamFinished = False; - RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); + RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)) - RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); + RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)) - XzBlock_ClearFlags(&block); - XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + // XzBlock_ClearFlags(&block) + XzBlock_ClearFlags_SetNumFilters(&block, 1 + (fp ? 1 : 0)) if (fp) { @@ -752,7 +786,7 @@ static SRes Xz_CompressBlock( else if (fp->ipDefined) { Byte *ptr = filter->props; - SetUi32(ptr, fp->ip); + SetUi32(ptr, fp->ip) filter->propsSize = 4; } } @@ -777,13 +811,13 @@ static SRes Xz_CompressBlock( if (props->blockSize != (UInt64)(Int64)-1) if (expectedSize > props->blockSize) block.unpackSize = props->blockSize; - XzBlock_SetHasUnpackSize(&block); + XzBlock_SetHasUnpackSize(&block) } */ if (outStream) { - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)) } checkInStream.vt.Read = SeqCheckInStream_Read; @@ -801,13 +835,13 @@ static SRes Xz_CompressBlock( if (fp->id == XZ_ID_Subblock) { lzmaf->sb.inStream = &checkInStream.vt; - RINOK(SbEncInStream_Init(&lzmaf->sb)); + RINOK(SbEncInStream_Init(&lzmaf->sb)) } else #endif { lzmaf->filter.realStream = &checkInStream.vt; - RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); + RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)) } } @@ -841,7 +875,7 @@ static SRes Xz_CompressBlock( #ifdef USE_SUBBLOCK (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: #endif - &lzmaf->filter.p) : + &lzmaf->filter.vt) : &checkInStream.vt) : NULL, useStream ? NULL : inBuf, @@ -852,13 +886,13 @@ static SRes Xz_CompressBlock( if (outBuf) seqSizeOutStream.processed += outSize; - RINOK(res); + RINOK(res) blockSizes->unpackSize = checkInStream.processed; } { - Byte buf[4 + 64]; - unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); - UInt64 packSize = seqSizeOutStream.processed; + Byte buf[4 + XZ_CHECK_SIZE_MAX]; + const unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + const UInt64 packSize = seqSizeOutStream.processed; buf[0] = 0; buf[1] = 0; @@ -866,7 +900,8 @@ static SRes Xz_CompressBlock( buf[3] = 0; SeqCheckInStream_GetDigest(&checkInStream, buf + 4); - RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), + padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))) blockSizes->totalSize = seqSizeOutStream.processed - padSize; @@ -877,12 +912,12 @@ static SRes Xz_CompressBlock( seqSizeOutStream.processed = 0; block.unpackSize = blockSizes->unpackSize; - XzBlock_SetHasUnpackSize(&block); + XzBlock_SetHasUnpackSize(&block) block.packSize = packSize; - XzBlock_SetHasPackSize(&block); + XzBlock_SetHasPackSize(&block) - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)) blockSizes->headerSize = (size_t)seqSizeOutStream.processed; blockSizes->totalSize += seqSizeOutStream.processed; @@ -906,15 +941,15 @@ static SRes Xz_CompressBlock( typedef struct { ICompressProgress vt; - ICompressProgress *progress; + ICompressProgressPtr progress; UInt64 inOffset; UInt64 outOffset; } CCompressProgress_XzEncOffset; -static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +static SRes CompressProgress_XzEncOffset_Progress(ICompressProgressPtr pp, UInt64 inSize, UInt64 outSize) { - const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); + const CCompressProgress_XzEncOffset *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CCompressProgress_XzEncOffset, vt); inSize += p->inOffset; outSize += p->outOffset; return ICompressProgress_Progress(p->progress, inSize, outSize); @@ -923,7 +958,7 @@ static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, U -typedef struct +struct CXzEnc { ISzAllocPtr alloc; ISzAllocPtr allocBig; @@ -933,20 +968,19 @@ typedef struct CXzEncIndex xzIndex; - CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; + CLzma2WithFilters lzmaf_Items[MTCODER_THREADS_MAX]; size_t outBufSize; /* size of allocated outBufs[i] */ - Byte *outBufs[MTCODER__BLOCKS_MAX]; + Byte *outBufs[MTCODER_BLOCKS_MAX]; - #ifndef _7ZIP_ST + #ifndef Z7_ST unsigned checkType; - ISeqOutStream *outStream; + ISeqOutStreamPtr outStream; BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; - CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; + CXzEncBlockInfo EncBlocks[MTCODER_BLOCKS_MAX]; #endif - -} CXzEnc; +}; static void XzEnc_Construct(CXzEnc *p) @@ -955,13 +989,13 @@ static void XzEnc_Construct(CXzEnc *p) XzEncIndex_Construct(&p->xzIndex); - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); - #ifndef _7ZIP_ST + #ifndef Z7_ST p->mtCoder_WasConstructed = False; { - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) p->outBufs[i] = NULL; p->outBufSize = 0; } @@ -972,7 +1006,7 @@ static void XzEnc_Construct(CXzEnc *p) static void XzEnc_FreeOutBufs(CXzEnc *p) { unsigned i; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + for (i = 0; i < MTCODER_BLOCKS_MAX; i++) if (p->outBufs[i]) { ISzAlloc_Free(p->alloc, p->outBufs[i]); @@ -988,10 +1022,10 @@ static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) XzEncIndex_Free(&p->xzIndex, alloc); - for (i = 0; i < MTCODER__THREADS_MAX; i++) + for (i = 0; i < MTCODER_THREADS_MAX; i++) Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); - #ifndef _7ZIP_ST + #ifndef Z7_ST if (p->mtCoder_WasConstructed) { MtCoder_Destruct(&p->mtCoder); @@ -1013,37 +1047,38 @@ CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) p->expectedDataSize = (UInt64)(Int64)-1; p->alloc = alloc; p->allocBig = allocBig; - return p; + return (CXzEncHandle)p; } +// #define GET_CXzEnc_p CXzEnc *p = (CXzEnc *)(void *)pp; -void XzEnc_Destroy(CXzEncHandle pp) +void XzEnc_Destroy(CXzEncHandle p) { - CXzEnc *p = (CXzEnc *)pp; + // GET_CXzEnc_p XzEnc_Free(p, p->alloc); ISzAlloc_Free(p->alloc, p); } -SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) +SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props) { - CXzEnc *p = (CXzEnc *)pp; + // GET_CXzEnc_p p->xzProps = *props; XzProps_Normalize(&p->xzProps); return SZ_OK; } -void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) +void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize) { - CXzEnc *p = (CXzEnc *)pp; + // GET_CXzEnc_p p->expectedDataSize = expectedDataSiize; } -#ifndef _7ZIP_ST +#ifndef Z7_ST static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, const Byte *src, size_t srcSize, int finished) @@ -1051,18 +1086,19 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf CXzEnc *me = (CXzEnc *)pp; SRes res; CMtProgressThunk progressThunk; - - Byte *dest = me->outBufs[outBufIndex]; - + Byte *dest; UNUSED_VAR(finished) - { CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; bInfo->totalSize = 0; bInfo->unpackSize = 0; bInfo->headerSize = 0; + // v23.02: we don't compress empty blocks + // also we must ignore that empty block in XzEnc_MtCallback_Write() + if (srcSize == 0) + return SZ_OK; } - + dest = me->outBufs[outBufIndex]; if (!dest) { dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); @@ -1073,7 +1109,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - MtProgressThunk_Init(&progressThunk); + MtProgressThunk_INIT(&progressThunk) { CXzEncBlockInfo blockSizes; @@ -1108,27 +1144,29 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) { CXzEnc *me = (CXzEnc *)pp; - const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; - const Byte *data = me->outBufs[outBufIndex]; - - RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); - + // v23.02: we don't write empty blocks + // note: if (bInfo->unpackSize == 0) then there is no compressed data of block + if (bInfo->unpackSize == 0) + return SZ_OK; { - UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); - RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); + const Byte *data = me->outBufs[outBufIndex]; + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)) + { + const UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)) + } + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); } - - return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); } #endif -SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, ICompressProgressPtr progress) { - CXzEnc *p = (CXzEnc *)pp; + // GET_CXzEnc_p const CXzProps *props = &p->xzProps; @@ -1137,7 +1175,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr UInt64 numBlocks = 1; UInt64 blockSize = props->blockSize; - if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID + if (blockSize != XZ_PROPS_BLOCK_SIZE_SOLID && props->reduceSize != (UInt64)(Int64)-1) { numBlocks = props->reduceSize / blockSize; @@ -1147,13 +1185,13 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr else blockSize = (UInt64)1 << 62; - RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); + RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)) } - RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); + RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)) - #ifndef _7ZIP_ST + #ifndef Z7_ST if (props->numBlockThreads_Reduced > 1) { IMtCoderCallback2 vt; @@ -1180,8 +1218,8 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr p->mtCoder.mtCallback = &vt; p->mtCoder.mtCallbackObject = p; - if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID - || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) + if ( props->blockSize == XZ_PROPS_BLOCK_SIZE_SOLID + || props->blockSize == XZ_PROPS_BLOCK_SIZE_AUTO) return SZ_ERROR_FAIL; p->mtCoder.blockSize = (size_t)props->blockSize; @@ -1200,7 +1238,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr p->mtCoder.numThreadsMax = (unsigned)props->numBlockThreads_Max; p->mtCoder.expectedDataSize = p->expectedDataSize; - RINOK(MtCoder_Code(&p->mtCoder)); + RINOK(MtCoder_Code(&p->mtCoder)) } else #endif @@ -1217,7 +1255,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr writeStartSizes = 0; - if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) + if (props->blockSize != XZ_PROPS_BLOCK_SIZE_SOLID) { writeStartSizes = (props->forceWriteSizesInHeader > 0); @@ -1274,18 +1312,18 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr &inStreamFinished, &blockSizes, p->alloc, - p->allocBig)); + p->allocBig)) { UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); if (writeStartSizes) { - RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); - RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); + RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)) + RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)) } - RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); + RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)) progress2.inOffset += blockSizes.unpackSize; progress2.outOffset += totalPackFull; @@ -1302,8 +1340,8 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr #include "Alloc.h" -SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress) +SRes Xz_Encode(ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, + const CXzProps *props, ICompressProgressPtr progress) { SRes res; CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); @@ -1317,7 +1355,7 @@ SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, } -SRes Xz_EncodeEmpty(ISeqOutStream *outStream) +SRes Xz_EncodeEmpty(ISeqOutStreamPtr outStream) { SRes res; CXzEncIndex xzIndex; diff --git a/multiarc/src/formats/7z/C/XzEnc.h b/multiarc/src/formats/7z/C/XzEnc.h old mode 100755 new mode 100644 index 0c29e7e1e..77b78c014 --- a/multiarc/src/formats/7z/C/XzEnc.h +++ b/multiarc/src/formats/7z/C/XzEnc.h @@ -1,8 +1,8 @@ /* XzEnc.h -- Xz Encode -2017-06-27 : Igor Pavlov : Public domain */ +2023-04-13 : Igor Pavlov : Public domain */ -#ifndef __XZ_ENC_H -#define __XZ_ENC_H +#ifndef ZIP7_INC_XZ_ENC_H +#define ZIP7_INC_XZ_ENC_H #include "Lzma2Enc.h" @@ -11,8 +11,8 @@ EXTERN_C_BEGIN -#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO -#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID +#define XZ_PROPS_BLOCK_SIZE_AUTO LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO +#define XZ_PROPS_BLOCK_SIZE_SOLID LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID typedef struct @@ -41,19 +41,20 @@ typedef struct void XzProps_Init(CXzProps *p); - -typedef void * CXzEncHandle; +typedef struct CXzEnc CXzEnc; +typedef CXzEnc * CXzEncHandle; +// Z7_DECLARE_HANDLE(CXzEncHandle) CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); void XzEnc_Destroy(CXzEncHandle p); SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); -SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, ICompressProgressPtr progress); -SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress); +SRes Xz_Encode(ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, + const CXzProps *props, ICompressProgressPtr progress); -SRes Xz_EncodeEmpty(ISeqOutStream *outStream); +SRes Xz_EncodeEmpty(ISeqOutStreamPtr outStream); EXTERN_C_END diff --git a/multiarc/src/formats/7z/C/XzIn.c b/multiarc/src/formats/7z/C/XzIn.c old mode 100755 new mode 100644 index 84f868ec6..b68af965c --- a/multiarc/src/formats/7z/C/XzIn.c +++ b/multiarc/src/formats/7z/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2021-09-04 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -15,27 +15,29 @@ #define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) -SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) +SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStreamPtr inStream) { Byte sig[XZ_STREAM_HEADER_SIZE]; - RINOK(SeqInStream_Read2(inStream, sig, XZ_STREAM_HEADER_SIZE, SZ_ERROR_NO_ARCHIVE)); - if (memcmp(sig, XZ_SIG, XZ_SIG_SIZE) != 0) + size_t processedSize = XZ_STREAM_HEADER_SIZE; + RINOK(SeqInStream_ReadMax(inStream, sig, &processedSize)) + if (processedSize != XZ_STREAM_HEADER_SIZE + || memcmp(sig, XZ_SIG, XZ_SIG_SIZE) != 0) return SZ_ERROR_NO_ARCHIVE; return Xz_ParseHeader(p, sig); } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ - { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ + { const unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; \ pos += s; } -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; unsigned headerSize; *headerSizeRes = 0; - RINOK(SeqInStream_ReadByte(inStream, &header[0])); - headerSize = (unsigned)header[0]; + RINOK(SeqInStream_ReadByte(inStream, &header[0])) + headerSize = header[0]; if (headerSize == 0) { *headerSizeRes = 1; @@ -45,20 +47,27 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, U *isIndex = False; headerSize = (headerSize << 2) + 4; - *headerSizeRes = headerSize; - RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); + *headerSizeRes = (UInt32)headerSize; + { + size_t processedSize = headerSize - 1; + RINOK(SeqInStream_ReadMax(inStream, header + 1, &processedSize)) + if (processedSize != headerSize - 1) + return SZ_ERROR_INPUT_EOF; + } return XzBlock_Parse(p, header); } #define ADD_SIZE_CHECK(size, val) \ - { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } + { const UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } UInt64 Xz_GetUnpackSize(const CXzStream *p) { UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); + { + ADD_SIZE_CHECK(size, p->blocks[i].unpackSize) + } return size; } @@ -67,12 +76,14 @@ UInt64 Xz_GetPackSize(const CXzStream *p) UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); + { + ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3) + } return size; } /* -SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStream *inStream) +SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStreamPtr inStream) { return SeqInStream_Read(inStream, p->check, XzFlags_GetCheckSize(f)); } @@ -93,7 +104,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt { UInt64 numBlocks64; - READ_VARINT_AND_CHECK(buf, pos, size, &numBlocks64); + READ_VARINT_AND_CHECK(buf, pos, size, &numBlocks64) numBlocks = (size_t)numBlocks64; if (numBlocks != numBlocks64 || numBlocks * 2 > size) return SZ_ERROR_ARCHIVE; @@ -110,8 +121,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt for (i = 0; i < numBlocks; i++) { CXzBlockSizes *block = &p->blocks[i]; - READ_VARINT_AND_CHECK(buf, pos, size, &block->totalSize); - READ_VARINT_AND_CHECK(buf, pos, size, &block->unpackSize); + READ_VARINT_AND_CHECK(buf, pos, size, &block->totalSize) + READ_VARINT_AND_CHECK(buf, pos, size, &block->unpackSize) if (block->totalSize == 0) return SZ_ERROR_ARCHIVE; } @@ -122,7 +133,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; } -static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAllocPtr alloc) +static SRes Xz_ReadIndex(CXzStream *p, ILookInStreamPtr stream, UInt64 indexSize, ISzAllocPtr alloc) { SRes res; size_t size; @@ -142,14 +153,14 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, return res; } -static SRes LookInStream_SeekRead_ForArc(ILookInStream *stream, UInt64 offset, void *buf, size_t size) +static SRes LookInStream_SeekRead_ForArc(ILookInStreamPtr stream, UInt64 offset, void *buf, size_t size) { - RINOK(LookInStream_SeekTo(stream, offset)); + RINOK(LookInStream_SeekTo(stream, offset)) return LookInStream_Read(stream, buf, size); /* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */ } -static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAllocPtr alloc) +static SRes Xz_ReadBackward(CXzStream *p, ILookInStreamPtr stream, Int64 *startOffset, ISzAllocPtr alloc) { UInt64 indexSize; Byte buf[XZ_STREAM_FOOTER_SIZE]; @@ -159,7 +170,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_NO_ARCHIVE; pos -= XZ_STREAM_FOOTER_SIZE; - RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); + RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)) if (!XZ_FOOTER_SIG_CHECK(buf + 10)) { @@ -174,7 +185,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff i = (pos > TEMP_BUF_SIZE) ? TEMP_BUF_SIZE : (size_t)pos; pos -= i; - RINOK(LookInStream_SeekRead_ForArc(stream, pos, temp, i)); + RINOK(LookInStream_SeekRead_ForArc(stream, pos, temp, i)) total += (UInt32)i; for (; i != 0; i--) if (temp[i - 1] != 0) @@ -193,7 +204,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff if (pos < XZ_STREAM_FOOTER_SIZE) return SZ_ERROR_NO_ARCHIVE; pos -= XZ_STREAM_FOOTER_SIZE; - RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); + RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)) if (!XZ_FOOTER_SIG_CHECK(buf + 10)) return SZ_ERROR_NO_ARCHIVE; } @@ -217,8 +228,8 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff return SZ_ERROR_ARCHIVE; pos -= indexSize; - RINOK(LookInStream_SeekTo(stream, pos)); - RINOK(Xz_ReadIndex(p, stream, indexSize, alloc)); + RINOK(LookInStream_SeekTo(stream, pos)) + RINOK(Xz_ReadIndex(p, stream, indexSize, alloc)) { UInt64 totalSize = Xz_GetPackSize(p); @@ -227,7 +238,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff || pos < totalSize + XZ_STREAM_HEADER_SIZE) return SZ_ERROR_ARCHIVE; pos -= (totalSize + XZ_STREAM_HEADER_SIZE); - RINOK(LookInStream_SeekTo(stream, pos)); + RINOK(LookInStream_SeekTo(stream, pos)) *startOffset = (Int64)pos; } { @@ -236,7 +247,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff SecToRead_CreateVTable(&secToRead); secToRead.realStream = stream; - RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)); + RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)) return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE; } } @@ -274,7 +285,9 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); + { + ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])) + } return size; } @@ -284,15 +297,17 @@ UInt64 Xzs_GetPackSize(const CXzs *p) UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); + { + ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])) + } return size; } */ -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc) +SRes Xzs_ReadBackward(CXzs *p, ILookInStreamPtr stream, Int64 *startOffset, ICompressProgressPtr progress, ISzAllocPtr alloc) { Int64 endOffset = 0; - RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)) *startOffset = endOffset; for (;;) { @@ -301,7 +316,7 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr Xz_Construct(&st); res = Xz_ReadBackward(&st, stream, startOffset, alloc); st.startOffset = (UInt64)*startOffset; - RINOK(res); + RINOK(res) if (p->num == p->numAllocated) { const size_t newNum = p->num + p->num / 4 + 1; @@ -317,7 +332,7 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr p->streams[p->num++] = st; if (*startOffset == 0) break; - RINOK(LookInStream_SeekTo(stream, (UInt64)*startOffset)); + RINOK(LookInStream_SeekTo(stream, (UInt64)*startOffset)) if (progress && ICompressProgress_Progress(progress, (UInt64)(endOffset - *startOffset), (UInt64)(Int64)-1) != SZ_OK) return SZ_ERROR_PROGRESS; } diff --git a/multiarc/src/formats/7z/C/ZstdDec.c b/multiarc/src/formats/7z/C/ZstdDec.c new file mode 100644 index 000000000..22165d966 --- /dev/null +++ b/multiarc/src/formats/7z/C/ZstdDec.c @@ -0,0 +1,4067 @@ +/* ZstdDec.c -- Zstd Decoder +2024-06-18 : the code was developed by Igor Pavlov, using Zstandard format + specification and original zstd decoder code as reference code. +original zstd decoder code: Copyright (c) Facebook, Inc. All rights reserved. +This source code is licensed under BSD 3-Clause License. +*/ + +#include "Precomp.h" + +#include +#include +// #include + +#include "Alloc.h" +#include "Xxh64.h" +#include "ZstdDec.h" +#include "CpuArch.h" + +#if defined(MY_CPU_ARM64) +#include +#endif + +/* original-zstd still doesn't support window larger than 2 GiB. + So we also limit our decoder for 2 GiB window: */ +#if defined(MY_CPU_64BIT) && 0 == 1 + #define MAX_WINDOW_SIZE_LOG 41 +#else + #define MAX_WINDOW_SIZE_LOG 31 +#endif + +typedef + #if MAX_WINDOW_SIZE_LOG < 32 + UInt32 + #else + size_t + #endif + CZstdDecOffset; + +// for debug: simpler and smaller code but slow: +// #define Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + +// #define SHOW_STAT +#ifdef SHOW_STAT +#include +static unsigned g_Num_Blocks_Compressed = 0; +static unsigned g_Num_Blocks_memcpy = 0; +static unsigned g_Num_Wrap_memmove_Num = 0; +static unsigned g_Num_Wrap_memmove_Bytes = 0; +static unsigned g_NumSeqs_total = 0; +// static unsigned g_NumCopy = 0; +static unsigned g_NumOver = 0; +static unsigned g_NumOver2 = 0; +static unsigned g_Num_Match = 0; +static unsigned g_Num_Lits = 0; +static unsigned g_Num_LitsBig = 0; +static unsigned g_Num_Lit0 = 0; +static unsigned g_Num_Rep0 = 0; +static unsigned g_Num_Rep1 = 0; +static unsigned g_Num_Rep2 = 0; +static unsigned g_Num_Rep3 = 0; +static unsigned g_Num_Threshold_0 = 0; +static unsigned g_Num_Threshold_1 = 0; +static unsigned g_Num_Threshold_0sum = 0; +static unsigned g_Num_Threshold_1sum = 0; +#define STAT_UPDATE(v) v +#else +#define STAT_UPDATE(v) +#endif +#define STAT_INC(v) STAT_UPDATE(v++;) + + +typedef struct +{ + const Byte *ptr; + size_t len; +} +CInBufPair; + + +#if defined(MY_CPU_ARM_OR_ARM64) || defined(MY_CPU_X86_OR_AMD64) + #if (defined(__clang__) && (__clang_major__ >= 6)) \ + || (defined(__GNUC__) && (__GNUC__ >= 6)) + // disable for debug: + #define Z7_ZSTD_DEC_USE_BSR + #elif defined(_MSC_VER) && (_MSC_VER >= 1300) + // #if defined(MY_CPU_ARM_OR_ARM64) + #if (_MSC_VER >= 1600) + #include + #endif + // disable for debug: + #define Z7_ZSTD_DEC_USE_BSR + #endif +#endif + +#ifdef Z7_ZSTD_DEC_USE_BSR + #if defined(__clang__) || defined(__GNUC__) + #define MY_clz(x) ((unsigned)__builtin_clz((UInt32)x)) + #else // #if defined(_MSC_VER) + #ifdef MY_CPU_ARM_OR_ARM64 + #define MY_clz _CountLeadingZeros + #endif // MY_CPU_X86_OR_AMD64 + #endif // _MSC_VER +#elif !defined(Z7_ZSTD_DEC_USE_LOG_TABLE) + #define Z7_ZSTD_DEC_USE_LOG_TABLE +#endif + + +static +Z7_FORCE_INLINE +unsigned GetHighestSetBit_32_nonzero_big(UInt32 num) +{ + // (num != 0) + #ifdef MY_clz + return 31 - MY_clz(num); + #elif defined(Z7_ZSTD_DEC_USE_BSR) + { + unsigned long zz; + _BitScanReverse(&zz, num); + return zz; + } + #else + { + int i = -1; + for (;;) + { + i++; + num >>= 1; + if (num == 0) + return (unsigned)i; + } + } + #endif +} + +#ifdef Z7_ZSTD_DEC_USE_LOG_TABLE + +#define R1(a) a, a +#define R2(a) R1(a), R1(a) +#define R3(a) R2(a), R2(a) +#define R4(a) R3(a), R3(a) +#define R5(a) R4(a), R4(a) +#define R6(a) R5(a), R5(a) +#define R7(a) R6(a), R6(a) +#define R8(a) R7(a), R7(a) +#define R9(a) R8(a), R8(a) + +#define Z7_ZSTD_FSE_MAX_ACCURACY 9 +// states[] values in FSE_Generate() can use (Z7_ZSTD_FSE_MAX_ACCURACY + 1) bits. +static const Byte k_zstd_LogTable[2 << Z7_ZSTD_FSE_MAX_ACCURACY] = +{ + R1(0), R1(1), R2(2), R3(3), R4(4), R5(5), R6(6), R7(7), R8(8), R9(9) +}; + +#define GetHighestSetBit_32_nonzero_small(num) (k_zstd_LogTable[num]) +#else +#define GetHighestSetBit_32_nonzero_small GetHighestSetBit_32_nonzero_big +#endif + + +#ifdef MY_clz + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + bitOffset -= (CBitCtr)(MY_clz(b) - 23); +#elif defined(Z7_ZSTD_DEC_USE_BSR) + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + { unsigned long zz; _BitScanReverse(&zz, b); bitOffset -= 8; bitOffset += zz; } +#else + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + for (;;) { bitOffset--; if (b & 0x80) { break; } b <<= 1; } +#endif + +#define SET_bitOffset_TO_PAD(bitOffset, src, srcLen) \ +{ \ + unsigned lastByte = (src)[(size_t)(srcLen) - 1]; \ + if (lastByte == 0) return SZ_ERROR_DATA; \ + bitOffset = (CBitCtr)((srcLen) * 8); \ + UPDATE_BIT_OFFSET_FOR_PADDING(lastByte, bitOffset) \ +} + +#ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + +#define SET_bitOffset_TO_PAD_and_SET_BIT_SIZE(bitOffset, src, srcLen_res) \ +{ \ + unsigned lastByte = (src)[(size_t)(srcLen_res) - 1]; \ + if (lastByte == 0) return SZ_ERROR_DATA; \ + srcLen_res *= 8; \ + bitOffset = (CBitCtr)srcLen_res; \ + UPDATE_BIT_OFFSET_FOR_PADDING(lastByte, bitOffset) \ +} + +#endif + +/* +typedef Int32 CBitCtr_signed; +typedef Int32 CBitCtr; +*/ +// /* +typedef ptrdiff_t CBitCtr_signed; +typedef ptrdiff_t CBitCtr; +// */ + + +#define MATCH_LEN_MIN 3 +#define kBlockSizeMax (1u << 17) + +// #define Z7_ZSTD_DEC_PRINT_TABLE + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE +#define NUM_OFFSET_SYMBOLS_PREDEF 29 +#endif +#define NUM_OFFSET_SYMBOLS_MAX (MAX_WINDOW_SIZE_LOG + 1) // 32 +#define NUM_LL_SYMBOLS 36 +#define NUM_ML_SYMBOLS 53 +#define FSE_NUM_SYMBOLS_MAX 53 // NUM_ML_SYMBOLS + +// /* +#if !defined(MY_CPU_X86) || defined(__PIC__) || defined(MY_CPU_64BIT) +#define Z7_ZSTD_DEC_USE_BASES_IN_OBJECT +#endif +// */ +// for debug: +// #define Z7_ZSTD_DEC_USE_BASES_LOCAL +// #define Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + +#define GLOBAL_TABLE(n) k_ ## n + +#if defined(Z7_ZSTD_DEC_USE_BASES_LOCAL) + #define BASES_TABLE(n) a_ ## n +#elif defined(Z7_ZSTD_DEC_USE_BASES_IN_OBJECT) + #define BASES_TABLE(n) p->m_ ## n +#else + #define BASES_TABLE(n) GLOBAL_TABLE(n) +#endif + +#define Z7_ZSTD_DEC_USE_ML_PLUS3 + +#if defined(Z7_ZSTD_DEC_USE_BASES_LOCAL) || \ + defined(Z7_ZSTD_DEC_USE_BASES_IN_OBJECT) + +#define SEQ_EXTRA_TABLES(n) \ + Byte n ## SEQ_LL_EXTRA [NUM_LL_SYMBOLS]; \ + Byte n ## SEQ_ML_EXTRA [NUM_ML_SYMBOLS]; \ + UInt32 n ## SEQ_LL_BASES [NUM_LL_SYMBOLS]; \ + UInt32 n ## SEQ_ML_BASES [NUM_ML_SYMBOLS]; \ + +#define Z7_ZSTD_DEC_USE_BASES_CALC + +#ifdef Z7_ZSTD_DEC_USE_BASES_CALC + + #define FILL_LOC_BASES(n, startSum) \ + { unsigned i; UInt32 sum = startSum; \ + for (i = 0; i != Z7_ARRAY_SIZE(GLOBAL_TABLE(n ## _EXTRA)); i++) \ + { const unsigned a = GLOBAL_TABLE(n ## _EXTRA)[i]; \ + BASES_TABLE(n ## _BASES)[i] = sum; \ + /* if (sum != GLOBAL_TABLE(n ## _BASES)[i]) exit(1); */ \ + sum += (UInt32)1 << a; \ + BASES_TABLE(n ## _EXTRA)[i] = (Byte)a; }} + + #define FILL_LOC_BASES_ALL \ + FILL_LOC_BASES (SEQ_LL, 0) \ + FILL_LOC_BASES (SEQ_ML, MATCH_LEN_MIN) \ + +#else + #define COPY_GLOBAL_ARR(n) \ + memcpy(BASES_TABLE(n), GLOBAL_TABLE(n), sizeof(GLOBAL_TABLE(n))); + #define FILL_LOC_BASES_ALL \ + COPY_GLOBAL_ARR (SEQ_LL_EXTRA) \ + COPY_GLOBAL_ARR (SEQ_ML_EXTRA) \ + COPY_GLOBAL_ARR (SEQ_LL_BASES) \ + COPY_GLOBAL_ARR (SEQ_ML_BASES) \ + +#endif + +#endif + + + +/// The sequence decoding baseline and number of additional bits to read/add +#if !defined(Z7_ZSTD_DEC_USE_BASES_CALC) +static const UInt32 GLOBAL_TABLE(SEQ_LL_BASES) [NUM_LL_SYMBOLS] = +{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 +}; +#endif + +static const Byte GLOBAL_TABLE(SEQ_LL_EXTRA) [NUM_LL_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16 +}; + +#if !defined(Z7_ZSTD_DEC_USE_BASES_CALC) +static const UInt32 GLOBAL_TABLE(SEQ_ML_BASES) [NUM_ML_SYMBOLS] = +{ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 +}; +#endif + +static const Byte GLOBAL_TABLE(SEQ_ML_EXTRA) [NUM_ML_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16 +}; + + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE + +static const Int16 SEQ_LL_PREDEF_DIST [NUM_LL_SYMBOLS] = +{ + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 +}; +static const Int16 SEQ_OFFSET_PREDEF_DIST [NUM_OFFSET_SYMBOLS_PREDEF] = +{ + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 +}; +static const Int16 SEQ_ML_PREDEF_DIST [NUM_ML_SYMBOLS] = +{ + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; + +#endif + +// typedef int FastInt; +// typedef Int32 FastInt32; +typedef unsigned FastInt; +typedef UInt32 FastInt32; +typedef FastInt32 CFseRecord; + + +#define FSE_REC_LEN_OFFSET 8 +#define FSE_REC_STATE_OFFSET 16 +#define GET_FSE_REC_SYM(st) ((Byte)(st)) +#define GET_FSE_REC_LEN(st) ((Byte)((st) >> FSE_REC_LEN_OFFSET)) +#define GET_FSE_REC_STATE(st) ((st) >> FSE_REC_STATE_OFFSET) + +// #define FSE_REC_SYM_MASK (0xff) +// #define GET_FSE_REC_SYM(st) (st & FSE_REC_SYM_MASK) + +#define W_BASE(state, len, sym) \ + (((UInt32)state << (4 + FSE_REC_STATE_OFFSET)) + \ + (len << FSE_REC_LEN_OFFSET) + (sym)) +#define W(state, len, sym) W_BASE(state, len, sym) +static const CFseRecord k_PredefRecords_LL[1 << 6] = { +W(0,4, 0),W(1,4, 0),W(2,5, 1),W(0,5, 3),W(0,5, 4),W(0,5, 6),W(0,5, 7),W(0,5, 9), +W(0,5,10),W(0,5,12),W(0,6,14),W(0,5,16),W(0,5,18),W(0,5,19),W(0,5,21),W(0,5,22), +W(0,5,24),W(2,5,25),W(0,5,26),W(0,6,27),W(0,6,29),W(0,6,31),W(2,4, 0),W(0,4, 1), +W(0,5, 2),W(2,5, 4),W(0,5, 5),W(2,5, 7),W(0,5, 8),W(2,5,10),W(0,5,11),W(0,6,13), +W(2,5,16),W(0,5,17),W(2,5,19),W(0,5,20),W(2,5,22),W(0,5,23),W(0,4,25),W(1,4,25), +W(2,5,26),W(0,6,28),W(0,6,30),W(3,4, 0),W(1,4, 1),W(2,5, 2),W(2,5, 3),W(2,5, 5), +W(2,5, 6),W(2,5, 8),W(2,5, 9),W(2,5,11),W(2,5,12),W(0,6,15),W(2,5,17),W(2,5,18), +W(2,5,20),W(2,5,21),W(2,5,23),W(2,5,24),W(0,6,35),W(0,6,34),W(0,6,33),W(0,6,32) +}; +static const CFseRecord k_PredefRecords_OF[1 << 5] = { +W(0,5, 0),W(0,4, 6),W(0,5, 9),W(0,5,15),W(0,5,21),W(0,5, 3),W(0,4, 7),W(0,5,12), +W(0,5,18),W(0,5,23),W(0,5, 5),W(0,4, 8),W(0,5,14),W(0,5,20),W(0,5, 2),W(1,4, 7), +W(0,5,11),W(0,5,17),W(0,5,22),W(0,5, 4),W(1,4, 8),W(0,5,13),W(0,5,19),W(0,5, 1), +W(1,4, 6),W(0,5,10),W(0,5,16),W(0,5,28),W(0,5,27),W(0,5,26),W(0,5,25),W(0,5,24) +}; +#if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) +#undef W +#define W(state, len, sym) W_BASE(state, len, (sym + MATCH_LEN_MIN)) +#endif +static const CFseRecord k_PredefRecords_ML[1 << 6] = { +W(0,6, 0),W(0,4, 1),W(2,5, 2),W(0,5, 3),W(0,5, 5),W(0,5, 6),W(0,5, 8),W(0,6,10), +W(0,6,13),W(0,6,16),W(0,6,19),W(0,6,22),W(0,6,25),W(0,6,28),W(0,6,31),W(0,6,33), +W(0,6,35),W(0,6,37),W(0,6,39),W(0,6,41),W(0,6,43),W(0,6,45),W(1,4, 1),W(0,4, 2), +W(2,5, 3),W(0,5, 4),W(2,5, 6),W(0,5, 7),W(0,6, 9),W(0,6,12),W(0,6,15),W(0,6,18), +W(0,6,21),W(0,6,24),W(0,6,27),W(0,6,30),W(0,6,32),W(0,6,34),W(0,6,36),W(0,6,38), +W(0,6,40),W(0,6,42),W(0,6,44),W(2,4, 1),W(3,4, 1),W(1,4, 2),W(2,5, 4),W(2,5, 5), +W(2,5, 7),W(2,5, 8),W(0,6,11),W(0,6,14),W(0,6,17),W(0,6,20),W(0,6,23),W(0,6,26), +W(0,6,29),W(0,6,52),W(0,6,51),W(0,6,50),W(0,6,49),W(0,6,48),W(0,6,47),W(0,6,46) +}; + + +// sum of freqs[] must be correct +// (numSyms != 0) +// (accuracy >= 5) +static +Z7_NO_INLINE +// Z7_FORCE_INLINE +void FSE_Generate(CFseRecord *table, + const Int16 *const freqs, const size_t numSyms, + const unsigned accuracy, UInt32 delta) +{ + size_t size = (size_t)1 << accuracy; + // max value in states[x] is ((1 << accuracy) * 2) + UInt16 states[FSE_NUM_SYMBOLS_MAX]; + { + /* Symbols with "less than 1" probability get a single cell, + starting from the end of the table. + These symbols define a full state reset, reading (accuracy) bits. */ + size_t threshold = size; + { + size_t s = 0; + do + if (freqs[s] == -1) + { + table[--threshold] = (CFseRecord)s; + states[s] = 1; + } + while (++s != numSyms); + } + + #ifdef SHOW_STAT + if (threshold == size) + { + STAT_INC(g_Num_Threshold_0) + STAT_UPDATE(g_Num_Threshold_0sum += (unsigned)size;) + } + else + { + STAT_INC(g_Num_Threshold_1) + STAT_UPDATE(g_Num_Threshold_1sum += (unsigned)size;) + } + #endif + + // { unsigned uuu; for (uuu = 0; uuu < 400; uuu++) + { + // Each (symbol) gets freqs[symbol] cells. + // Cell allocation is spread, not linear. + const size_t step = (size >> 1) + (size >> 3) + 3; + size_t pos = 0; + // const unsigned mask = size - 1; + /* + if (threshold == size) + { + size_t s = 0; + size--; + do + { + int freq = freqs[s]; + if (freq <= 0) + continue; + states[s] = (UInt16)freq; + do + { + table[pos] (CFseRecord)s; + pos = (pos + step) & size; // & mask; + } + while (--freq); + } + while (++s != numSyms); + } + else + */ + { + size_t s = 0; + size--; + do + { + int freq = freqs[s]; + if (freq <= 0) + continue; + states[s] = (UInt16)freq; + do + { + table[pos] = (CFseRecord)s; + // we skip position, if it's already occupied by a "less than 1" probability symbol. + // (step) is coprime to table size, so the cycle will visit each position exactly once + do + pos = (pos + step) & size; // & mask; + while (pos >= threshold); + } + while (--freq); + } + while (++s != numSyms); + } + size++; + // (pos != 0) is unexpected case that means that freqs[] are not correct. + // so it's some failure in code (for example, incorrect predefined freq[] table) + // if (pos != 0) return SZ_ERROR_FAIL; + } + // } + } + { + const CFseRecord * const limit = table + size; + delta = ((UInt32)size << FSE_REC_STATE_OFFSET) - delta; + /* State increases by symbol over time, decreasing number of bits. + Baseline increases until the bit threshold is passed, at which point it resets to 0 */ + do + { + #define TABLE_ITER(a) \ + { \ + const FastInt sym = (FastInt)table[a]; \ + const unsigned nextState = states[sym]; \ + unsigned nb; \ + states[sym] = (UInt16)(nextState + 1); \ + nb = accuracy - GetHighestSetBit_32_nonzero_small(nextState); \ + table[a] = (CFseRecord)(sym - delta \ + + ((UInt32)nb << FSE_REC_LEN_OFFSET) \ + + ((UInt32)nextState << FSE_REC_STATE_OFFSET << nb)); \ + } + TABLE_ITER(0) + TABLE_ITER(1) + table += 2; + } + while (table != limit); + } +} + + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE + +static void Print_Predef(unsigned predefAccuracy, + const unsigned numSymsPredef, + const Int16 * const predefFreqs, + const CFseRecord *checkTable) +{ + CFseRecord table[1 << 6]; + unsigned i; + FSE_Generate(table, predefFreqs, numSymsPredef, predefAccuracy, + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + numSymsPredef == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : + #endif + 0 + ); + if (memcmp(table, checkTable, sizeof(UInt32) << predefAccuracy) != 0) + _exit(1); + for (i = 0; i < (1u << predefAccuracy); i++) + { + const UInt32 v = table[i]; + const unsigned state = (unsigned)(GET_FSE_REC_STATE(v)); + if (state & 0xf) + _exit(1); + if (i != 0) + { + printf(","); + if (i % 8 == 0) + printf("\n"); + } + printf("W(%d,%d,%2d)", + (unsigned)(state >> 4), + (unsigned)((v >> FSE_REC_LEN_OFFSET) & 0xff), + (unsigned)GET_FSE_REC_SYM(v)); + } + printf("\n\n"); +} + +#endif + + +#define GET16(dest, p) { const Byte *ptr = p; dest = GetUi16(ptr); } +#define GET32(dest, p) { const Byte *ptr = p; dest = GetUi32(ptr); } + +// (1 <= numBits <= 9) +#define FORWARD_READ_BITS(destVal, numBits, mask) \ + { const CBitCtr_signed bos3 = (bitOffset) >> 3; \ + if (bos3 >= 0) return SZ_ERROR_DATA; \ + GET16(destVal, src + bos3) \ + destVal >>= (bitOffset) & 7; \ + bitOffset += (CBitCtr_signed)(numBits); \ + mask = (1u << (numBits)) - 1; \ + destVal &= mask; \ + } + +#define FORWARD_READ_1BIT(destVal) \ + { const CBitCtr_signed bos3 = (bitOffset) >> 3; \ + if (bos3 >= 0) return SZ_ERROR_DATA; \ + destVal = *(src + bos3); \ + destVal >>= (bitOffset) & 7; \ + (bitOffset)++; \ + destVal &= 1; \ + } + + +// in: (accuracyMax <= 9) +// at least 2 bytes will be processed from (in) stream. +// at return: (in->len > 0) +static +Z7_NO_INLINE +SRes FSE_DecodeHeader(CFseRecord *const table, + CInBufPair *const in, + const unsigned accuracyMax, + Byte *const accuracyRes, + unsigned numSymbolsMax) +{ + unsigned accuracy; + unsigned remain1; + unsigned syms; + Int16 freqs[FSE_NUM_SYMBOLS_MAX + 3]; // +3 for overwrite (repeat) + const Byte *src = in->ptr; + CBitCtr_signed bitOffset = (CBitCtr_signed)in->len - 1; + if (bitOffset <= 0) + return SZ_ERROR_DATA; + accuracy = *src & 0xf; + accuracy += 5; + if (accuracy > accuracyMax) + return SZ_ERROR_DATA; + *accuracyRes = (Byte)accuracy; + remain1 = (1u << accuracy) + 1; // (it's remain_freqs_sum + 1) + syms = 0; + src += bitOffset; // src points to last byte + bitOffset = 4 - (bitOffset << 3); + + for (;;) + { + // (2 <= remain1) + const unsigned bits = GetHighestSetBit_32_nonzero_small((unsigned)remain1); + // (1 <= bits <= accuracy) + unsigned val; // it must be unsigned or int + unsigned mask; + FORWARD_READ_BITS(val, bits, mask) + { + const unsigned val2 = remain1 + val - mask; + if (val2 > mask) + { + unsigned bit; + FORWARD_READ_1BIT(bit) + if (bit) + val = val2; + } + } + { + // (remain1 >= 2) + // (0 <= (int)val <= remain1) + val = (unsigned)((int)val - 1); + // val now is "probability" of symbol + // (probability == -1) means "less than 1" frequency. + // (-1 <= (int)val <= remain1 - 1) + freqs[syms++] = (Int16)(int)val; + if (val != 0) + { + remain1 -= (int)val < 0 ? 1u : (unsigned)val; + // remain1 -= val; + // val >>= (sizeof(val) * 8 - 2); + // remain1 -= val & 2; + // freqs[syms++] = (Int16)(int)val; + // syms++; + if (remain1 == 1) + break; + if (syms >= FSE_NUM_SYMBOLS_MAX) + return SZ_ERROR_DATA; + } + else // if (val == 0) + { + // freqs[syms++] = 0; + // syms++; + for (;;) + { + unsigned repeat; + FORWARD_READ_BITS(repeat, 2, mask) + freqs[syms ] = 0; + freqs[syms + 1] = 0; + freqs[syms + 2] = 0; + syms += repeat; + if (syms >= FSE_NUM_SYMBOLS_MAX) + return SZ_ERROR_DATA; + if (repeat != 3) + break; + } + } + } + } + + if (syms > numSymbolsMax) + return SZ_ERROR_DATA; + bitOffset += 7; + bitOffset >>= 3; + if (bitOffset > 0) + return SZ_ERROR_DATA; + in->ptr = src + bitOffset; + in->len = (size_t)(1 - bitOffset); + { + // unsigned uuu; for (uuu = 0; uuu < 50; uuu++) + FSE_Generate(table, freqs, syms, accuracy, + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + numSymbolsMax == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : + #endif + 0 + ); + } + return SZ_OK; +} + + +// ---------- HUFFMAN ---------- + +#define HUF_MAX_BITS 12 +#define HUF_MAX_SYMBS 256 +#define HUF_DUMMY_SIZE (128 + 8 * 2) // it must multiple of 8 +// #define HUF_DUMMY_SIZE 0 +#define HUF_TABLE_SIZE ((2 << HUF_MAX_BITS) + HUF_DUMMY_SIZE) +#define HUF_GET_SYMBOLS(table) ((table) + (1 << HUF_MAX_BITS) + HUF_DUMMY_SIZE) +// #define HUF_GET_LENS(table) (table) + +typedef struct +{ + // Byte table[HUF_TABLE_SIZE]; + UInt64 table64[HUF_TABLE_SIZE / sizeof(UInt64)]; +} +CZstdDecHufTable; + +/* +Input: + numSyms != 0 + (bits) array size must be aligned for 2 + if (numSyms & 1), then bits[numSyms] == 0, + Huffman tree must be correct before Huf_Build() call: + (sum (1/2^bits[i]) == 1). + && (bits[i] <= HUF_MAX_BITS) +*/ +static +Z7_FORCE_INLINE +void Huf_Build(Byte * const table, + const Byte *bits, const unsigned numSyms) +{ + unsigned counts0[HUF_MAX_BITS + 1]; + unsigned counts1[HUF_MAX_BITS + 1]; + const Byte * const bitsEnd = bits + numSyms; + // /* + { + unsigned t; + for (t = 0; t < Z7_ARRAY_SIZE(counts0); t++) counts0[t] = 0; + for (t = 0; t < Z7_ARRAY_SIZE(counts1); t++) counts1[t] = 0; + } + // */ + // memset(counts0, 0, sizeof(counts0)); + // memset(counts1, 0, sizeof(counts1)); + { + const Byte *bits2 = bits; + // we access additional bits[symbol] if (numSyms & 1) + do + { + counts0[bits2[0]]++; + counts1[bits2[1]]++; + } + while ((bits2 += 2) < bitsEnd); + } + { + unsigned r = 0; + unsigned i = HUF_MAX_BITS; + // Byte *lens = HUF_GET_LENS(symbols); + do + { + const unsigned num = (counts0[i] + counts1[i]) << (HUF_MAX_BITS - i); + counts0[i] = r; + if (num) + { + Byte *lens = &table[r]; + r += num; + memset(lens, (int)i, num); + } + } + while (--i); + counts0[0] = 0; // for speculated loads + // no need for check: + // if (r != (UInt32)1 << HUF_MAX_BITS) exit(0); + } + { + #ifdef MY_CPU_64BIT + UInt64 + #else + UInt32 + #endif + v = 0; + Byte *symbols = HUF_GET_SYMBOLS(table); + do + { + const unsigned nb = *bits++; + if (nb) + { + const unsigned code = counts0[nb]; + const unsigned num = (1u << HUF_MAX_BITS) >> nb; + counts0[nb] = code + num; + // memset(&symbols[code], i, num); + // /* + { + Byte *s2 = &symbols[code]; + if (num <= 2) + { + s2[0] = (Byte)v; + s2[(size_t)num - 1] = (Byte)v; + } + else if (num <= 8) + { + *(UInt32 *)(void *)s2 = (UInt32)v; + *(UInt32 *)(void *)(s2 + (size_t)num - 4) = (UInt32)v; + } + else + { + #ifdef MY_CPU_64BIT + UInt64 *s = (UInt64 *)(void *)s2; + const UInt64 *lim = (UInt64 *)(void *)(s2 + num); + do + { + s[0] = v; s[1] = v; s += 2; + } + while (s != lim); + #else + UInt32 *s = (UInt32 *)(void *)s2; + const UInt32 *lim = (const UInt32 *)(const void *)(s2 + num); + do + { + s[0] = v; s[1] = v; s += 2; + s[0] = v; s[1] = v; s += 2; + } + while (s != lim); + #endif + } + } + // */ + } + v += + #ifdef MY_CPU_64BIT + 0x0101010101010101; + #else + 0x01010101; + #endif + } + while (bits != bitsEnd); + } +} + + + +// how many bytes (src) was moved back from original value. +// we need (HUF_SRC_OFFSET == 3) for optimized 32-bit memory access +#define HUF_SRC_OFFSET 3 + +// v <<= 8 - (bitOffset & 7) + numBits; +// v >>= 32 - HUF_MAX_BITS; +#define HUF_GET_STATE(v, bitOffset, numBits) \ + GET32(v, src + (HUF_SRC_OFFSET - 3) + ((CBitCtr_signed)bitOffset >> 3)) \ + v >>= 32 - HUF_MAX_BITS - 8 + ((unsigned)bitOffset & 7) - numBits; \ + v &= (1u << HUF_MAX_BITS) - 1; \ + + +#ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS +#if defined(MY_CPU_AMD64) && defined(_MSC_VER) && _MSC_VER == 1400 \ + || !defined(MY_CPU_X86_OR_AMD64) \ + // || 1 == 1 /* for debug : to force STREAM4_PRELOAD mode */ + // we need big number (>=16) of registers for PRELOAD4 + #define Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4 + // #define Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2 // for debug +#endif +#endif + +// for debug: simpler and smaller code but slow: +// #define Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE + +#if defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE) || \ + !defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS) + +#define HUF_DECODE(bitOffset, dest) \ +{ \ + UInt32 v; \ + HUF_GET_STATE(v, bitOffset, 0) \ + bitOffset -= table[v]; \ + *(dest) = symbols[v]; \ + if ((CBitCtr_signed)bitOffset < 0) return SZ_ERROR_DATA; \ +} + +#endif + +#if !defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE) || \ + defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4) || \ + defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2) \ + +#define HUF_DECODE_2_INIT(v, bitOffset) \ + HUF_GET_STATE(v, bitOffset, 0) + +#define HUF_DECODE_2(v, bitOffset, dest) \ +{ \ + unsigned numBits; \ + numBits = table[v]; \ + *(dest) = symbols[v]; \ + HUF_GET_STATE(v, bitOffset, numBits) \ + bitOffset -= (CBitCtr)numBits; \ + if ((CBitCtr_signed)bitOffset < 0) return SZ_ERROR_DATA; \ +} + +#endif + + +// src == ptr - HUF_SRC_OFFSET +// we are allowed to access 3 bytes before start of input buffer +static +Z7_NO_INLINE +SRes Huf_Decompress_1stream(const Byte * const table, + const Byte *src, const size_t srcLen, + Byte *dest, const size_t destLen) +{ + CBitCtr bitOffset; + if (srcLen == 0) + return SZ_ERROR_DATA; + SET_bitOffset_TO_PAD (bitOffset, src + HUF_SRC_OFFSET, srcLen) + if (destLen) + { + const Byte *symbols = HUF_GET_SYMBOLS(table); + const Byte *destLim = dest + destLen; + #ifdef Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE + { + do + { + HUF_DECODE (bitOffset, dest) + } + while (++dest != destLim); + } + #else + { + UInt32 v; + HUF_DECODE_2_INIT (v, bitOffset) + do + { + HUF_DECODE_2 (v, bitOffset, dest) + } + while (++dest != destLim); + } + #endif + } + return bitOffset == 0 ? SZ_OK : SZ_ERROR_DATA; +} + + +// for debug : it reduces register pressure : by array copy can be slow : +// #define Z7_ZSTD_DEC_USE_HUF_LOCAL + +// src == ptr + (6 - HUF_SRC_OFFSET) +// srcLen >= 10 +// we are allowed to access 3 bytes before start of input buffer +static +Z7_NO_INLINE +SRes Huf_Decompress_4stream(const Byte * const + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + table2, + #else + table, + #endif + const Byte *src, size_t srcLen, + Byte *dest, size_t destLen) +{ + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + Byte table[HUF_TABLE_SIZE]; + #endif + UInt32 sizes[3]; + const size_t delta = (destLen + 3) / 4; + if ((sizes[0] = GetUi16(src + (0 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + if ((sizes[1] = GetUi16(src + (2 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + sizes[1] += sizes[0]; + if ((sizes[2] = GetUi16(src + (4 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + sizes[2] += sizes[1]; + srcLen -= 6; + if (srcLen <= sizes[2]) + return SZ_ERROR_DATA; + + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + { + // unsigned i = 0; for(; i < 1000; i++) + memcpy(table, table2, HUF_TABLE_SIZE); + } + #endif + + #ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + { + CBitCtr bitOffset_0, + bitOffset_1, + bitOffset_2, + bitOffset_3; + { + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_0, src + HUF_SRC_OFFSET, sizes[0]) + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_1, src + HUF_SRC_OFFSET, sizes[1]) + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_2, src + HUF_SRC_OFFSET, sizes[2]) + SET_bitOffset_TO_PAD (bitOffset_3, src + HUF_SRC_OFFSET, srcLen) + } + { + const Byte * const symbols = HUF_GET_SYMBOLS(table); + Byte *destLim = dest + destLen - delta * 3; + + if (dest != destLim) + #ifdef Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4 + { + UInt32 v_0, v_1, v_2, v_3; + HUF_DECODE_2_INIT (v_0, bitOffset_0) + HUF_DECODE_2_INIT (v_1, bitOffset_1) + HUF_DECODE_2_INIT (v_2, bitOffset_2) + HUF_DECODE_2_INIT (v_3, bitOffset_3) + // #define HUF_DELTA (1 << 17) / 4 + do + { + HUF_DECODE_2 (v_3, bitOffset_3, dest + delta * 3) + HUF_DECODE_2 (v_2, bitOffset_2, dest + delta * 2) + HUF_DECODE_2 (v_1, bitOffset_1, dest + delta) + HUF_DECODE_2 (v_0, bitOffset_0, dest) + } + while (++dest != destLim); + /* + {// unsigned y = 0; for (;y < 1; y++) + { + const size_t num = destLen - delta * 3; + Byte *orig = dest - num; + memmove (orig + delta , orig + HUF_DELTA, num); + memmove (orig + delta * 2, orig + HUF_DELTA * 2, num); + memmove (orig + delta * 3, orig + HUF_DELTA * 3, num); + }} + */ + } + #elif defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2) + { + UInt32 v_0, v_1, v_2, v_3; + HUF_DECODE_2_INIT (v_0, bitOffset_0) + HUF_DECODE_2_INIT (v_1, bitOffset_1) + do + { + HUF_DECODE_2 (v_0, bitOffset_0, dest) + HUF_DECODE_2 (v_1, bitOffset_1, dest + delta) + } + while (++dest != destLim); + dest = destLim - (destLen - delta * 3); + dest += delta * 2; + destLim += delta * 2; + HUF_DECODE_2_INIT (v_2, bitOffset_2) + HUF_DECODE_2_INIT (v_3, bitOffset_3) + do + { + HUF_DECODE_2 (v_2, bitOffset_2, dest) + HUF_DECODE_2 (v_3, bitOffset_3, dest + delta) + } + while (++dest != destLim); + dest -= delta * 2; + destLim -= delta * 2; + } + #else + { + do + { + HUF_DECODE (bitOffset_3, dest + delta * 3) + HUF_DECODE (bitOffset_2, dest + delta * 2) + HUF_DECODE (bitOffset_1, dest + delta) + HUF_DECODE (bitOffset_0, dest) + } + while (++dest != destLim); + } + #endif + + if (bitOffset_3 != (CBitCtr)sizes[2]) + return SZ_ERROR_DATA; + if (destLen &= 3) + { + destLim = dest + 4 - destLen; + do + { + HUF_DECODE (bitOffset_2, dest + delta * 2) + HUF_DECODE (bitOffset_1, dest + delta) + HUF_DECODE (bitOffset_0, dest) + } + while (++dest != destLim); + } + if ( bitOffset_0 != 0 + || bitOffset_1 != (CBitCtr)sizes[0] + || bitOffset_2 != (CBitCtr)sizes[1]) + return SZ_ERROR_DATA; + } + } + #else // Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + { + unsigned i; + for (i = 0; i < 4; i++) + { + size_t d = destLen; + size_t size = srcLen; + if (i != 3) + { + d = delta; + size = sizes[i]; + } + if (i != 0) + size -= sizes[i - 1]; + destLen -= d; + RINOK(Huf_Decompress_1stream(table, src, size, dest, d)) + dest += d; + src += size; + } + } + #endif + + return SZ_OK; +} + + + +// (in->len != 0) +// we are allowed to access in->ptr[-3] +// at least 2 bytes in (in->ptr) will be processed +static SRes Huf_DecodeTable(CZstdDecHufTable *const p, CInBufPair *const in) +{ + Byte weights[HUF_MAX_SYMBS + 1]; // +1 for extra write for loop unroll + unsigned numSyms; + const unsigned header = *(in->ptr)++; + in->len--; + // memset(weights, 0, sizeof(weights)); + if (header >= 128) + { + // direct representation: 4 bits field (0-15) per weight + numSyms = header - 127; + // numSyms != 0 + { + const size_t numBytes = (numSyms + 1) / 2; + const Byte *const ws = in->ptr; + size_t i = 0; + if (in->len < numBytes) + return SZ_ERROR_DATA; + in->ptr += numBytes; + in->len -= numBytes; + do + { + const unsigned b = ws[i]; + weights[i * 2 ] = (Byte)(b >> 4); + weights[i * 2 + 1] = (Byte)(b & 0xf); + } + while (++i != numBytes); + /* 7ZIP: we can restore correct zero value for weights[numSyms], + if we want to use zero values starting from numSyms in code below. */ + // weights[numSyms] = 0; + } + } + else + { + #define MAX_ACCURACY_LOG_FOR_WEIGHTS 6 + CFseRecord table[1 << MAX_ACCURACY_LOG_FOR_WEIGHTS]; + + Byte accuracy; + const Byte *src; + size_t srcLen; + if (in->len < header) + return SZ_ERROR_DATA; + { + CInBufPair fse_stream; + fse_stream.len = header; + fse_stream.ptr = in->ptr; + in->ptr += header; + in->len -= header; + RINOK(FSE_DecodeHeader(table, &fse_stream, + MAX_ACCURACY_LOG_FOR_WEIGHTS, + &accuracy, + 16 // num weight symbols max (max-symbol is 15) + )) + // at least 2 bytes were processed in fse_stream. + // (srcLen > 0) after FSE_DecodeHeader() + // if (srcLen == 0) return SZ_ERROR_DATA; + src = fse_stream.ptr; + srcLen = fse_stream.len; + } + // we are allowed to access src[-5] + { + // unsigned yyy = 200; do { + CBitCtr bitOffset; + FastInt32 state1, state2; + SET_bitOffset_TO_PAD (bitOffset, src, srcLen) + state1 = accuracy; + src -= state1 >> 2; // src -= 1; // for GET16() optimization + state1 <<= FSE_REC_LEN_OFFSET; + state2 = state1; + numSyms = 0; + for (;;) + { + #define FSE_WEIGHT_DECODE(st) \ + { \ + const unsigned bits = GET_FSE_REC_LEN(st); \ + FastInt r; \ + GET16(r, src + (bitOffset >> 3)) \ + r >>= (unsigned)bitOffset & 7; \ + if ((CBitCtr_signed)(bitOffset -= (CBitCtr)bits) < 0) \ + { if (bitOffset + (CBitCtr)bits != 0) \ + return SZ_ERROR_DATA; \ + break; } \ + r &= 0xff; \ + r >>= 8 - bits; \ + st = table[GET_FSE_REC_STATE(st) + r]; \ + weights[numSyms++] = (Byte)GET_FSE_REC_SYM(st); \ + } + FSE_WEIGHT_DECODE (state1) + FSE_WEIGHT_DECODE (state2) + if (numSyms == HUF_MAX_SYMBS) + return SZ_ERROR_DATA; + } + // src += (unsigned)accuracy >> 2; } while (--yyy); + } + } + + // Build using weights: + { + UInt32 sum = 0; + { + // numSyms >= 1 + unsigned i = 0; + weights[numSyms] = 0; + do + { + sum += ((UInt32)1 << weights[i ]) & ~(UInt32)1; + sum += ((UInt32)1 << weights[i + 1]) & ~(UInt32)1; + i += 2; + } + while (i < numSyms); + if (sum == 0) + return SZ_ERROR_DATA; + } + { + const unsigned maxBits = GetHighestSetBit_32_nonzero_big(sum) + 1; + { + const UInt32 left = ((UInt32)1 << maxBits) - sum; + // (left != 0) + // (left) must be power of 2 in correct stream + if (left & (left - 1)) + return SZ_ERROR_DATA; + weights[numSyms++] = (Byte)GetHighestSetBit_32_nonzero_big(left); + } + // if (numSyms & 1) + weights[numSyms] = 0; // for loop unroll + // numSyms >= 2 + { + unsigned i = 0; + do + { + /* + #define WEIGHT_ITER(a) \ + { unsigned w = weights[i + (a)]; \ + const unsigned t = maxBits - w; \ + w = w ? t: w; \ + if (w > HUF_MAX_BITS) return SZ_ERROR_DATA; \ + weights[i + (a)] = (Byte)w; } + */ + // /* + #define WEIGHT_ITER(a) \ + { unsigned w = weights[i + (a)]; \ + if (w) { \ + w = maxBits - w; \ + if (w > HUF_MAX_BITS) return SZ_ERROR_DATA; \ + weights[i + (a)] = (Byte)w; }} + // */ + WEIGHT_ITER(0) + // WEIGHT_ITER(1) + // i += 2; + } + while (++i != numSyms); + } + } + } + { + // unsigned yyy; for (yyy = 0; yyy < 100; yyy++) + Huf_Build((Byte *)(void *)p->table64, weights, numSyms); + } + return SZ_OK; +} + + +typedef enum +{ + k_SeqMode_Predef = 0, + k_SeqMode_RLE = 1, + k_SeqMode_FSE = 2, + k_SeqMode_Repeat = 3 +} +z7_zstd_enum_SeqMode; + +// predefAccuracy == 5 for OFFSET symbols +// predefAccuracy == 6 for MATCH/LIT LEN symbols +static +SRes +Z7_NO_INLINE +// Z7_FORCE_INLINE +FSE_Decode_SeqTable(CFseRecord * const table, + CInBufPair * const in, + unsigned predefAccuracy, + Byte * const accuracyRes, + unsigned numSymbolsMax, + const CFseRecord * const predefs, + const unsigned seqMode) +{ + // UNUSED_VAR(numSymsPredef) + // UNUSED_VAR(predefFreqs) + if (seqMode == k_SeqMode_FSE) + { + // unsigned y = 50; CInBufPair in2 = *in; do { *in = in2; RINOK( + return + FSE_DecodeHeader(table, in, + predefAccuracy + 3, // accuracyMax + accuracyRes, + numSymbolsMax) + ; + // )} while (--y); return SZ_OK; + } + // numSymsMax = numSymsPredef + ((predefAccuracy & 1) * (32 - 29))); // numSymsMax + // numSymsMax == 32 for offsets + + if (seqMode == k_SeqMode_Predef) + { + *accuracyRes = (Byte)predefAccuracy; + memcpy(table, predefs, sizeof(UInt32) << predefAccuracy); + return SZ_OK; + } + + // (seqMode == k_SeqMode_RLE) + if (in->len == 0) + return SZ_ERROR_DATA; + in->len--; + { + const Byte *ptr = in->ptr; + const unsigned sym = ptr[0]; + in->ptr = ptr + 1; + if (sym >= numSymbolsMax) + return SZ_ERROR_DATA; + table[0] = (FastInt32)sym + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + + (numSymbolsMax == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : 0) + #endif + ; + *accuracyRes = 0; + } + return SZ_OK; +} + + +typedef struct +{ + CFseRecord of[1 << 8]; + CFseRecord ll[1 << 9]; + CFseRecord ml[1 << 9]; +} +CZstdDecFseTables; + + +typedef struct +{ + Byte *win; + SizeT cycSize; + /* + if (outBuf_fromCaller) : cycSize = outBufSize_fromCaller + else { + if ( isCyclicMode) : cycSize = cyclic_buffer_size = (winSize + extra_space) + if (!isCyclicMode) : cycSize = ContentSize, + (isCyclicMode == true) if (ContetSize >= winSize) or ContetSize is unknown + } + */ + SizeT winPos; + + CZstdDecOffset reps[3]; + + Byte ll_accuracy; + Byte of_accuracy; + Byte ml_accuracy; + // Byte seqTables_wereSet; + Byte litHuf_wasSet; + + Byte *literalsBase; + + size_t winSize; // from header + size_t totalOutCheck; // totalOutCheck <= winSize + + #ifdef Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + SEQ_EXTRA_TABLES(m_) + #endif + // UInt64 _pad_Alignment; // is not required now + CZstdDecFseTables fse; + CZstdDecHufTable huf; +} +CZstdDec1; + +#define ZstdDec1_GET_BLOCK_SIZE_LIMIT(p) \ + ((p)->winSize < kBlockSizeMax ? (UInt32)(p)->winSize : kBlockSizeMax) + +#define SEQ_TABLES_WERE_NOT_SET_ml_accuracy 1 // accuracy=1 is not used by zstd +#define IS_SEQ_TABLES_WERE_SET(p) (((p)->ml_accuracy != SEQ_TABLES_WERE_NOT_SET_ml_accuracy)) +// #define IS_SEQ_TABLES_WERE_SET(p) ((p)->seqTables_wereSet) + + +static void ZstdDec1_Construct(CZstdDec1 *p) +{ + #ifdef Z7_ZSTD_DEC_PRINT_TABLE + Print_Predef(6, NUM_LL_SYMBOLS, SEQ_LL_PREDEF_DIST, k_PredefRecords_LL); + Print_Predef(5, NUM_OFFSET_SYMBOLS_PREDEF, SEQ_OFFSET_PREDEF_DIST, k_PredefRecords_OF); + Print_Predef(6, NUM_ML_SYMBOLS, SEQ_ML_PREDEF_DIST, k_PredefRecords_ML); + #endif + + p->win = NULL; + p->cycSize = 0; + p->literalsBase = NULL; + #ifdef Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + FILL_LOC_BASES_ALL + #endif +} + + +static void ZstdDec1_Init(CZstdDec1 *p) +{ + p->reps[0] = 1; + p->reps[1] = 4; + p->reps[2] = 8; + // p->seqTables_wereSet = False; + p->ml_accuracy = SEQ_TABLES_WERE_NOT_SET_ml_accuracy; + p->litHuf_wasSet = False; + p->totalOutCheck = 0; +} + + + +#ifdef MY_CPU_LE_UNALIGN + #define Z7_ZSTD_DEC_USE_UNALIGNED_COPY +#endif + +#ifdef Z7_ZSTD_DEC_USE_UNALIGNED_COPY + + #define COPY_CHUNK_SIZE 16 + + #define COPY_CHUNK_4_2(dest, src) \ + { \ + ((UInt32 *)(void *)dest)[0] = ((const UInt32 *)(const void *)src)[0]; \ + ((UInt32 *)(void *)dest)[1] = ((const UInt32 *)(const void *)src)[1]; \ + src += 4 * 2; \ + dest += 4 * 2; \ + } + + /* sse2 doesn't help here in GCC and CLANG. + so we disabled sse2 here */ + /* + #if defined(MY_CPU_AMD64) + #define Z7_ZSTD_DEC_USE_SSE2 + #elif defined(MY_CPU_X86) + #if defined(_MSC_VER) && _MSC_VER >= 1300 && defined(_M_IX86_FP) && (_M_IX86_FP >= 2) \ + || defined(__SSE2__) \ + // || 1 == 1 // for debug only + #define Z7_ZSTD_DEC_USE_SSE2 + #endif + #endif + */ + + #if defined(MY_CPU_ARM64) + #define COPY_OFFSET_MIN 16 + #define COPY_CHUNK1(dest, src) \ + { \ + vst1q_u8((uint8_t *)(void *)dest, \ + vld1q_u8((const uint8_t *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if ((len -= COPY_CHUNK_SIZE) == 0) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(Z7_ZSTD_DEC_USE_SSE2) + #include // sse2 + #define COPY_OFFSET_MIN 16 + + #define COPY_CHUNK1(dest, src) \ + { \ + _mm_storeu_si128((__m128i *)(void *)dest, \ + _mm_loadu_si128((const __m128i *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if ((len -= COPY_CHUNK_SIZE) == 0) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(MY_CPU_64BIT) + #define COPY_OFFSET_MIN 8 + + #define COPY_CHUNK(dest, src) \ + { \ + ((UInt64 *)(void *)dest)[0] = ((const UInt64 *)(const void *)src)[0]; \ + ((UInt64 *)(void *)dest)[1] = ((const UInt64 *)(const void *)src)[1]; \ + src += 8 * 2; \ + dest += 8 * 2; \ + } + + #else + #define COPY_OFFSET_MIN 4 + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_4_2(dest, src); \ + COPY_CHUNK_4_2(dest, src); \ + } + + #endif +#endif + + +#ifndef COPY_CHUNK_SIZE + #define COPY_OFFSET_MIN 4 + #define COPY_CHUNK_SIZE 8 + #define COPY_CHUNK_2(dest, src) \ + { \ + const Byte a0 = src[0]; \ + const Byte a1 = src[1]; \ + dest[0] = a0; \ + dest[1] = a1; \ + src += 2; \ + dest += 2; \ + } + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + } +#endif + + +#define COPY_PREPARE \ + len += (COPY_CHUNK_SIZE - 1); \ + len &= ~(size_t)(COPY_CHUNK_SIZE - 1); \ + { if (len > rem) \ + { len = rem; \ + rem &= (COPY_CHUNK_SIZE - 1); \ + if (rem) { \ + len -= rem; \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do *dest++ = *src++; while (--rem); \ + if (len == 0) return; }}} + +#define COPY_CHUNKS \ +{ \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do { COPY_CHUNK(dest, src) } \ + while (len -= COPY_CHUNK_SIZE); \ +} + +// (len != 0) +// (len <= rem) +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyLiterals(Byte *dest, Byte const *src, size_t len, size_t rem) +{ + COPY_PREPARE + COPY_CHUNKS +} + + +/* we can define Z7_STD_DEC_USE_AFTER_CYC_BUF, if we want to use additional + space after cycSize that can be used to reduce the code in CopyMatch(): */ +// for debug: +// #define Z7_STD_DEC_USE_AFTER_CYC_BUF + +/* +CopyMatch() +if wrap (offset > winPos) +{ + then we have at least (COPY_CHUNK_SIZE) avail in (dest) before we will overwrite (src): + (cycSize >= offset + COPY_CHUNK_SIZE) + if defined(Z7_STD_DEC_USE_AFTER_CYC_BUF) + we are allowed to read win[cycSize + COPY_CHUNK_SIZE - 1], +} +(len != 0) +*/ +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyMatch(size_t offset, size_t len, + Byte *win, size_t winPos, size_t rem, const size_t cycSize) +{ + Byte *dest = win + winPos; + const Byte *src; + // STAT_INC(g_NumCopy) + + if (offset > winPos) + { + size_t back = offset - winPos; + // src = win + cycSize - back; + // cycSize -= offset; + STAT_INC(g_NumOver) + src = dest + (cycSize - offset); + // (src >= dest) here + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + if (back < len) + { + #else + if (back < len + (COPY_CHUNK_SIZE - 1)) + { + if (back >= len) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--len); + return; + } + #endif + // back < len + STAT_INC(g_NumOver2) + len -= back; + rem -= back; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--back); + src = dest - offset; + // src = win; + // we go to MAIN-COPY + } + } + else + src = dest - offset; + + // len != 0 + // do *dest++ = *src++; while (--len); return; + + // --- MAIN COPY --- + // if (src >= dest), then ((size_t)(src - dest) >= COPY_CHUNK_SIZE) + // so we have at least COPY_CHUNK_SIZE space before overlap for writing. + COPY_PREPARE + + /* now (len == COPY_CHUNK_SIZE * x) + so we can unroll for aligned copy */ + { + // const unsigned b0 = src[0]; + // (COPY_OFFSET_MIN >= 4) + + if (offset >= COPY_OFFSET_MIN) + { + COPY_CHUNKS + // return; + } + else + #if (COPY_OFFSET_MIN > 4) + #if COPY_CHUNK_SIZE < 8 + #error Stop_Compiling_Bad_COPY_CHUNK_SIZE + #endif + if (offset >= 4) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + COPY_CHUNK_4_2(dest, src) + #if COPY_CHUNK_SIZE != 16 + if (len == 8) break; + #endif + COPY_CHUNK_4_2(dest, src) + } + while (len -= 16); + // return; + } + else + #endif + { + // (offset < 4) + const unsigned b0 = src[0]; + if (offset < 2) + { + #if defined(Z7_ZSTD_DEC_USE_UNALIGNED_COPY) && (COPY_CHUNK_SIZE == 16) + #if defined(MY_CPU_64BIT) + { + const UInt64 v64 = (UInt64)b0 * 0x0101010101010101; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + ((UInt64 *)(void *)dest)[0] = v64; + ((UInt64 *)(void *)dest)[1] = v64; + dest += 16; + } + while (len -= 16); + } + #else + { + UInt32 v = b0; + v |= v << 8; + v |= v << 16; + do + { + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + } + while (len -= 16); + } + #endif + #else + do + { + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + } + while (len -= 4); + #endif + } + else if (offset == 2) + { + const Byte b1 = src[1]; + { + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest += 2; + } + while (len -= 2); + } + } + else // (offset == 3) + { + const Byte *lim = dest + len - 2; + const Byte b1 = src[1]; + const Byte b2 = src[2]; + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest[2] = b2; + dest += 3; + } + while (dest < lim); + lim++; // points to last byte that must be written + if (dest <= lim) + { + *dest = (Byte)b0; + if (dest != lim) + dest[1] = b1; + } + } + } + } +} + + + +#define UPDATE_TOTAL_OUT(p, size) \ +{ \ + size_t _toc = (p)->totalOutCheck + (size); \ + const size_t _ws = (p)->winSize; \ + if (_toc >= _ws) _toc = _ws; \ + (p)->totalOutCheck = _toc; \ +} + + +#if defined(MY_CPU_64BIT) && defined(MY_CPU_LE_UNALIGN) +// we can disable it for debug: +#define Z7_ZSTD_DEC_USE_64BIT_LOADS +#endif +// #define Z7_ZSTD_DEC_USE_64BIT_LOADS // for debug : slow in 32-bit + +// SEQ_SRC_OFFSET: how many bytes (src) (seqSrc) was moved back from original value. +// we need (SEQ_SRC_OFFSET != 0) for optimized memory access +#ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define SEQ_SRC_OFFSET 7 +#else + #define SEQ_SRC_OFFSET 3 +#endif +#define SRC_PLUS_FOR_4BYTES(bitOffset) (SEQ_SRC_OFFSET - 3) + ((CBitCtr_signed)(bitOffset) >> 3) +#define BIT_OFFSET_7BITS(bitOffset) ((unsigned)(bitOffset) & 7) +/* + if (BIT_OFFSET_DELTA_BITS == 0) : bitOffset == number_of_unprocessed_bits + if (BIT_OFFSET_DELTA_BITS == 1) : bitOffset == number_of_unprocessed_bits - 1 + and we can read 1 bit more in that mode : (8 * n + 1). +*/ +// #define BIT_OFFSET_DELTA_BITS 0 +#define BIT_OFFSET_DELTA_BITS 1 +#if BIT_OFFSET_DELTA_BITS == 1 + #define GET_SHIFT_FROM_BOFFS7(boff7) (7 ^ (boff7)) +#else + #define GET_SHIFT_FROM_BOFFS7(boff7) (8 - BIT_OFFSET_DELTA_BITS - (boff7)) +#endif + +#define UPDATE_BIT_OFFSET(bitOffset, numBits) \ + (bitOffset) -= (CBitCtr)(numBits); + +#define GET_SHIFT(bitOffset) GET_SHIFT_FROM_BOFFS7(BIT_OFFSET_7BITS(bitOffset)) + + +#if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) + #if (NUM_OFFSET_SYMBOLS_MAX - BIT_OFFSET_DELTA_BITS < 32) + /* if (NUM_OFFSET_SYMBOLS_MAX == 32 && BIT_OFFSET_DELTA_BITS == 1), + we have depth 31 + 9 + 9 + 8 = 57 bits that can b read with single read. */ + #define Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + #endif + #ifndef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + #if (BIT_OFFSET_DELTA_BITS == 1) + /* if (winLimit - winPos <= (kBlockSizeMax = (1 << 17))) + { + the case (16 bits literal extra + 16 match extra) is not possible + in correct stream. So error will be detected for (16 + 16) case. + And longest correct sequence after offset reading is (31 + 9 + 9 + 8 = 57 bits). + So we can use just one 64-bit load here in that case. + } + */ + #define Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML + #endif + #endif +#endif + + +#if !defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) || \ + (!defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML)) +// in : (0 < bits <= (24 or 25)): +#define STREAM_READ_BITS(dest, bits) \ +{ \ + GET32(dest, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + dest <<= GET_SHIFT(bitOffset); \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + dest >>= 32 - bits; \ +} +#endif + + +#define FSE_Peek_1(table, state) table[state] + +#define STATE_VAR(name) state_ ## name + +// in : (0 <= accuracy <= (24 or 25)) +#define FSE_INIT_STATE(name, cond) \ +{ \ + UInt32 r; \ + const unsigned bits = p->name ## _accuracy; \ + GET32(r, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + r <<= GET_SHIFT(bitOffset); \ + r >>= 1; \ + r >>= 31 ^ bits; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + STATE_VAR(name) = FSE_Peek_1(FSE_TABLE(name), r); \ + /* STATE_VAR(name) = dest << 16; */ \ +} + + +#define FSE_Peek_Plus(name, r) \ + STATE_VAR(name) = FSE_Peek_1(FSE_TABLE(name), \ + GET_FSE_REC_STATE(STATE_VAR(name)) + r); + +#define LZ_LOOP_ERROR_EXIT { return SZ_ERROR_DATA; } + +#define BO_OVERFLOW_CHECK \ + { if ((CBitCtr_signed)bitOffset < 0) LZ_LOOP_ERROR_EXIT } + + +#ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + +#define GET64(dest, p) { const Byte *ptr = p; dest = GetUi64(ptr); } + +#define FSE_PRELOAD \ +{ \ + GET64(v, src - 4 + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + v <<= GET_SHIFT(bitOffset); \ +} + +#define FSE_UPDATE_STATE_2(name, cond) \ +{ \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + UInt64 r = v; \ + v <<= bits; \ + r >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + r >>= 63 ^ bits; \ + FSE_Peek_Plus(name, r); \ +} + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_2 (ll, {} ) \ + FSE_UPDATE_STATE_2 (ml, {} ) \ + FSE_UPDATE_STATE_2 (of, BO_OVERFLOW_CHECK) \ + +#else // Z7_ZSTD_DEC_USE_64BIT_LOADS + +// it supports 8 bits accuracy for any code +// it supports 9 bits accuracy, if (BIT_OFFSET_DELTA_BITS == 1) +#define FSE_UPDATE_STATE_0(name, cond) \ +{ \ + UInt32 r; \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + GET16(r, src + 2 + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + r >>= (bitOffset & 7); \ + r &= (1 << (8 + BIT_OFFSET_DELTA_BITS)) - 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + r >>= (8 + BIT_OFFSET_DELTA_BITS) - bits; \ + FSE_Peek_Plus(name, r); \ +} + +// for debug (slow): +// #define Z7_ZSTD_DEC_USE_FSE_FUSION_FORCE +#if BIT_OFFSET_DELTA_BITS == 0 || defined(Z7_ZSTD_DEC_USE_FSE_FUSION_FORCE) + #define Z7_ZSTD_DEC_USE_FSE_FUSION +#endif + +#ifdef Z7_ZSTD_DEC_USE_FSE_FUSION +#define FSE_UPDATE_STATE_1(name) \ +{ UInt32 rest2; \ +{ \ + UInt32 r; \ + unsigned bits; \ + GET32(r, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + r <<= GET_SHIFT(bitOffset); \ + rest2 = r << bits; \ + r >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + r >>= 31 ^ bits; \ + FSE_Peek_Plus(name, r); \ +} + +#define FSE_UPDATE_STATE_3(name) \ +{ \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + rest2 >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + rest2 >>= 31 ^ bits; \ + FSE_Peek_Plus(name, rest2); \ +}} + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_1 (ll) \ + FSE_UPDATE_STATE_3 (ml) \ + FSE_UPDATE_STATE_0 (of, BO_OVERFLOW_CHECK) \ + +#else // Z7_ZSTD_DEC_USE_64BIT_LOADS + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_0 (ll, {} ) \ + FSE_UPDATE_STATE_0 (ml, {} ) \ + FSE_UPDATE_STATE_0 (of, BO_OVERFLOW_CHECK) \ + +#endif // Z7_ZSTD_DEC_USE_FSE_FUSION +#endif // Z7_ZSTD_DEC_USE_64BIT_LOADS + + + +typedef struct +{ + UInt32 numSeqs; + UInt32 literalsLen; + const Byte *literals; +} +CZstdDec1_Vars; + + +// if (BIT_OFFSET_DELTA_BITS != 0), we need (BIT_OFFSET_DELTA_BYTES > 0) +#define BIT_OFFSET_DELTA_BYTES BIT_OFFSET_DELTA_BITS + +/* if (NUM_OFFSET_SYMBOLS_MAX == 32) + max_seq_bit_length = (31) + 16 + 16 + 9 + 8 + 9 = 89 bits + if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) we have longest backward + lookahead offset, and we read UInt64 after literal_len reading. + if (BIT_OFFSET_DELTA_BITS == 1 && NUM_OFFSET_SYMBOLS_MAX == 32) + MAX_BACKWARD_DEPTH = 16 bytes +*/ +#define MAX_BACKWARD_DEPTH \ + ((NUM_OFFSET_SYMBOLS_MAX - 1 + 16 + 16 + 7) / 8 + 7 + BIT_OFFSET_DELTA_BYTES) + +/* srcLen != 0 + src == real_data_ptr - SEQ_SRC_OFFSET - BIT_OFFSET_DELTA_BYTES + if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) then + (winLimit - p->winPos <= (1 << 17)) is required +*/ +static +Z7_NO_INLINE +// Z7_ATTRIB_NO_VECTOR +SRes Decompress_Sequences(CZstdDec1 * const p, + const Byte *src, const size_t srcLen, + const size_t winLimit, + const CZstdDec1_Vars * const vars) +{ +#ifdef Z7_ZSTD_DEC_USE_BASES_LOCAL + SEQ_EXTRA_TABLES(a_) +#endif + + // for debug: + // #define Z7_ZSTD_DEC_USE_LOCAL_FSE_TABLES +#ifdef Z7_ZSTD_DEC_USE_LOCAL_FSE_TABLES + #define FSE_TABLE(n) fse. n + const CZstdDecFseTables fse = p->fse; + /* + CZstdDecFseTables fse; + #define COPY_FSE_TABLE(n) \ + memcpy(fse. n, p->fse. n, (size_t)4 << p-> n ## _accuracy); + COPY_FSE_TABLE(of) + COPY_FSE_TABLE(ll) + COPY_FSE_TABLE(ml) + */ +#else + #define FSE_TABLE(n) (p->fse. n) +#endif + +#ifdef Z7_ZSTD_DEC_USE_BASES_LOCAL + FILL_LOC_BASES_ALL +#endif + + { + unsigned numSeqs = vars->numSeqs; + const Byte *literals = vars->literals; + ptrdiff_t literalsLen = (ptrdiff_t)vars->literalsLen; + Byte * const win = p->win; + size_t winPos = p->winPos; + const size_t cycSize = p->cycSize; + size_t totalOutCheck = p->totalOutCheck; + const size_t winSize = p->winSize; + size_t reps_0 = p->reps[0]; + size_t reps_1 = p->reps[1]; + size_t reps_2 = p->reps[2]; + UInt32 STATE_VAR(ll), STATE_VAR(of), STATE_VAR(ml); + CBitCtr bitOffset; + + SET_bitOffset_TO_PAD (bitOffset, src + SEQ_SRC_OFFSET, srcLen + BIT_OFFSET_DELTA_BYTES) + + bitOffset -= BIT_OFFSET_DELTA_BITS; + + FSE_INIT_STATE(ll, {} ) + FSE_INIT_STATE(of, {} ) + FSE_INIT_STATE(ml, BO_OVERFLOW_CHECK) + + for (;;) + { + size_t matchLen; + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + UInt64 v; + #endif + + #ifdef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + FSE_PRELOAD + #endif + + // if (of_code == 0) + if ((Byte)STATE_VAR(of) == 0) + { + if (GET_FSE_REC_SYM(STATE_VAR(ll)) == 0) + { + const size_t offset = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep1) + } + STAT_UPDATE(else g_Num_Rep0++;) + } + else + { + const unsigned of_code = (Byte)STATE_VAR(of); + + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #if !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #endif + #else + UInt32 v; + { + const Byte *src4 = src + SRC_PLUS_FOR_4BYTES(bitOffset); + const unsigned skip = GET_SHIFT(bitOffset); + GET32(v, src4) + v <<= skip; + v |= (UInt32)src4[-1] >> (8 - skip); + } + #endif + + UPDATE_BIT_OFFSET(bitOffset, of_code) + + if (of_code == 1) + { + // read 1 bit + #if defined(Z7_MSC_VER_ORIGINAL) || defined(MY_CPU_X86_OR_AMD64) + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define CHECK_HIGH_BIT_64(a) ((Int64)(UInt64)(a) < 0) + #else + #define CHECK_HIGH_BIT_32(a) ((Int32)(UInt32)(a) < 0) + #endif + #else + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define CHECK_HIGH_BIT_64(a) ((UInt64)(a) & ((UInt64)1 << 63)) + #else + #define CHECK_HIGH_BIT_32(a) ((UInt32)(a) & ((UInt32)1 << 31)) + #endif + #endif + + if + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + CHECK_HIGH_BIT_64 (((UInt64)GET_FSE_REC_SYM(STATE_VAR(ll)) - 1) ^ v) + #else + CHECK_HIGH_BIT_32 (((UInt32)GET_FSE_REC_SYM(STATE_VAR(ll)) - 1) ^ v) + #endif + { + v <<= 1; + { + const size_t offset = reps_2; + reps_2 = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep2) + } + } + else + { + if (GET_FSE_REC_SYM(STATE_VAR(ll)) == 0) + { + // litLen == 0 && bit == 1 + STAT_INC(g_Num_Rep3) + v <<= 1; + reps_2 = reps_1; + reps_1 = reps_0; + if (--reps_0 == 0) + { + // LZ_LOOP_ERROR_EXIT + // original-zstd decoder : input is corrupted; force offset to 1 + // reps_0 = 1; + reps_0++; + } + } + else + { + // litLen != 0 && bit == 0 + v <<= 1; + { + const size_t offset = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep1) + } + } + } + } + else + { + // (2 <= of_code) + // if (of_code >= 32) LZ_LOOP_ERROR_EXIT // optional check + // we don't allow (of_code >= 32) cases in another code + reps_2 = reps_1; + reps_1 = reps_0; + reps_0 = ((size_t)1 << of_code) - 3 + (size_t) + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + (v >> (64 - of_code)); + v <<= of_code; + #else + (v >> (32 - of_code)); + #endif + } + } + + #ifdef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML + FSE_PRELOAD + #endif + + matchLen = (size_t)GET_FSE_REC_SYM(STATE_VAR(ml)) + #ifndef Z7_ZSTD_DEC_USE_ML_PLUS3 + + MATCH_LEN_MIN + #endif + ; + { + { + if (matchLen >= 32 + MATCH_LEN_MIN) // if (state_ml & 0x20) + { + const unsigned extra = BASES_TABLE(SEQ_ML_EXTRA) [(size_t)matchLen - MATCH_LEN_MIN]; + matchLen = BASES_TABLE(SEQ_ML_BASES) [(size_t)matchLen - MATCH_LEN_MIN]; + #if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) && \ + (defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) || \ + defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF)) + { + UPDATE_BIT_OFFSET(bitOffset, extra) + matchLen += (size_t)(v >> (64 - extra)); + #if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #else + v <<= extra; + #endif + } + #else + { + UInt32 v32; + STREAM_READ_BITS(v32, extra) + matchLen += v32; + } + #endif + STAT_INC(g_Num_Match) + } + } + } + + #if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) + FSE_PRELOAD + #endif + + { + size_t litLen = GET_FSE_REC_SYM(STATE_VAR(ll)); + if (litLen) + { + // if (STATE_VAR(ll) & 0x70) + if (litLen >= 16) + { + const unsigned extra = BASES_TABLE(SEQ_LL_EXTRA) [litLen]; + litLen = BASES_TABLE(SEQ_LL_BASES) [litLen]; + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + { + UPDATE_BIT_OFFSET(bitOffset, extra) + litLen += (size_t)(v >> (64 - extra)); + #if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #else + v <<= extra; + #endif + } + #else + { + UInt32 v32; + STREAM_READ_BITS(v32, extra) + litLen += v32; + } + #endif + STAT_INC(g_Num_LitsBig) + } + + if ((literalsLen -= (ptrdiff_t)litLen) < 0) + LZ_LOOP_ERROR_EXIT + totalOutCheck += litLen; + { + const size_t rem = winLimit - winPos; + if (litLen > rem) + LZ_LOOP_ERROR_EXIT + { + const Byte *literals_temp = literals; + Byte *d = win + winPos; + literals += litLen; + winPos += litLen; + CopyLiterals(d, literals_temp, litLen, rem); + } + } + } + STAT_UPDATE(else g_Num_Lit0++;) + } + + #define COPY_MATCH \ + { if (reps_0 > winSize || reps_0 > totalOutCheck) LZ_LOOP_ERROR_EXIT \ + totalOutCheck += matchLen; \ + { const size_t rem = winLimit - winPos; \ + if (matchLen > rem) LZ_LOOP_ERROR_EXIT \ + { const size_t winPos_temp = winPos; \ + winPos += matchLen; \ + CopyMatch(reps_0, matchLen, win, winPos_temp, rem, cycSize); }}} + + if (--numSeqs == 0) + { + COPY_MATCH + break; + } + FSE_UPDATE_STATES + COPY_MATCH + } // for + + if ((CBitCtr_signed)bitOffset != BIT_OFFSET_DELTA_BYTES * 8 - BIT_OFFSET_DELTA_BITS) + return SZ_ERROR_DATA; + + if (literalsLen) + { + const size_t rem = winLimit - winPos; + if ((size_t)literalsLen > rem) + return SZ_ERROR_DATA; + { + Byte *d = win + winPos; + winPos += (size_t)literalsLen; + totalOutCheck += (size_t)literalsLen; + CopyLiterals + // memcpy + (d, literals, (size_t)literalsLen, rem); + } + } + if (totalOutCheck >= winSize) + totalOutCheck = winSize; + p->totalOutCheck = totalOutCheck; + p->winPos = winPos; + p->reps[0] = (CZstdDecOffset)reps_0; + p->reps[1] = (CZstdDecOffset)reps_1; + p->reps[2] = (CZstdDecOffset)reps_2; + } + return SZ_OK; +} + + +// for debug: define to check that ZstdDec1_NeedTempBufferForInput() works correctly: +// #define Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP // define it for debug only +#ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP +static unsigned g_numSeqs; +#endif + + +#define k_LitBlockType_Flag_RLE_or_Treeless 1 +#define k_LitBlockType_Flag_Compressed 2 + +// outLimit : is strong limit +// outLimit <= ZstdDec1_GET_BLOCK_SIZE_LIMIT(p) +// inSize != 0 +static +Z7_NO_INLINE +SRes ZstdDec1_DecodeBlock(CZstdDec1 *p, + const Byte *src, SizeT inSize, SizeT afterAvail, + const size_t outLimit) +{ + CZstdDec1_Vars vars; + vars.literals = p->literalsBase; + { + const unsigned b0 = *src++; + UInt32 numLits, compressedSize; + const Byte *litStream; + Byte *literalsDest; + inSize--; + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + // we need at least one additional byte for (numSeqs). + // so we check for that additional byte in conditions. + numLits = b0 >> 3; + if (b0 & 4) + { + UInt32 v; + if (inSize < 1 + 1) // we need at least 1 byte here and 1 byte for (numSeqs). + return SZ_ERROR_DATA; + numLits >>= 1; + v = GetUi16(src); + src += 2; + inSize -= 2; + if ((b0 & 8) == 0) + { + src--; + inSize++; + v = (Byte)v; + } + numLits += v << 4; + } + compressedSize = 1; + if ((b0 & k_LitBlockType_Flag_RLE_or_Treeless) == 0) + compressedSize = numLits; + } + else if (inSize < 4) + return SZ_ERROR_DATA; + else + { + const unsigned mode4Streams = b0 & 0xc; + const unsigned numBytes = (3 * mode4Streams + 32) >> 4; + const unsigned numBits = 4 * numBytes - 2; + const UInt32 mask = ((UInt32)16 << numBits) - 1; + compressedSize = GetUi32(src); + numLits = (( + #ifdef MY_CPU_LE_UNALIGN + GetUi32(src - 1) + #else + ((compressedSize << 8) + b0) + #endif + ) >> 4) & mask; + src += numBytes; + inSize -= numBytes; + compressedSize >>= numBits; + compressedSize &= mask; + /* + if (numLits != 0) printf("inSize = %7u num_lits=%7u compressed=%7u ratio = %u ratio2 = %u\n", + i1, numLits, (unsigned)compressedSize * 1, (unsigned)compressedSize * 100 / numLits, + (unsigned)numLits * 100 / (unsigned)inSize); + } + */ + if (compressedSize == 0) + return SZ_ERROR_DATA; // (compressedSize == 0) is not allowed + } + + STAT_UPDATE(g_Num_Lits += numLits;) + + vars.literalsLen = numLits; + + if (compressedSize >= inSize) + return SZ_ERROR_DATA; + litStream = src; + src += compressedSize; + inSize -= compressedSize; + // inSize != 0 + { + UInt32 numSeqs = *src++; + inSize--; + if (numSeqs > 127) + { + UInt32 b1; + if (inSize == 0) + return SZ_ERROR_DATA; + numSeqs -= 128; + b1 = *src++; + inSize--; + if (numSeqs == 127) + { + if (inSize == 0) + return SZ_ERROR_DATA; + numSeqs = (UInt32)(*src++) + 127; + inSize--; + } + numSeqs = (numSeqs << 8) + b1; + } + if (numSeqs * MATCH_LEN_MIN + numLits > outLimit) + return SZ_ERROR_DATA; + vars.numSeqs = numSeqs; + + STAT_UPDATE(g_NumSeqs_total += numSeqs;) + /* + #ifdef SHOW_STAT + printf("\n %5u : %8u, %8u : %5u", (int)g_Num_Blocks_Compressed, (int)numSeqs, (int)g_NumSeqs_total, + (int)g_NumSeqs_total / g_Num_Blocks_Compressed); + #endif + // printf("\nnumSeqs2 = %d", numSeqs); + */ + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + if (numSeqs != g_numSeqs) return SZ_ERROR_DATA; // for debug + #endif + if (numSeqs == 0) + { + if (inSize != 0) + return SZ_ERROR_DATA; + literalsDest = p->win + p->winPos; + } + else + literalsDest = p->literalsBase; + } + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + if (b0 & k_LitBlockType_Flag_RLE_or_Treeless) + { + memset(literalsDest, litStream[0], numLits); + if (vars.numSeqs) + { + // literalsDest == p->literalsBase == vars.literals + #if COPY_CHUNK_SIZE > 1 + memset(p->literalsBase + numLits, 0, COPY_CHUNK_SIZE); + #endif + } + } + else + { + // unsigned y; + // for (y = 0; y < 10000; y++) + memcpy(literalsDest, litStream, numLits); + if (vars.numSeqs) + { + /* we need up to (15 == COPY_CHUNK_SIZE - 1) space for optimized CopyLiterals(). + If we have additional space in input stream after literals stream, + we use direct copy of rar literals in input stream */ + if ((size_t)(src + inSize - litStream) - numLits + afterAvail >= (COPY_CHUNK_SIZE - 1)) + vars.literals = litStream; + else + { + // literalsDest == p->literalsBase == vars.literals + #if COPY_CHUNK_SIZE > 1 + /* CopyLiterals(): + 1) we don't want reading non-initialized data + 2) we will copy only zero byte after literals buffer */ + memset(p->literalsBase + numLits, 0, COPY_CHUNK_SIZE); + #endif + } + } + } + } + else + { + CInBufPair hufStream; + hufStream.ptr = litStream; + hufStream.len = compressedSize; + + if ((b0 & k_LitBlockType_Flag_RLE_or_Treeless) == 0) + { + // unsigned y = 100; CInBufPair hs2 = hufStream; do { hufStream = hs2; + RINOK(Huf_DecodeTable(&p->huf, &hufStream)) + p->litHuf_wasSet = True; + // } while (--y); + } + else if (!p->litHuf_wasSet) + return SZ_ERROR_DATA; + + { + // int yyy; for (yyy = 0; yyy < 34; yyy++) { + SRes sres; + if ((b0 & 0xc) == 0) // mode4Streams + sres = Huf_Decompress_1stream((const Byte *)(const void *)p->huf.table64, + hufStream.ptr - HUF_SRC_OFFSET, hufStream.len, literalsDest, numLits); + else + { + // 6 bytes for the jump table + 4x1 bytes of end-padding Bytes) + if (hufStream.len < 6 + 4) + return SZ_ERROR_DATA; + // the condition from original-zstd decoder: + #define Z7_ZSTD_MIN_LITERALS_FOR_4_STREAMS 6 + if (numLits < Z7_ZSTD_MIN_LITERALS_FOR_4_STREAMS) + return SZ_ERROR_DATA; + sres = Huf_Decompress_4stream((const Byte *)(const void *)p->huf.table64, + hufStream.ptr + (6 - HUF_SRC_OFFSET), hufStream.len, literalsDest, numLits); + } + RINOK(sres) + // } + } + } + + if (vars.numSeqs == 0) + { + p->winPos += numLits; + UPDATE_TOTAL_OUT(p, numLits) + return SZ_OK; + } + } + { + CInBufPair in; + unsigned mode; + unsigned seqMode; + + in.ptr = src; + in.len = inSize; + if (in.len == 0) + return SZ_ERROR_DATA; + in.len--; + mode = *in.ptr++; + if (mode & 3) // Reserved bits + return SZ_ERROR_DATA; + + seqMode = (mode >> 6); + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else RINOK(FSE_Decode_SeqTable( + p->fse.ll, + &in, + 6, // predefAccuracy + &p->ll_accuracy, + NUM_LL_SYMBOLS, + k_PredefRecords_LL, + seqMode)) + + seqMode = (mode >> 4) & 3; + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else RINOK(FSE_Decode_SeqTable( + p->fse.of, + &in, + 5, // predefAccuracy + &p->of_accuracy, + NUM_OFFSET_SYMBOLS_MAX, + k_PredefRecords_OF, + seqMode)) + + seqMode = (mode >> 2) & 3; + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else + { + RINOK(FSE_Decode_SeqTable( + p->fse.ml, + &in, + 6, // predefAccuracy + &p->ml_accuracy, + NUM_ML_SYMBOLS, + k_PredefRecords_ML, + seqMode)) + /* + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + // { unsigned y = 1 << 10; do + { + const unsigned accuracy = p->ml_accuracy; + if (accuracy == 0) + p->fse.ml[0] += 3; + else + #ifdef MY_CPU_64BIT + { + // alignemt (UInt64 _pad_Alignment) in fse.ml is required for that code + UInt64 *table = (UInt64 *)(void *)p->fse.ml; + const UInt64 *end = (const UInt64 *)(const void *) + ((const Byte *)(const void *)table + ((size_t)sizeof(CFseRecord) << accuracy)); + do + { + table[0] += ((UInt64)MATCH_LEN_MIN << 32) + MATCH_LEN_MIN; + table[1] += ((UInt64)MATCH_LEN_MIN << 32) + MATCH_LEN_MIN; + table += 2; + } + while (table != end); + } + #else + { + UInt32 *table = p->fse.ml; + const UInt32 *end = (const UInt32 *)(const void *) + ((const Byte *)(const void *)table + ((size_t)sizeof(CFseRecord) << accuracy)); + do + { + table[0] += MATCH_LEN_MIN; + table[1] += MATCH_LEN_MIN; + table += 2; + table[0] += MATCH_LEN_MIN; + table[1] += MATCH_LEN_MIN; + table += 2; + } + while (table != end); + } + #endif + } + // while (--y); } + #endif + */ + } + + // p->seqTables_wereSet = True; + if (in.len == 0) + return SZ_ERROR_DATA; + return Decompress_Sequences(p, + in.ptr - SEQ_SRC_OFFSET - BIT_OFFSET_DELTA_BYTES, in.len, + p->winPos + outLimit, &vars); + } +} + + + + +// inSize != 0 +// it must do similar to ZstdDec1_DecodeBlock() +static size_t ZstdDec1_NeedTempBufferForInput( + const SizeT beforeSize, const Byte * const src, const SizeT inSize) +{ + unsigned b0; + UInt32 pos; + + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + g_numSeqs = 1 << 24; + #else + // we have at least 3 bytes before seq data: litBlockType, numSeqs, seqMode + #define MIN_BLOCK_LZ_HEADERS_SIZE 3 + if (beforeSize >= MAX_BACKWARD_DEPTH - MIN_BLOCK_LZ_HEADERS_SIZE) + return 0; + #endif + + b0 = src[0]; + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + UInt32 numLits = b0 >> 3; + pos = 1; + if (b0 & 4) + { + UInt32 v; + if (inSize < 3) + return 0; + numLits >>= 1; + v = GetUi16(src + 1); + pos = 3; + if ((b0 & 8) == 0) + { + pos = 2; + v = (Byte)v; + } + numLits += v << 4; + } + if (b0 & k_LitBlockType_Flag_RLE_or_Treeless) + numLits = 1; + pos += numLits; + } + else if (inSize < 5) + return 0; + else + { + const unsigned mode4Streams = b0 & 0xc; + const unsigned numBytes = (3 * mode4Streams + 48) >> 4; + const unsigned numBits = 4 * numBytes - 6; + UInt32 cs = GetUi32(src + 1); + cs >>= numBits; + cs &= ((UInt32)16 << numBits) - 1; + if (cs == 0) + return 0; + pos = numBytes + cs; + } + + if (pos >= inSize) + return 0; + { + UInt32 numSeqs = src[pos++]; + if (numSeqs > 127) + { + UInt32 b1; + if (pos >= inSize) + return 0; + numSeqs -= 128; + b1 = src[pos++]; + if (numSeqs == 127) + { + if (pos >= inSize) + return 0; + numSeqs = (UInt32)(src[pos++]) + 127; + } + numSeqs = (numSeqs << 8) + b1; + } + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + g_numSeqs = numSeqs; // for debug + #endif + if (numSeqs == 0) + return 0; + } + /* + if (pos >= inSize) + return 0; + pos++; + */ + // we will have one additional byte for seqMode: + if (beforeSize + pos >= MAX_BACKWARD_DEPTH - 1) + return 0; + return 1; +} + + + +// ---------- ZSTD FRAME ---------- + +#define kBlockType_Raw 0 +#define kBlockType_RLE 1 +#define kBlockType_Compressed 2 +#define kBlockType_Reserved 3 + +typedef enum +{ + // begin: states that require 4 bytes: + ZSTD2_STATE_SIGNATURE, + ZSTD2_STATE_HASH, + ZSTD2_STATE_SKIP_HEADER, + // end of states that require 4 bytes + + ZSTD2_STATE_SKIP_DATA, + ZSTD2_STATE_FRAME_HEADER, + ZSTD2_STATE_AFTER_HEADER, + ZSTD2_STATE_BLOCK, + ZSTD2_STATE_DATA, + ZSTD2_STATE_FINISHED +} EZstd2State; + + +struct CZstdDec +{ + EZstd2State frameState; + unsigned tempSize; + + Byte temp[14]; // 14 is required + + Byte descriptor; + Byte windowDescriptor; + Byte isLastBlock; + Byte blockType; + Byte isErrorState; + Byte hashError; + Byte disableHash; + Byte isCyclicMode; + + UInt32 blockSize; + UInt32 dictionaryId; + UInt32 curBlockUnpackRem; // for compressed blocks only + UInt32 inTempPos; + + UInt64 contentSize; + UInt64 contentProcessed; + CXxh64State xxh64; + + Byte *inTemp; + SizeT winBufSize_Allocated; + Byte *win_Base; + + ISzAllocPtr alloc_Small; + ISzAllocPtr alloc_Big; + + CZstdDec1 decoder; +}; + +#define ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p) \ + ((unsigned)(p)->contentProcessed & (Z7_XXH64_BLOCK_SIZE - 1)) + +#define ZSTD_DEC_IS_LAST_BLOCK(p) ((p)->isLastBlock) + + +static void ZstdDec_FreeWindow(CZstdDec * const p) +{ + if (p->win_Base) + { + ISzAlloc_Free(p->alloc_Big, p->win_Base); + p->win_Base = NULL; + // p->decoder.win = NULL; + p->winBufSize_Allocated = 0; + } +} + + +CZstdDecHandle ZstdDec_Create(ISzAllocPtr alloc_Small, ISzAllocPtr alloc_Big) +{ + CZstdDec *p = (CZstdDec *)ISzAlloc_Alloc(alloc_Small, sizeof(CZstdDec)); + if (!p) + return NULL; + p->alloc_Small = alloc_Small; + p->alloc_Big = alloc_Big; + // ZstdDec_CONSTRUCT(p) + p->inTemp = NULL; + p->win_Base = NULL; + p->winBufSize_Allocated = 0; + p->disableHash = False; + ZstdDec1_Construct(&p->decoder); + return p; +} + +void ZstdDec_Destroy(CZstdDecHandle p) +{ + #ifdef SHOW_STAT + #define PRINT_STAT1(name, v) \ + printf("\n%25s = %9u", name, v); + PRINT_STAT1("g_Num_Blocks_Compressed", g_Num_Blocks_Compressed) + PRINT_STAT1("g_Num_Blocks_memcpy", g_Num_Blocks_memcpy) + PRINT_STAT1("g_Num_Wrap_memmove_Num", g_Num_Wrap_memmove_Num) + PRINT_STAT1("g_Num_Wrap_memmove_Bytes", g_Num_Wrap_memmove_Bytes) + if (g_Num_Blocks_Compressed) + { + #define PRINT_STAT(name, v) \ + printf("\n%17s = %9u, per_block = %8u", name, v, v / g_Num_Blocks_Compressed); + PRINT_STAT("g_NumSeqs", g_NumSeqs_total) + // PRINT_STAT("g_NumCopy", g_NumCopy) + PRINT_STAT("g_NumOver", g_NumOver) + PRINT_STAT("g_NumOver2", g_NumOver2) + PRINT_STAT("g_Num_Match", g_Num_Match) + PRINT_STAT("g_Num_Lits", g_Num_Lits) + PRINT_STAT("g_Num_LitsBig", g_Num_LitsBig) + PRINT_STAT("g_Num_Lit0", g_Num_Lit0) + PRINT_STAT("g_Num_Rep_0", g_Num_Rep0) + PRINT_STAT("g_Num_Rep_1", g_Num_Rep1) + PRINT_STAT("g_Num_Rep_2", g_Num_Rep2) + PRINT_STAT("g_Num_Rep_3", g_Num_Rep3) + PRINT_STAT("g_Num_Threshold_0", g_Num_Threshold_0) + PRINT_STAT("g_Num_Threshold_1", g_Num_Threshold_1) + PRINT_STAT("g_Num_Threshold_0sum", g_Num_Threshold_0sum) + PRINT_STAT("g_Num_Threshold_1sum", g_Num_Threshold_1sum) + } + printf("\n"); + #endif + + ISzAlloc_Free(p->alloc_Small, p->decoder.literalsBase); + // p->->decoder.literalsBase = NULL; + ISzAlloc_Free(p->alloc_Small, p->inTemp); + // p->inTemp = NULL; + ZstdDec_FreeWindow(p); + ISzAlloc_Free(p->alloc_Small, p); +} + + + +#define kTempBuffer_PreSize (1u << 6) +#if kTempBuffer_PreSize < MAX_BACKWARD_DEPTH + #error Stop_Compiling_Bad_kTempBuffer_PreSize +#endif + +static SRes ZstdDec_AllocateMisc(CZstdDec *p) +{ + #define k_Lit_AfterAvail (1u << 6) + #if k_Lit_AfterAvail < (COPY_CHUNK_SIZE - 1) + #error Stop_Compiling_Bad_k_Lit_AfterAvail + #endif + // return ZstdDec1_Allocate(&p->decoder, p->alloc_Small); + if (!p->decoder.literalsBase) + { + p->decoder.literalsBase = (Byte *)ISzAlloc_Alloc(p->alloc_Small, + kBlockSizeMax + k_Lit_AfterAvail); + if (!p->decoder.literalsBase) + return SZ_ERROR_MEM; + } + if (!p->inTemp) + { + // we need k_Lit_AfterAvail here for owerread from raw literals stream + p->inTemp = (Byte *)ISzAlloc_Alloc(p->alloc_Small, + kBlockSizeMax + kTempBuffer_PreSize + k_Lit_AfterAvail); + if (!p->inTemp) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + + +static void ZstdDec_Init_ForNewFrame(CZstdDec *p) +{ + p->frameState = ZSTD2_STATE_SIGNATURE; + p->tempSize = 0; + + p->isErrorState = False; + p->hashError = False; + p->isCyclicMode = False; + p->contentProcessed = 0; + Xxh64State_Init(&p->xxh64); + ZstdDec1_Init(&p->decoder); +} + + +void ZstdDec_Init(CZstdDec *p) +{ + ZstdDec_Init_ForNewFrame(p); + p->decoder.winPos = 0; + memset(p->temp, 0, sizeof(p->temp)); +} + + +#define DESCRIPTOR_Get_DictionaryId_Flag(d) ((d) & 3) +#define DESCRIPTOR_FLAG_CHECKSUM (1 << 2) +#define DESCRIPTOR_FLAG_RESERVED (1 << 3) +// #define DESCRIPTOR_FLAG_UNUSED (1 << 4) +#define DESCRIPTOR_FLAG_SINGLE (1 << 5) +#define DESCRIPTOR_Get_ContentSize_Flag3(d) ((d) >> 5) +#define DESCRIPTOR_Is_ContentSize_Defined(d) (((d) & 0xe0) != 0) + + +static EZstd2State ZstdDec_UpdateState(CZstdDec * const p, const Byte b, CZstdDecInfo * const info) +{ + unsigned tempSize = p->tempSize; + p->temp[tempSize++] = b; + p->tempSize = tempSize; + + if (p->frameState == ZSTD2_STATE_BLOCK) + { + if (tempSize < 3) + return ZSTD2_STATE_BLOCK; + { + UInt32 b0 = GetUi32(p->temp); + const unsigned type = ((unsigned)b0 >> 1) & 3; + if (type == kBlockType_RLE && tempSize == 3) + return ZSTD2_STATE_BLOCK; + // info->num_Blocks_forType[type]++; + info->num_Blocks++; + if (type == kBlockType_Reserved) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED + return ZSTD2_STATE_BLOCK; + } + p->blockType = (Byte)type; + p->isLastBlock = (Byte)(b0 & 1); + p->inTempPos = 0; + p->tempSize = 0; + b0 >>= 3; + b0 &= 0x1fffff; + // info->num_BlockBytes_forType[type] += b0; + if (b0 == 0) + { + // empty RAW/RLE blocks are allowed in original-zstd decoder + if (type == kBlockType_Compressed) + { + p->isErrorState = True; + return ZSTD2_STATE_BLOCK; + } + if (!ZSTD_DEC_IS_LAST_BLOCK(p)) + return ZSTD2_STATE_BLOCK; + if (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) + return ZSTD2_STATE_HASH; + return ZSTD2_STATE_FINISHED; + } + p->blockSize = b0; + { + UInt32 blockLim = ZstdDec1_GET_BLOCK_SIZE_LIMIT(&p->decoder); + // compressed and uncompressed block sizes cannot be larger than min(kBlockSizeMax, window_size) + if (b0 > blockLim) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED; + return ZSTD2_STATE_BLOCK; + } + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor)) + { + const UInt64 rem = p->contentSize - p->contentProcessed; + if (blockLim > rem) + blockLim = (UInt32)rem; + } + p->curBlockUnpackRem = blockLim; + // uncompressed block size cannot be larger than remain data size: + if (type != kBlockType_Compressed) + { + if (b0 > blockLim) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED; + return ZSTD2_STATE_BLOCK; + } + } + } + } + return ZSTD2_STATE_DATA; + } + + if ((unsigned)p->frameState < ZSTD2_STATE_SKIP_DATA) + { + UInt32 v; + if (tempSize != 4) + return p->frameState; + v = GetUi32(p->temp); + if ((unsigned)p->frameState < ZSTD2_STATE_HASH) // == ZSTD2_STATE_SIGNATURE + { + if (v == 0xfd2fb528) + { + p->tempSize = 0; + info->num_DataFrames++; + return ZSTD2_STATE_FRAME_HEADER; + } + if ((v & 0xfffffff0) == 0x184d2a50) + { + p->tempSize = 0; + info->num_SkipFrames++; + return ZSTD2_STATE_SKIP_HEADER; + } + p->isErrorState = True; + return ZSTD2_STATE_SIGNATURE; + // return ZSTD2_STATE_ERROR; // is not ZSTD stream + } + if (p->frameState == ZSTD2_STATE_HASH) + { + info->checksum_Defined = True; + info->checksum = v; + // #ifndef DISABLE_XXH_CHECK + if (!p->disableHash) + { + if (p->decoder.winPos < ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p)) + { + // unexpected code failure + p->isErrorState = True; + // SZ_ERROR_FAIL; + } + else + if ((UInt32)Xxh64State_Digest(&p->xxh64, + p->decoder.win + (p->decoder.winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p)), + p->contentProcessed) != v) + { + p->hashError = True; + // return ZSTD2_STATE_ERROR; // hash error + } + } + // #endif + return ZSTD2_STATE_FINISHED; + } + // (p->frameState == ZSTD2_STATE_SKIP_HEADER) + { + p->blockSize = v; + info->skipFrames_Size += v; + p->tempSize = 0; + /* we want the caller could know that there was finished frame + finished frame. So we allow the case where + we have ZSTD2_STATE_SKIP_DATA state with (blockSize == 0). + */ + // if (v == 0) return ZSTD2_STATE_SIGNATURE; + return ZSTD2_STATE_SKIP_DATA; + } + } + + // if (p->frameState == ZSTD2_STATE_FRAME_HEADER) + { + unsigned descriptor; + const Byte *h; + descriptor = p->temp[0]; + p->descriptor = (Byte)descriptor; + if (descriptor & DESCRIPTOR_FLAG_RESERVED) // reserved bit + { + p->isErrorState = True; + return ZSTD2_STATE_FRAME_HEADER; + // return ZSTD2_STATE_ERROR; + } + { + const unsigned n = DESCRIPTOR_Get_ContentSize_Flag3(descriptor); + // tempSize -= 1 + ((1u << (n >> 1)) | ((n + 1) & 1)); + tempSize -= (0x9a563422u >> (n * 4)) & 0xf; + } + if (tempSize != (4u >> (3 - DESCRIPTOR_Get_DictionaryId_Flag(descriptor)))) + return ZSTD2_STATE_FRAME_HEADER; + + info->descriptor_OR = (Byte)(info->descriptor_OR | descriptor); + info->descriptor_NOT_OR = (Byte)(info->descriptor_NOT_OR | ~descriptor); + + h = &p->temp[1]; + { + Byte w = 0; + if ((descriptor & DESCRIPTOR_FLAG_SINGLE) == 0) + { + w = *h++; + if (info->windowDescriptor_MAX < w) + info->windowDescriptor_MAX = w; + // info->are_WindowDescriptors = True; + // info->num_WindowDescriptors++; + } + else + { + // info->are_SingleSegments = True; + // info->num_SingleSegments++; + } + p->windowDescriptor = w; + } + { + unsigned n = DESCRIPTOR_Get_DictionaryId_Flag(descriptor); + UInt32 d = 0; + if (n) + { + n = 1u << (n - 1); + d = GetUi32(h) & ((UInt32)(Int32)-1 >> (32 - 8u * n)); + h += n; + } + p->dictionaryId = d; + // info->dictionaryId_Cur = d; + if (d != 0) + { + if (info->dictionaryId == 0) + info->dictionaryId = d; + else if (info->dictionaryId != d) + info->are_DictionaryId_Different = True; + } + } + { + unsigned n = DESCRIPTOR_Get_ContentSize_Flag3(descriptor); + UInt64 v = 0; + if (n) + { + n >>= 1; + if (n == 1) + v = 256; + v += GetUi64(h) & ((UInt64)(Int64)-1 >> (64 - (8u << n))); + // info->are_ContentSize_Known = True; + // info->num_Frames_with_ContentSize++; + if (info->contentSize_MAX < v) + info->contentSize_MAX = v; + info->contentSize_Total += v; + } + else + { + info->are_ContentSize_Unknown = True; + // info->num_Frames_without_ContentSize++; + } + p->contentSize = v; + } + // if ((size_t)(h - p->temp) != headerSize) return ZSTD2_STATE_ERROR; // it's unexpected internal code failure + p->tempSize = 0; + + info->checksum_Defined = False; + /* + if (descriptor & DESCRIPTOR_FLAG_CHECKSUM) + info->are_Checksums = True; + else + info->are_Non_Checksums = True; + */ + + return ZSTD2_STATE_AFTER_HEADER; // ZSTD2_STATE_BLOCK; + } +} + + +static void ZstdDec_Update_XXH(CZstdDec * const p, size_t xxh64_winPos) +{ + /* + #ifdef DISABLE_XXH_CHECK + UNUSED_VAR(data) + #else + */ + if (!p->disableHash && (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM)) + { + // const size_t pos = p->xxh64_winPos; + const size_t size = (p->decoder.winPos - xxh64_winPos) & ~(size_t)31; + if (size) + { + // p->xxh64_winPos = pos + size; + Xxh64State_UpdateBlocks(&p->xxh64, + p->decoder.win + xxh64_winPos, + p->decoder.win + xxh64_winPos + size); + } + } +} + + +/* +in: + (winLimit) : is relaxed limit, where this function is allowed to stop writing of decoded data (if possible). + - this function uses (winLimit) for RAW/RLE blocks only, + because this function can decode single RAW/RLE block in several different calls. + - this function DOESN'T use (winLimit) for Compressed blocks, + because this function decodes full compressed block in single call. + (CZstdDec1::winPos <= winLimit) + (winLimit <= CZstdDec1::cycSize). + Note: if (ds->outBuf_fromCaller) mode is used, then + { + (strong_limit) is stored in CZstdDec1::cycSize. + So (winLimit) is more strong than (strong_limit). + } + +exit: + Note: (CZstdDecState::winPos) will be set by caller after exit of this function. + + This function can exit for any of these conditions: + - (frameState == ZSTD2_STATE_AFTER_HEADER) + - (frameState == ZSTD2_STATE_FINISHED) : frame was finished : (status == ZSTD_STATUS_FINISHED_FRAME) is set + - finished non-empty non-last block. So (CZstdDec1::winPos_atExit != winPos_atFuncStart). + - ZSTD_STATUS_NEEDS_MORE_INPUT in src + - (CZstdDec1::winPos) have reached (winLimit) in non-finished RAW/RLE block + + This function decodes no more than one non-empty block. + So it fulfills the condition at exit: + (CZstdDec1::winPos_atExit - winPos_atFuncStart <= block_size_max) + Note: (winPos_atExit > winLimit) is possible in some cases after compressed block decoding. + + if (ds->outBuf_fromCaller) mode (useAdditionalWinLimit medo) + { + then this function uses additional strong limit from (CZstdDec1::cycSize). + So this function will not write any data after (CZstdDec1::cycSize) + And it fulfills the condition at exit: + (CZstdDec1::winPos_atExit <= CZstdDec1::cycSize) + } +*/ +static SRes ZstdDec_DecodeBlock(CZstdDec * const p, CZstdDecState * const ds, + SizeT winLimitAdd) +{ + const Byte *src = ds->inBuf; + SizeT * const srcLen = &ds->inPos; + const SizeT inSize = ds->inLim; + // const int useAdditionalWinLimit = ds->outBuf_fromCaller ? 1 : 0; + enum_ZstdStatus * const status = &ds->status; + CZstdDecInfo * const info = &ds->info; + SizeT winLimit; + + const SizeT winPos_atFuncStart = p->decoder.winPos; + src += *srcLen; + *status = ZSTD_STATUS_NOT_SPECIFIED; + + // finishMode = ZSTD_FINISH_ANY; + if (ds->outSize_Defined) + { + if (ds->outSize < ds->outProcessed) + { + // p->isAfterSizeMode = 2; // we have extra bytes already + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + // size = 0; + } + else + { + // p->outSize >= p->outProcessed + const UInt64 rem = ds->outSize - ds->outProcessed; + /* + if (rem == 0) + p->isAfterSizeMode = 1; // we have reached exact required size + */ + if (winLimitAdd >= rem) + { + winLimitAdd = (SizeT)rem; + // if (p->finishMode) finishMode = ZSTD_FINISH_END; + } + } + } + + winLimit = p->decoder.winPos + winLimitAdd; + // (p->decoder.winPos <= winLimit) + + // while (p->frameState != ZSTD2_STATE_ERROR) + while (!p->isErrorState) + { + SizeT inCur = inSize - *srcLen; + + if (p->frameState == ZSTD2_STATE_DATA) + { + /* (p->decoder.winPos == winPos_atFuncStart) is expected, + because this function doesn't start new block. + if it have finished some non-empty block in this call. */ + if (p->decoder.winPos != winPos_atFuncStart) + return SZ_ERROR_FAIL; // it's unexpected + + /* + if (p->decoder.winPos > winLimit) + { + // we can be here, if in this function call + // - we have extracted non-empty compressed block, and (winPos > winLimit) after that. + // - we have started new block decoding after that. + // It's unexpected case, because we exit after non-empty non-last block. + *status = (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED; + return SZ_OK; + } + */ + // p->decoder.winPos <= winLimit + + if (p->blockType != kBlockType_Compressed) + { + // it's RLE or RAW block. + // p->BlockSize != 0_ + // winLimit <= p->decoder.cycSize + /* So here we use more strong (winLimit), even for + (ds->outBuf_fromCaller) mode. */ + SizeT outCur = winLimit - p->decoder.winPos; + { + const UInt32 rem = p->blockSize; + if (outCur > rem) + outCur = rem; + } + if (p->blockType == kBlockType_Raw) + { + if (outCur > inCur) + outCur = inCur; + /* output buffer is better aligned for XXH code. + So we use hash for output buffer data */ + // ZstdDec_Update_XXH(p, src, outCur); // for debug: + memcpy(p->decoder.win + p->decoder.winPos, src, outCur); + src += outCur; + *srcLen += outCur; + } + else // kBlockType_RLE + { + #define RLE_BYTE_INDEX_IN_temp 3 + memset(p->decoder.win + p->decoder.winPos, + p->temp[RLE_BYTE_INDEX_IN_temp], outCur); + } + { + const SizeT xxh64_winPos = p->decoder.winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p); + p->decoder.winPos += outCur; + UPDATE_TOTAL_OUT(&p->decoder, outCur) + p->contentProcessed += outCur; + ZstdDec_Update_XXH(p, xxh64_winPos); + } + // ds->winPos = p->decoder.winPos; // the caller does it instead. for debug: + ds->outProcessed += outCur; + if (p->blockSize -= (UInt32)outCur) + { + /* + if (ds->outSize_Defined) + { + if (ds->outSize <= ds->outProcessed) ds->isAfterSizeMode = (enum_ZstdStatus) + (ds->outSize == ds->outProcessed ? 1u: 2u); + } + */ + *status = (enum_ZstdStatus) + (ds->outSize_Defined && ds->outSize <= ds->outProcessed ? + ZSTD_STATUS_OUT_REACHED : (p->blockType == kBlockType_Raw && inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED); + return SZ_OK; + } + } + else // kBlockType_Compressed + { + // p->blockSize != 0 + // (uncompressed_size_of_block == 0) is allowed + // (p->curBlockUnpackRem == 0) is allowed + /* + if (p->decoder.winPos >= winLimit) + { + if (p->decoder.winPos != winPos_atFuncStart) + { + // it's unexpected case + // We already have some data in finished blocks in this function call. + // So we don't decompress new block after (>=winLimit), + // even if it's empty block. + *status = (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED; + return SZ_OK; + } + // (p->decoder.winPos == winLimit == winPos_atFuncStart) + // we will decode current block, because that current + // block can be empty block and we want to make some visible + // change of (src) stream after function start. + } + */ + /* + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + // we don't want to start new block, if we have more extra decoded bytes already + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + } + */ + { + const Byte *comprStream; + size_t afterAvail; + UInt32 inTempPos = p->inTempPos; + const UInt32 rem = p->blockSize - inTempPos; + // rem != 0 + if (inTempPos != 0 // (inTemp) buffer already contains some input data + || inCur < rem // available input data size is smaller than compressed block size + || ZstdDec1_NeedTempBufferForInput(*srcLen, src, rem)) + { + if (inCur > rem) + inCur = rem; + if (inCur) + { + STAT_INC(g_Num_Blocks_memcpy) + // we clear data for backward lookahead reading + if (inTempPos == 0) + memset(p->inTemp + kTempBuffer_PreSize - MAX_BACKWARD_DEPTH, 0, MAX_BACKWARD_DEPTH); + // { unsigned y = 0; for(;y < 1000; y++) + memcpy(p->inTemp + inTempPos + kTempBuffer_PreSize, src, inCur); + // } + src += inCur; + *srcLen += inCur; + inTempPos += (UInt32)inCur; + p->inTempPos = inTempPos; + } + if (inTempPos != p->blockSize) + { + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + #if COPY_CHUNK_SIZE > 1 + memset(p->inTemp + kTempBuffer_PreSize + inTempPos, 0, COPY_CHUNK_SIZE); + #endif + comprStream = p->inTemp + kTempBuffer_PreSize; + afterAvail = k_Lit_AfterAvail; + // we don't want to read non-initialized data or junk in CopyMatch(): + } + else + { + // inCur >= rem + // we use direct decoding from (src) buffer: + afterAvail = inCur - rem; + comprStream = src; + src += rem; + *srcLen += rem; + } + + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + ZstdDec1_NeedTempBufferForInput(*srcLen, comprStream, p->blockSize); + #endif + // printf("\nblockSize=%u", p->blockSize); + // printf("%x\n", (unsigned)p->contentProcessed); + STAT_INC(g_Num_Blocks_Compressed) + { + SRes sres; + const size_t winPos = p->decoder.winPos; + /* + if ( useAdditionalWinLimit), we use strong unpack limit: smallest from + - limit from stream : (curBlockUnpackRem) + - limit from caller : (cycSize - winPos) + if (!useAdditionalWinLimit), we use only relaxed limit: + - limit from stream : (curBlockUnpackRem) + */ + SizeT outLimit = p->curBlockUnpackRem; + if (ds->outBuf_fromCaller) + // if (useAdditionalWinLimit) + { + const size_t limit = p->decoder.cycSize - winPos; + if (outLimit > limit) + outLimit = limit; + } + sres = ZstdDec1_DecodeBlock(&p->decoder, + comprStream, p->blockSize, afterAvail, outLimit); + // ds->winPos = p->decoder.winPos; // the caller does it instead. for debug: + if (sres) + { + p->isErrorState = True; + return sres; + } + { + const SizeT xxh64_winPos = winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p); + const size_t num = p->decoder.winPos - winPos; + ds->outProcessed += num; + p->contentProcessed += num; + ZstdDec_Update_XXH(p, xxh64_winPos); + } + } + // printf("\nwinPos=%x", (int)(unsigned)p->decoder.winPos); + } + } + + /* + if (ds->outSize_Defined) + { + if (ds->outSize <= ds->outProcessed) ds->isAfterSizeMode = (enum_ZstdStatus) + (ds->outSize == ds->outProcessed ? 1u: 2u); + } + */ + + if (!ZSTD_DEC_IS_LAST_BLOCK(p)) + { + p->frameState = ZSTD2_STATE_BLOCK; + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + } + // we exit only if (winPos) was changed in this function call: + if (p->decoder.winPos != winPos_atFuncStart) + { + // decoded block was not empty. So we exit: + *status = (enum_ZstdStatus)( + (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED); + return SZ_OK; + } + // (p->decoder.winPos == winPos_atFuncStart) + // so current decoded block was empty. + // we will try to decode more blocks in this function. + continue; + } + + // decoded block was last in frame + if (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) + { + p->frameState = ZSTD2_STATE_HASH; + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; // disable if want to + /* We want to get same return codes for any input buffer sizes. + We want to get faster ZSTD_STATUS_OUT_REACHED status. + So we exit with ZSTD_STATUS_OUT_REACHED here, + instead of ZSTD2_STATE_HASH and ZSTD2_STATE_FINISHED processing. + that depends from input buffer size and that can set + ZSTD_STATUS_NEEDS_MORE_INPUT or return SZ_ERROR_DATA or SZ_ERROR_CRC. + */ + } + } + else + { + /* ZSTD2_STATE_FINISHED proccesing doesn't depend from input buffer */ + p->frameState = ZSTD2_STATE_FINISHED; + } + /* + p->frameState = (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) ? + ZSTD2_STATE_HASH : + ZSTD2_STATE_FINISHED; + */ + /* it's required to process ZSTD2_STATE_FINISHED state in this function call, + because we must check contentSize and hashError in ZSTD2_STATE_FINISHED code, + while the caller can reinit full state for ZSTD2_STATE_FINISHED + So we can't exit from function here. */ + continue; + } + + if (p->frameState == ZSTD2_STATE_FINISHED) + { + *status = ZSTD_STATUS_FINISHED_FRAME; + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor) + && p->contentSize != p->contentProcessed) + return SZ_ERROR_DATA; + if (p->hashError) // for debug + return SZ_ERROR_CRC; + return SZ_OK; + // p->frameState = ZSTD2_STATE_SIGNATURE; + // continue; + } + + if (p->frameState == ZSTD2_STATE_AFTER_HEADER) + return SZ_OK; // we need memory allocation for that state + + if (p->frameState == ZSTD2_STATE_SKIP_DATA) + { + UInt32 blockSize = p->blockSize; + // (blockSize == 0) is possible + if (inCur > blockSize) + inCur = blockSize; + src += inCur; + *srcLen += inCur; + blockSize -= (UInt32)inCur; + p->blockSize = blockSize; + if (blockSize == 0) + { + p->frameState = ZSTD2_STATE_SIGNATURE; + // continue; // for debug: we can continue without return to caller. + // we notify the caller that skip frame was finished: + *status = ZSTD_STATUS_FINISHED_FRAME; + return SZ_OK; + } + // blockSize != 0 + // (inCur) was smaller than previous value of p->blockSize. + // (inSize == *srcLen) now + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + if (inCur == 0) + { + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + { + (*srcLen)++; + p->frameState = ZstdDec_UpdateState(p, *src++, info); + } + } + + *status = ZSTD_STATUS_NOT_SPECIFIED; + p->isErrorState = True; + // p->frameState = ZSTD2_STATE_ERROR; + // if (p->frameState = ZSTD2_STATE_SIGNATURE) return SZ_ERROR_NO_ARCHIVE + return SZ_ERROR_DATA; +} + + + + +SRes ZstdDec_Decode(CZstdDecHandle dec, CZstdDecState *p) +{ + p->needWrite_Size = 0; + p->status = ZSTD_STATUS_NOT_SPECIFIED; + dec->disableHash = p->disableHash; + + if (p->outBuf_fromCaller) + { + dec->decoder.win = p->outBuf_fromCaller; + dec->decoder.cycSize = p->outBufSize_fromCaller; + } + + // p->winPos = dec->decoder.winPos; + + for (;;) + { + SizeT winPos, size; + // SizeT outProcessed; + SRes res; + + if (p->wrPos > dec->decoder.winPos) + return SZ_ERROR_FAIL; + + if (dec->frameState == ZSTD2_STATE_FINISHED) + { + if (!p->outBuf_fromCaller) + { + // we need to set positions to zero for new frame. + if (p->wrPos != dec->decoder.winPos) + { + /* We have already asked the caller to flush all data + with (p->needWrite_Size) and (ZSTD_STATUS_FINISHED_FRAME) status. + So it's unexpected case */ + // p->winPos = dec->decoder.winPos; + // p->needWrite_Size = dec->decoder.winPos - p->wrPos; // flush size asking + // return SZ_OK; // ask to flush again + return SZ_ERROR_FAIL; + } + // (p->wrPos == dec->decoder.winPos), and we wrap to zero: + dec->decoder.winPos = 0; + p->winPos = 0; + p->wrPos = 0; + } + ZstdDec_Init_ForNewFrame(dec); + // continue; + } + + winPos = dec->decoder.winPos; + { + SizeT next = dec->decoder.cycSize; + /* cycSize == 0, if no buffer was allocated still, + or, if (outBuf_fromCaller) mode and (outBufSize_fromCaller == 0) */ + if (!p->outBuf_fromCaller + && next + && next <= winPos + && dec->isCyclicMode) + { + // (0 < decoder.cycSize <= winPos) in isCyclicMode. + // so we need to wrap (winPos) and (wrPos) over (cycSize). + const size_t delta = next; + // (delta) is how many bytes we remove from buffer. + /* + // we don't need data older than last (cycSize) bytes. + size_t delta = winPos - next; // num bytes after (cycSize) + if (delta <= next) // it's expected case + delta = next; + // delta == Max(cycSize, winPos - cycSize) + */ + if (p->wrPos < delta) + { + // (wrPos < decoder.cycSize) + // We have asked already the caller to flush required data + // p->status = ZSTD_STATUS_NOT_SPECIFIED; + // p->winPos = winPos; + // p->needWrite_Size = delta - p->wrPos; // flush size asking + // return SZ_OK; // ask to flush again + return SZ_ERROR_FAIL; + } + // p->wrPos >= decoder.cycSize + // we move extra data after (decoder.cycSize) to start of cyclic buffer: + winPos -= delta; + if (winPos) + { + if (winPos >= delta) + return SZ_ERROR_FAIL; + memmove(dec->decoder.win, dec->decoder.win + delta, winPos); + // printf("\nmemmove processed=%8x winPos=%8x\n", (unsigned)p->outProcessed, (unsigned)dec->decoder.winPos); + STAT_INC(g_Num_Wrap_memmove_Num) + STAT_UPDATE(g_Num_Wrap_memmove_Bytes += (unsigned)winPos;) + } + dec->decoder.winPos = winPos; + p->winPos = winPos; + p->wrPos -= delta; + // dec->xxh64_winPos -= delta; + + // (winPos < delta) + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + /* we set the data after cycSize, because + we don't want to read non-initialized data or junk in CopyMatch(). */ + memset(dec->decoder.win + next, 0, COPY_CHUNK_SIZE); + #endif + + /* + if (winPos == next) + { + if (winPos != p->wrPos) + { + // we already requested before to flush full data for that case. + // but we give the caller a second chance to flush data: + p->needWrite_Size = winPos - p->wrPos; + return SZ_OK; + } + // (decoder.cycSize == winPos == p->wrPos) + // so we do second wrapping to zero: + winPos = 0; + dec->decoder.winPos = 0; + p->winPos = 0; + p->wrPos = 0; + } + */ + // (winPos < next) + } + + if (winPos > next) + return SZ_ERROR_FAIL; // it's unexpected case + /* + if (!outBuf_fromCaller && isCyclicMode && cycSize != 0) + then (winPos < cycSize) + else (winPos <= cycSize) + */ + if (!p->outBuf_fromCaller) + { + // that code is optional. We try to optimize write chunk sizes. + /* (next2) is expected next write position in the caller, + if the caller writes by kBlockSizeMax chunks. + */ + /* + const size_t next2 = (winPos + kBlockSizeMax) & (kBlockSizeMax - 1); + if (winPos < next2 && next2 < next) + next = next2; + */ + } + size = next - winPos; + } + + // note: ZstdDec_DecodeBlock() uses (winLimit = winPos + size) only for RLE and RAW blocks + res = ZstdDec_DecodeBlock(dec, p, size); + /* + after one block decoding: + if (!outBuf_fromCaller && isCyclicMode && cycSize != 0) + then (winPos < cycSize + max_block_size) + else (winPos <= cycSize) + */ + + if (!p->outBuf_fromCaller) + p->win = dec->decoder.win; + p->winPos = dec->decoder.winPos; + + // outProcessed = dec->decoder.winPos - winPos; + // p->outProcessed += outProcessed; + + if (res != SZ_OK) + return res; + + if (dec->frameState != ZSTD2_STATE_AFTER_HEADER) + { + if (p->outBuf_fromCaller) + return SZ_OK; + { + // !p->outBuf_fromCaller + /* + if (ZSTD_STATUS_FINISHED_FRAME), we request full flushing here because + 1) it's simpler to work with allocation and extracting of next frame, + 2) it's better to start writing to next new frame with aligned memory + for faster xxh 64-bit reads. + */ + size_t end = dec->decoder.winPos; // end pos for all data flushing + if (p->status != ZSTD_STATUS_FINISHED_FRAME) + { + // we will request flush here only for cases when wrap in cyclic buffer can be required in next call. + if (!dec->isCyclicMode) + return SZ_OK; + // isCyclicMode + { + const size_t delta = dec->decoder.cycSize; + if (end < delta) + return SZ_OK; // (winPos < cycSize). no need for flush + // cycSize <= winPos + // So we ask the caller to flush of (cycSize - wrPos) bytes, + // and then we will wrap cylicBuffer in next call + end = delta; + } + } + p->needWrite_Size = end - p->wrPos; + } + return SZ_OK; + } + + // ZSTD2_STATE_AFTER_HEADER + { + BoolInt useCyclic = False; + size_t cycSize; + + // p->status = ZSTD_STATUS_NOT_FINISHED; + if (dec->dictionaryId != 0) + { + /* actually we can try to decode some data, + because it's possible that some data doesn't use dictionary */ + // p->status = ZSTD_STATUS_NOT_SPECIFIED; + return SZ_ERROR_UNSUPPORTED; + } + + { + UInt64 winSize = dec->contentSize; + UInt64 winSize_Allocate = winSize; + const unsigned descriptor = dec->descriptor; + + if ((descriptor & DESCRIPTOR_FLAG_SINGLE) == 0) + { + const Byte wd = dec->windowDescriptor; + winSize = (UInt64)(8 + (wd & 7)) << ((wd >> 3) + 10 - 3); + if (!DESCRIPTOR_Is_ContentSize_Defined(descriptor) + || winSize_Allocate > winSize) + { + winSize_Allocate = winSize; + useCyclic = True; + } + } + /* + else + { + if (p->info.singleSegment_ContentSize_MAX < winSize) + p->info.singleSegment_ContentSize_MAX = winSize; + // p->info.num_SingleSegments++; + } + */ + if (p->info.windowSize_MAX < winSize) + p->info.windowSize_MAX = winSize; + if (p->info.windowSize_Allocate_MAX < winSize_Allocate) + p->info.windowSize_Allocate_MAX = winSize_Allocate; + /* + winSize_Allocate is MIN(content_size, window_size_from_descriptor). + Wven if (content_size < (window_size_from_descriptor)) + original-zstd still uses (window_size_from_descriptor) to check that decoding is allowed. + We try to follow original-zstd, and here we check (winSize) instead of (winSize_Allocate)) + */ + if ( + // winSize_Allocate // it's relaxed check + winSize // it's more strict check to be compatible with original-zstd + > ((UInt64)1 << MAX_WINDOW_SIZE_LOG)) + return SZ_ERROR_UNSUPPORTED; // SZ_ERROR_MEM + cycSize = (size_t)winSize_Allocate; + if (cycSize != winSize_Allocate) + return SZ_ERROR_MEM; + // cycSize <= winSize + /* later we will use (CZstdDec1::winSize) to check match offsets and check block sizes. + if (there is window descriptor) + { + We will check block size with (window_size_from_descriptor) instead of (winSize_Allocate). + Does original-zstd do it that way also? + } + Here we must reduce full real 64-bit (winSize) to size_t for (CZstdDec1::winSize). + Also we don't want too big values for (CZstdDec1::winSize). + our (CZstdDec1::winSize) will meet the condition: + (CZstdDec1::winSize < kBlockSizeMax || CZstdDec1::winSize <= cycSize). + */ + dec->decoder.winSize = (winSize < kBlockSizeMax) ? (size_t)winSize: cycSize; + // note: (CZstdDec1::winSize > cycSize) is possible, if (!useCyclic) + } + + RINOK(ZstdDec_AllocateMisc(dec)) + + if (p->outBuf_fromCaller) + dec->isCyclicMode = False; + else + { + size_t d = cycSize; + + if (dec->decoder.winPos != p->wrPos) + return SZ_ERROR_FAIL; + + dec->decoder.winPos = 0; + p->wrPos = 0; + p->winPos = dec->decoder.winPos; + + /* + const size_t needWrite = dec->decoder.winPos - p->wrPos; + if (!needWrite) + { + dec->decoder.winPos = 0; + p->wrPos = 0; + p->winPos = dec->decoder.winPos; + } + */ + /* if (!useCyclic) we allocate only cycSize = ContentSize. + But if we want to support the case where new frame starts with winPos != 0, + then we will wrap over zero, and we still need + to set (useCyclic) and allocate additional buffer spaces. + Now we don't allow new frame starting with (winPos != 0). + so (dec->decoder->winPos == 0) + can use (!useCyclic) with reduced buffer sizes. + */ + /* + if (dec->decoder->winPos != 0) + useCyclic = True; + */ + + if (useCyclic) + { + /* cyclyc buffer size must be at least (COPY_CHUNK_SIZE - 1) bytes + larger than window size, because CopyMatch() can write additional + (COPY_CHUNK_SIZE - 1) bytes and overwrite oldests data in cyclyc buffer. + But for performance reasons we align (cycSize) for (kBlockSizeMax). + also we must provide (cycSize >= max_decoded_data_after_cycSize), + because after data move wrapping over zero we must provide (winPos < cycSize). + */ + const size_t alignSize = kBlockSizeMax; + /* here we add (1 << 7) instead of (COPY_CHUNK_SIZE - 1), because + we want to get same (cycSize) for different COPY_CHUNK_SIZE values. */ + // cycSize += (COPY_CHUNK_SIZE - 1) + (alignSize - 1); // for debug : we can get smallest (cycSize) + cycSize += (1 << 7) + alignSize; + cycSize &= ~(size_t)(alignSize - 1); + // cycSize must be aligned for 32, because xxh requires 32-bytes blocks. + // cycSize += 12345; // for debug + // cycSize += 1 << 10; // for debug + // cycSize += 32; // for debug + // cycSize += kBlockSizeMax; // for debug + if (cycSize < d) + return SZ_ERROR_MEM; + /* + in cyclic buffer mode we allow to decode one additional block + that exceeds (cycSize). + So we must allocate additional (kBlockSizeMax) bytes after (cycSize). + if defined(Z7_STD_DEC_USE_AFTER_CYC_BUF) + { + we can read (COPY_CHUNK_SIZE - 1) bytes after (cycSize) + but we aready allocate additional kBlockSizeMax that + is larger than COPY_CHUNK_SIZE. + So we don't need additional space of COPY_CHUNK_SIZE after (cycSize). + } + */ + /* + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + d = cycSize + (1 << 7); // we must add at least (COPY_CHUNK_SIZE - 1) + #endif + */ + d = cycSize + kBlockSizeMax; + if (d < cycSize) + return SZ_ERROR_MEM; + } + + { + const size_t kMinWinAllocSize = 1 << 12; + if (d < kMinWinAllocSize) + d = kMinWinAllocSize; + } + + if (d > dec->winBufSize_Allocated) + { + /* + if (needWrite) + { + p->needWrite_Size = needWrite; + return SZ_OK; + // return SZ_ERROR_FAIL; + } + */ + + if (dec->winBufSize_Allocated != 0) + { + const size_t k_extra = (useCyclic || d >= (1u << 20)) ? + 2 * kBlockSizeMax : 0; + unsigned i = useCyclic ? 17 : 12; + for (; i < sizeof(size_t) * 8; i++) + { + const size_t d2 = ((size_t)1 << i) + k_extra; + if (d2 >= d) + { + d = d2; + break; + } + } + } + // RINOK(ZstdDec_AllocateWindow(dec, d)) + ZstdDec_FreeWindow(dec); + dec->win_Base = (Byte *)ISzAlloc_Alloc(dec->alloc_Big, d); + if (!dec->win_Base) + return SZ_ERROR_MEM; + dec->decoder.win = dec->win_Base; + dec->winBufSize_Allocated = d; + } + /* + else + { + // for non-cyclycMode we want flush data, and set winPos = 0 + if (needWrite) + { + if (!useCyclic || dec->decoder.winPos >= cycSize) + { + p->needWrite_Size = needWrite; + return SZ_OK; + // return SZ_ERROR_FAIL; + } + } + } + */ + + dec->decoder.cycSize = cycSize; + p->win = dec->decoder.win; + // p->cycSize = dec->decoder.cycSize; + dec->isCyclicMode = (Byte)useCyclic; + } // (!p->outBuf_fromCaller) end + + // p->winPos = dec->decoder.winPos; + dec->frameState = ZSTD2_STATE_BLOCK; + // continue; + } // ZSTD2_STATE_AFTER_HEADER end + } +} + + +void ZstdDec_GetResInfo(const CZstdDec *dec, + const CZstdDecState *p, + SRes res, + CZstdDecResInfo *stat) +{ + // ZstdDecInfo_CLEAR(stat); + stat->extraSize = 0; + stat->is_NonFinishedFrame = False; + if (dec->frameState != ZSTD2_STATE_FINISHED) + { + if (dec->frameState == ZSTD2_STATE_SIGNATURE) + { + stat->extraSize = (Byte)dec->tempSize; + if (ZstdDecInfo_GET_NUM_FRAMES(&p->info) == 0) + res = SZ_ERROR_NO_ARCHIVE; + } + else + { + stat->is_NonFinishedFrame = True; + if (res == SZ_OK && p->status == ZSTD_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + } + } + stat->decode_SRes = res; +} + + +size_t ZstdDec_ReadUnusedFromInBuf( + CZstdDecHandle dec, + size_t afterDecoding_tempPos, + void *data, size_t size) +{ + size_t processed = 0; + if (dec->frameState == ZSTD2_STATE_SIGNATURE) + { + Byte *dest = (Byte *)data; + const size_t tempSize = dec->tempSize; + while (afterDecoding_tempPos < tempSize) + { + if (size == 0) + break; + size--; + *dest++ = dec->temp[afterDecoding_tempPos++]; + processed++; + } + } + return processed; +} + + +void ZstdDecState_Clear(CZstdDecState *p) +{ + memset(p, 0 , sizeof(*p)); +} diff --git a/multiarc/src/formats/7z/C/ZstdDec.h b/multiarc/src/formats/7z/C/ZstdDec.h new file mode 100644 index 000000000..cd2613183 --- /dev/null +++ b/multiarc/src/formats/7z/C/ZstdDec.h @@ -0,0 +1,173 @@ +/* ZstdDec.h -- Zstd Decoder interfaces +2024-01-21 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_ZSTD_DEC_H +#define ZIP7_INC_ZSTD_DEC_H + +EXTERN_C_BEGIN + +typedef struct CZstdDec CZstdDec; +typedef CZstdDec * CZstdDecHandle; + +CZstdDecHandle ZstdDec_Create(ISzAllocPtr alloc_Small, ISzAllocPtr alloc_Big); +void ZstdDec_Destroy(CZstdDecHandle p); + +typedef enum +{ + ZSTD_STATUS_NOT_SPECIFIED, /* use main error code instead */ + ZSTD_STATUS_FINISHED_FRAME, /* data frame or skip frame was finished */ + ZSTD_STATUS_NOT_FINISHED, /* just finished non-empty block or unfinished RAW/RLE block */ + ZSTD_STATUS_NEEDS_MORE_INPUT, /* the callee needs more input bytes. It has more priority over ZSTD_STATUS_NOT_FINISHED */ + ZSTD_STATUS_OUT_REACHED /* is not finihed frame and ((outProcessed > outSize) || (outProcessed == outSize && unfinished RAW/RLE block) */ +} enum_ZstdStatus_Dummy; + +#define ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(p) \ + ((p)->status & ZSTD_STATUS_FINISHED_FRAME) +/* + ((p)->status == ZSTD_STATUS_NEEDS_MORE_INPUT || \ + (p)->status == ZSTD_STATUS_FINISHED_FRAME) +*/ + +typedef Byte enum_ZstdStatus; + + +void ZstdDec_Init(CZstdDecHandle p); + +typedef struct +{ + UInt64 num_Blocks; + Byte descriptor_OR; + Byte descriptor_NOT_OR; + Byte are_ContentSize_Unknown; + Byte windowDescriptor_MAX; + + // Byte are_ContentSize_Known; + // Byte are_SingleSegments; + // Byte are_WindowDescriptors; + Byte checksum_Defined; + // Byte are_Checksums; + // Byte are_Non_Checksums; + + // Byte are_DictionaryId; + Byte are_DictionaryId_Different; + + // Byte reserved[3]; + + UInt32 checksum; // checksum of last data frame + /// UInt32 dictionaryId_Cur; + UInt32 dictionaryId; // if there are non-zero dictionary IDs, then it's first dictionaryId + + UInt64 num_DataFrames; + UInt64 num_SkipFrames; + UInt64 skipFrames_Size; + UInt64 contentSize_Total; + UInt64 contentSize_MAX; + // UInt64 num_Checksums; + // UInt64 num_Non_Checksums; // frames without checksum + // UInt64 num_WindowDescriptors; + // UInt64 num_SingleSegments; + // UInt64 num_Frames_with_ContentSize; + // UInt64 num_Frames_without_ContentSize; + UInt64 windowSize_MAX; + UInt64 windowSize_Allocate_MAX; + // UInt64 num_DictionaryIds; + // UInt64 num_Blocks_forType[4]; + // UInt64 num_BlockBytes_forType[4]; + // UInt64 num_SingleSegments; + // UInt64 singleSegment_ContentSize_MAX; +} CZstdDecInfo; + +#define ZstdDecInfo_CLEAR(p) { memset(p, 0, sizeof(*(p))); } + +#define ZstdDecInfo_GET_NUM_FRAMES(p) ((p)->num_DataFrames + (p)->num_SkipFrames) + + +typedef struct CZstdDecState +{ + enum_ZstdStatus status; // out + Byte disableHash; + // Byte mustBeFinished; + Byte outSize_Defined; + // Byte isAfterSizeMode; + // UInt64 inProcessed; + // SRes codeRes; + // Byte needWrite_IsStrong; + + const Byte *inBuf; + size_t inPos; // in/out + size_t inLim; + + const Byte *win; // out + size_t winPos; // out + size_t wrPos; // in/out + // size_t cycSize; // out : if (!outBuf_fromCaller) + size_t needWrite_Size; // out + + Byte *outBuf_fromCaller; + size_t outBufSize_fromCaller; + /* (outBufSize_fromCaller >= full_uncompressed_size_of_all_frames) is required + for success decoding. + If outBufSize_fromCaller < full_uncompressed_size_of_all_frames), + decoding can give error message, because we decode per block basis. + */ + + // size_t outStep; + UInt64 outSize; // total in all frames + UInt64 outProcessed; // out decoded in all frames (it can be >= outSize) + + CZstdDecInfo info; +} CZstdDecState; + +void ZstdDecState_Clear(CZstdDecState *p); + +/* +ZstdDec_Decode() +return: + SZ_OK - no error + SZ_ERROR_DATA - Data Error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported method or method properties + SZ_ERROR_CRC - XXH hash Error + // SZ_ERROR_ARCHIVE - Headers error (not used now) +*/ +SRes ZstdDec_Decode(CZstdDecHandle dec, CZstdDecState *p); + +/* +ZstdDec_ReadUnusedFromInBuf(): +returns: the number of bytes that were read from InBuf +(*afterDecoding_tempPos) must be set to zero before first call of ZstdDec_ReadUnusedFromInBuf() +*/ +size_t ZstdDec_ReadUnusedFromInBuf( + CZstdDecHandle dec, + size_t afterDecoding_tempPos, // in/out + void *data, size_t size); + +typedef struct +{ + SRes decode_SRes; // error code of data decoding + Byte is_NonFinishedFrame; // there is unfinished decoding for data frame or skip frame + Byte extraSize; +} CZstdDecResInfo; + +/* +#define ZstdDecResInfo_CLEAR(p) \ +{ (p)->decode_SRes = 0; \ + (p)->is_NonFinishedFrame; \ + (p)->extraSize = 0; \ +} +// memset(p, 0, sizeof(*p)); +*/ + +/* +additional error codes for CZstdDecResInfo::decode_SRes: + SZ_ERROR_NO_ARCHIVE - is not zstd stream (no frames) + SZ_ERROR_INPUT_EOF - need more data in input stream +*/ +void ZstdDec_GetResInfo(const CZstdDec *dec, + const CZstdDecState *p, + SRes res, // it's result from ZstdDec_Decode() + CZstdDecResInfo *info); + +EXTERN_C_END + +#endif From 3d045a5585bac75e5d1c40a1d979823ca7f0cb62 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 15:10:59 +0300 Subject: [PATCH 15/22] fixing build --- multiarc/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/multiarc/CMakeLists.txt b/multiarc/CMakeLists.txt index fc4c9cd8f..1974fd3c8 100644 --- a/multiarc/CMakeLists.txt +++ b/multiarc/CMakeLists.txt @@ -70,7 +70,10 @@ set(SOURCES src/formats/7z/C/Ppmd7aDec.c src/formats/7z/C/Sha1Opt.c src/formats/7z/C/Sha256Opt.c - + src/formats/7z/C/Sha256Opt.c + src/formats/7z/C/Xxh64.c + src/formats/7z/C/SwapBytes.c + src/formats/7z/C/ZstdDec.c src/formats/ha/ha/acoder.c src/formats/ha/ha/archive.c src/formats/ha/ha/asc.c From f2e6f0ea34bbb496d77ec4a827c8b167a5257f7a Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 15:28:15 +0300 Subject: [PATCH 16/22] fixing build --- multiarc/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/multiarc/CMakeLists.txt b/multiarc/CMakeLists.txt index 1974fd3c8..b7972a5ac 100644 --- a/multiarc/CMakeLists.txt +++ b/multiarc/CMakeLists.txt @@ -59,6 +59,7 @@ set(SOURCES src/formats/7z/C/XzIn.c src/formats/7z/C/CpuArch.c src/formats/7z/C/LzFind.c + src/formats/7z/C/LzFindMt.c src/formats/7z/C/Ppmd7.c src/formats/7z/C/Lzma2Enc.c src/formats/7z/C/Sha256.c From 48cf3969ae0ec2e8b792078f2bf564a29f7f3102 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 15:33:54 +0300 Subject: [PATCH 17/22] fixing build --- multiarc/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/multiarc/CMakeLists.txt b/multiarc/CMakeLists.txt index b7972a5ac..17fbd913b 100644 --- a/multiarc/CMakeLists.txt +++ b/multiarc/CMakeLists.txt @@ -60,6 +60,7 @@ set(SOURCES src/formats/7z/C/CpuArch.c src/formats/7z/C/LzFind.c src/formats/7z/C/LzFindMt.c + src/formats/7z/C/Threads.c src/formats/7z/C/Ppmd7.c src/formats/7z/C/Lzma2Enc.c src/formats/7z/C/Sha256.c From fbf2958e2bac09697a068a32632d21f343f442f6 Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 15:40:52 +0300 Subject: [PATCH 18/22] fixing build --- WinPort/WinCompat.h | 1 + 1 file changed, 1 insertion(+) diff --git a/WinPort/WinCompat.h b/WinPort/WinCompat.h index 0e5bbf274..58d2a9a9a 100644 --- a/WinPort/WinCompat.h +++ b/WinPort/WinCompat.h @@ -1139,6 +1139,7 @@ typedef void *HKL; #define ERROR_INSUFFICIENT_BUFFER ENOBUFS #define ERROR_NO_UNICODE_TRANSLATION EILSEQ #define ERROR_DIRECTORY EISDIR +#define ERROR_TOO_MANY_POSTS E2BIG #define ERROR_INVALID_NAME ENAMETOOLONG #define ERROR_FILE_EXISTS EEXIST #define ERROR_OUTOFMEMORY ENOMEM From 76a42526999e533f478464acdf79ff2fe37be34b Mon Sep 17 00:00:00 2001 From: elfmz Date: Sun, 3 Nov 2024 15:51:36 +0300 Subject: [PATCH 19/22] fixing build --- multiarc/CMakeLists.txt | 90 +++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/multiarc/CMakeLists.txt b/multiarc/CMakeLists.txt index 17fbd913b..ad85f5b01 100644 --- a/multiarc/CMakeLists.txt +++ b/multiarc/CMakeLists.txt @@ -19,63 +19,67 @@ set(SOURCES src/formats/zip/zip.cpp src/formats/lzh/lzh.cpp - src/formats/7z/7z.cpp - src/formats/7z/7zMain.cpp - src/formats/7z/C/Lzma86Dec.c - src/formats/7z/C/7zDec.c - src/formats/7z/C/Ppmd7Dec.c - src/formats/7z/C/Sha1.c - src/formats/7z/C/BraIA64.c - src/formats/7z/C/Xz.c - src/formats/7z/C/Lzma2Dec.c - src/formats/7z/C/AesOpt.c - src/formats/7z/C/BwtSort.c - src/formats/7z/C/Bra86.c - src/formats/7z/C/7zCrcOpt.c src/formats/7z/C/7zAlloc.c - src/formats/7z/C/LzmaLib.c - src/formats/7z/C/HuffEnc.c - src/formats/7z/C/7zBuf.c - src/formats/7z/C/XzDec.c + src/formats/7z/C/7zArcIn.c src/formats/7z/C/7zBuf2.c - src/formats/7z/C/Lzma86Enc.c - src/formats/7z/C/Alloc.c - src/formats/7z/C/XzCrc64.c - src/formats/7z/C/XzEnc.c - src/formats/7z/C/Ppmd8.c - src/formats/7z/C/Aes.c + src/formats/7z/C/7zBuf.c src/formats/7z/C/7zCrc.c - src/formats/7z/C/Bcj2.c - src/formats/7z/C/Delta.c + src/formats/7z/C/7zCrcOpt.c + src/formats/7z/C/7zDec.c src/formats/7z/C/7zFile.c - src/formats/7z/C/Bcj2Enc.c - src/formats/7z/C/Ppmd8Enc.c - src/formats/7z/C/LzmaEnc.c - src/formats/7z/C/Ppmd7Enc.c src/formats/7z/C/7zStream.c - src/formats/7z/C/Ppmd8Dec.c - src/formats/7z/C/XzCrc64Opt.c - src/formats/7z/C/LzmaDec.c - src/formats/7z/C/XzIn.c + src/formats/7z/C/Aes.c + src/formats/7z/C/AesOpt.c + src/formats/7z/C/Alloc.c + src/formats/7z/C/Bcj2.c + src/formats/7z/C/Bcj2Enc.c + src/formats/7z/C/Blake2s.c + src/formats/7z/C/Bra86.c + src/formats/7z/C/Bra.c + src/formats/7z/C/BraIA64.c + src/formats/7z/C/BwtSort.c src/formats/7z/C/CpuArch.c + src/formats/7z/C/Delta.c + src/formats/7z/C/DllSecur.c + src/formats/7z/C/HuffEnc.c src/formats/7z/C/LzFind.c src/formats/7z/C/LzFindMt.c - src/formats/7z/C/Threads.c - src/formats/7z/C/Ppmd7.c - src/formats/7z/C/Lzma2Enc.c - src/formats/7z/C/Sha256.c - src/formats/7z/C/Sort.c - src/formats/7z/C/7zArcIn.c - src/formats/7z/C/Bra.c - src/formats/7z/C/Blake2s.c src/formats/7z/C/LzFindOpt.c + src/formats/7z/C/Lzma2Dec.c + src/formats/7z/C/Lzma2DecMt.c + src/formats/7z/C/Lzma2Enc.c + src/formats/7z/C/Lzma86Dec.c + src/formats/7z/C/Lzma86Enc.c + src/formats/7z/C/LzmaDec.c + src/formats/7z/C/LzmaEnc.c + src/formats/7z/C/LzmaLib.c + src/formats/7z/C/MtCoder.c + src/formats/7z/C/MtDec.c src/formats/7z/C/Ppmd7aDec.c + src/formats/7z/C/Ppmd7.c + src/formats/7z/C/Ppmd7Dec.c + src/formats/7z/C/Ppmd7Enc.c + src/formats/7z/C/Ppmd8.c + src/formats/7z/C/Ppmd8Dec.c + src/formats/7z/C/Ppmd8Enc.c + src/formats/7z/C/Sha1.c src/formats/7z/C/Sha1Opt.c + src/formats/7z/C/Sha256.c src/formats/7z/C/Sha256Opt.c - src/formats/7z/C/Sha256Opt.c - src/formats/7z/C/Xxh64.c + src/formats/7z/C/Sort.c src/formats/7z/C/SwapBytes.c + src/formats/7z/C/Threads.c + src/formats/7z/C/Xxh64.c + src/formats/7z/C/Xz.c + src/formats/7z/C/XzCrc64.c + src/formats/7z/C/XzCrc64Opt.c + src/formats/7z/C/XzDec.c + src/formats/7z/C/XzEnc.c + src/formats/7z/C/XzIn.c src/formats/7z/C/ZstdDec.c + src/formats/7z/7z.cpp + src/formats/7z/7zMain.cpp + src/formats/ha/ha/acoder.c src/formats/ha/ha/archive.c src/formats/ha/ha/asc.c From 3267a8b048dfd90df20438811fe17c65379a4da7 Mon Sep 17 00:00:00 2001 From: Ivan Sorokin Date: Sun, 3 Nov 2024 16:20:17 +0100 Subject: [PATCH 20/22] include wxWidgets version information into the log --- WinPort/src/Backend/WX/wxMain.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/WinPort/src/Backend/WX/wxMain.cpp b/WinPort/src/Backend/WX/wxMain.cpp index f37990fa7..18ea2f548 100644 --- a/WinPort/src/Backend/WX/wxMain.cpp +++ b/WinPort/src/Backend/WX/wxMain.cpp @@ -131,6 +131,12 @@ extern "C" __attribute__ ((visibility("default"))) bool WinPortMainBackend(WinPo if (!wxInitialize()) return false; + fprintf(stderr, "FAR2L wxWidgets build version %d.%d.%d\n", wxMAJOR_VERSION, wxMINOR_VERSION, wxRELEASE_NUMBER ); +#if wxCHECK_VERSION(2, 9, 2) + wxVersionInfo wxv = wxGetLibraryVersionInfo(); + fprintf(stderr, "FAR2L wxWidgets use version %d.%d.%d\n", wxv.GetMajor(), wxv.GetMinor(), wxv.GetMicro() ); +#endif + wxSetAssertHandler(WinPortWxAssertHandler); DetectHostAbilities(); From 4762dbff8bf532509c5c839a7ccaa908a954eca0 Mon Sep 17 00:00:00 2001 From: Aleksey Dobrunov Date: Sun, 3 Nov 2024 22:20:55 +0500 Subject: [PATCH 21/22] tmppanel: remove Unicode define --- tmppanel/src/TmpCfg.cpp | 44 +------- tmppanel/src/TmpCfg.hpp | 3 - tmppanel/src/TmpClass.cpp | 230 ++++++-------------------------------- tmppanel/src/TmpMix.cpp | 58 +--------- tmppanel/src/TmpPanel.cpp | 114 +++---------------- tmppanel/src/TmpPanel.hpp | 8 +- 6 files changed, 56 insertions(+), 401 deletions(-) diff --git a/tmppanel/src/TmpCfg.cpp b/tmppanel/src/TmpCfg.cpp index a369b61a0..998371143 100644 --- a/tmppanel/src/TmpCfg.cpp +++ b/tmppanel/src/TmpCfg.cpp @@ -9,13 +9,8 @@ Temporary panel configuration #include #include -#ifndef UNICODE -#define GetCheck(i) DialogItems[i].Selected -#define GetDataPtr(i) DialogItems[i].Data -#else #define GetCheck(i) (int)Info.SendDlgMessage(hDlg, DM_GETCHECK, i, 0) #define GetDataPtr(i) ((const TCHAR *)Info.SendDlgMessage(hDlg, DM_GETCONSTTEXTPTR, i, 0)) -#endif #define INI_LOCATION InMyConfig("plugins/tmppanel/config.ini") #define INI_SECTION "Settings" @@ -36,9 +31,6 @@ enum ColumnWidths, StatusColumnTypes, StatusColumnWidths, -#ifndef UNICODE - DisksMenuDigit, -#endif Mask, Prefix }; @@ -46,11 +38,7 @@ enum options_t Opt; static const char REGStr[][9] = {("InDisks"), ("InPlug"), ("Common"), ("Safe"), ("Any"), ("Contents"), - ("Mode"), ("Menu"), ("NewP"), ("Full"), ("ColT"), ("ColW"), ("StatT"), ("StatW"), -#ifndef UNICODE - ("DigitV"), -#endif - ("Mask"), ("Prefix")}; + ("Mode"), ("Menu"), ("NewP"), ("Full"), ("ColT"), ("ColW"), ("StatT"), ("StatW"), ("Mask"), ("Prefix")}; struct COptionsList { @@ -79,9 +67,6 @@ static const struct COptionsList OptionsList[] = { {Opt.StatusColumnTypes, _T("NR,SC,D,T"), 19, sizeof(Opt.StatusColumnTypes) }, {Opt.StatusColumnWidths, _T("0,8,0,5"), 21, sizeof(Opt.StatusColumnWidths) }, -#ifndef UNICODE - {Opt.DisksMenuDigit, _T("1"), 2, sizeof(Opt.DisksMenuDigit) }, -#endif {Opt.Mask, _T("*.temp"), 25, sizeof(Opt.Mask) }, {Opt.Prefix, _T("tmp"), 27, sizeof(Opt.Mask) }, }; @@ -116,21 +101,9 @@ int Config() /* 1*/ {DI_CHECKBOX, 5, 2, 0, 0, 0, MConfigAddToDisksMenu}, /* 2*/ - {DI_FIXEDIT, 7, 3, 7, 3, - 0 -#ifdef UNICODE - | DIF_HIDDEN -#endif - , - -1}, + {DI_FIXEDIT, 7, 3, 7, 3,DIF_HIDDEN, -1}, /* 3*/ - {DI_TEXT, 9, 3, 0, 0, - 0 -#ifdef UNICODE - | DIF_HIDDEN -#endif - , - MConfigDisksMenuDigit}, + {DI_TEXT, 9, 3, 0, 0,DIF_HIDDEN,MConfigDisksMenuDigit}, /* 4*/ {DI_CHECKBOX, DC, 2, 0, 0, 0, MConfigAddToPluginsMenu}, /* 5*/ {DI_TEXT, 5, 4, 0, 0, DIF_BOXCOLOR | DIF_SEPARATOR, -1}, @@ -178,23 +151,14 @@ int Config() if (i < ColumnTypes) DialogItems[OptionsList[i].DialogItem].Selected = *(int *)(OptionsList[i].Option); else -#ifndef UNICODE - lstrcpy(DialogItems[OptionsList[i].DialogItem].Data, (char *)OptionsList[i].Option); -#else DialogItems[OptionsList[i].DialogItem].PtrData = (TCHAR *)OptionsList[i].Option; -#endif -#ifndef UNICODE - int Ret = Info.Dialog(Info.ModuleNumber, -1, -1, DIALOG_WIDTH, DIALOG_HEIGHT, "Config", DialogItems, - ARRAYSIZE(DialogItems)); -#else HANDLE hDlg = Info.DialogInit(Info.ModuleNumber, -1, -1, DIALOG_WIDTH, DIALOG_HEIGHT, L"Config", DialogItems, ARRAYSIZE(DialogItems), 0, 0, NULL, 0); if (hDlg == INVALID_HANDLE_VALUE) return FALSE; int Ret = Info.DialogRun(hDlg); -#endif KeyFileHelper kfh(INI_LOCATION); if ((unsigned)Ret >= ARRAYSIZE(InitItems) - 1) @@ -220,8 +184,6 @@ int Config() Info.PluginsControl(NULL, PCTL_CACHEFORGET, PLT_PATH, (LONG_PTR)GetTmpPanelModule()); } done: -#ifdef UNICODE Info.DialogFree(hDlg); -#endif return ((unsigned)Ret < ARRAYSIZE(InitItems)); } diff --git a/tmppanel/src/TmpCfg.hpp b/tmppanel/src/TmpCfg.hpp index de5168439..f13a7b2ea 100644 --- a/tmppanel/src/TmpCfg.hpp +++ b/tmppanel/src/TmpCfg.hpp @@ -21,9 +21,6 @@ typedef struct options_s TCHAR ColumnWidths[64]; TCHAR StatusColumnTypes[64]; TCHAR StatusColumnWidths[64]; -#ifndef UNICODE - TCHAR DisksMenuDigit[1]; -#endif TCHAR Mask[512]; TCHAR Prefix[16]; } options_t; diff --git a/tmppanel/src/TmpClass.cpp b/tmppanel/src/TmpClass.cpp index 63c7cb95e..219d3004a 100644 --- a/tmppanel/src/TmpClass.cpp +++ b/tmppanel/src/TmpClass.cpp @@ -27,17 +27,12 @@ TmpPanel::~TmpPanel() int TmpPanel::GetFindData(PluginPanelItem **pPanelItem, int *pItemsNumber, int OpMode) { IfOptCommonPanel(); -#ifndef UNICODE - struct PanelInfo PInfo; - Info.Control(this, FCTL_GETPANELINFO, &PInfo); - UpdateItems(IsOwnersDisplayed(PInfo.ColumnTypes), IsLinksDisplayed(PInfo.ColumnTypes)); -#else + int Size = Info.Control(this, FCTL_GETCOLUMNTYPES, 0, 0); wchar_t *ColumnTypes = new wchar_t[Size]; Info.Control(this, FCTL_GETCOLUMNTYPES, Size, (LONG_PTR)ColumnTypes); UpdateItems(IsOwnersDisplayed(ColumnTypes), IsLinksDisplayed(ColumnTypes)); delete[] ColumnTypes; -#endif *pPanelItem = TmpPanelItem; *pItemsNumber = TmpItemsNumber; @@ -95,18 +90,11 @@ int TmpPanel::SetDirectory(const TCHAR *Dir, int OpMode) if ((OpMode & OPM_FIND) /* || lstrcmp(Dir,_T("\\"))==0*/) return (FALSE); if (lstrcmp(Dir, WGOOD_SLASH) == 0) -#ifndef UNICODE - Info.Control(this, FCTL_CLOSEPLUGIN, (void *)NULL); -#else Info.Control(this, FCTL_CLOSEPLUGIN, 0, 0); -#endif else -#ifndef UNICODE - Info.Control(this, FCTL_CLOSEPLUGIN, (void *)Dir); -#else Info.Control(this, FCTL_CLOSEPLUGIN, 0, (LONG_PTR)Dir); -#endif - return (TRUE); + + return (TRUE); } int TmpPanel::PutFiles(struct PluginPanelItem *PanelItem, int ItemsNumber, int, const TCHAR *SrcPath, int) @@ -138,12 +126,7 @@ HANDLE TmpPanel::BeginPutFiles() static inline int cmp_names(const WIN32_FIND_DATA &wfd, const FAR_FIND_DATA &ffd) { -#ifndef UNICODE -#define FILE_NAME cFileName -#else -#define FILE_NAME lpwszFileName -#endif - return lstrcmp(wfd.cFileName, FSF.PointToName(ffd.FILE_NAME)); + return lstrcmp(wfd.cFileName, FSF.PointToName(ffd.lpwszFileName)); } int TmpPanel::PutDirectoryContents(const TCHAR *Path) @@ -154,11 +137,8 @@ int TmpPanel::PutDirectoryContents(const TCHAR *Path) ARRAYSIZE(MsgItems), 0); } if (Opt.SelectedCopyContents) { -#ifdef UNICODE FAR_FIND_DATA *DirItems; -#else - struct PluginPanelItem *DirItems; -#endif + int DirItemsNumber; if (!Info.GetDirList(Path, &DirItems, &DirItemsNumber)) { FreePanelItems(TmpPanelItem, TmpItemsNumber); @@ -171,37 +151,16 @@ int TmpPanel::PutDirectoryContents(const TCHAR *Path) return FALSE; TmpPanelItem = NewPanelItem; memset(&TmpPanelItem[TmpItemsNumber], 0, sizeof(*TmpPanelItem) * DirItemsNumber); -#ifndef UNICODE - size_t PathLen = lstrlen(Path); - const TCHAR *lpSlash = _tcsrchr(Path, GOOD_SLASH); - if (lpSlash) - PathLen = lpSlash - Path; -#endif + for (int i = 0; i < DirItemsNumber; i++) { struct PluginPanelItem *CurPanelItem = &TmpPanelItem[TmpItemsNumber]; CurPanelItem->UserData = TmpItemsNumber; TmpItemsNumber++; -#ifdef UNICODE + CurPanelItem->FindData = DirItems[i]; CurPanelItem->FindData.lpwszFileName = wcsdup(DirItems[i].lpwszFileName); -#else - if ((PathLen + 1 + strlen(DirItems[i].FindData.cFileName)) - < ARRAYSIZE(CurPanelItem->FindData.cFileName)) { - CurPanelItem->FindData = DirItems[i].FindData; - strncpy(CurPanelItem->FindData.cFileName, Path); - CurPanelItem->FindData.cFileName[PathLen] = 0; - FSF.AddEndSlash(CurPanelItem->FindData.cFileName); - lstrcat(CurPanelItem->FindData.cFileName, DirItems[i].FindData.cFileName); - } else - strcpy(CurPanelItem->FindData.cFileName, "Oh long Johnson!"); -#endif } - Info.FreeDirList(DirItems -#ifdef UNICODE - , - DirItemsNumber -#endif - ); + Info.FreeDirList(DirItems, DirItemsNumber); } return TRUE; } @@ -217,22 +176,19 @@ int TmpPanel::PutOneFile(const TCHAR *SrcPath, PluginPanelItem &PanelItem) memset(CurPanelItem, 0, sizeof(*CurPanelItem)); CurPanelItem->FindData = PanelItem.FindData; CurPanelItem->UserData = TmpItemsNumber; -#ifdef UNICODE CurPanelItem->FindData.lpwszFileName = reinterpret_cast( malloc((lstrlen(SrcPath) + 1 + lstrlen(PanelItem.FindData.lpwszFileName) + 1) * sizeof(wchar_t))); if (CurPanelItem->FindData.lpwszFileName == NULL) return FALSE; -#else - *CurPanelItem->FindData.cFileName = 0; -#endif - lstrcpy((TCHAR *)CurPanelItem->FindData.FILE_NAME, SrcPath); + + lstrcpy((TCHAR *)CurPanelItem->FindData.lpwszFileName, SrcPath); if (*SrcPath) { - FSF.AddEndSlash((TCHAR *)CurPanelItem->FindData.FILE_NAME); + FSF.AddEndSlash((TCHAR *)CurPanelItem->FindData.lpwszFileName); } - lstrcat((TCHAR *)CurPanelItem->FindData.FILE_NAME, PanelItem.FindData.FILE_NAME); + lstrcat((TCHAR *)CurPanelItem->FindData.lpwszFileName, PanelItem.FindData.lpwszFileName); TmpItemsNumber++; if (Opt.SelectedCopyContents && (CurPanelItem->FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) - return PutDirectoryContents(CurPanelItem->FindData.FILE_NAME); + return PutDirectoryContents(CurPanelItem->FindData.lpwszFileName); return TRUE; } @@ -249,7 +205,7 @@ int TmpPanel::PutOneFile(const TCHAR *FilePath) if (GetFileInfoAndValidate(FilePath, &CurPanelItem->FindData, Opt.AnyInPanel)) { TmpItemsNumber++; if (Opt.SelectedCopyContents && (CurPanelItem->FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) - return PutDirectoryContents(CurPanelItem->FindData.FILE_NAME); + return PutDirectoryContents(CurPanelItem->FindData.lpwszFileName); } return TRUE; } @@ -274,11 +230,8 @@ int TmpPanel::SetFindList(const struct PluginPanelItem *PanelItem, int ItemsNumb for (int i = 0; i < ItemsNumber; ++i) { TmpPanelItem[i].UserData = i; TmpPanelItem[i].FindData = PanelItem[i].FindData; -#ifdef UNICODE if (TmpPanelItem[i].FindData.lpwszFileName) TmpPanelItem[i].FindData.lpwszFileName = wcsdup(TmpPanelItem[i].FindData.lpwszFileName); -#else -#endif } } CommitPutFiles(hScreen, TRUE); @@ -322,7 +275,7 @@ int _cdecl SortListCmp(const void *el1, const void *el2, void *userparam) PluginPanelItem *TmpPanelItem = reinterpret_cast(userparam); int idx1 = *reinterpret_cast(el1); int idx2 = *reinterpret_cast(el2); - int res = lstrcmp(TmpPanelItem[idx1].FindData.FILE_NAME, TmpPanelItem[idx2].FindData.FILE_NAME); + int res = lstrcmp(TmpPanelItem[idx1].FindData.lpwszFileName, TmpPanelItem[idx2].FindData.lpwszFileName); if (res == 0) { if (idx1 < idx2) return -1; @@ -343,8 +296,8 @@ void TmpPanel::RemoveDups() indices[i] = i; FSF.qsortex(indices, TmpItemsNumber, sizeof(int), SortListCmp, TmpPanelItem); for (int i = 0; i + 1 < TmpItemsNumber; i++) - if (lstrcmp(TmpPanelItem[indices[i]].FindData.FILE_NAME, - TmpPanelItem[indices[i + 1]].FindData.FILE_NAME) + if (lstrcmp(TmpPanelItem[indices[i]].FindData.lpwszFileName, + TmpPanelItem[indices[i + 1]].FindData.lpwszFileName) == 0) TmpPanelItem[indices[i + 1]].Flags|= REMOVE_FLAG; free(indices); @@ -361,12 +314,10 @@ void TmpPanel::RemoveEmptyItems() free((void *)CurItem->Owner); CurItem->Owner = NULL; } -#ifdef UNICODE if (CurItem->FindData.lpwszFileName) { free((wchar_t *)CurItem->FindData.lpwszFileName); CurItem->FindData.lpwszFileName = NULL; } -#endif EmptyCount++; } else if (EmptyCount) { CurItem->UserData-= EmptyCount; @@ -400,7 +351,7 @@ void TmpPanel::UpdateItems(int ShowOwners, int ShowLinks) for (int i = 0; i < TmpItemsNumber; i++, CurItem++) { HANDLE FindHandle; - const TCHAR *lpFullName = CurItem->FindData.FILE_NAME; + const TCHAR *lpFullName = CurItem->FindData.lpwszFileName; const TCHAR *lpSlash = _tcsrchr(lpFullName, GOOD_SLASH); int Length = lpSlash ? (int)(lpSlash - lpFullName + 1) : 0; @@ -414,8 +365,8 @@ void TmpPanel::UpdateItems(int ShowOwners, int ShowLinks) if (Length > 0 && Length > (int)lstrlen(lpFullName)) /* DJ $ */ { for (int j = 1; i + j < TmpItemsNumber; j++) { - if (memcmp(lpFullName, CurItem[j].FindData.FILE_NAME, Length * sizeof(TCHAR)) == 0 - && _tcschr((const TCHAR *)CurItem[j].FindData.FILE_NAME + Length, GOOD_SLASH) + if (memcmp(lpFullName, CurItem[j].FindData.lpwszFileName, Length * sizeof(TCHAR)) == 0 + && _tcschr((const TCHAR *)CurItem[j].FindData.lpwszFileName + Length, GOOD_SLASH) == NULL) { SameFolderItems++; } else { @@ -434,12 +385,9 @@ void TmpPanel::UpdateItems(int ShowOwners, int ShowLinks) StrBuf FindFile((int)(lpSlash - lpFullName) + 1 + 1 + 1); lstrcpyn(FindFile, lpFullName, (int)(lpSlash - lpFullName) + 1); lstrcpy(FindFile + (lpSlash + 1 - lpFullName), _T("*")); -#ifdef UNICODE + StrBuf NtPath; FormNtPath(FindFile, NtPath); -#else - const char *NtPath = FindFile; -#endif for (int J = 0; J < SameFolderItems; J++) CurItem[J].Flags|= REMOVE_FLAG; @@ -450,21 +398,11 @@ void TmpPanel::UpdateItems(int ShowOwners, int ShowLinks) if ((CurItem[J].Flags & 1) && cmp_names(FindData, CurItem[J].FindData) == 0) { CurItem[J].Flags&= ~REMOVE_FLAG; -#ifndef UNICODE - char save[sizeof(CurItem[J].FindData.cFileName)]; - lstrcpy(save, CurItem[J].FindData.cFileName); -#else const wchar_t *save = CurItem[J].FindData.lpwszFileName; -#endif - WFD2FFD(FindData, CurItem[J].FindData); -#ifndef UNICODE - lstrcpy(CurItem[J].FindData.cFileName, save); -#else free((wchar_t *)CurItem[J].FindData.lpwszFileName); CurItem[J].FindData.lpwszFileName = save; -#endif break; } } @@ -491,16 +429,11 @@ void TmpPanel::UpdateItems(int ShowOwners, int ShowLinks) free((void *)CurItem->Owner); CurItem->Owner = NULL; } - if (FSF.GetFileOwner(NULL, CurItem->FindData.FILE_NAME, Owner -#ifdef UNICODE - , - 80 -#endif - )) + if (FSF.GetFileOwner(NULL, CurItem->FindData.lpwszFileName, Owner,80)) CurItem->Owner = _tcsdup(Owner); } if (ShowLinks) - CurItem->NumberOfLinks = FSF.GetNumberOfLinks(CurItem->FindData.FILE_NAME); + CurItem->NumberOfLinks = FSF.GetNumberOfLinks(CurItem->FindData.lpwszFileName); } } Info.RestoreScreen(hScreen); @@ -510,29 +443,20 @@ int TmpPanel::ProcessEvent(int Event, void *) { if (Event == FE_CHANGEVIEWMODE) { IfOptCommonPanel(); -#ifndef UNICODE - struct PanelInfo PInfo; - Info.Control(this, FCTL_GETPANELINFO, &PInfo); - int UpdateOwners = IsOwnersDisplayed(PInfo.ColumnTypes) && !LastOwnersRead; - int UpdateLinks = IsLinksDisplayed(PInfo.ColumnTypes) && !LastLinksRead; -#else + int Size = Info.Control(this, FCTL_GETCOLUMNTYPES, 0, 0); wchar_t *ColumnTypes = new wchar_t[Size]; Info.Control(this, FCTL_GETCOLUMNTYPES, Size, (LONG_PTR)ColumnTypes); int UpdateOwners = IsOwnersDisplayed(ColumnTypes) && !LastOwnersRead; int UpdateLinks = IsLinksDisplayed(ColumnTypes) && !LastLinksRead; delete[] ColumnTypes; -#endif + if (UpdateOwners || UpdateLinks) { UpdateItems(UpdateOwners, UpdateLinks); -#ifndef UNICODE - Info.Control(this, FCTL_UPDATEPANEL, (void *)TRUE); - Info.Control(this, FCTL_REDRAWPANEL, NULL); -#else + Info.Control(this, FCTL_UPDATEPANEL, TRUE, 0); Info.Control(this, FCTL_REDRAWPANEL, 0, 0); -#endif } } return (FALSE); @@ -546,10 +470,6 @@ bool TmpPanel::IsCurrentFileCorrect(TCHAR **pCurFileName) if (pCurFileName) *pCurFileName = NULL; -#ifndef UNICODE - Info.Control(this, FCTL_GETPANELINFO, &PInfo); - CurFileName = PInfo.PanelItems[PInfo.CurrentItem].FindData.cFileName; -#else Info.Control(this, FCTL_GETPANELINFO, 0, (LONG_PTR)&PInfo); PluginPanelItem *PPI = (PluginPanelItem *)malloc(Info.Control(this, FCTL_GETPANELITEM, PInfo.CurrentItem, 0)); @@ -559,7 +479,6 @@ bool TmpPanel::IsCurrentFileCorrect(TCHAR **pCurFileName) } else { return false; } -#endif bool IsCorrectFile = false; if (lstrcmp(CurFileName, _T("..")) == 0) { @@ -574,9 +493,7 @@ bool TmpPanel::IsCurrentFileCorrect(TCHAR **pCurFileName) lstrcpy(*pCurFileName, CurFileName); } -#ifdef UNICODE free(PPI); -#endif return IsCorrectFile; } @@ -597,17 +514,9 @@ int TmpPanel::ProcessKey(int Key, unsigned int ControlState) PtrGuard CurFileName; if (IsCurrentFileCorrect(CurFileName.PtrPtr())) { struct PanelInfo PInfo; -#ifndef UNICODE - Info.Control(this, FCTL_GETPANELINFO, &PInfo); -#else Info.Control(this, FCTL_GETPANELINFO, 0, (LONG_PTR)&PInfo); -#endif if (lstrcmp(CurFileName, _T("..")) != 0) { -#ifndef UNICODE - if (PInfo.PanelItems[PInfo.CurrentItem].FindData.dwFileAttributes - & FILE_ATTRIBUTE_DIRECTORY) { -#else PluginPanelItem *PPI = (PluginPanelItem *)malloc( Info.Control(this, FCTL_GETPANELITEM, PInfo.CurrentItem, 0)); DWORD attributes = 0; @@ -618,21 +527,12 @@ int TmpPanel::ProcessKey(int Key, unsigned int ControlState) } if (attributes & FILE_ATTRIBUTE_DIRECTORY) { -#endif -#ifndef UNICODE - Info.Control(INVALID_HANDLE_VALUE, FCTL_SETANOTHERPANELDIR, CurFileName.Ptr()); -#else Info.Control(PANEL_PASSIVE, FCTL_SETPANELDIR, 0, (LONG_PTR)CurFileName.Ptr()); -#endif } else { GoToFile(CurFileName, true); } -#ifndef UNICODE - Info.Control(INVALID_HANDLE_VALUE, FCTL_REDRAWANOTHERPANEL, NULL); -#else Info.Control(PANEL_PASSIVE, FCTL_REDRAWPANEL, 0, 0); -#endif return (TRUE); } @@ -647,12 +547,7 @@ int TmpPanel::ProcessKey(int Key, unsigned int ControlState) if (ControlState == 0 && Key == VK_RETURN && Opt.AnyInPanel) { PtrGuard CurFileName; if (!IsCurrentFileCorrect(CurFileName.PtrPtr())) { -#ifndef UNICODE - Info.Control(this, FCTL_SETCMDLINE, CurFileName.Ptr()); -#else Info.Control(this, FCTL_SETCMDLINE, 0, (LONG_PTR)CurFileName.Ptr()); -#endif - return (TRUE); } } @@ -697,51 +592,28 @@ void TmpPanel::ProcessRemoveKey() IfOptCommonPanel(); struct PanelInfo PInfo; -#ifndef UNICODE - Info.Control(this, FCTL_GETPANELINFO, &PInfo); -#else + Info.Control(this, FCTL_GETPANELINFO, 0, (LONG_PTR)&PInfo); -#endif for (int i = 0; i < PInfo.SelectedItemsNumber; i++) { -#ifndef UNICODE - struct PluginPanelItem *RemovedItem = TmpPanelItem + PInfo.SelectedItems[i].UserData; -#else struct PluginPanelItem *RemovedItem = NULL; PluginPanelItem *PPI = (PluginPanelItem *)malloc(Info.Control(this, FCTL_GETSELECTEDPANELITEM, i, 0)); if (PPI) { Info.Control(this, FCTL_GETSELECTEDPANELITEM, i, (LONG_PTR)PPI); RemovedItem = TmpPanelItem + PPI->UserData; } -#endif if (RemovedItem != NULL) RemovedItem->Flags|= REMOVE_FLAG; -#ifdef UNICODE free(PPI); -#endif } RemoveEmptyItems(); -#ifndef UNICODE - Info.Control(this, FCTL_UPDATEPANEL, NULL); - Info.Control(this, FCTL_REDRAWPANEL, NULL); -#else Info.Control(this, FCTL_UPDATEPANEL, 0, 0); Info.Control(this, FCTL_REDRAWPANEL, 0, 0); -#endif - -#ifndef UNICODE - Info.Control(this, FCTL_GETANOTHERPANELSHORTINFO, (void *)&PInfo); -#else Info.Control(PANEL_PASSIVE, FCTL_GETPANELINFO, 0, (LONG_PTR)&PInfo); -#endif + if (PInfo.PanelType == PTYPE_QVIEWPANEL) { -#ifndef UNICODE - Info.Control(this, FCTL_UPDATEANOTHERPANEL, NULL); - Info.Control(this, FCTL_REDRAWANOTHERPANEL, NULL); -#else Info.Control(PANEL_PASSIVE, FCTL_UPDATEPANEL, 0, 0); Info.Control(PANEL_PASSIVE, FCTL_REDRAWPANEL, 0, 0); -#endif } } @@ -753,13 +625,7 @@ void TmpPanel::ProcessSaveListKey() // default path: opposite panel directory\panel. StrBuf ListPath(NT_MAX_PATH + 20 + 512); -#ifndef UNICODE - PanelInfo PInfo; - Info.Control(this, FCTL_GETANOTHERPANELINFO, &PInfo); - lstrcpy(ListPath, PInfo.CurDir); -#else Info.Control(PANEL_PASSIVE, FCTL_GETPANELDIR, NT_MAX_PATH, (LONG_PTR)ListPath.Ptr()); -#endif FSF.AddEndSlash(ListPath); lstrcat(ListPath, _T("panel")); @@ -778,13 +644,8 @@ void TmpPanel::ProcessSaveListKey() if (Info.InputBox(GetMsg(MTempPanel), GetMsg(MListFilePath), _T("TmpPanel.SaveList"), ListPath, ListPath, ListPath.Size() - 1, NULL, FIB_BUTTONS)) { SaveListFile(ListPath); -#ifndef UNICODE - Info.Control(this, FCTL_UPDATEANOTHERPANEL, NULL); - Info.Control(this, FCTL_REDRAWANOTHERPANEL, NULL); -#else Info.Control(PANEL_PASSIVE, FCTL_UPDATEPANEL, 0, 0); Info.Control(PANEL_PASSIVE, FCTL_REDRAWPANEL, 0, 0); -#endif } #undef _HANDLE #undef _UPDATE @@ -799,14 +660,10 @@ void TmpPanel::SaveListFile(const TCHAR *Path) if (!TmpItemsNumber) return; -#ifdef UNICODE StrBuf FullPath; GetFullPath(Path, FullPath); StrBuf NtPath; FormNtPath(FullPath, NtPath); -#else - const char *NtPath = Path; -#endif HANDLE hFile = CreateFile(NtPath, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { @@ -816,14 +673,14 @@ void TmpPanel::SaveListFile(const TCHAR *Path) } DWORD BytesWritten; -#ifdef UNICODE + static const unsigned short bom = BOM_UCS2; WriteFile(hFile, &bom, sizeof(bom), &BytesWritten, NULL); -#endif + int i = 0; do { static const TCHAR *CRLF = _T("\r\n"); - const TCHAR *FName = TmpPanelItem[i].FindData.FILE_NAME; + const TCHAR *FName = TmpPanelItem[i].FindData.lpwszFileName; WriteFile(hFile, FName, sizeof(TCHAR) * lstrlen(FName), &BytesWritten, NULL); WriteFile(hFile, CRLF, 2 * sizeof(TCHAR), &BytesWritten, NULL); } while (++i < TmpItemsNumber); @@ -841,13 +698,8 @@ void TmpPanel::SwitchToPanel(int NewPanelIndex) } if (CommonPanels[NewPanelIndex].Items) { CurrentCommonPanel = PanelIndex = NewPanelIndex; -#ifndef UNICODE - Info.Control(this, FCTL_UPDATEPANEL, NULL); - Info.Control(this, FCTL_REDRAWPANEL, NULL); -#else Info.Control(this, FCTL_UPDATEPANEL, 0, 0); Info.Control(this, FCTL_REDRAWPANEL, 0, 0); -#endif } } } @@ -857,19 +709,13 @@ void TmpPanel::ProcessPanelSwitchMenu() FarMenuItem fmi[COMMONPANELSNUMBER]; memset(&fmi, 0, sizeof(FarMenuItem) * COMMONPANELSNUMBER); const TCHAR *txt = GetMsg(MSwitchMenuTxt); -#ifdef UNICODE + wchar_t tmpstr[COMMONPANELSNUMBER][128]; static const TCHAR fmt1[] = _T("&%c. %ls %d"); -#else - static const TCHAR fmt1[] = _T("&%c. %s %d"); -#endif for (unsigned int i = 0; i < COMMONPANELSNUMBER; ++i) { -#ifndef UNICODE -#define _OUT fmi[i].Text -#else + #define _OUT tmpstr[i] fmi[i].Text = tmpstr[i]; -#endif if (i < 10) FSF.snprintf(_OUT, sizeof(_OUT) - 1, fmt1, _T('0') + i, txt, CommonPanels[i].ItemsNumber); else if (i < 36) @@ -924,27 +770,19 @@ bool TmpPanel::GetFileInfoAndValidate(const TCHAR *FilePath, FAR_FIND_DATA *Find TCHAR *FileName = ExpFilePath; ParseParam(FileName); -#ifdef UNICODE StrBuf FullPath; GetFullPath(FileName, FullPath); StrBuf NtPath; FormNtPath(FullPath, NtPath); -#else - char *FullPath = FileName; - const char *NtPath = FileName; -#endif if (!wcscmp(FileName, L"/")) { copy_name_set_attr: FindData->dwFileAttributes = FILE_ATTRIBUTE_ARCHIVE; copy_name: -#ifndef UNICODE - lstrcpyn(FindData->cFileName, FileName, sizeof(FindData->cFileName) - 1); -#else + if (FindData->lpwszFileName) free((void *)FindData->lpwszFileName); FindData->lpwszFileName = wcsdup(FileName); -#endif return (TRUE); } diff --git a/tmppanel/src/TmpMix.cpp b/tmppanel/src/TmpMix.cpp index a184676dc..ad0d43155 100644 --- a/tmppanel/src/TmpMix.cpp +++ b/tmppanel/src/TmpMix.cpp @@ -29,14 +29,8 @@ void InitDialogItems(const MyInitDialogItem *Init, struct FarDialogItem *Item, i PItem->Focus = 0; PItem->History = 0; PItem->DefaultButton = 0; -#ifdef UNICODE PItem->MaxLen = 0; -#endif -#ifndef UNICODE - lstrcpy(PItem->Data, PInit->Data != -1 ? GetMsg(PInit->Data) : ""); -#else PItem->PtrData = PInit->Data != -1 ? GetMsg(PInit->Data) : L""; -#endif } } @@ -46,10 +40,8 @@ void FreePanelItems(PluginPanelItem *Items, DWORD Total) for (DWORD I = 0; I < Total; I++) { if (Items[I].Owner) free((void *)Items[I].Owner); -#ifdef UNICODE if (Items[I].FindData.lpwszFileName) free((void *)Items[I].FindData.lpwszFileName); -#endif } free(Items); } @@ -74,17 +66,10 @@ TCHAR *ParseParam(TCHAR *&str) void GoToFile(const TCHAR *Target, BOOL AnotherPanel) { -#ifndef UNICODE - int FCTL_SetPanelDir = AnotherPanel ? FCTL_SETANOTHERPANELDIR : FCTL_SETPANELDIR; - int FCTL_GetPanelInfo = AnotherPanel ? FCTL_GETANOTHERPANELINFO : FCTL_GETPANELINFO; - int FCTL_RedrawPanel = AnotherPanel ? FCTL_REDRAWANOTHERPANEL : FCTL_REDRAWPANEL; -#define _PANEL_HANDLE INVALID_HANDLE_VALUE -#else #define FCTL_SetPanelDir FCTL_SETPANELDIR #define FCTL_GetPanelInfo FCTL_GETPANELINFO #define FCTL_RedrawPanel FCTL_REDRAWPANEL HANDLE _PANEL_HANDLE = AnotherPanel ? PANEL_PASSIVE : PANEL_ACTIVE; -#endif PanelRedrawInfo PRI; PanelInfo PInfo; @@ -105,54 +90,34 @@ void GoToFile(const TCHAR *Target, BOOL AnotherPanel) FSF.Unquote(Dir); if (*Dir.Ptr()) { -#ifndef UNICODE - Info.Control(_PANEL_HANDLE, FCTL_SetPanelDir, Dir.Ptr()); -#else Info.Control(_PANEL_HANDLE, FCTL_SetPanelDir, 0, (LONG_PTR)Dir.Ptr()); -#endif } -#ifndef UNICODE - Info.Control(_PANEL_HANDLE, FCTL_GetPanelInfo, &PInfo); -#else Info.Control(_PANEL_HANDLE, FCTL_GetPanelInfo, 0, (LONG_PTR)&PInfo); -#endif PRI.CurrentItem = PInfo.CurrentItem; PRI.TopPanelItem = PInfo.TopPanelItem; for (int J = 0; J < PInfo.ItemsNumber; J++) { -#ifndef UNICODE -#define FileName PInfo.PanelItems[J].FindData.cFileName -#else #define FileName (PPI ? PPI->FindData.lpwszFileName : NULL) PluginPanelItem *PPI = (PluginPanelItem *)malloc(Info.Control(_PANEL_HANDLE, FCTL_GETPANELITEM, J, 0)); if (PPI) { Info.Control(_PANEL_HANDLE, FCTL_GETPANELITEM, J, (LONG_PTR)PPI); } -#endif if (!FSF.LStricmp(Name, FSF.PointToName(FileName))) #undef FileName { PRI.CurrentItem = J; PRI.TopPanelItem = J; -#ifdef UNICODE free(PPI); -#endif break; } -#ifdef UNICODE free(PPI); -#endif } -#ifndef UNICODE - Info.Control(_PANEL_HANDLE, FCTL_RedrawPanel, &PRI); -#else Info.Control(_PANEL_HANDLE, FCTL_RedrawPanel, 0, (LONG_PTR)&PRI); -#endif #undef _PANEL_HANDLE #undef FCTL_SetPanelDir #undef FCTL_GetPanelInfo @@ -168,14 +133,9 @@ void WFD2FFD(WIN32_FIND_DATA &wfd, FAR_FIND_DATA &ffd) ffd.ftLastWriteTime = wfd.ftLastWriteTime; ffd.nFileSize = wfd.nFileSize; ffd.nPhysicalSize = wfd.nPhysicalSize; -#ifndef UNICODE - strncpy(ffd.cFileName, wfd.cFileName, ARRAYSIZE(ffd.cFileName)); -#else ffd.lpwszFileName = wcsdup(wfd.cFileName); -#endif } -#ifdef UNICODE wchar_t *FormNtPath(const wchar_t *path, StrBuf &buf) { int l = lstrlen(path); @@ -183,11 +143,10 @@ wchar_t *FormNtPath(const wchar_t *path, StrBuf &buf) lstrcpy(buf, path); return buf; } -#endif TCHAR *ExpandEnvStrs(const TCHAR *input, StrBuf &output) { -#ifdef UNICODE + std::string s; Wide2MB(input, s); Environment::ExpandString(s, false); @@ -196,10 +155,6 @@ TCHAR *ExpandEnvStrs(const TCHAR *input, StrBuf &output) output.Grow(w.size() + 1); lstrcpy(output, w.c_str()); -#else - output.Grow(MAX_PATH); - FSF.ExpandEnvironmentStr(input, output, output.Size()); -#endif return output; } @@ -227,23 +182,14 @@ bool FindListFile(const TCHAR *FileName, StrBuf &output) StrBuf Path; DWORD dwSize; -#ifdef UNICODE StrBuf FullPath; GetFullPath(FileName, FullPath); StrBuf NtPath; FormNtPath(FullPath, NtPath); -#else - const char *FullPath = FileName; - const char *NtPath = FileName; -#endif const TCHAR *final = NULL; if (GetFileAttributes(NtPath) != INVALID_FILE_ATTRIBUTES) { -#ifdef UNICODE output.Grow(FullPath.Size()); -#else - output.Grow(lstrlen(FullPath) + 1); -#endif lstrcpy(output, FullPath); return true; } @@ -290,7 +236,6 @@ bool FindListFile(const TCHAR *FileName, StrBuf &output) return true; } -#ifdef UNICODE wchar_t *GetFullPath(const wchar_t *input, StrBuf &output) { output.Grow(MAX_PATH); @@ -301,4 +246,3 @@ wchar_t *GetFullPath(const wchar_t *input, StrBuf &output) } return output; } -#endif diff --git a/tmppanel/src/TmpPanel.cpp b/tmppanel/src/TmpPanel.cpp index 9d6bb932d..5fbab0390 100644 --- a/tmppanel/src/TmpPanel.cpp +++ b/tmppanel/src/TmpPanel.cpp @@ -35,13 +35,8 @@ BOOL WINAPI DllMainCRTStartup(HANDLE hDll, DWORD dwReason, LPVOID lpReserved) } #endif -#ifndef UNICODE -#define WITH_ANSI_ARG , ANSI -#define WITH_ANSI_PARAM , int ANSI -#else #define WITH_ANSI_ARG #define WITH_ANSI_PARAM -#endif static void ProcessList(HANDLE hPlugin, TCHAR *Name, int Mode WITH_ANSI_PARAM); static void ShowMenuFromList(TCHAR *Name); static HANDLE OpenPanelFromOutput(TCHAR *argv WITH_ANSI_PARAM); @@ -82,21 +77,10 @@ SHAREDSYMBOL HANDLE WINAPI EXP_NAME(OpenPlugin)(int OpenFrom, INT_PTR Item) StartupOpenFrom = OpenFrom; if (OpenFrom == OPEN_COMMANDLINE) { TCHAR *argv = (TCHAR *)Item; -#ifndef UNICODE - int ANSI = FALSE; -#define OPT_COUNT 6 -#else + #define OPT_COUNT 5 -#endif - static const TCHAR ParamsStr[OPT_COUNT][8] = { -#ifndef UNICODE - _T("ansi"), -#endif - _T("safe"), _T("any"), _T("replace"), _T("menu"), _T("full")}; + static const TCHAR ParamsStr[OPT_COUNT][8] = {_T("safe"), _T("any"), _T("replace"), _T("menu"), _T("full")}; const int *ParamsOpt[OPT_COUNT] = { -#ifndef UNICODE - &ANSI, -#endif &Opt.SafeModePanel, &Opt.AnyInPanel, &Opt.Mode, &Opt.MenuForFilelist, &Opt.FullScreenPanel}; while (*argv == _T(' ')) @@ -173,12 +157,7 @@ static HANDLE OpenPanelFromOutput(TCHAR *argv WITH_ANSI_PARAM) StrBuf cmdparams(NT_MAX_PATH); // BUGBUG StrBuf fullcmd; - FSF.MkTemp(tempfilename, -#ifdef UNICODE - tempfilename.Size(), -#endif - _T("FARTMP")); - + FSF.MkTemp(tempfilename, tempfilename.Size(), _T("FARTMP")); lstrcpy(cmdparams, _T("%COMSPEC% /c ")); lstrcat(cmdparams, argv); ExpandEnvStrs(cmdparams, fullcmd); @@ -212,16 +191,11 @@ static HANDLE OpenPanelFromOutput(TCHAR *argv WITH_ANSI_PARAM) workDir.Reset(NT_MAX_PATH); // BUGBUG ExpandEnvStrs(tempDir, workDir); } else { -#ifdef UNICODE DWORD Size = FSF.GetCurrentDirectory(0, NULL); if (Size) { workDir.Reset(Size); FSF.GetCurrentDirectory(Size, workDir); } -#else - workDir.Reset(MAX_PATH); - GetCurrentDirectory(workDir.Size(), workDir); -#endif } TCHAR consoleTitle[255]; @@ -276,7 +250,6 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n StrBuf TMP(NT_MAX_PATH); // BUGBUG DWORD Len, Pos = 0, Size = FileSizeLow; -#ifdef UNICODE bool ucs2 = false; if (Size >= 2) switch (*(LPWORD)FileData) { @@ -292,12 +265,9 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n Pos+= 2; break; } -#endif while (Pos < Size) { -#ifdef UNICODE if (!ucs2) { -#endif char c; while (Pos < Size && ((c = FileData[Pos]) == '\r' || c == '\n')) Pos++; @@ -305,12 +275,8 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n while (Pos < Size && (c = FileData[Pos]) != '\r' && c != '\n') Pos++; Len = Pos - Off; -#ifndef UNICODE - if (Len >= (DWORD)TMP.Size()) - Len = TMP.Size() - 1; - memcpy(TMP.Ptr(), &FileData[Off], Len); -#else // UNICODE - Len = MultiByteToWideChar(CP_OEMCP, MB_PRECOMPOSED | MB_ERR_INVALID_CHARS, &FileData[Off], Len, TMP, + + Len = MultiByteToWideChar(CP_OEMCP, MB_PRECOMPOSED | MB_ERR_INVALID_CHARS, &FileData[Off], Len, TMP, TMP.Size() - 1); } else @@ -329,7 +295,6 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n Len = TMP.Size() - 1; memcpy(TMP.Ptr(), &FileData[Off], Len * sizeof(wchar_t)); } -#endif if (!Len) continue; @@ -338,11 +303,6 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n FSF.Trim(TMP); FSF.Unquote(TMP); -#ifndef UNICODE - if (ANSI) - CharToOem(TMP, TMP); -#endif - Len = lstrlen(TMP); if (!Len) continue; @@ -358,9 +318,8 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n (*numchars)+= Len + 1; ++*numargs; } -#ifdef UNICODE + done: -#endif munmap((LPVOID)FileData, FileSizeLow); } @@ -369,14 +328,10 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n *argc = 0; *argv = NULL; -#ifdef UNICODE StrBuf FullPath; GetFullPath(filename, FullPath); StrBuf NtPath; FormNtPath(FullPath, NtPath); -#else - const char *NtPath = filename; -#endif HANDLE hFile = CreateFile(NtPath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); @@ -422,9 +377,7 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n { int argc; TCHAR **argv = 0; -#ifndef UNICODE - static const int ANSI = FALSE; -#endif + ReadFileList(Name, &argc, &argv WITH_ANSI_ARG); FarMenuItem *fmi = (FarMenuItem *)malloc(argc * sizeof(FarMenuItem)); @@ -436,11 +389,7 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n param = ParseParam(p); FSF.TruncStr(param ? param : p, 67); -#ifndef UNICODE - strncpy(fmi[i].Text, param ? param : p, sizeof(fmi[i].Text) - 1); -#else - fmi[i].Text = wcsdup(param ? param : p); -#endif + fmi[i].Text = wcsdup(param ? param : p); fmi[i].Separator = !lstrcmp(param, _T("-")); fmi[i].Selected = FALSE; @@ -450,10 +399,7 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n TCHAR Title[128]; // BUGBUG FSF.ProcessName(FSF.PointToName(Name), lstrcpy(Title, _T("*.")), -#ifdef UNICODE - ARRAYSIZE(Title), -#endif - PN_GENERATENAME); + ARRAYSIZE(Title),PN_GENERATENAME); FSF.TruncPathStr(Title, 64); int BreakCode; @@ -462,11 +408,9 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n int ExitCode = Info.Menu(Info.ModuleNumber, -1, -1, 0, FMENU_WRAPMODE, Title, NULL, _T("Contents"), &BreakKeys[0], &BreakCode, fmi, argc); -#ifdef UNICODE for (int i = 0; i < argc; ++i) if (fmi[i].Text) free((void *)fmi[i].Text); -#endif free(fmi); @@ -481,20 +425,13 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n if (!bShellExecute) { if (TmpPanel::GetFileInfoAndValidate(p, &FindData, FALSE)) { if (FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { -#ifndef UNICODE - Info.Control(INVALID_HANDLE_VALUE, FCTL_SETPANELDIR, p); -#else Info.Control(INVALID_HANDLE_VALUE, FCTL_SETPANELDIR, 0, (LONG_PTR)p); -#endif } else { bShellExecute = TRUE; } } else { -#ifndef UNICODE - Info.Control(INVALID_HANDLE_VALUE, FCTL_SETCMDLINE, p); -#else + Info.Control(PANEL_ACTIVE, FCTL_SETCMDLINE, 0, (LONG_PTR)p); -#endif } } @@ -507,13 +444,8 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n free(argv); } -#ifndef UNICODE -#define _CONST -#define _OPARG -#else #define _CONST const #define _OPARG , int -#endif SHAREDSYMBOL HANDLE WINAPI EXP_NAME(OpenFilePlugin)(_CONST TCHAR * Name, const unsigned char *, int DataSize, int OpMode) #undef _OPARG @@ -521,14 +453,10 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n { if (!Name) return INVALID_HANDLE_VALUE; -#ifndef UNICODE -#define PNAME_ARG Name -#define pName Name -#else + StrBuf pName(NT_MAX_PATH); // BUGBUG lstrcpy(pName, Name); #define PNAME_ARG pName, pName.Size() -#endif if (!DataSize || !FSF.ProcessName(Opt.Mask, PNAME_ARG, PN_CMPNAMELIST)) return INVALID_HANDLE_VALUE; @@ -538,9 +466,7 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n HANDLE hPlugin = new TmpPanel(); if (hPlugin == NULL) return INVALID_HANDLE_VALUE; -#ifndef UNICODE - static const int ANSI = FALSE; -#endif + ProcessList(hPlugin, pName, Opt.Mode WITH_ANSI_ARG); return hPlugin; } else { @@ -577,11 +503,7 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n static const TCHAR *DiskMenuStrings[1]; DiskMenuStrings[0] = GetMsg(MDiskMenuString); Info->DiskMenuStrings = DiskMenuStrings; -#ifndef UNICODE - static int DiskMenuNumbers[1]; - DiskMenuNumbers[0] = FSF.atoi(Opt.DisksMenuDigit); - Info->DiskMenuNumbers = DiskMenuNumbers; -#endif + Info->DiskMenuStringsNumber = Opt.AddToDisksMenu ? ARRAYSIZE(DiskMenuStrings) : 0; static const TCHAR *PluginMenuStrings[1]; PluginMenuStrings[0] = GetMsg(MTempPanel); @@ -608,15 +530,9 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n SHAREDSYMBOL int WINAPI EXP_NAME(PutFiles)(HANDLE hPlugin, struct PluginPanelItem * PanelItem, int ItemsNumber, int Move, -#ifdef UNICODE - const wchar_t *SrcPath, -#endif - int OpMode) + const wchar_t *SrcPath, int OpMode) { -#ifndef UNICODE - StrBuf SrcPath(MAX_PATH); - GetCurrentDirectory(SrcPath.Size(), SrcPath); -#endif + TmpPanel *Panel = (TmpPanel *)hPlugin; return Panel->PutFiles(PanelItem, ItemsNumber, Move, SrcPath, OpMode); } diff --git a/tmppanel/src/TmpPanel.hpp b/tmppanel/src/TmpPanel.hpp index 15a95cae2..01116e978 100644 --- a/tmppanel/src/TmpPanel.hpp +++ b/tmppanel/src/TmpPanel.hpp @@ -53,12 +53,10 @@ TCHAR *ParseParam(TCHAR *&str); void GetOptions(void); void WFD2FFD(WIN32_FIND_DATA &wfd, FAR_FIND_DATA &ffd); -#ifdef UNICODE + #define BOM_UCS2 0xFEFF #define BOM_UCS2_BE 0xFFFE #define BOM_UTF8 0xBFBBEF -#endif - #define NT_MAX_PATH 32768 class StrBuf @@ -123,11 +121,11 @@ class PtrGuard ~PtrGuard() { free(ptr); } }; -#ifdef UNICODE + wchar_t *FormNtPath(const wchar_t *path, StrBuf &buf); wchar_t *GetFullPath(const wchar_t *input, StrBuf &output); -#endif TCHAR *ExpandEnvStrs(const TCHAR *input, StrBuf &output); bool FindListFile(const TCHAR *FileName, StrBuf &output); const wchar_t *GetTmpPanelModule(); + #endif /* __TMPPANEL_HPP__ */ From 285ed92badd0b32ca8a81551e3a36563b4d17c49 Mon Sep 17 00:00:00 2001 From: Aleksey Dobrunov Date: Mon, 4 Nov 2024 00:20:07 +0500 Subject: [PATCH 22/22] fix #2462: saving file list from tmppanel is broken use UTF8 with BOM for list of files --- tmppanel/src/TmpClass.cpp | 19 +++++--- tmppanel/src/TmpMix.cpp | 6 +++ tmppanel/src/TmpPanel.cpp | 92 ++++++++++++++------------------------- tmppanel/src/TmpPanel.hpp | 2 +- 4 files changed, 53 insertions(+), 66 deletions(-) diff --git a/tmppanel/src/TmpClass.cpp b/tmppanel/src/TmpClass.cpp index 219d3004a..5ab379928 100644 --- a/tmppanel/src/TmpClass.cpp +++ b/tmppanel/src/TmpClass.cpp @@ -674,15 +674,22 @@ void TmpPanel::SaveListFile(const TCHAR *Path) DWORD BytesWritten; - static const unsigned short bom = BOM_UCS2; - WriteFile(hFile, &bom, sizeof(bom), &BytesWritten, NULL); + static const char bom_utf8[] = {'\xEF', '\xBB', '\xBF'}; + WriteFile(hFile, &bom_utf8, sizeof(bom_utf8), &BytesWritten, NULL); int i = 0; do { - static const TCHAR *CRLF = _T("\r\n"); - const TCHAR *FName = TmpPanelItem[i].FindData.lpwszFileName; - WriteFile(hFile, FName, sizeof(TCHAR) * lstrlen(FName), &BytesWritten, NULL); - WriteFile(hFile, CRLF, 2 * sizeof(TCHAR), &BytesWritten, NULL); + static const char *LF = "\n"; + const wchar_t *FName = TmpPanelItem[i].FindData.lpwszFileName; + + size_t Size = 4 * wcslen(FName) + 1; + LPSTR FNameA = new char[Size]; + PWZ_to_PZ(FName, FNameA, Size); + + WriteFile(hFile, FNameA, sizeof(char) * strlen(FNameA), &BytesWritten, NULL); + WriteFile(hFile, LF, sizeof(char) * strlen(LF), &BytesWritten, NULL); + + delete[] FNameA; } while (++i < TmpItemsNumber); CloseHandle(hFile); } diff --git a/tmppanel/src/TmpMix.cpp b/tmppanel/src/TmpMix.cpp index ad0d43155..d18a1252c 100644 --- a/tmppanel/src/TmpMix.cpp +++ b/tmppanel/src/TmpMix.cpp @@ -246,3 +246,9 @@ wchar_t *GetFullPath(const wchar_t *input, StrBuf &output) } return output; } + +int PWZ_to_PZ(const wchar_t *src, char *dst, int lendst) +{ + ErrnoSaver ErSr; + return WINPORT(WideCharToMultiByte)(CP_UTF8, 0, (src), -1, (dst), (int)(lendst), nullptr, nullptr); +} \ No newline at end of file diff --git a/tmppanel/src/TmpPanel.cpp b/tmppanel/src/TmpPanel.cpp index 5fbab0390..72dad44ad 100644 --- a/tmppanel/src/TmpPanel.cpp +++ b/tmppanel/src/TmpPanel.cpp @@ -250,79 +250,53 @@ void ReadFileLines(int fd, DWORD FileSizeLow, TCHAR **argv, TCHAR *args, UINT *n StrBuf TMP(NT_MAX_PATH); // BUGBUG DWORD Len, Pos = 0, Size = FileSizeLow; - bool ucs2 = false; - if (Size >= 2) - switch (*(LPWORD)FileData) { - case (BOM_UTF8 & 0xFFFF): - if (Size >= 3 && ((LPBYTE)FileData)[2] == (BYTE)(BOM_UTF8 >> 16)) { - case BOM_UCS2_BE: - goto done; - } - default: - break; - case BOM_UCS2: - ucs2 = true; - Pos+= 2; - break; + if (Size >= 3) { + if ( ((LPBYTE)FileData)[0]==0xEF && ((LPBYTE)FileData)[1]==0xBB && ((LPBYTE)FileData)[2]==0xBF + //*(LPWORD)FileData == BOM_UTF8 & 0xFFFF && ((LPBYTE)FileData)[2] == (BYTE)(BOM_UTF8 >> 16) + ){ + Pos+= 3; } + } while (Pos < Size) { - if (!ucs2) { - char c; - while (Pos < Size && ((c = FileData[Pos]) == '\r' || c == '\n')) - Pos++; - DWORD Off = Pos; - while (Pos < Size && (c = FileData[Pos]) != '\r' && c != '\n') - Pos++; - Len = Pos - Off; - - Len = MultiByteToWideChar(CP_OEMCP, MB_PRECOMPOSED | MB_ERR_INVALID_CHARS, &FileData[Off], Len, TMP, - TMP.Size() - 1); - } - else - { - wchar_t c; - --Size; - while (Pos < Size && ((c = *(wchar_t *)&FileData[Pos]) == L'\r' || c == L'\n')) - Pos+= sizeof(wchar_t); + + char c; + while (Pos < Size && ((c = FileData[Pos]) == '\r' || c == '\n')) + Pos++; DWORD Off = Pos; - while (Pos < Size && (c = *(wchar_t *)&FileData[Pos]) != L'\r' && c != L'\n') - Pos+= sizeof(wchar_t); - if (Pos < Size) - ++Size; - Len = (Pos - Off) / sizeof(wchar_t); - if (Len >= (DWORD)TMP.Size()) - Len = TMP.Size() - 1; - memcpy(TMP.Ptr(), &FileData[Off], Len * sizeof(wchar_t)); - } - if (!Len) - continue; + while (Pos < Size && (c = FileData[Pos]) != '\r' && c != '\n') + Pos++; + Len = Pos - Off; - TMP.Ptr()[Len] = 0; + Len = MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED | MB_ERR_INVALID_CHARS, &FileData[Off], Len, TMP, + TMP.Size() - 1); - FSF.Trim(TMP); - FSF.Unquote(TMP); + if (!Len) + continue; - Len = lstrlen(TMP); - if (!Len) - continue; + TMP.Ptr()[Len] = 0; - if (argv) - *argv++ = args; + FSF.Trim(TMP); + FSF.Unquote(TMP); - if (args) { - lstrcpy(args, TMP); - args+= Len + 1; - } + Len = lstrlen(TMP); + if (!Len) + continue; + + if (argv) + *argv++ = args; - (*numchars)+= Len + 1; - ++*numargs; + if (args) { + lstrcpy(args, TMP); + args+= Len + 1; } - done: - munmap((LPVOID)FileData, FileSizeLow); + (*numchars)+= Len + 1; + ++*numargs; } +} + static void ReadFileList(TCHAR * filename, int *argc, TCHAR ***argv WITH_ANSI_PARAM) { *argc = 0; diff --git a/tmppanel/src/TmpPanel.hpp b/tmppanel/src/TmpPanel.hpp index 01116e978..81f6bf7e0 100644 --- a/tmppanel/src/TmpPanel.hpp +++ b/tmppanel/src/TmpPanel.hpp @@ -52,7 +52,7 @@ void FreePanelItems(PluginPanelItem *Items, DWORD Total); TCHAR *ParseParam(TCHAR *&str); void GetOptions(void); void WFD2FFD(WIN32_FIND_DATA &wfd, FAR_FIND_DATA &ffd); - +int PWZ_to_PZ(const wchar_t *src, char *dst, int lendst); #define BOM_UCS2 0xFEFF #define BOM_UCS2_BE 0xFFFE