diff --git a/.DS_Store b/.DS_Store new file mode 100755 index 0000000..2853aae Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 0a0dc82..cbc7a38 100755 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,7 @@ *.zip .vs x64 -FelicityEgtb.xcodeproj/project.xcworkspace/xcshareddata -FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/ \ No newline at end of file +Debug +FelicityEgtb.xcodeproj/project.xcworkspace/xcshareddata/* +FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/* +FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata diff --git a/FelicityEgtb.xcodeproj/project.pbxproj b/FelicityEgtb.xcodeproj/project.pbxproj old mode 100644 new mode 100755 diff --git a/FelicityEgtb.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/FelicityEgtb.xcodeproj/project.xcworkspace/contents.xcworkspacedata old mode 100644 new mode 100755 diff --git a/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/IDEFindNavigatorScopes.plist b/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/IDEFindNavigatorScopes.plist old mode 100644 new mode 100755 diff --git a/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/UserInterfaceState.xcuserstate b/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/UserInterfaceState.xcuserstate old mode 100644 new mode 100755 index 5bb2a48..43d1431 Binary files a/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/UserInterfaceState.xcuserstate and b/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/xcdebugger/Expressions.xcexplist b/FelicityEgtb.xcodeproj/project.xcworkspace/xcuserdata/TonyPham.xcuserdatad/xcdebugger/Expressions.xcexplist old mode 100644 new mode 100755 diff --git a/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist b/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist old mode 100644 new mode 100755 index fe2b454..bedf664 --- a/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist +++ b/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist @@ -2,4 +2,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcschemes/FelicityEgtb.xcscheme b/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcschemes/FelicityEgtb.xcscheme old mode 100644 new mode 100755 index c7e0ee5..dc7422e --- a/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcschemes/FelicityEgtb.xcscheme +++ b/FelicityEgtb.xcodeproj/xcuserdata/TonyPham.xcuserdatad/xcschemes/FelicityEgtb.xcscheme @@ -26,7 +26,6 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - language = "" shouldUseLaunchSchemeArgsEnv = "YES"> @@ -43,10 +42,9 @@ EGTB_SMALL_COMPRESS_SIZE ? 5 : 4; for(int i = 0; i < blocknum; i++) { - int blocksz = (blocktable[i] & ~EGTB_UNCOMPRESS_BIT) - (i == 0 ? 0 : (blocktable[i - 1] & ~EGTB_UNCOMPRESS_BIT)); - int uncompressed = blocktable[i] & EGTB_UNCOMPRESS_BIT; + bool uncompressed = false; + int blocksz; + if (blockTableItemSize == 4) { + const u32* p = (u32*)blocktable; + blocksz = (p[i] & EGTB_SMALL_COMPRESS_SIZE) - (i == 0 ? 0 : (p[i - 1] & EGTB_SMALL_COMPRESS_SIZE)); + uncompressed = (p[i] & EGTB_UNCOMPRESS_BIT) != 0; + } else { + const u8* p = blocktable + i * blockTableItemSize; + i64 sz1 = *((i64*)p) & EGTB_LARGE_COMPRESS_SIZE; + i64 sz0 = i == 0 ? 0 : (*((i64*)(p - blockTableItemSize)) & EGTB_LARGE_COMPRESS_SIZE); + blocksz = (int)(sz1 - sz0); + uncompressed = (*((i64*)p) & EGTB_UNCOMPRESS_BIT_FOR_LARGE_COMPRESSTABLE) != 0; + } if (uncompressed) { + assert(i + 1 == blocknum || blocksz == blocksize); memcpy(p, s, blocksz); p += blocksz; } else { - auto left = uncompressedlen - (i64)(p - dest); + // Size of uncompressed data (dest, not src) + auto left = uncompressedlen - (i64)(p - dest); assert(left > 0); auto curBlockSize = (int)MIN(left, (i64)blocksize); auto originSz = decompress((char*)p, curBlockSize, s, blocksz); + assert(originSz == curBlockSize || (i + 1 == blocknum && originSz > 0)); p += originSz; } s += blocksz; diff --git a/source/Egtb.h b/source/Egtb.h old mode 100644 new mode 100755 index 34d37bc..12dc113 --- a/source/Egtb.h +++ b/source/Egtb.h @@ -83,6 +83,9 @@ namespace egtb { const int W = 1; const int EGTB_UNCOMPRESS_BIT = 1 << 31; + const int64_t EGTB_SMALL_COMPRESS_SIZE = (1LL << 31) - 1; + const int64_t EGTB_UNCOMPRESS_BIT_FOR_LARGE_COMPRESSTABLE = (1LL << 39); + const int64_t EGTB_LARGE_COMPRESS_SIZE = (EGTB_UNCOMPRESS_BIT_FOR_LARGE_COMPRESSTABLE - 1); // 7fffffffff enum EgtbMemMode { tiny, // load minimum data into memory @@ -121,7 +124,7 @@ namespace egtb { }; enum class EgtbType { - dtm, newdtm, lookup, none + dtm, newdtm, lookup, tmp, none }; enum EgtbLoadStatus { @@ -141,20 +144,22 @@ namespace egtb { std::vector listdir(std::string dirname); int decompress(char *dst, int uncompresslen, const char *src, int slen); - i64 decompressAllBlocks(int blocksize, int blocknum, u32* blocktable, char *dest, i64 uncompressedlen, const char *src, i64 slen); +// i64 decompressAllBlocks(int blocksize, int blocknum, u32* blocktable, char *dest, i64 uncompressedlen, const char *src, i64 slen); + i64 decompressAllBlocks(int blocksize, int blocknum, u8* blocktable, char *dest, i64 uncompressedlen, const char *src, i64 slen); extern const int egtbPieceListStartIdxByType[7]; extern const PieceType egtbPieceListIdxToType[16]; // set it to true if you want to print out more messages extern bool egtbVerbose; + extern int availableAttackerTotal; class Piece; class Move; class MoveList; class EgtbFile; class EgtbLookup; - class EgtbDb; + class _EgtbDb; class EgtbBoard; class EgtbKeyRec; class EgtbKey; diff --git a/source/EgtbBoard.cpp b/source/EgtbBoard.cpp old mode 100644 new mode 100755 diff --git a/source/EgtbBoard.h b/source/EgtbBoard.h old mode 100644 new mode 100755 index 9dc5ae7..224147f --- a/source/EgtbBoard.h +++ b/source/EgtbBoard.h @@ -168,11 +168,14 @@ namespace egtb { } return stringStream.str(); } + bool isEmpty() const { + return end == 0; + } }; class EgtbBoard { - private: + protected: Piece pieces[90]; public: @@ -231,7 +234,7 @@ namespace egtb { return pieceList_isThereAttacker((const int *)pieceList); } - private: + protected: std::string toString() const; void gen_addMove(MoveList& moveList, int from, int dest, bool capOnly) const; diff --git a/source/EgtbDb.cpp b/source/EgtbDb.cpp old mode 100644 new mode 100755 index da3f2af..b1460e4 --- a/source/EgtbDb.cpp +++ b/source/EgtbDb.cpp @@ -31,7 +31,7 @@ using namespace egtb; EgtbDb::EgtbDb() { } -EgtbDb::EgtbDb(const std::string& folder, EgtbMemMode egtbMemMode) { +EgtbDb::EgtbDb(const std::string& folder, EgtbMemMode egtbMemMode){ preload(folder, egtbMemMode); } @@ -64,6 +64,10 @@ void EgtbDb::addFolder(const std::string& folderName) { folders.push_back(folderName); } +EgtbFile* EgtbDb::createEgtbFile() const { + return new EgtbFile(); +} + EgtbFile* EgtbDb::getEgtbFile(const std::string& name) { return nameMap[name]; } @@ -82,7 +86,7 @@ void EgtbDb::preload(EgtbMemMode egtbMemMode, EgtbLoadMode loadMode) { for (auto && path : vec) { auto p = EgtbFile::getExtensionType(path); if (p.first == EgtbType::dtm || p.first == EgtbType::newdtm) { - EgtbFile *egtbFile = new EgtbFile(); + EgtbFile *egtbFile = createEgtbFile(); //new EgtbFile(); if (egtbFile->preload(path, egtbMemMode, loadMode)) { auto pos = map.find(egtbFile->materialsignWB); if (pos == map.end()) { @@ -146,12 +150,12 @@ int EgtbDb::getScore(const int* pieceList, Side side, AcceptScore accept) { return EGTB_SCORE_MISSING; } - pEgtbFile->checkToLoadHeaderAndTable(side); + pEgtbFile->checkToLoadHeaderAndTable(Side::none); // load all sides auto r = pEgtbFile->getKey(pieceList); - auto querySide = r.second ? getXSide(side) : side; + auto querySide = r.flipSide ? getXSide(side) : side; if (pEgtbFile->header->isSide(querySide)) { - auto score = pEgtbFile->getScore(r.first, querySide); + auto score = pEgtbFile->getScore(r.key, querySide); if ((score == EGTB_SCORE_WINNING && accept == AcceptScore::real) || score == EGTB_SCORE_UNKNOWN) { score = pEgtbFile->lookup((const int *)pieceList, querySide); } @@ -196,6 +200,10 @@ int EgtbDb::getScore(const EgtbBoard& board, AcceptScore accept) { return getScore((const int*)board.pieceList, board.side, accept); } +int EgtbDb::getScore(const EgtbBoard& board, Side side, AcceptScore accept) { + return getScore((const int*)board.pieceList, side, accept); +} + int EgtbDb::getScore(const char* fenString, AcceptScore accept) { EgtbBoard board; board.setFen(fenString); diff --git a/source/EgtbDb.h b/source/EgtbDb.h old mode 100644 new mode 100755 index 7255548..a237e91 --- a/source/EgtbDb.h +++ b/source/EgtbDb.h @@ -57,6 +57,7 @@ namespace egtb { // Scores int getScore(const EgtbBoard& board, AcceptScore accept = AcceptScore::real); + int getScore(const EgtbBoard& board, Side side, AcceptScore accept = AcceptScore::real); int getScore(const char* fenString, AcceptScore accept=AcceptScore::real); int getScore(const std::vector pieceVec, Side side, AcceptScore accept=AcceptScore::real); @@ -65,17 +66,27 @@ namespace egtb { int probe(const std::vector pieceVec, Side side, MoveList& moveList); int probe(const char* fenString, MoveList& moveList); - private: - int getScore(const int* pieceList, Side side, AcceptScore accept = AcceptScore::real); + std::vector& getEgtbFileVec() { + return egtbFileVec; + } + const std::vector& getEgtbFileVec() const { + return egtbFileVec; + } EgtbFile* getEgtbFile(const std::string& name); + int getScore(const int* pieceList, Side side, AcceptScore accept = AcceptScore::real); + + protected: + virtual EgtbFile* getEgtbFile(const int* pieceList) const; void closeAll(); void addEgtbFile(EgtbFile *egtbFile); - private: + virtual EgtbFile* createEgtbFile() const; + + protected: std::vector folders; std::map map; std::map nameMap; diff --git a/source/EgtbFile.cpp b/source/EgtbFile.cpp index a9f53fa..2ac9494 100644 --- a/source/EgtbFile.cpp +++ b/source/EgtbFile.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include "Egtb.h" #include "EgtbKey.h" @@ -34,6 +35,15 @@ using namespace egtb; +const char* EgtbFile::egtbFileExtensions[] = { + ".xtb", ".ntb", ".ltb", nullptr +}; + +const char* EgtbFile::egtbCompressFileExtensions[] = { + ".ztb", ".znb", ".zlt", nullptr +}; + + ////////////////////////////////////////////////////////////////////// #define EGTB_SIZE_KAAEE 1410 #define EGTB_SIZE_KAEE 810 @@ -82,14 +92,6 @@ static const int m_pieceSign[7][6] = { }; ////////////////////////////////////////////////////////////////////// -static const char* egtbFileExtensions[] = { - ".xtb", ".ntb", ".ltb", nullptr -}; - -static const char* egtbCompressFileExtensions[] = { - ".ztb", ".znb", ".zlt", nullptr -}; - std::pair EgtbFile::getExtensionType(const std::string& path) { std::pair p; p.first = EgtbType::none; p.second = false; @@ -135,6 +137,7 @@ void EgtbFile::reset() { path[0] = path[1]= ""; startpos[0] = 0; endpos[0] = 0; startpos[1] = 0; endpos[1] = 0; + subPawnRank = -1; }; void EgtbFile::removeBuffers() { @@ -241,11 +244,13 @@ bool EgtbFile::preload(const std::string& path, EgtbMemMode _memMode, EgtbLoadMo memMode = _memMode; loadMode = _loadMode; + subPawnRank = -1; loadStatus = EgtbLoadStatus::none; if (loadMode == EgtbLoadMode::onrequest) { auto theName = getFileName(path); if (theName.length() < 4) { + assert(false); return false; } toLower(theName); @@ -265,90 +270,118 @@ bool EgtbFile::preload(const std::string& path, EgtbMemMode _memMode, EgtbLoadMo return r; } -// Load all data too if requested +// Load all data if requested bool EgtbFile::loadHeaderAndTable(const std::string& path) { - + assert(path.size() > 6); std::ifstream file(path, std::ios::binary); - // if there are files for both sides, header has been created already - auto oldSide = Side::none; - if (header == nullptr) { - header = new EgtbFileHeader(); - } else { - oldSide = header->isSide(Side::black) ? Side::black : Side::white; - } - auto loadingSide = Side::none; - bool r = false; - if (file && header->readFile(file) && header->isValid()) { - loadingSide = header->isSide(Side::white) ? Side::white : Side::black; - toLower(header->name); - egtbName = header->name; - egtbType = (header->property & EGTB_PROP_NEW) ? EgtbType::newdtm : EgtbType::dtm; - - setPath(path, static_cast(loadingSide)); - header->setOnlySide(loadingSide); - r = loadingSide != Side::none; - - if (r) { - setupIdxComputing(getName(), header->order); - if (oldSide != Side::none) { - header->addSide(oldSide); + if (file.is_open()) { + + // if there are files for both sides, header has been created already + auto oldSide = Side::none; + u32 oldProperty = 0; + if (header == nullptr) { + header = new EgtbFileHeader(); + } else { + oldSide = header->isSide(Side::black) ? Side::black : Side::white; + oldProperty = header->property; + } + auto loadingSide = Side::none; + + if (readHeader(file)) { + header->property |= oldProperty; + loadingSide = path.find("w.") != std::string::npos ? Side::white : Side::black; + toLower(header->name); + egtbName = header->name; + egtbType = (header->property & EGTB_PROP_NEW) ? EgtbType::newdtm : EgtbType::dtm; + + setPath(path, static_cast(loadingSide)); + header->setOnlySide(loadingSide); + assert(loadingSide == (path.find("w.") != std::string::npos ? Side::white : Side::black)); + r = loadingSide != Side::none; + + if (r) { + setupIdxComputing(getName(), header->order); + + if (oldSide != Side::none) { + header->addSide(oldSide); + } + + auto sd = static_cast(loadingSide); + startpos[sd] = endpos[sd] = 0; } } - } - auto sd = static_cast(loadingSide); - startpos[sd] = endpos[sd] = 0; + if (r && isCompressed()) { + auto sd = static_cast(loadingSide); + // Create & read compress block table + auto blockCnt = getCompresseBlockCount(); - if (r && isCompressed()) { - // Create & read compress block table - auto blockCnt = getCompresseBlockCount(); - int blockTableSz = blockCnt * sizeof(u32); + int blockTableItemSize = (header->property & (EGTB_PROP_LARGE_COMPRESSTABLE_B << sd)) != 0 ? 5 : 4; - compressBlockTables[sd] = (u32*) malloc(blockTableSz + 64); + int blockTableSz = blockCnt * blockTableItemSize; //sizeof(u32); - if (!file.read((char *)compressBlockTables[sd], blockTableSz)) { - if (egtbVerbose) { - std::cerr << "Error: cannot read " << path << std::endl; + compressBlockTables[sd] = (u8*)malloc(blockTableSz + 64); + + if (!file.read((char *)compressBlockTables[sd], blockTableSz)) { + if (egtbVerbose) { + std::cerr << "Error: cannot read " << path << std::endl; + } + file.close(); + free(compressBlockTables[sd]); + compressBlockTables[sd] = nullptr; + return false; } - file.close(); - free(compressBlockTables[sd]); - compressBlockTables[sd] = nullptr; - return false; + } - } - if (r && memMode == EgtbMemMode::all) { - r = loadAllData(file, loadingSide); + if (r && memMode == EgtbMemMode::all) { + r = loadAllData(file, loadingSide); + + } + file.close(); } - file.close(); - if (!r && egtbVerbose) { + if (egtbVerbose && !r) { std::cerr << "Error: cannot read " << path << std::endl; } return r; } -bool EgtbFile::loadAllData(std::ifstream& file, Side side) -{ +bool EgtbFile::loadAllData(std::ifstream& file, Side side) { + + assert(file.is_open()); auto sd = static_cast(side); + startpos[sd] = endpos[sd] = 0; if (isCompressed()) { auto blockCnt = getCompresseBlockCount(); - int blockTableSz = blockCnt * sizeof(u32); + int blockTableItemSize = (header->property & (EGTB_PROP_LARGE_COMPRESSTABLE_B << sd)) != 0 ? 5 : 4; + int blockTableSz = blockCnt * blockTableItemSize; i64 seekpos = EGTB_HEADER_SIZE + blockTableSz; file.seekg(seekpos, std::ios::beg); - createBuf(getSize(), sd); + createBuf(getSize(), sd); assert(pBuf[sd]); + + i64 compDataSz; + + u8 *p = compressBlockTables[sd] + blockTableItemSize * (blockCnt - 1); + if (blockTableItemSize == 4) { + compDataSz = *(u32*)p & EGTB_SMALL_COMPRESS_SIZE; + } else { + compDataSz = *(i64*)p & EGTB_LARGE_COMPRESS_SIZE; + } + assert(compDataSz > 0 && compDataSz <= getSize()); - auto compDataSz = compressBlockTables[sd][blockCnt - 1] & ~EGTB_UNCOMPRESS_BIT; char* tempBuf = (char*) malloc(compDataSz + 64); if (file.read(tempBuf, compDataSz)) { auto originSz = decompressAllBlocks(EGTB_SIZE_COMPRESS_BLOCK, blockCnt, compressBlockTables[sd], (char*)pBuf[sd], getSize(), tempBuf, compDataSz); + assert(originSz == getSize()); + endpos[sd] = originSz; } @@ -369,27 +402,45 @@ bool EgtbFile::loadAllData(std::ifstream& file, Side side) return startpos[sd] < endpos[sd]; } + void EgtbFile::checkToLoadHeaderAndTable(Side side) { - if (loadStatus != EgtbLoadStatus::none) { + auto sd = static_cast(side); + + if (loadStatus != EgtbLoadStatus::none || (sd < 2 && path[sd].empty())) { return; } std::lock_guard thelock(mtx); - if (loadStatus != EgtbLoadStatus::none) { + + if (loadStatus != EgtbLoadStatus::none || (sd < 2 && path[sd].empty())) { return; } - bool r = false; - if (!path[0].empty() && !path[1].empty()) { - r = loadHeaderAndTable(path[0]) && loadHeaderAndTable(path[1]); + bool r = true; + if (sd < 2) { + assert(!path[sd].empty()); + r = loadHeaderAndTable(path[sd]); } else { - auto thepath = path[0].empty() ? path[1] : path[0]; - r = loadHeaderAndTable(thepath); + if (!path[0].empty()) { + r = loadHeaderAndTable(path[0]); + } + if (r && !path[1].empty()) { + r = loadHeaderAndTable(path[1]); + } } loadStatus = r ? EgtbLoadStatus::loaded : EgtbLoadStatus::error; } +bool EgtbFile::forceLoadHeaderAndTable(Side side) { + std::lock_guard thelock(mtx); + + int sd = static_cast(side); + bool r = !path[sd].empty() && loadHeaderAndTable(path[sd]); + loadStatus = r ? EgtbLoadStatus::loaded : EgtbLoadStatus::error; + return r; +} + bool EgtbFile::readBuf(i64 idx, int sd) { if (!pBuf[sd]) { @@ -480,6 +531,10 @@ bool EgtbFile::readCompressedBlock(std::ifstream& file, i64 idx, int sd, char* p #define EGTB_START_LOSING 130 int EgtbFile::cellToScore(char cell) { + return _cellToScore(cell); +} + +int EgtbFile::_cellToScore(char cell) { u8 s = (u8)cell; if (s >= TB_DRAW) { if (s == TB_DRAW) return EGTB_SCORE_DRAW; @@ -530,7 +585,7 @@ char EgtbFile::getCell(i64 idx, Side side) } char EgtbFile::getCell(const int* pieceList, Side side) { - i64 key = getKey(pieceList).first; + auto key = getKey(pieceList).key; return getCell(key, side); } @@ -545,10 +600,10 @@ int EgtbFile::getScoreNoLock(const EgtbBoard& board, Side side) { int EgtbFile::getScoreNoLock(const int* pieceList, Side side) { auto r = getKey(pieceList); - if (r.second) { + if (r.flipSide) { side = getXSide(side); } - return getScoreNoLock(r.first, side); + return getScoreNoLock(r.key, side); } int EgtbFile::getScoreNoLock(i64 idx, Side side) @@ -559,7 +614,10 @@ int EgtbFile::getScoreNoLock(i64 idx, Side side) int EgtbFile::getScore(i64 idx, Side side, bool useLock) { - checkToLoadHeaderAndTable(side); + checkToLoadHeaderAndTable(Side::none); // load all sides + if (loadStatus == EgtbLoadStatus::error) { + return EGTB_SCORE_MISSING; + } if (useLock && (memMode != EgtbMemMode::all || !isDataReady(idx, static_cast(side)))) { std::lock_guard thelock(sdmtx[static_cast(side)]); return getScoreNoLock(idx, side); @@ -569,10 +627,10 @@ int EgtbFile::getScore(i64 idx, Side side, bool useLock) int EgtbFile::getScore(const int* pieceList, Side side, bool useLock) { auto r = getKey(pieceList); - if (r.second) { + if (r.flipSide) { side = getXSide(side); } - return getScore(r.first, side, useLock); + return getScore(r.key, side, useLock); } int EgtbFile::lookup(const int *pieceList, Side side) { @@ -1045,16 +1103,206 @@ u64 EgtbFile::pieceListToMaterialSign(const int* pieceList) } ///////////////////////////////////////////////////////////////////////// -std::pair EgtbFile::getKey(const EgtbBoard& board) const { +EgtbKeyRec EgtbFile::getKey(const EgtbBoard& board) const { return getKey((const int*)board.pieceList); } -std::pair EgtbFile::getKey(const int* pieceList) const { +EgtbKeyRec EgtbFile::getKey(const int* pieceList) const { EgtbKeyRec rec; EgtbKey::getKey(rec, pieceList, getEgtbType(), false, idxArr, idxMult, header->order); + return rec; +} + + +//std::pair EgtbFile::getKey(const EgtbBoard& board) const { +// return getKey((const int*)board.pieceList); +//} +// +//std::pair EgtbFile::getKey(const int* pieceList) const { +// EgtbKeyRec rec; +// EgtbKey::getKey(rec, pieceList, getEgtbType(), false, idxArr, idxMult, header->order); +// +// std::pair r; +// r.first = rec.key; +// r.second = rec.flipSide; +// return r; +//} + +bool EgtbFile::merge(EgtbFile* otherEgtbFile) +{ + for(int sd = 0; sd < 2; sd++) { + Side side = static_cast(sd); + if (header == nullptr) { + auto path = otherEgtbFile->getPath(sd); + if (!path.empty()) { + setPath(path, sd); + } + continue; + } + + if (otherEgtbFile->header->isSide(side)) { - std::pair r; - r.first = rec.key; - r.second = rec.flipSide; + assert(getName() == otherEgtbFile->getName()); + assert(!header->isSide(side)); + header->addSide(side); + setPath(otherEgtbFile->getPath(sd), sd); + + if (compressBlockTables[sd]) { + free(compressBlockTables[sd]); + } + compressBlockTables[sd] = otherEgtbFile->compressBlockTables[sd]; + + if (pBuf[sd] == nullptr && otherEgtbFile->pBuf[sd] != nullptr) { + pBuf[sd] = otherEgtbFile->pBuf[sd]; + startpos[sd] = otherEgtbFile->startpos[sd]; + endpos[sd] = otherEgtbFile->endpos[sd]; + + otherEgtbFile->pBuf[sd] = nullptr; + otherEgtbFile->startpos[sd] = 0; + otherEgtbFile->endpos[sd] = 0; + otherEgtbFile->compressBlockTables[sd] = nullptr; + } + } + } + + assert(header == nullptr || (header->isSide(Side::white) && header->isSide(Side::black))); + + return true; +} + + +///////////////////////////////////////////////////////////////////////// +EgtbPawnFiles::EgtbPawnFiles(EgtbFile *egtbFile) +: EgtbFile() +{ + memset(subFiles, 0, sizeof(subFiles)); + + auto s1 = egtbFile->getName(); + + char s2[256]; + std::copy_if(s1.begin(), s1.end() + 1, s2, [](char c){ return c < '0' || c > '9'; }); + + egtbName = s2; + + header = new EgtbFileHeader(); + header->reset(); + + egtbType = egtbFile->getEgtbType(); + // if (egtbType == EgtbType::newdtm) { + // addProperty(EGTB_PROP_NEW); + // } + + size = setupIdxComputing(egtbName, 0); + merge(egtbFile); + + loadStatus = EgtbLoadStatus::loaded; + subPawnRank = egtbFile->subPawnRank; +} + +bool EgtbPawnFiles::merge(EgtbFile* otherEgtbFile) { + assert(otherEgtbFile->subPawnRank >= 0 && otherEgtbFile->subPawnRank < 7); + + std::cout << "EgtbPawnFiles merge, name: " << getName() << ", other " << otherEgtbFile->getName() << ", subPawnRank " << otherEgtbFile->subPawnRank << std::endl; + + bool r = true; + if (subFiles[otherEgtbFile->subPawnRank]) { + std::cout << " merge with existed: " << subFiles[otherEgtbFile->subPawnRank]->getName() << std::endl; + subFiles[otherEgtbFile->subPawnRank]->merge(otherEgtbFile); + } else { + subFiles[otherEgtbFile->subPawnRank] = (EgtbFile *)otherEgtbFile; + r = false; // keep but don't delete otherEgtbFile + } + + assert(!subFiles[otherEgtbFile->subPawnRank]->getName().empty()); return r; } + +void EgtbPawnFiles::checkToLoadHeaderAndTable(Side side) +{ + for(int i = 0; i < 7; i++) { + if (subFiles[i]) { + subFiles[i]->checkToLoadHeaderAndTable(side); + } + } +} + +std::pair EgtbPawnFiles::getSubPawnKey(i64 idx) const +{ + i64 subIdx = 1; + int rank = -1; + + i64 rest = idx; + + for(int i = 0; ; i++) { + auto arr = idxArr[i]; + if (arr == EGTB_IDX_NONE) { + break; + } + auto key = (int)(rest / idxMult[i]); + rest = rest % idxMult[i]; + + if (arr == EGTB_IDX_PPP) { + auto sub = egtbKey.pppKeyToSubKey[key]; + subIdx *= sub.subIdx; + rank = sub.rank; + } else { + subIdx *= key; + } + } + + assert(subIdx >= 0 && rank >= 0 && rank < 7); + std::pair p; + p.first = subIdx; + p.second = rank; + return p; +} + +void EgtbPawnFiles::removeBuffers() +{ + for(int i = 0; i < 7; i++) { + if (subFiles[i]) { + subFiles[i]->removeBuffers(); + } + } +} + + +int EgtbPawnFiles::getSubRank(const int* pieceList) +{ + int pawnPos[10], x = 0; + auto t = egtbPieceListStartIdxByType[static_cast(PieceType::pawn)]; + for (int i = 0; i < 5; i++) { + int k = t + i; + if (pieceList[k] >= 0) { + pawnPos[x++] = EgtbBoard::flip(pieceList[k], FlipMode::vertical); + } + if (pieceList[k + 16] >= 0) { + pawnPos[x++] = pieceList[k + 16]; + } + } + + assert(x == 3); + + int p0 = MIN(MIN(pawnPos[0], pawnPos[1]), pawnPos[2]); + int p1 = pawnPos[0] == p0 ? MIN(pawnPos[1], pawnPos[2]) : pawnPos[1] == p0 ? MIN(pawnPos[0], pawnPos[2]) : MIN(pawnPos[0], pawnPos[1]); + + auto rr = getRow(p0) << 8 | getRow(p1); + auto subRank = egtbKey.subpppRankToSubIdx[rr]; + assert(subRank >= 0 && subRank < 7); + + return subRank; +} + +int EgtbPawnFiles::getScore(const int* pieceList, Side side, bool useLock) +{ + auto subRank = getSubRank(pieceList); + return subFiles[subRank]->getScore(pieceList, side, useLock); +} + +EgtbKeyRec EgtbPawnFiles::getKey(const int* pieceList) const +{ + auto subRank = getSubRank(pieceList); + auto rec = subFiles[subRank]->getKey(pieceList); + rec.subpawnFileIdx = subRank; + return rec; +} diff --git a/source/EgtbFile.h b/source/EgtbFile.h old mode 100644 new mode 100755 index e255e7d..972e797 --- a/source/EgtbFile.h +++ b/source/EgtbFile.h @@ -39,6 +39,9 @@ namespace egtb { const int EGTB_HEADER_SIZE = 128; const int EGTB_ID_MAIN = 556682; const int EGTB_PROP_COMPRESSED = (1 << 2); + const int EGTB_PROP_LARGE_COMPRESSTABLE_B = (1 << 4); + const int EGTB_PROP_LARGE_COMPRESSTABLE_W = (1 << 5); + const int EGTB_PROP_NEW = (1 << 9); const int EGTB_SIZE_COMPRESS_BLOCK = 4 * 1024; @@ -82,6 +85,16 @@ namespace egtb { EGTB_IDX_HPP, EGTB_IDX_PPP, + EGTB_IDX_PPP0, + EGTB_IDX_PPP1, + EGTB_IDX_PPP2, + EGTB_IDX_PPP3, + EGTB_IDX_PPP4, + EGTB_IDX_PPP5, + EGTB_IDX_PPP6, + + EGTB_IDX_LAST = EGTB_IDX_PPP6, + EGTB_IDX_NONE = 254 }; @@ -105,13 +118,13 @@ namespace egtb { memset(this, 0, EGTB_HEADER_SIZE); }; - bool isValid() const { - return signature == EGTB_ID_MAIN; - } - - bool readFile(std::ifstream& file) { - return file.read((char*)&signature, EGTB_HEADER_SIZE) && isValid(); - } +// bool isValid() const { +// return signature == EGTB_ID_MAIN; +// } +// +// bool readFile(std::ifstream& file) { +// return file.read((char*)&signature, EGTB_HEADER_SIZE) ? true : false; +// } bool isSide(Side side) const { return property & (1 << static_cast(side)); @@ -142,8 +155,9 @@ namespace egtb { u32 materialsignWB, materialsignBW; static u64 pieceListToMaterialSign(const int* pieceList); EgtbLoadStatus loadStatus; + int subPawnRank; - private: + protected: i64 startpos[2], endpos[2]; std::mutex mtx, sdmtx[2]; @@ -154,7 +168,7 @@ namespace egtb { static std::pair getExtensionType(const std::string& path); void removeBuffers(); - + i64 getSize() const { return size; } int getCompresseBlockCount() const { @@ -177,7 +191,12 @@ namespace egtb { EgtbType getEgtbType() const { return egtbType; } - void checkToLoadHeaderAndTable(Side side); + virtual void checkToLoadHeaderAndTable(Side side); + virtual bool forceLoadHeaderAndTable(Side side); + virtual bool merge(EgtbFile* otherEgtbFile); + + static const char* egtbFileExtensions[]; + static const char* egtbCompressFileExtensions[]; public: int getScore(i64 idx, Side side, bool useLock = true); @@ -191,14 +210,17 @@ namespace egtb { void merge(EgtbFile& otherEgtbFile); bool addLookup(EgtbLookup* lookup); - static int cellToScore(char cell); + virtual int cellToScore(char cell); + static int _cellToScore(char cell); - std::pair getKey(const int* pieceList) const; +// std::pair getKey(const int* pieceList) const; + virtual EgtbKeyRec getKey(const int* pieceList) const; + virtual EgtbKeyRec getKey(const EgtbBoard& board) const; - private: + protected: u32 pieceCount[2][7]; char* pBuf[2]; - u32* compressBlockTables[2]; + u8* compressBlockTables[2]; char* pCompressBuf; EgtbLookup *lookups[2]; std::string path[2]; @@ -209,9 +231,20 @@ namespace egtb { std::string egtbName; EgtbType egtbType; - std::pair getKey(const EgtbBoard& board) const; +// std::pair getKey(const EgtbBoard& board) const; void reset(); + virtual bool readHeader(std::ifstream& file) { + if (header && file.read((char*) & header->signature, EGTB_HEADER_SIZE)) { + return header->signature == EGTB_ID_MAIN; + } + return false; + } + +// virtual bool isValidHeader() const { +// return header && header->signature == EGTB_ID_MAIN; +// } + bool readBuf(i64 startpos, int sd); bool isDataReady(i64 pos, int sd) const { return pos >= startpos[sd] && pos < endpos[sd] && pBuf[sd]; } @@ -232,6 +265,52 @@ namespace egtb { bool readCompressedBlock(std::ifstream& file, i64 idx, int sd, char* pDest); }; + + class EgtbPawnFiles : public EgtbFile { + public: + EgtbFile* subFiles[7]; + + EgtbPawnFiles(EgtbFile *egtbFile); +// : EgtbFile(); +// { +// memset(subFiles, 0, sizeof(subFiles)); +// +// auto s1 = egtbFile->getName(); +// +// char s2[256]; +// std::copy_if(s1.begin(), s1.end() + 1, s2, [](char c){ return c < '0' || c > '9'; }); +// +// egtbName = s2; +// +// header = new EgtbFileHeader(); +// header->reset(); +// +// egtbType = egtbFile->getEgtbType(); +//// if (egtbType == EgtbType::newdtm) { +//// addProperty(EGTB_PROP_NEW); +//// } +// +// setSize(setupIdxComputing(egtbName, order, header->getVersion());); +// merge(egtbFile); +// +// loadStatus = EgtbLoadStatus::loaded; +// subPawnRank = egtbFile->subPawnRank; +// } + + bool merge(EgtbFile* otherEgtbFile); + void checkToLoadHeaderAndTable(Side side); + std::pair getSubPawnKey(i64 idx) const; + + void removeBuffers(); + + EgtbKeyRec getKey(const int* pieceList) const; + + int getScore(i64 idx, Side side, bool useLock = true); + int getScore(const int* pieceList, Side side, bool useLock = true); + + static int getSubRank(const int* pieceList); + }; + } // namespace egtb #endif /* EgtbFile_hpp */ diff --git a/source/EgtbKey.cpp b/source/EgtbKey.cpp old mode 100644 new mode 100755 diff --git a/source/EgtbKey.h b/source/EgtbKey.h old mode 100644 new mode 100755 index 7c8bd2b..51fccd4 --- a/source/EgtbKey.h +++ b/source/EgtbKey.h @@ -34,14 +34,26 @@ namespace egtb { class EgtbKeyRec { public: + EgtbKeyRec() {} + ~EgtbKeyRec() {} + i64 key; - bool flipSide; + bool flipSide; // if strong side is black, we need to flip query side - // for loop-up + // for loopup int subKey; int groupIdx; + + // for subpawn + int subpawnFileIdx; }; + class SubPawnKey { + public: + int rank, subIdx; + }; + + class EgtbKey { public: EgtbKey(); @@ -68,6 +80,9 @@ namespace egtb { std::map pppPos2KeyMap; int pppKeyToPos[EGTB_SIZE_PPP_HALF]; + public: + std::mapsubpppRankToSubIdx; + SubPawnKey pppKeyToSubKey[EGTB_SIZE_PPP_HALF + 1]; }; extern EgtbKey egtbKey; diff --git a/source/EgtbLookup.cpp b/source/EgtbLookup.cpp old mode 100644 new mode 100755 index d26482a..9b9c76e --- a/source/EgtbLookup.cpp +++ b/source/EgtbLookup.cpp @@ -30,11 +30,11 @@ using namespace egtb; -static const int luGroupSizes[2] = { 7, 22 }; /////////////////////////////////////////////////////////////////////////////// // Constructors/Destructors /////////////////////////////////////////////////////////////////////////////// +const int EgtbLookup::luGroupSizes[2] = { 7, 22 }; EgtbLookup::EgtbLookup() { reset(); @@ -267,7 +267,7 @@ int EgtbLookup::getCell(int groupIdx, int itemidx, int subIdx) { seekpos += compTableSz; if (memMode == EgtbMemMode::all) { - int compressSz0 = blockTable[blockCnt[0] - 1] & ~EGTB_UNCOMPRESS_BIT; + int compressSz0 = blockCnt[0] > 0 ? blockTable[blockCnt[0] - 1] & ~EGTB_UNCOMPRESS_BIT : 0; int compressSz = groupIdx == 0 ? compressSz0 : (blockTable[blockCnt[0] + blockCnt[1] - 1] & ~EGTB_UNCOMPRESS_BIT); pCompressBuf = (char*)malloc(compressSz + 64); @@ -279,7 +279,7 @@ int EgtbLookup::getCell(int groupIdx, int itemidx, int subIdx) { if (file.read(pCompressBuf, compressSz)) { int addBCnt = groupIdx == 0 ? 0 : blockCnt[0]; - auto originSz = decompressAllBlocks(EGTBLU_COMPRESS_BLOCK_SIZE, blockCnt[groupIdx], blockTable + addBCnt, (char*)data[groupIdx], sz[groupIdx], pCompressBuf, compressSz); + auto originSz = decompressAllBlocks(EGTBLU_COMPRESS_BLOCK_SIZE, blockCnt[groupIdx], (u8*)(blockTable + addBCnt), (char*)data[groupIdx], sz[groupIdx], pCompressBuf, compressSz); if (originSz > 0) { startpos[groupIdx] = 0; endpos[groupIdx] = itemCnt[groupIdx]; @@ -292,7 +292,7 @@ int EgtbLookup::getCell(int groupIdx, int itemidx, int subIdx) { int addBCnt = 0; if (groupIdx == 1) { addBCnt = blockCnt[0]; - int compressSz0 = blockTable[blockCnt[0] - 1] & ~EGTB_UNCOMPRESS_BIT; + int compressSz0 = blockCnt[0] > 0 ? blockTable[blockCnt[0] - 1] & ~EGTB_UNCOMPRESS_BIT : 0; seekpos += compressSz0; } @@ -340,7 +340,7 @@ int EgtbLookup::getCell(int groupIdx, int itemidx, int subIdx) { int score = (int)pScore[idx]; if (isVersion2()) { - score = EgtbFile::cellToScore(score); + score = EgtbFile::_cellToScore(score); } else if (score != 0) { score += score > 0 ? EGTB_SCORE_BASE : -EGTB_SCORE_BASE; } diff --git a/source/EgtbLookup.h b/source/EgtbLookup.h old mode 100644 new mode 100755 index 797730d..26d7918 --- a/source/EgtbLookup.h +++ b/source/EgtbLookup.h @@ -27,13 +27,13 @@ #define EgtbLookup_h #include "Egtb.h" -#include +#include namespace egtb { class EgtbLookup { - private: + protected: const int EGTBLU_HEADER_SIZE = 48; const int EGTBLU_HEADER_SIGN = 1765; const int EGTBLU_PROPERTY_COMPRESS = (1 << 4); @@ -72,7 +72,13 @@ namespace egtb { return property & EGTBLU_PROPERTY_VERSION_2; } - private: + std::string getPath() const { + return path; + } + + static const int luGroupSizes[2]; + + protected: u32* keyTable; u32* blockTable; diff --git a/source/lzma/.DS_Store b/source/lzma/.DS_Store new file mode 100755 index 0000000..8376248 Binary files /dev/null and b/source/lzma/.DS_Store differ diff --git a/source/lzma/7zTypes.h b/source/lzma/7zTypes.h old mode 100644 new mode 100755 diff --git a/source/lzma/Compiler.h b/source/lzma/Compiler.h old mode 100644 new mode 100755 diff --git a/source/lzma/LzFind.c b/source/lzma/LzFind.c old mode 100644 new mode 100755 diff --git a/source/lzma/LzFind.h b/source/lzma/LzFind.h old mode 100644 new mode 100755 diff --git a/source/lzma/LzHash.h b/source/lzma/LzHash.h old mode 100644 new mode 100755 diff --git a/source/lzma/LzmaDec.c b/source/lzma/LzmaDec.c old mode 100644 new mode 100755 diff --git a/source/lzma/LzmaDec.h b/source/lzma/LzmaDec.h old mode 100644 new mode 100755 diff --git a/source/lzma/LzmaEnc.c b/source/lzma/LzmaEnc.c new file mode 100755 index 0000000..9c45a21 --- /dev/null +++ b/source/lzma/LzmaEnc.c @@ -0,0 +1,2370 @@ +/* LzmaEnc.c -- LZMA Encoder +2017-06-22 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +/* #define SHOW_STAT */ +/* #define SHOW_STAT2 */ + +#if defined(SHOW_STAT) || defined(SHOW_STAT2) +#include +#endif + +#include "LzmaEnc.h" + +#include "LzFind.h" +#ifndef _7ZIP_ST +#include "LzFindMt.h" +#endif + +#ifdef SHOW_STAT +static unsigned g_STAT_OFFSET = 0; +#endif + +#define kLzmaMaxHistorySize ((UInt32)3 << 29) +/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ + +#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) + +#define kBlockSize (9 << 10) +#define kUnpackBlockSize (1 << 18) +#define kMatchArraySize (1 << 21) +#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) + +#define kNumMaxDirectBits (31) + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 +#define kProbInitValue (kBitModelTotal >> 1) + +#define kNumMoveReducingBits 4 +#define kNumBitPriceShiftBits 4 +#define kBitPrice (1 << kNumBitPriceShiftBits) + +void LzmaEncProps_Init(CLzmaEncProps *p) +{ + p->level = 5; + p->dictSize = p->mc = 0; + p->reduceSize = (UInt64)(Int64)-1; + p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; + p->writeEndMark = 0; +} + +void LzmaEncProps_Normalize(CLzmaEncProps *p) +{ + int level = p->level; + if (level < 0) level = 5; + p->level = level; + + if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); + if (p->dictSize > p->reduceSize) + { + unsigned i; + UInt32 reduceSize = (UInt32)p->reduceSize; + for (i = 11; i <= 30; i++) + { + if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } + if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } + } + } + + if (p->lc < 0) p->lc = 3; + if (p->lp < 0) p->lp = 0; + if (p->pb < 0) p->pb = 2; + + if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); + if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); + if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); + if (p->numHashBytes < 0) p->numHashBytes = 4; + if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); + + if (p->numThreads < 0) + p->numThreads = + #ifndef _7ZIP_ST + ((p->btMode && p->algo) ? 2 : 1); + #else + 1; + #endif +} + +UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) +{ + CLzmaEncProps props = *props2; + LzmaEncProps_Normalize(&props); + return props.dictSize; +} + +#if (_MSC_VER >= 1400) +/* BSR code is fast for some new CPUs */ +/* #define LZMA_LOG_BSR */ +#endif + +#ifdef LZMA_LOG_BSR + +#define kDicLogSizeMaxCompress 32 + +#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } + +static UInt32 GetPosSlot1(UInt32 pos) +{ + UInt32 res; + BSR2_RET(pos, res); + return res; +} +#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } + +#else + +#define kNumLogBits (9 + sizeof(size_t) / 2) +/* #define kNumLogBits (11 + sizeof(size_t) / 8 * 3) */ + +#define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) + +static void LzmaEnc_FastPosInit(Byte *g_FastPos) +{ + unsigned slot; + g_FastPos[0] = 0; + g_FastPos[1] = 1; + g_FastPos += 2; + + for (slot = 2; slot < kNumLogBits * 2; slot++) + { + size_t k = ((size_t)1 << ((slot >> 1) - 1)); + size_t j; + for (j = 0; j < k; j++) + g_FastPos[j] = (Byte)slot; + g_FastPos += k; + } +} + +/* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ +/* +#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ + (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } +*/ + +/* +#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ + (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } +*/ + +#define BSR2_RET(pos, res) { UInt32 zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } + +/* +#define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \ + p->g_FastPos[pos >> 6] + 12 : \ + p->g_FastPos[pos >> (6 + kNumLogBits - 1)] + (6 + (kNumLogBits - 1)) * 2; } +*/ + +#define GetPosSlot1(pos) p->g_FastPos[pos] +#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } + +#endif + + +#define LZMA_NUM_REPS 4 + +typedef unsigned CState; + +typedef struct +{ + UInt32 price; + + CState state; + int prev1IsChar; + int prev2; + + UInt32 posPrev2; + UInt32 backPrev2; + + UInt32 posPrev; + UInt32 backPrev; + UInt32 backs[LZMA_NUM_REPS]; +} COptimal; + +#define kNumOpts (1 << 12) + +#define kNumLenToPosStates 4 +#define kNumPosSlotBits 6 +#define kDicLogSizeMin 0 +#define kDicLogSizeMax 32 +#define kDistTableSizeMax (kDicLogSizeMax * 2) + + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) +#define kAlignMask (kAlignTableSize - 1) + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) + +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#ifdef _LZMA_PROB32 +#define CLzmaProb UInt32 +#else +#define CLzmaProb UInt16 +#endif + +#define LZMA_PB_MAX 4 +#define LZMA_LC_MAX 8 +#define LZMA_LP_MAX 4 + +#define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) + + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) + +#define LZMA_MATCH_LEN_MIN 2 +#define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) + +#define kNumStates 12 + + +typedef struct +{ + CLzmaProb choice; + CLzmaProb choice2; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; + CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; + CLzmaProb high[kLenNumHighSymbols]; +} CLenEnc; + + +typedef struct +{ + CLenEnc p; + UInt32 tableSize; + UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; + UInt32 counters[LZMA_NUM_PB_STATES_MAX]; +} CLenPriceEnc; + + +typedef struct +{ + UInt32 range; + Byte cache; + UInt64 low; + UInt64 cacheSize; + Byte *buf; + Byte *bufLim; + Byte *bufBase; + ISeqOutStream *outStream; + UInt64 processed; + SRes res; +} CRangeEnc; + + +typedef struct +{ + CLzmaProb *litProbs; + + UInt32 state; + UInt32 reps[LZMA_NUM_REPS]; + + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb isRep[kNumStates]; + CLzmaProb isRepG0[kNumStates]; + CLzmaProb isRepG1[kNumStates]; + CLzmaProb isRepG2[kNumStates]; + CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + + CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; + CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + + CLenPriceEnc lenEnc; + CLenPriceEnc repLenEnc; +} CSaveState; + + +typedef struct +{ + void *matchFinderObj; + IMatchFinder matchFinder; + + UInt32 optimumEndIndex; + UInt32 optimumCurrentIndex; + + UInt32 longestMatchLength; + UInt32 numPairs; + UInt32 numAvail; + + UInt32 numFastBytes; + UInt32 additionalOffset; + UInt32 reps[LZMA_NUM_REPS]; + UInt32 state; + + unsigned lc, lp, pb; + unsigned lpMask, pbMask; + unsigned lclp; + + CLzmaProb *litProbs; + + Bool fastMode; + Bool writeEndMark; + Bool finished; + Bool multiThread; + Bool needInit; + + UInt64 nowPos64; + + UInt32 matchPriceCount; + UInt32 alignPriceCount; + + UInt32 distTableSize; + + UInt32 dictSize; + SRes result; + + CRangeEnc rc; + + #ifndef _7ZIP_ST + Bool mtMode; + CMatchFinderMt matchFinderMt; + #endif + + CMatchFinder matchFinderBase; + + #ifndef _7ZIP_ST + Byte pad[128]; + #endif + + COptimal opt[kNumOpts]; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + + UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; + UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + + UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; + UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; + UInt32 alignPrices[kAlignTableSize]; + + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb isRep[kNumStates]; + CLzmaProb isRepG0[kNumStates]; + CLzmaProb isRepG1[kNumStates]; + CLzmaProb isRepG2[kNumStates]; + CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + + CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; + CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + + CLenPriceEnc lenEnc; + CLenPriceEnc repLenEnc; + + CSaveState saveState; + + #ifndef _7ZIP_ST + Byte pad2[128]; + #endif +} CLzmaEnc; + + +void LzmaEnc_SaveState(CLzmaEncHandle pp) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + CSaveState *dest = &p->saveState; + int i; + dest->lenEnc = p->lenEnc; + dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + for (i = 0; i < kNumStates; i++) + { + memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); + memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); + } + for (i = 0; i < kNumLenToPosStates; i++) + memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); + memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); + memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); + memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); + memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); + memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); + memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); + memcpy(dest->reps, p->reps, sizeof(p->reps)); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); +} + +void LzmaEnc_RestoreState(CLzmaEncHandle pp) +{ + CLzmaEnc *dest = (CLzmaEnc *)pp; + const CSaveState *p = &dest->saveState; + int i; + dest->lenEnc = p->lenEnc; + dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + for (i = 0; i < kNumStates; i++) + { + memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); + memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); + } + for (i = 0; i < kNumLenToPosStates; i++) + memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); + memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); + memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); + memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); + memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); + memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); + memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); + memcpy(dest->reps, p->reps, sizeof(p->reps)); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); +} + +SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + CLzmaEncProps props = *props2; + LzmaEncProps_Normalize(&props); + + if (props.lc > LZMA_LC_MAX + || props.lp > LZMA_LP_MAX + || props.pb > LZMA_PB_MAX + || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) + || props.dictSize > kLzmaMaxHistorySize) + return SZ_ERROR_PARAM; + + p->dictSize = props.dictSize; + { + unsigned fb = props.fb; + if (fb < 5) + fb = 5; + if (fb > LZMA_MATCH_LEN_MAX) + fb = LZMA_MATCH_LEN_MAX; + p->numFastBytes = fb; + } + p->lc = props.lc; + p->lp = props.lp; + p->pb = props.pb; + p->fastMode = (props.algo == 0); + p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); + { + UInt32 numHashBytes = 4; + if (props.btMode) + { + if (props.numHashBytes < 2) + numHashBytes = 2; + else if (props.numHashBytes < 4) + numHashBytes = props.numHashBytes; + } + p->matchFinderBase.numHashBytes = numHashBytes; + } + + p->matchFinderBase.cutValue = props.mc; + + p->writeEndMark = props.writeEndMark; + + #ifndef _7ZIP_ST + /* + if (newMultiThread != _multiThread) + { + ReleaseMatchFinder(); + _multiThread = newMultiThread; + } + */ + p->multiThread = (props.numThreads > 1); + #endif + + return SZ_OK; +} + + +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.expectedDataSize = expectedDataSiize; +} + + + +static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +#define IsCharState(s) ((s) < 7) + +#define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) + +#define kInfinityPrice (1 << 30) + +static void RangeEnc_Construct(CRangeEnc *p) +{ + p->outStream = NULL; + p->bufBase = NULL; +} + +#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) + +#define RC_BUF_SIZE (1 << 16) +static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) +{ + if (!p->bufBase) + { + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); + if (!p->bufBase) + return 0; + p->bufLim = p->bufBase + RC_BUF_SIZE; + } + return 1; +} + +static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->bufBase); + p->bufBase = 0; +} + +static void RangeEnc_Init(CRangeEnc *p) +{ + /* Stream.Init(); */ + p->low = 0; + p->range = 0xFFFFFFFF; + p->cacheSize = 1; + p->cache = 0; + + p->buf = p->bufBase; + + p->processed = 0; + p->res = SZ_OK; +} + +static void RangeEnc_FlushStream(CRangeEnc *p) +{ + size_t num; + if (p->res != SZ_OK) + return; + num = p->buf - p->bufBase; + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) + p->res = SZ_ERROR_WRITE; + p->processed += num; + p->buf = p->bufBase; +} + +static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +{ + if ((UInt32)p->low < (UInt32)0xFF000000 || (unsigned)(p->low >> 32) != 0) + { + Byte temp = p->cache; + do + { + Byte *buf = p->buf; + *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + temp = 0xFF; + } + while (--p->cacheSize != 0); + p->cache = (Byte)((UInt32)p->low >> 24); + } + p->cacheSize++; + p->low = (UInt32)p->low << 8; +} + +static void RangeEnc_FlushData(CRangeEnc *p) +{ + int i; + for (i = 0; i < 5; i++) + RangeEnc_ShiftLow(p); +} + +static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, unsigned numBits) +{ + do + { + p->range >>= 1; + p->low += p->range & (0 - ((value >> --numBits) & 1)); + if (p->range < kTopValue) + { + p->range <<= 8; + RangeEnc_ShiftLow(p); + } + } + while (numBits != 0); +} + +static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) +{ + UInt32 ttt = *prob; + UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; + if (symbol == 0) + { + p->range = newBound; + ttt += (kBitModelTotal - ttt) >> kNumMoveBits; + } + else + { + p->low += newBound; + p->range -= newBound; + ttt -= ttt >> kNumMoveBits; + } + *prob = (CLzmaProb)ttt; + if (p->range < kTopValue) + { + p->range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) +{ + symbol |= 0x100; + do + { + RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); + symbol <<= 1; + } + while (symbol < 0x10000); +} + +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) +{ + UInt32 offs = 0x100; + symbol |= 0x100; + do + { + matchByte <<= 1; + RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); + symbol <<= 1; + offs &= ~(matchByte ^ symbol); + } + while (symbol < 0x10000); +} + +static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) +{ + UInt32 i; + for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) + { + const int kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = i; + UInt32 bitCount = 0; + int j; + for (j = 0; j < kCyclesBits; j++) + { + w = w * w; + bitCount <<= 1; + while (w >= ((UInt32)1 << 16)) + { + w >>= 1; + bitCount++; + } + } + ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + } +} + + +#define GET_PRICE(prob, symbol) \ + p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + +#define GET_PRICEa(prob, symbol) \ + ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + +#define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] + +#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const UInt32 *ProbPrices) +{ + UInt32 price = 0; + symbol |= 0x100; + do + { + price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); + symbol <<= 1; + } + while (symbol < 0x10000); + return price; +} + +static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const UInt32 *ProbPrices) +{ + UInt32 price = 0; + UInt32 offs = 0x100; + symbol |= 0x100; + do + { + matchByte <<= 1; + price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); + symbol <<= 1; + offs &= ~(matchByte ^ symbol); + } + while (symbol < 0x10000); + return price; +} + + +static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +{ + UInt32 m = 1; + int i; + for (i = numBitLevels; i != 0;) + { + UInt32 bit; + i--; + bit = (symbol >> i) & 1; + RangeEnc_EncodeBit(rc, probs + m, bit); + m = (m << 1) | bit; + } +} + +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +{ + UInt32 m = 1; + int i; + for (i = 0; i < numBitLevels; i++) + { + UInt32 bit = symbol & 1; + RangeEnc_EncodeBit(rc, probs + m, bit); + m = (m << 1) | bit; + symbol >>= 1; + } +} + +static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) +{ + UInt32 price = 0; + symbol |= (1 << numBitLevels); + while (symbol != 1) + { + price += GET_PRICEa(probs[symbol >> 1], symbol & 1); + symbol >>= 1; + } + return price; +} + +static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) +{ + UInt32 price = 0; + UInt32 m = 1; + int i; + for (i = numBitLevels; i != 0; i--) + { + UInt32 bit = symbol & 1; + symbol >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) | bit; + } + return price; +} + + +static void LenEnc_Init(CLenEnc *p) +{ + unsigned i; + p->choice = p->choice2 = kProbInitValue; + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) + p->low[i] = kProbInitValue; + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) + p->mid[i] = kProbInitValue; + for (i = 0; i < kLenNumHighSymbols; i++) + p->high[i] = kProbInitValue; +} + +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +{ + if (symbol < kLenNumLowSymbols) + { + RangeEnc_EncodeBit(rc, &p->choice, 0); + RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); + } + else + { + RangeEnc_EncodeBit(rc, &p->choice, 1); + if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) + { + RangeEnc_EncodeBit(rc, &p->choice2, 0); + RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); + } + else + { + RangeEnc_EncodeBit(rc, &p->choice2, 1); + RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); + } + } +} + +static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, const UInt32 *ProbPrices) +{ + UInt32 a0 = GET_PRICE_0a(p->choice); + UInt32 a1 = GET_PRICE_1a(p->choice); + UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); + UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); + UInt32 i = 0; + for (i = 0; i < kLenNumLowSymbols; i++) + { + if (i >= numSymbols) + return; + prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); + } + for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) + { + if (i >= numSymbols) + return; + prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); + } + for (; i < numSymbols; i++) + prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); +} + +static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, const UInt32 *ProbPrices) +{ + LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); + p->counters[posState] = p->tableSize; +} + +static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, const UInt32 *ProbPrices) +{ + UInt32 posState; + for (posState = 0; posState < numPosStates; posState++) + LenPriceEnc_UpdateTable(p, posState, ProbPrices); +} + +static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, const UInt32 *ProbPrices) +{ + LenEnc_Encode(&p->p, rc, symbol, posState); + if (updatePrice) + if (--p->counters[posState] == 0) + LenPriceEnc_UpdateTable(p, posState, ProbPrices); +} + + + + +static void MovePos(CLzmaEnc *p, UInt32 num) +{ + #ifdef SHOW_STAT + g_STAT_OFFSET += num; + printf("\n MovePos %u", num); + #endif + + if (num != 0) + { + p->additionalOffset += num; + p->matchFinder.Skip(p->matchFinderObj, num); + } +} + +static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) +{ + UInt32 lenRes = 0, numPairs; + p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); + numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + + #ifdef SHOW_STAT + printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); + g_STAT_OFFSET++; + { + UInt32 i; + for (i = 0; i < numPairs; i += 2) + printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); + } + #endif + + if (numPairs > 0) + { + lenRes = p->matches[(size_t)numPairs - 2]; + if (lenRes == p->numFastBytes) + { + UInt32 numAvail = p->numAvail; + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + { + const Byte *pbyCur = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *pby = pbyCur + lenRes; + ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; + const Byte *pbyLim = pbyCur + numAvail; + for (; pby != pbyLim && *pby == pby[dif]; pby++); + lenRes = (UInt32)(pby - pbyCur); + } + } + } + p->additionalOffset++; + *numDistancePairsRes = numPairs; + return lenRes; +} + + +#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; +#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; +#define IsShortRep(p) ((p)->backPrev == 0) + +static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) +{ + return + GET_PRICE_0(p->isRepG0[state]) + + GET_PRICE_0(p->isRep0Long[state][posState]); +} + +static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +{ + UInt32 price; + if (repIndex == 0) + { + price = GET_PRICE_0(p->isRepG0[state]); + price += GET_PRICE_1(p->isRep0Long[state][posState]); + } + else + { + price = GET_PRICE_1(p->isRepG0[state]); + if (repIndex == 1) + price += GET_PRICE_0(p->isRepG1[state]); + else + { + price += GET_PRICE_1(p->isRepG1[state]); + price += GET_PRICE(p->isRepG2[state], repIndex - 2); + } + } + return price; +} + +static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) +{ + return p->repLenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN] + + GetPureRepPrice(p, repIndex, state, posState); +} + +static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) +{ + UInt32 posMem = p->opt[cur].posPrev; + UInt32 backMem = p->opt[cur].backPrev; + p->optimumEndIndex = cur; + do + { + if (p->opt[cur].prev1IsChar) + { + MakeAsChar(&p->opt[posMem]) + p->opt[posMem].posPrev = posMem - 1; + if (p->opt[cur].prev2) + { + p->opt[(size_t)posMem - 1].prev1IsChar = False; + p->opt[(size_t)posMem - 1].posPrev = p->opt[cur].posPrev2; + p->opt[(size_t)posMem - 1].backPrev = p->opt[cur].backPrev2; + } + } + { + UInt32 posPrev = posMem; + UInt32 backCur = backMem; + + backMem = p->opt[posPrev].backPrev; + posMem = p->opt[posPrev].posPrev; + + p->opt[posPrev].backPrev = backCur; + p->opt[posPrev].posPrev = cur; + cur = posPrev; + } + } + while (cur != 0); + *backRes = p->opt[0].backPrev; + p->optimumCurrentIndex = p->opt[0].posPrev; + return p->optimumCurrentIndex; +} + +#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * (UInt32)0x300) + +static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) +{ + UInt32 lenEnd, cur; + UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; + UInt32 *matches; + + { + + UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, len; + UInt32 matchPrice, repMatchPrice, normalMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + if (p->optimumEndIndex != p->optimumCurrentIndex) + { + const COptimal *opt = &p->opt[p->optimumCurrentIndex]; + UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; + *backRes = opt->backPrev; + p->optimumCurrentIndex = opt->posPrev; + return lenRes; + } + p->optimumCurrentIndex = p->optimumEndIndex = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else + { + mainLen = p->longestMatchLength; + numPairs = p->numPairs; + } + + numAvail = p->numAvail; + if (numAvail < 2) + { + *backRes = (UInt32)(-1); + return 1; + } + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) + { + UInt32 lenTest; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i] - 1; + if (data[0] != data2[0] || data[1] != data2[1]) + { + repLens[i] = 0; + continue; + } + for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); + repLens[i] = lenTest; + if (lenTest > repLens[repMaxIndex]) + repMaxIndex = i; + } + if (repLens[repMaxIndex] >= p->numFastBytes) + { + UInt32 lenRes; + *backRes = repMaxIndex; + lenRes = repLens[repMaxIndex]; + MovePos(p, lenRes - 1); + return lenRes; + } + + matches = p->matches; + if (mainLen >= p->numFastBytes) + { + *backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MovePos(p, mainLen - 1); + return mainLen; + } + curByte = *data; + matchByte = *(data - (reps[0] + 1)); + + if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) + { + *backRes = (UInt32)-1; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsCharState(p->state) ? + LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } + + MakeAsChar(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + if (matchByte == curByte) + { + UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) + { + p->opt[1].price = shortRepPrice; + MakeAsShortRep(&p->opt[1]); + } + } + lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); + + if (lenEnd < 2) + { + *backRes = p->opt[1].backPrev; + return 1; + } + + p->opt[1].posPrev = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) + p->opt[0].backs[i] = reps[i]; + + len = lenEnd; + do + p->opt[len--].price = kInfinityPrice; + while (len >= 2); + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + UInt32 repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); + do + { + UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][(size_t)repLen - 2]; + COptimal *opt = &p->opt[repLen]; + if (curAndLenPrice < opt->price) + { + opt->price = curAndLenPrice; + opt->posPrev = 0; + opt->backPrev = i; + opt->prev1IsChar = False; + } + } + while (--repLen >= 2); + } + + normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); + if (len <= mainLen) + { + UInt32 offs = 0; + while (len > matches[offs]) + offs += 2; + for (; ; len++) + { + COptimal *opt; + UInt32 distance = matches[(size_t)offs + 1]; + + UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; + UInt32 lenToPosState = GetLenToPosState(len); + if (distance < kNumFullDistances) + curAndLenPrice += p->distancesPrices[lenToPosState][distance]; + else + { + UInt32 slot; + GetPosSlot2(distance, slot); + curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; + } + opt = &p->opt[len]; + if (curAndLenPrice < opt->price) + { + opt->price = curAndLenPrice; + opt->posPrev = 0; + opt->backPrev = distance + LZMA_NUM_REPS; + opt->prev1IsChar = False; + } + if (len == matches[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } + } + + cur = 0; + + #ifdef SHOW_STAT2 + /* if (position >= 0) */ + { + unsigned i; + printf("\n pos = %4X", position); + for (i = cur; i <= lenEnd; i++) + printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); + } + #endif + + } + + for (;;) + { + UInt32 numAvail; + UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; + UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; + Bool nextIsChar; + Byte curByte, matchByte; + const Byte *data; + COptimal *curOpt; + COptimal *nextOpt; + + cur++; + if (cur == lenEnd) + return Backward(p, backRes, cur); + + newLen = ReadMatchDistances(p, &numPairs); + if (newLen >= p->numFastBytes) + { + p->numPairs = numPairs; + p->longestMatchLength = newLen; + return Backward(p, backRes, cur); + } + position++; + curOpt = &p->opt[cur]; + posPrev = curOpt->posPrev; + if (curOpt->prev1IsChar) + { + posPrev--; + if (curOpt->prev2) + { + state = p->opt[curOpt->posPrev2].state; + if (curOpt->backPrev2 < LZMA_NUM_REPS) + state = kRepNextStates[state]; + else + state = kMatchNextStates[state]; + } + else + state = p->opt[posPrev].state; + state = kLiteralNextStates[state]; + } + else + state = p->opt[posPrev].state; + if (posPrev == cur - 1) + { + if (IsShortRep(curOpt)) + state = kShortRepNextStates[state]; + else + state = kLiteralNextStates[state]; + } + else + { + UInt32 pos; + const COptimal *prevOpt; + if (curOpt->prev1IsChar && curOpt->prev2) + { + posPrev = curOpt->posPrev2; + pos = curOpt->backPrev2; + state = kRepNextStates[state]; + } + else + { + pos = curOpt->backPrev; + if (pos < LZMA_NUM_REPS) + state = kRepNextStates[state]; + else + state = kMatchNextStates[state]; + } + prevOpt = &p->opt[posPrev]; + if (pos < LZMA_NUM_REPS) + { + UInt32 i; + reps[0] = prevOpt->backs[pos]; + for (i = 1; i <= pos; i++) + reps[i] = prevOpt->backs[(size_t)i - 1]; + for (; i < LZMA_NUM_REPS; i++) + reps[i] = prevOpt->backs[i]; + } + else + { + UInt32 i; + reps[0] = (pos - LZMA_NUM_REPS); + for (i = 1; i < LZMA_NUM_REPS; i++) + reps[i] = prevOpt->backs[(size_t)i - 1]; + } + } + curOpt->state = (CState)state; + + curOpt->backs[0] = reps[0]; + curOpt->backs[1] = reps[1]; + curOpt->backs[2] = reps[2]; + curOpt->backs[3] = reps[3]; + + curPrice = curOpt->price; + nextIsChar = False; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + curByte = *data; + matchByte = *(data - (reps[0] + 1)); + + posState = (position & p->pbMask); + + curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + curAnd1Price += + (!IsCharState(state) ? + LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } + + nextOpt = &p->opt[(size_t)cur + 1]; + + if (curAnd1Price < nextOpt->price) + { + nextOpt->price = curAnd1Price; + nextOpt->posPrev = cur; + MakeAsChar(nextOpt); + nextIsChar = True; + } + + matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); + + if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) + { + UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); + if (shortRepPrice <= nextOpt->price) + { + nextOpt->price = shortRepPrice; + nextOpt->posPrev = cur; + MakeAsShortRep(nextOpt); + nextIsChar = True; + } + } + numAvailFull = p->numAvail; + { + UInt32 temp = kNumOpts - 1 - cur; + if (temp < numAvailFull) + numAvailFull = temp; + } + + if (numAvailFull < 2) + continue; + numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); + + if (!nextIsChar && matchByte != curByte) /* speed optimization */ + { + /* try Literal + rep0 */ + UInt32 temp; + UInt32 lenTest2; + const Byte *data2 = data - reps[0] - 1; + UInt32 limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + + for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); + lenTest2 = temp - 1; + if (lenTest2 >= 2) + { + UInt32 state2 = kLiteralNextStates[state]; + UInt32 posStateNext = (position + 1) & p->pbMask; + UInt32 nextRepMatchPrice = curAnd1Price + + GET_PRICE_1(p->isMatch[state2][posStateNext]) + + GET_PRICE_1(p->isRep[state2]); + /* for (; lenTest2 >= 2; lenTest2--) */ + { + UInt32 curAndLenPrice; + COptimal *opt; + UInt32 offset = cur + 1 + lenTest2; + while (lenEnd < offset) + p->opt[++lenEnd].price = kInfinityPrice; + curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + opt = &p->opt[offset]; + if (curAndLenPrice < opt->price) + { + opt->price = curAndLenPrice; + opt->posPrev = cur + 1; + opt->backPrev = 0; + opt->prev1IsChar = True; + opt->prev2 = False; + } + } + } + } + + startLen = 2; /* speed optimization */ + { + UInt32 repIndex; + for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) + { + UInt32 lenTest; + UInt32 lenTestTemp; + UInt32 price; + const Byte *data2 = data - reps[repIndex] - 1; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); + while (lenEnd < cur + lenTest) + p->opt[++lenEnd].price = kInfinityPrice; + lenTestTemp = lenTest; + price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); + do + { + UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][(size_t)lenTest - 2]; + COptimal *opt = &p->opt[cur + lenTest]; + if (curAndLenPrice < opt->price) + { + opt->price = curAndLenPrice; + opt->posPrev = cur; + opt->backPrev = repIndex; + opt->prev1IsChar = False; + } + } + while (--lenTest >= 2); + lenTest = lenTestTemp; + + if (repIndex == 0) + startLen = lenTest + 1; + + /* if (_maxMode) */ + { + UInt32 lenTest2 = lenTest + 1; + UInt32 limit = lenTest2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); + lenTest2 -= lenTest + 1; + if (lenTest2 >= 2) + { + UInt32 nextRepMatchPrice; + UInt32 state2 = kRepNextStates[state]; + UInt32 posStateNext = (position + lenTest) & p->pbMask; + UInt32 curAndLenCharPrice = + price + p->repLenEnc.prices[posState][(size_t)lenTest - 2] + + GET_PRICE_0(p->isMatch[state2][posStateNext]) + + LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[(size_t)lenTest - 1]), + data[lenTest], data2[lenTest], p->ProbPrices); + state2 = kLiteralNextStates[state2]; + posStateNext = (position + lenTest + 1) & p->pbMask; + nextRepMatchPrice = curAndLenCharPrice + + GET_PRICE_1(p->isMatch[state2][posStateNext]) + + GET_PRICE_1(p->isRep[state2]); + + /* for (; lenTest2 >= 2; lenTest2--) */ + { + UInt32 curAndLenPrice; + COptimal *opt; + UInt32 offset = cur + lenTest + 1 + lenTest2; + while (lenEnd < offset) + p->opt[++lenEnd].price = kInfinityPrice; + curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + opt = &p->opt[offset]; + if (curAndLenPrice < opt->price) + { + opt->price = curAndLenPrice; + opt->posPrev = cur + lenTest + 1; + opt->backPrev = 0; + opt->prev1IsChar = True; + opt->prev2 = True; + opt->posPrev2 = cur; + opt->backPrev2 = repIndex; + } + } + } + } + } + } + /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ + if (newLen > numAvail) + { + newLen = numAvail; + for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); + matches[numPairs] = newLen; + numPairs += 2; + } + if (newLen >= startLen) + { + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); + UInt32 offs, curBack, posSlot; + UInt32 lenTest; + while (lenEnd < cur + newLen) + p->opt[++lenEnd].price = kInfinityPrice; + + offs = 0; + while (startLen > matches[offs]) + offs += 2; + curBack = matches[(size_t)offs + 1]; + GetPosSlot2(curBack, posSlot); + for (lenTest = /*2*/ startLen; ; lenTest++) + { + UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][(size_t)lenTest - LZMA_MATCH_LEN_MIN]; + { + UInt32 lenToPosState = GetLenToPosState(lenTest); + COptimal *opt; + if (curBack < kNumFullDistances) + curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; + else + curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; + + opt = &p->opt[cur + lenTest]; + if (curAndLenPrice < opt->price) + { + opt->price = curAndLenPrice; + opt->posPrev = cur; + opt->backPrev = curBack + LZMA_NUM_REPS; + opt->prev1IsChar = False; + } + } + + if (/*_maxMode && */lenTest == matches[offs]) + { + /* Try Match + Literal + Rep0 */ + const Byte *data2 = data - curBack - 1; + UInt32 lenTest2 = lenTest + 1; + UInt32 limit = lenTest2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); + lenTest2 -= lenTest + 1; + if (lenTest2 >= 2) + { + UInt32 nextRepMatchPrice; + UInt32 state2 = kMatchNextStates[state]; + UInt32 posStateNext = (position + lenTest) & p->pbMask; + UInt32 curAndLenCharPrice = curAndLenPrice + + GET_PRICE_0(p->isMatch[state2][posStateNext]) + + LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[(size_t)lenTest - 1]), + data[lenTest], data2[lenTest], p->ProbPrices); + state2 = kLiteralNextStates[state2]; + posStateNext = (posStateNext + 1) & p->pbMask; + nextRepMatchPrice = curAndLenCharPrice + + GET_PRICE_1(p->isMatch[state2][posStateNext]) + + GET_PRICE_1(p->isRep[state2]); + + /* for (; lenTest2 >= 2; lenTest2--) */ + { + UInt32 offset = cur + lenTest + 1 + lenTest2; + UInt32 curAndLenPrice2; + COptimal *opt; + while (lenEnd < offset) + p->opt[++lenEnd].price = kInfinityPrice; + curAndLenPrice2 = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + opt = &p->opt[offset]; + if (curAndLenPrice2 < opt->price) + { + opt->price = curAndLenPrice2; + opt->posPrev = cur + lenTest + 1; + opt->backPrev = 0; + opt->prev1IsChar = True; + opt->prev2 = True; + opt->posPrev2 = cur; + opt->backPrev2 = curBack + LZMA_NUM_REPS; + } + } + } + offs += 2; + if (offs == numPairs) + break; + curBack = matches[(size_t)offs + 1]; + if (curBack >= kNumFullDistances) + GetPosSlot2(curBack, posSlot); + } + } + } + } +} + +#define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) + +static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) +{ + UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; + const Byte *data; + const UInt32 *matches; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else + { + mainLen = p->longestMatchLength; + numPairs = p->numPairs; + } + + numAvail = p->numAvail; + *backRes = (UInt32)-1; + if (numAvail < 2) + return 1; + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + + repLen = repIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) + { + UInt32 len; + const Byte *data2 = data - p->reps[i] - 1; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + for (len = 2; len < numAvail && data[len] == data2[len]; len++); + if (len >= p->numFastBytes) + { + *backRes = i; + MovePos(p, len - 1); + return len; + } + if (len > repLen) + { + repIndex = i; + repLen = len; + } + } + + matches = p->matches; + if (mainLen >= p->numFastBytes) + { + *backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MovePos(p, mainLen - 1); + return mainLen; + } + + mainDist = 0; /* for GCC */ + if (mainLen >= 2) + { + mainDist = matches[(size_t)numPairs - 1]; + while (numPairs > 2 && mainLen == matches[(size_t)numPairs - 4] + 1) + { + if (!ChangePair(matches[(size_t)numPairs - 3], mainDist)) + break; + numPairs -= 2; + mainLen = matches[(size_t)numPairs - 2]; + mainDist = matches[(size_t)numPairs - 1]; + } + if (mainLen == 2 && mainDist >= 0x80) + mainLen = 1; + } + + if (repLen >= 2 && ( + (repLen + 1 >= mainLen) || + (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || + (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) + { + *backRes = repIndex; + MovePos(p, repLen - 1); + return repLen; + } + + if (mainLen < 2 || numAvail <= 2) + return 1; + + p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); + if (p->longestMatchLength >= 2) + { + UInt32 newDistance = matches[(size_t)p->numPairs - 1]; + if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || + (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || + (p->longestMatchLength > mainLen + 1) || + (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) + return 1; + } + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + for (i = 0; i < LZMA_NUM_REPS; i++) + { + UInt32 len, limit; + const Byte *data2 = data - p->reps[i] - 1; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + limit = mainLen - 1; + for (len = 2; len < limit && data[len] == data2[len]; len++); + if (len >= limit) + return 1; + } + *backRes = mainDist + LZMA_NUM_REPS; + MovePos(p, mainLen - 2); + return mainLen; +} + +static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) +{ + UInt32 len; + RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); + RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + p->state = kMatchNextStates[p->state]; + len = LZMA_MATCH_LEN_MIN; + LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); + RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); + RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); +} + +static SRes CheckErrors(CLzmaEnc *p) +{ + if (p->result != SZ_OK) + return p->result; + if (p->rc.res != SZ_OK) + p->result = SZ_ERROR_WRITE; + if (p->matchFinderBase.result != SZ_OK) + p->result = SZ_ERROR_READ; + if (p->result != SZ_OK) + p->finished = True; + return p->result; +} + +static SRes Flush(CLzmaEnc *p, UInt32 nowPos) +{ + /* ReleaseMFStream(); */ + p->finished = True; + if (p->writeEndMark) + WriteEndMarker(p, nowPos & p->pbMask); + RangeEnc_FlushData(&p->rc); + RangeEnc_FlushStream(&p->rc); + return CheckErrors(p); +} + +static void FillAlignPrices(CLzmaEnc *p) +{ + UInt32 i; + for (i = 0; i < kAlignTableSize; i++) + p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + p->alignPriceCount = 0; +} + +static void FillDistancesPrices(CLzmaEnc *p) +{ + UInt32 tempPrices[kNumFullDistances]; + UInt32 i, lenToPosState; + for (i = kStartPosModelIndex; i < kNumFullDistances; i++) + { + UInt32 posSlot = GetPosSlot1(i); + UInt32 footerBits = ((posSlot >> 1) - 1); + UInt32 base = ((2 | (posSlot & 1)) << footerBits); + tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); + } + + for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) + { + UInt32 posSlot; + const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; + UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; + for (posSlot = 0; posSlot < p->distTableSize; posSlot++) + posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); + for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) + posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + + { + UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; + for (i = 0; i < kStartPosModelIndex; i++) + distancesPrices[i] = posSlotPrices[i]; + for (; i < kNumFullDistances; i++) + distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; + } + } + p->matchPriceCount = 0; +} + +void LzmaEnc_Construct(CLzmaEnc *p) +{ + RangeEnc_Construct(&p->rc); + MatchFinder_Construct(&p->matchFinderBase); + + #ifndef _7ZIP_ST + MatchFinderMt_Construct(&p->matchFinderMt); + p->matchFinderMt.MatchFinder = &p->matchFinderBase; + #endif + + { + CLzmaEncProps props; + LzmaEncProps_Init(&props); + LzmaEnc_SetProps(p, &props); + } + + #ifndef LZMA_LOG_BSR + LzmaEnc_FastPosInit(p->g_FastPos); + #endif + + LzmaEnc_InitPriceTables(p->ProbPrices); + p->litProbs = NULL; + p->saveState.litProbs = NULL; +} + +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) +{ + void *p; + p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); + if (p) + LzmaEnc_Construct((CLzmaEnc *)p); + return p; +} + +void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->litProbs); + ISzAlloc_Free(alloc, p->saveState.litProbs); + p->litProbs = NULL; + p->saveState.litProbs = NULL; +} + +void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + #ifndef _7ZIP_ST + MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); + #endif + + MatchFinder_Free(&p->matchFinderBase, allocBig); + LzmaEnc_FreeLits(p, alloc); + RangeEnc_Free(&p->rc, alloc); +} + +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); + ISzAlloc_Free(alloc, p); +} + +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) +{ + UInt32 nowPos32, startPos32; + if (p->needInit) + { + p->matchFinder.Init(p->matchFinderObj); + p->needInit = 0; + } + + if (p->finished) + return p->result; + RINOK(CheckErrors(p)); + + nowPos32 = (UInt32)p->nowPos64; + startPos32 = nowPos32; + + if (p->nowPos64 == 0) + { + UInt32 numPairs; + Byte curByte; + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) + return Flush(p, nowPos32); + ReadMatchDistances(p, &numPairs); + RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); + p->state = kLiteralNextStates[p->state]; + curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); + LitEnc_Encode(&p->rc, p->litProbs, curByte); + p->additionalOffset--; + nowPos32++; + } + + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + for (;;) + { + UInt32 pos, len, posState; + + if (p->fastMode) + len = GetOptimumFast(p, &pos); + else + len = GetOptimum(p, nowPos32, &pos); + + #ifdef SHOW_STAT2 + printf("\n pos = %4X, len = %u pos = %u", nowPos32, len, pos); + #endif + + posState = nowPos32 & p->pbMask; + if (len == 1 && pos == (UInt32)-1) + { + Byte curByte; + CLzmaProb *probs; + const Byte *data; + + RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; + curByte = *data; + probs = LIT_PROBS(nowPos32, *(data - 1)); + if (IsCharState(p->state)) + LitEnc_Encode(&p->rc, probs, curByte); + else + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); + p->state = kLiteralNextStates[p->state]; + } + else + { + RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); + if (pos < LZMA_NUM_REPS) + { + RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); + if (pos == 0) + { + RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); + RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); + } + else + { + UInt32 distance = p->reps[pos]; + RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); + if (pos == 1) + RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); + else + { + RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); + RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); + if (pos == 3) + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + } + p->reps[1] = p->reps[0]; + p->reps[0] = distance; + } + if (len == 1) + p->state = kShortRepNextStates[p->state]; + else + { + LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + p->state = kRepNextStates[p->state]; + } + } + else + { + UInt32 posSlot; + RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + p->state = kMatchNextStates[p->state]; + LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + pos -= LZMA_NUM_REPS; + GetPosSlot(pos, posSlot); + RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); + + if (posSlot >= kStartPosModelIndex) + { + UInt32 footerBits = ((posSlot >> 1) - 1); + UInt32 base = ((2 | (posSlot & 1)) << footerBits); + UInt32 posReduced = pos - base; + + if (posSlot < kEndPosModelIndex) + RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); + else + { + RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + p->alignPriceCount++; + } + } + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + p->reps[1] = p->reps[0]; + p->reps[0] = pos; + p->matchPriceCount++; + } + } + p->additionalOffset -= len; + nowPos32 += len; + if (p->additionalOffset == 0) + { + UInt32 processed; + if (!p->fastMode) + { + if (p->matchPriceCount >= (1 << 7)) + FillDistancesPrices(p); + if (p->alignPriceCount >= kAlignTableSize) + FillAlignPrices(p); + } + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) + break; + processed = nowPos32 - startPos32; + if (useLimits) + { + if (processed + kNumOpts + 300 >= maxUnpackSize || + RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) + break; + } + else if (processed >= (1 << 17)) + { + p->nowPos64 += nowPos32 - startPos32; + return CheckErrors(p); + } + } + } + p->nowPos64 += nowPos32 - startPos32; + return Flush(p, nowPos32); +} + +#define kBigHashDicLimit ((UInt32)1 << 24) + +static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + UInt32 beforeSize = kNumOpts; + if (!RangeEnc_Alloc(&p->rc, alloc)) + return SZ_ERROR_MEM; + + #ifndef _7ZIP_ST + p->mtMode = (p->multiThread && !p->fastMode && (p->matchFinderBase.btMode != 0)); + #endif + + { + unsigned lclp = p->lc + p->lp; + if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) + { + LzmaEnc_FreeLits(p, alloc); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + if (!p->litProbs || !p->saveState.litProbs) + { + LzmaEnc_FreeLits(p, alloc); + return SZ_ERROR_MEM; + } + p->lclp = lclp; + } + } + + p->matchFinderBase.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + + if (beforeSize + p->dictSize < keepWindowSize) + beforeSize = keepWindowSize - p->dictSize; + + #ifndef _7ZIP_ST + if (p->mtMode) + { + RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); + p->matchFinderObj = &p->matchFinderMt; + p->matchFinderBase.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); + MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); + } + else + #endif + { + if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) + return SZ_ERROR_MEM; + p->matchFinderObj = &p->matchFinderBase; + MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); + } + + return SZ_OK; +} + +void LzmaEnc_Init(CLzmaEnc *p) +{ + UInt32 i; + p->state = 0; + for (i = 0 ; i < LZMA_NUM_REPS; i++) + p->reps[i] = 0; + + RangeEnc_Init(&p->rc); + + + for (i = 0; i < kNumStates; i++) + { + UInt32 j; + for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) + { + p->isMatch[i][j] = kProbInitValue; + p->isRep0Long[i][j] = kProbInitValue; + } + p->isRep[i] = kProbInitValue; + p->isRepG0[i] = kProbInitValue; + p->isRepG1[i] = kProbInitValue; + p->isRepG2[i] = kProbInitValue; + } + + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + CLzmaProb *probs = p->litProbs; + for (i = 0; i < num; i++) + probs[i] = kProbInitValue; + } + + { + for (i = 0; i < kNumLenToPosStates; i++) + { + CLzmaProb *probs = p->posSlotEncoder[i]; + UInt32 j; + for (j = 0; j < (1 << kNumPosSlotBits); j++) + probs[j] = kProbInitValue; + } + } + { + for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + p->posEncoders[i] = kProbInitValue; + } + + LenEnc_Init(&p->lenEnc.p); + LenEnc_Init(&p->repLenEnc.p); + + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; + + p->optimumEndIndex = 0; + p->optimumCurrentIndex = 0; + p->additionalOffset = 0; + + p->pbMask = (1 << p->pb) - 1; + p->lpMask = (1 << p->lp) - 1; +} + +void LzmaEnc_InitPrices(CLzmaEnc *p) +{ + if (!p->fastMode) + { + FillDistancesPrices(p); + FillAlignPrices(p); + } + + p->lenEnc.tableSize = + p->repLenEnc.tableSize = + p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); +} + +static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + UInt32 i; + for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) + if (p->dictSize <= ((UInt32)1 << i)) + break; + p->distTableSize = i * 2; + + p->finished = False; + p->result = SZ_OK; + RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); + LzmaEnc_Init(p); + LzmaEnc_InitPrices(p); + p->nowPos64 = 0; + return SZ_OK; +} + +static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, + ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.stream = inStream; + p->needInit = 1; + p->rc.outStream = outStream; + return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); +} + +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, + ISeqInStream *inStream, UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.stream = inStream; + p->needInit = 1; + return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); +} + +static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) +{ + p->matchFinderBase.directInput = 1; + p->matchFinderBase.bufferBase = (Byte *)src; + p->matchFinderBase.directInputRem = srcLen; +} + +SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + LzmaEnc_SetInputBuf(p, src, srcLen); + p->needInit = 1; + + LzmaEnc_SetDataSize(pp, srcLen); + return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); +} + +void LzmaEnc_Finish(CLzmaEncHandle pp) +{ + #ifndef _7ZIP_ST + CLzmaEnc *p = (CLzmaEnc *)pp; + if (p->mtMode) + MatchFinderMt_ReleaseStream(&p->matchFinderMt); + #else + UNUSED_VAR(pp); + #endif +} + + +typedef struct +{ + ISeqOutStream vt; + Byte *data; + SizeT rem; + Bool overflow; +} CLzmaEnc_SeqOutStreamBuf; + +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) +{ + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); + if (p->rem < size) + { + size = p->rem; + p->overflow = True; + } + memcpy(p->data, data, size); + p->rem -= size; + p->data += size; + return size; +} + + +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); +} + + +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; +} + + +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, + Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + UInt64 nowPos64; + SRes res; + CLzmaEnc_SeqOutStreamBuf outStream; + + outStream.vt.Write = SeqOutStreamBuf_Write; + outStream.data = dest; + outStream.rem = *destLen; + outStream.overflow = False; + + p->writeEndMark = False; + p->finished = False; + p->result = SZ_OK; + + if (reInit) + LzmaEnc_Init(p); + LzmaEnc_InitPrices(p); + nowPos64 = p->nowPos64; + RangeEnc_Init(&p->rc); + p->rc.outStream = &outStream.vt; + + res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); + + *unpackSize = (UInt32)(p->nowPos64 - nowPos64); + *destLen -= outStream.rem; + if (outStream.overflow) + return SZ_ERROR_OUTPUT_EOF; + + return res; +} + + +static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) +{ + SRes res = SZ_OK; + + #ifndef _7ZIP_ST + Byte allocaDummy[0x300]; + allocaDummy[0] = 0; + allocaDummy[1] = allocaDummy[0]; + #endif + + for (;;) + { + res = LzmaEnc_CodeOneBlock(p, False, 0, 0); + if (res != SZ_OK || p->finished) + break; + if (progress) + { + res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); + if (res != SZ_OK) + { + res = SZ_ERROR_PROGRESS; + break; + } + } + } + + LzmaEnc_Finish(p); + + /* + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) + res = SZ_ERROR_FAIL; + } + */ + + return res; +} + + +SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, + ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); + return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); +} + + +SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + unsigned i; + UInt32 dictSize = p->dictSize; + if (*size < LZMA_PROPS_SIZE) + return SZ_ERROR_PARAM; + *size = LZMA_PROPS_SIZE; + props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); + + if (dictSize >= ((UInt32)1 << 22)) + { + UInt32 kDictMask = ((UInt32)1 << 20) - 1; + if (dictSize < (UInt32)0xFFFFFFFF - kDictMask) + dictSize = (dictSize + kDictMask) & ~kDictMask; + } + else for (i = 11; i <= 30; i++) + { + if (dictSize <= ((UInt32)2 << i)) { dictSize = (2 << i); break; } + if (dictSize <= ((UInt32)3 << i)) { dictSize = (3 << i); break; } + } + + for (i = 0; i < 4; i++) + props[1 + i] = (Byte)(dictSize >> (8 * i)); + return SZ_OK; +} + + +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +{ + return ((CLzmaEnc *)pp)->writeEndMark; +} + + +SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + SRes res; + CLzmaEnc *p = (CLzmaEnc *)pp; + + CLzmaEnc_SeqOutStreamBuf outStream; + + outStream.vt.Write = SeqOutStreamBuf_Write; + outStream.data = dest; + outStream.rem = *destLen; + outStream.overflow = False; + + p->writeEndMark = writeEndMark; + p->rc.outStream = &outStream.vt; + + res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); + + if (res == SZ_OK) + { + res = LzmaEnc_Encode2(p, progress); + if (res == SZ_OK && p->nowPos64 != srcLen) + res = SZ_ERROR_FAIL; + } + + *destLen -= outStream.rem; + if (outStream.overflow) + return SZ_ERROR_OUTPUT_EOF; + return res; +} + + +SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); + SRes res; + if (!p) + return SZ_ERROR_MEM; + + res = LzmaEnc_SetProps(p, props); + if (res == SZ_OK) + { + res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); + if (res == SZ_OK) + res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, + writeEndMark, progress, alloc, allocBig); + } + + LzmaEnc_Destroy(p, alloc, allocBig); + return res; +} diff --git a/source/lzma/LzmaEnc.h b/source/lzma/LzmaEnc.h new file mode 100755 index 0000000..c9938f0 --- /dev/null +++ b/source/lzma/LzmaEnc.h @@ -0,0 +1,76 @@ +/* LzmaEnc.h -- LZMA Encoder +2017-07-27 : Igor Pavlov : Public domain */ + +#ifndef __LZMA_ENC_H +#define __LZMA_ENC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define LZMA_PROPS_SIZE 5 + +typedef struct _CLzmaEncProps +{ + int level; /* 0 <= level <= 9 */ + UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ + int lc; /* 0 <= lc <= 8, default = 3 */ + int lp; /* 0 <= lp <= 4, default = 0 */ + int pb; /* 0 <= pb <= 4, default = 2 */ + int algo; /* 0 - fast, 1 - normal, default = 1 */ + int fb; /* 5 <= fb <= 273, default = 32 */ + int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ + int numHashBytes; /* 2, 3 or 4, default = 4 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ + int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ +} CLzmaEncProps; + +void LzmaEncProps_Init(CLzmaEncProps *p); +void LzmaEncProps_Normalize(CLzmaEncProps *p); +UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); + + +/* ---------- CLzmaEncHandle Interface ---------- */ + +/* LzmaEnc* functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzmaEncHandle; + +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + +SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); +SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + +SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + + +/* ---------- One Call Interface ---------- */ + +SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + +EXTERN_C_END + +#endif diff --git a/source/lzma/Precomp.h b/source/lzma/Precomp.h old mode 100644 new mode 100755 diff --git a/source/main.cpp b/source/main.cpp old mode 100644 new mode 100755