From b3c317c914e2b999c6f6ed45c6dd7e2749743eb4 Mon Sep 17 00:00:00 2001 From: yanczi Date: Fri, 14 Nov 2025 18:00:10 +0100 Subject: [PATCH] Podmiana LZ4 na ZSTD raw block. Podmiana CompressManager na ChunkManager --- ChunkManager.cpp | 177 +++++++++++++++++++++++++ CompressingManager.h => ChunkManager.h | 25 +++- CompressingManager.cpp | 119 ----------------- CreateCargo.cpp | 128 +++--------------- CreateCargo.h | 12 +- ExtractCargo.cpp | 16 +-- ExtractCargo.h | 4 +- voidcmd.cpp | 10 +- voidcmd.vcxproj | 16 +-- voidcmd.vcxproj.filters | 4 +- 10 files changed, 244 insertions(+), 267 deletions(-) create mode 100644 ChunkManager.cpp rename CompressingManager.h => ChunkManager.h (58%) delete mode 100644 CompressingManager.cpp diff --git a/ChunkManager.cpp b/ChunkManager.cpp new file mode 100644 index 0000000..e6b28ab --- /dev/null +++ b/ChunkManager.cpp @@ -0,0 +1,177 @@ +#include "ChunkManager.h" + +ChunkManager::ChunkManager(EncryptionManager& em) + :eman(em) +{ } + +ChunkManager::~ChunkManager() +{ } + +//----------------------------------------------------------------------------- +// Kompresja blokowa +// +// Dzielenie vectora na chunki dokładnie po 128KB +// Kompresowanie chunków bez nagłówka +//----------------------------------------------------------------------------- +std::vector ChunkManager::chunked(const std::vector& raw, const bool& encrypt) +{ + //std::vector blockSizes; + + // Maksymalny rozmiar chunka + const size_t maxBlockSize = BLOCK_SIZE; + const size_t rawSize = raw.size(); + + uint16_t blockLen = 0; + uint32_t lastChunkRawSize; + std::vector compressedBlocks; + for (size_t offset = 0; offset < rawSize; offset += maxBlockSize) + { + // Rozmiar chunka + const size_t chunkSize = std::min(maxBlockSize, rawSize - offset); + + auto begin = raw.begin() + offset; + auto end = begin + chunkSize; + + // Skopiuj fragment danych do chunka + std::vector chunk(begin, end); + + std::vector outChunk = encrypt ? eman.encrypt(compress_data(chunk)) : compress_data(chunk); + + uint32_t chs = chunk.size(); + uint32_t zch = outChunk.size(); + + //addIntToVector(compressedBlocks, chs); + lastChunkRawSize = chs; + addIntToVector(compressedBlocks, zch); + compressedBlocks.insert(compressedBlocks.end(), outChunk.begin(), outChunk.end()); + + blockLen++; + } + + std::vector zip; + // Wstaw liczbę o ilości bloków do vectora; + // Przekonpwertuj usigned int32 na ciąg znkaów + // uint16_t blockLen = blockSizes .size(); + addIntToVector(zip, blockLen); + addIntToVector(zip, maxBlockSize); + addIntToVector(zip, lastChunkRawSize); + + // Dodaj skompresowane dane + zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end()); + + return zip; +} + +// Kompresja +std::vector ChunkManager::compress_data(const std::vector& input) +{ + const int level = 3; + + // Obsługa pustego chunku: zwracamy pusty wynik (0 bajtów). + if (input.empty()) return {}; + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + if (!cctx) { + std::cerr << "ZSTD_createCCtx failed\n"; + return {}; + } + + // Ustawienia „bez ramek” + size_t rc = 0; + rc |= ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless); + rc |= ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0); + rc |= ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0); + rc |= ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, 0); + rc |= ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); + + if (ZSTD_isError(rc)) { + std::cerr << "ZSTD_CCtx_setParameter error\n"; + ZSTD_freeCCtx(cctx); + return {}; + } + + const size_t srcSize = input.size(); + + // Szacowanie rozmiaru skompresowanego vectoru + const size_t maxDst = ZSTD_compressBound(srcSize); + + std::vector out(maxDst); + + // Faktyczna kompresja + size_t written = ZSTD_compress2(cctx, out.data(), maxDst, + input.data(), srcSize); + + ZSTD_freeCCtx(cctx); + + if (ZSTD_isError(written)) { + std::cerr << "ZSTD_compress2 error: " << ZSTD_getErrorName(written) << "\n"; + return {}; + } + + out.resize(written); + return out; +} + +////////////////////////////////////////////////////////////// + +//----------------------------------------------------------------------------- +// Dekompresja blokowa +//----------------------------------------------------------------------------- +std::vector ChunkManager::dechunked(const std::vector& zip, const bool& encrypt) +{ + size_t offset = 0; + const uint16_t chunkLen = getIntFromVector(zip, offset); + const uint32_t chunkBeforeSize = getIntFromVector(zip, offset); + const uint32_t chunkLastSize = getIntFromVector(zip, offset); + + std::vector chunksString; + + // Dekompresja bloków + for (size_t i = 0; i < chunkLen; ++i) + { + // Pobierz rozmiar chunków przed i po skompresowaniem + uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; + uint32_t chunkZipSize = getIntFromVector(zip, offset); + + // Pobierz blok chunka + std::vector inChunk(chunkZipSize); + std::memcpy(inChunk.data(), zip.data() + offset, chunkZipSize); + offset += chunkZipSize; + + // Jeśli flaga encrypt jest aktywna najpierw zdeszyfruj blok + std::vector zipChunk = encrypt ? eman.decrypt(inChunk) : std::move(inChunk); + + // Zdeklarój pusty chunk + std::vector chunk = decompress_data(zipChunk, chunkSize); + + // Scal chunki + chunksString.insert(chunksString.end(), chunk.begin(), chunk.end()); + } + + return chunksString; +} + +// Dekompresja +std::vector ChunkManager::decompress_data(const std::vector& input, const size_t& expected) +{ + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t r = 0; + r |= ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless); + if (ZSTD_isError(r)) + { + std::cerr << "ZSTD_DCtx_setParameter error" << std::endl; + ZSTD_freeDCtx(dctx); + } + + std::vector output(expected); + + size_t dsize = ZSTD_decompressDCtx(dctx, output.data(), expected, input.data(), input.size()); + ZSTD_freeDCtx(dctx); + + if (ZSTD_isError(dsize)) { + std::cerr << "ZSTD_decompressDCtx error: " << ZSTD_getErrorName(dsize) << "\n"; + return {}; + } + + return output; +} \ No newline at end of file diff --git a/CompressingManager.h b/ChunkManager.h similarity index 58% rename from CompressingManager.h rename to ChunkManager.h index e3fd839..03cafa5 100644 --- a/CompressingManager.h +++ b/ChunkManager.h @@ -7,6 +7,14 @@ #include #include #include +#define ZSTD_STATIC_LINKING_ONLY +#include + +#if ZSTD_VERSION_NUMBER < 10400 +#error "Wymagane zstd >= 1.4.0 dla ZSTD_c_format / ZSTD_f_zstd1_magicless" +#endif + +#include "EncryptionManager.h" #define BLOCK_SIZE 131072 // 128KB @@ -16,19 +24,20 @@ struct BlockSize uint32_t zip; }; -class CompressingManager +class ChunkManager { public: - CompressingManager(); - ~CompressingManager(); + ChunkManager(EncryptionManager& em); + ~ChunkManager(); // Kompresja danych - std::vector compress(const std::vector&); + std::vector chunked(const std::vector&, const bool&); // Dekompresja - std::vector decompress(const std::vector&); + std::vector dechunked(const std::vector&, const bool&); private: + EncryptionManager eman; std::vector blockSizes; // Przekonwertuj zmienną na ciąg na vector @@ -50,5 +59,11 @@ private: return tmp; } + + // Kompresja + std::vector compress_data(const std::vector&); + + // Dekompresja + std::vector decompress_data(const std::vector&, const size_t&); }; diff --git a/CompressingManager.cpp b/CompressingManager.cpp deleted file mode 100644 index 552a5bb..0000000 --- a/CompressingManager.cpp +++ /dev/null @@ -1,119 +0,0 @@ -#include "CompressingManager.h" - -CompressingManager::CompressingManager() -{ } - -CompressingManager::~CompressingManager() -{ } - -//----------------------------------------------------------------------------- -// Kompresja blokowa -// -// Dzielenie vectora na chunki dokładnie po 128KB -// Kompresowanie chunków bez nagłówka -//----------------------------------------------------------------------------- -std::vector CompressingManager::compress(const std::vector& raw) -{ - //std::vector blockSizes; - - // Maksymalny rozmiar chunka - const size_t maxBlockSize = BLOCK_SIZE; - const size_t rawSize = raw.size(); - - uint16_t blockLen = 0; - uint32_t lastChunkRawSize; - std::vector compressedBlocks; - for (size_t offset = 0; offset < rawSize; offset += maxBlockSize) - { - // Rozmiar chunka - const size_t chunkSize = std::min(maxBlockSize, rawSize - offset); - - auto begin = raw.begin() + offset; - auto end = begin + chunkSize; - - // Skopiuj fragment danych do chunka - std::vector chunk(begin, end); - - // Obliczanie rozmiaru skompresowanego bloku - int maxZipChunkSize = LZ4_compressBound(chunkSize); - - // Buffor wyjściowy nadpisany skompresowanymi danymi - std::vector zipChunk(maxZipChunkSize); - - // Kompresja - int zipSize = LZ4_compress_default(chunk.data(), zipChunk.data(), chunkSize, maxZipChunkSize); - - // Zmiana rozmiaru do faktycznego rozmiaru po kompresji - zipChunk.resize(zipSize); - - uint32_t chs = chunk.size(); - uint32_t zch = zipChunk.size(); - - //addIntToVector(compressedBlocks, chs); - lastChunkRawSize = chs; - addIntToVector(compressedBlocks, zch); - compressedBlocks.insert(compressedBlocks.end(), zipChunk.begin(), zipChunk.end()); - - blockLen++; - } - - std::vector zip; - // Wstaw liczbę o ilości bloków do vectora; - // Przekonpwertuj usigned int32 na ciąg znkaów - // uint16_t blockLen = blockSizes .size(); - addIntToVector(zip, blockLen); - addIntToVector(zip, maxBlockSize); - addIntToVector(zip, lastChunkRawSize); - - // Dodaj skompresowane dane - zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end()); - - return zip; -} - -////////////////////////////////////////////////////////////// - -//----------------------------------------------------------------------------- -// Dekompresja blokowa -//----------------------------------------------------------------------------- -std::vector CompressingManager::decompress(const std::vector& zip) -{ - size_t offset = 0; - const uint16_t chunkLen = getIntFromVector(zip, offset); - const uint32_t chunkBeforeSize = getIntFromVector(zip, offset); - const uint32_t chunkLastSize = getIntFromVector(zip, offset); - - std::vector chunksString; - - // Dekompresja bloków - for (size_t i = 0; i < chunkLen; ++i) - { - // Pobierz rozmiar chunków przed i po skompresowaniem - uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; - uint32_t chunkZipSize = getIntFromVector(zip, offset); - - // Pobierz blok chunka - std::vector zipChunk(chunkZipSize); - std::memcpy(zipChunk.data(), zip.data() + offset, chunkZipSize); - offset += chunkZipSize; - - // Zdeklarój pusty chunk - std::vector chunk(chunkSize); - - // Dekompresja chunka - int sizeData = LZ4_decompress_safe(zipChunk.data(), chunk.data(), static_cast(chunkZipSize), static_cast(chunkSize)); - - if (sizeData < 0) - { - throw std::runtime_error("LZ4 Decompressing Error"); - } - - // Dostosowanie rozmiaru vectora po skompresowaniu - chunk.resize(sizeData); - - // Scal chunki - chunksString.insert(chunksString.end(), chunk.begin(), chunk.end()); - } - - return chunksString; -} \ No newline at end of file diff --git a/CreateCargo.cpp b/CreateCargo.cpp index 4476699..ac99a67 100644 --- a/CreateCargo.cpp +++ b/CreateCargo.cpp @@ -110,31 +110,28 @@ bool CreateCargo::GetFileList(const std::string& path) else { std::string fileRef = RemoveStartPath(PathToUnixLike(tmpPath)); - if (CheckIgnorePath(tmpPath)) + PathConf pc; + if (methodFlags > -1) { - PathConf pc; - if (methodFlags > -1) + pc.path = PathToUnixLike(tmpPath); + pc.parameter = methodFlags; + filesPaths.push_back(pc); + } + else + { + if (!FindOnTheList(ignoreList, fileRef) || !CheckFileExtension(fileRef, ignoreList)) { - pc.path = PathToUnixLike(tmpPath); - pc.parameter = methodFlags; - filesPaths.push_back(pc); - } - else - { - if (!FindOnTheList(ignoreList, fileRef) || !CheckFileExtension(fileRef, ignoreList)) + if (FindOnTheList(zipList, fileRef) || CheckFileExtension(fileRef, zipList)) { - if (FindOnTheList(zipList, fileRef) || CheckFileExtension(fileRef, zipList)) - { - pc.parameter = FindOnTheList(encList, fileRef) || CheckFileExtension(fileRef, encList) ? 3 : 1; - } - else - { - pc.parameter = FindOnTheList(encList, fileRef) || CheckFileExtension(fileRef, encList) ? 2 : 0; - } - pc.path = PathToUnixLike(tmpPath); - std::cout << pc.path << " - " << pc.parameter << std::endl; - filesPaths.push_back(pc); + pc.parameter = FindOnTheList(encList, fileRef) || CheckFileExtension(fileRef, encList) ? 3 : 1; } + else + { + pc.parameter = FindOnTheList(encList, fileRef) || CheckFileExtension(fileRef, encList) ? 2 : 0; + } + pc.path = PathToUnixLike(tmpPath); + std::cout << pc.path << " - " << pc.parameter << std::endl; + filesPaths.push_back(pc); } } } @@ -192,12 +189,12 @@ void CreateCargo::computingBytes(const int16_t& flag, std::vector& input, { //Flaga aktywna sprawdza czy plik jest na liście. Jeśli jest to zwraca surowedane //Przeciwnie kompresuje dane - CompressingManager cm; + ChunkManager cm(crypt); switch (flag) { case 1: - output = cm.compress(input); + output = cm.chunked(input, false); break; case 2: @@ -205,7 +202,7 @@ void CreateCargo::computingBytes(const int16_t& flag, std::vector& input, break; case 3: - output = crypt.encrypt(cm.compress(input)); + output = crypt.encrypt(cm.chunked(input, false)); break; default: @@ -301,26 +298,6 @@ bool CreateCargo::FindOnTheList(const std::vector& list, const std: return it == list.end() ? false : true; } -//----------------------------------------------------------------------------- -// Rozdzielanie paternu od ścieżki -//----------------------------------------------------------------------------- -void CreateCargo::ExtPatternAndPathDetection(const std::vector& data, std::vector& pattern, std::vector& path) -{ - for (const auto& d : data) - { - if (d.front() == '*') - { - std::string tmpPattern = d; - tmpPattern.erase(tmpPattern.begin()); - pattern.push_back(UpperString(tmpPattern)); - } - else - { - path.push_back(d); - } - } -} - //----------------------------------------------------------------------------- // Sprawdzanie rozszeżeń plików //----------------------------------------------------------------------------- @@ -367,69 +344,6 @@ uint64_t CreateCargo::fnv64(const std::string& data) return hash; } -//----------------------------------------------------------------------------- -// Sprawdzanie czy plik znajduje się na liście -//----------------------------------------------------------------------------- -bool CreateCargo::FilteringData(const std::string& path) -{ - std::vector cmPatterns; - std::vector cmPaths; - - // Rozdziel ścieżki i patterny na osobne listy - ExtPatternAndPathDetection(zipList, cmPatterns, cmPaths); - - if (FindOnTheList(cmPatterns, ALL_FILE)) - { - return true; - } - - // Sprawdż czy istnieje plik o danym rozszeżeniu - if (CheckFileExtension(path, cmPatterns)) - { - return true; - } - - // Sprawdź czy instnieje dany plik w danej lokalizacji - if (FindOnTheList(cmPaths, path)) - { - return true; - } - - return false; -} - -//----------------------------------------------------------------------------- -// Kasowanie z listy plików ignorow -//----------------------------------------------------------------------------- -bool CreateCargo::CheckIgnorePath(const std::string& path) -{ - std::vector igPatterns; - std::vector igPaths; - - ExtPatternAndPathDetection(ignoreList, igPatterns, igPaths); - - // Sprawdż czy istnieje plik o danym rozszeżeniu - if (CheckFileExtension(path, igPatterns)) - { - return false; - } - - // Obrubka ścierzki - // Usuwanie katalogu root - std::string cleanPath = RemoveStartPath(path); - - // Przekształcenie ścierzki na format unixowy - std::string unixPath = PathToUnixLike(cleanPath); - - // Sprawdź czy instnieje dany plik w danej lokalizacji - if (FindOnTheList(igPaths, unixPath)) - { - return false; - } - - return true; -} - //----------------------------------------------------------------------------- // Trworzenie archiwum //----------------------------------------------------------------------------- diff --git a/CreateCargo.h b/CreateCargo.h index 12fda09..1d3c8b6 100644 --- a/CreateCargo.h +++ b/CreateCargo.h @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -34,7 +33,7 @@ #include #include "DataStruct.h" -#include "CompressingManager.h" +#include "ChunkManager.h" #include "EncryptionManager.h" @@ -107,18 +106,12 @@ private: // Przygotowanie nagłówków i plików std::vector ComputingHeadFiles(); - // Sprawdzanie czy plik znajduje się na liście - bool FilteringData(const std::string&); - // Wczytanie filtrów wyjątków void GetFilters(const std::string&); // Sprawdza czy plik znajduje się na liście void computingBytes(const int16_t&, std::vector&, std::vector&); - // Kasowanie z listy plików ignorow - bool CheckIgnorePath(const std::string&); - // Sprawdzanie rozszeżeń plików bool CheckFileExtension(const std::string&, const std::vector&); @@ -128,9 +121,6 @@ private: // Wygenerój FNV-1a HASH uint64_t fnv64(const std::string& data); - // Rozdzielanie paternu od ścieżki - void ExtPatternAndPathDetection(const std::vector&, std::vector&, std::vector&); - // Znajdź wskazany element na liście bool FindOnTheList(const std::vector&, const std::string&); }; diff --git a/ExtractCargo.cpp b/ExtractCargo.cpp index b98f174..5237e44 100644 --- a/ExtractCargo.cpp +++ b/ExtractCargo.cpp @@ -137,24 +137,24 @@ bool ExtractCargo::HashValid(const std::vector& data, const uint64_t& crc) //----------------------------------------------------------------------------- // Magiczna funkcja do dekompresji i deszyfracji danych //----------------------------------------------------------------------------- -void ExtractCargo::computingBytes(const std::vector& input, std::vector& output, const int8_t& flag) +void ExtractCargo::computingBytes(const std::vector& input, std::vector& output, const int16_t& flag) { - CompressingManager cm; - + CompressingManager cm(eman); + switch (flag) { case 1: - output = cm.decompress(input); + output = cm.decompress(input, false); break; - + case 2: output = eman.decrypt(input); break; - + case 3: - output = cm.decompress(eman.decrypt(input)); + output = cm.decompress(input, false); break; - + default: output = input; break; diff --git a/ExtractCargo.h b/ExtractCargo.h index 4c356ae..0681494 100644 --- a/ExtractCargo.h +++ b/ExtractCargo.h @@ -31,7 +31,7 @@ #include #include "DataStruct.h" -#include "CompressingManager.h" +#include "ChunkManager.h" #include "EncryptionManager.h" class ExtractCargo { @@ -76,6 +76,6 @@ private: void CreateDirections(std::filesystem::path); // Magiczna funkcja do dekompresji i deszyfracji danych - void computingBytes(const std::vector&, std::vector&, const int8_t&); + void computingBytes(const std::vector&, std::vector&, const int16_t&); }; diff --git a/voidcmd.cpp b/voidcmd.cpp index cdb7f23..c719bae 100644 --- a/voidcmd.cpp +++ b/voidcmd.cpp @@ -40,11 +40,11 @@ void RenderHelp() const std::string HelpInstruction = "pakcmd \n" " \n" - " \-c Compressing \n" - " \-r Raw files \n" - " \-e Encrypted \n" - " \-s Compressing and Encrypted \n" - " \-f Pack the files according to the guidelines given in the \.json \n" + " -c Compressing \n" + " -r Raw files \n" + " -e Encrypted \n" + " -s Compressing and Encrypted \n" + " -f Pack the files according to the guidelines given in the .json \n" " \n" "Extracting: \n" " -x Extract files from the specified container \n" diff --git a/voidcmd.vcxproj b/voidcmd.vcxproj index e5abb79..ede51e0 100644 --- a/voidcmd.vcxproj +++ b/voidcmd.vcxproj @@ -104,13 +104,13 @@ _DEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp17 - 3rd\ftxui\include;3rd\libsodium\include;3rd\json\include;3rd\lz4\include;3rd\xxhash\include + 3rd\ftxui\include;3rd\libsodium\include;3rd\json\include;3rd\zstd\include;3rd\lz4\include;3rd\xxhash\include Console true - 3rd\ftxui\Debug;3rd\lz4\lib;3rd\libsodium\lib\Debug - liblz4_static.lib;ftxui-component.lib;ftxui-dom.lib;ftxui-screen.lib;libsodium.lib + 3rd\zstd\lib\Debug;3rd\ftxui\Debug;3rd\lz4\lib;3rd\libsodium\lib\Debug + liblz4_static.lib;ftxui-component.lib;ftxui-dom.lib;ftxui-screen.lib;libsodium.lib;zstd_static.lib @@ -122,17 +122,17 @@ NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp17 - 3rd\ftxui\include;3rd\libsodium\include;3rd\json\include;3rd\lz4\include; + 3rd\ftxui\include;3rd\libsodium\include;3rd\json\include;3rd\zstd\include;3rd\lz4\include; Console true - 3rd\libsodium\lib\Release;3rd\ftxui\Release;3rd\lz4\lib - liblz4_static.lib;ftxui-component.lib;ftxui-dom.lib;ftxui-screen.lib;libsodium.lib + 3rd\zstd\lib\Release;3rd\libsodium\lib\Release;3rd\ftxui\Release;3rd\lz4\lib + liblz4_static.lib;ftxui-component.lib;ftxui-dom.lib;ftxui-screen.lib;libsodium.lib;zstd_static.lib - + @@ -141,7 +141,7 @@ - + diff --git a/voidcmd.vcxproj.filters b/voidcmd.vcxproj.filters index eb76d84..e26190f 100644 --- a/voidcmd.vcxproj.filters +++ b/voidcmd.vcxproj.filters @@ -30,7 +30,7 @@ Pliki ĹşrĂłdĹ‚owe - + Pliki ĹşrĂłdĹ‚owe @@ -53,7 +53,7 @@ Pliki nagĹ‚Ăłwkowe - + Pliki nagĹ‚Ăłwkowe