From 022bc0d06918723e5ca5f226ad77fdf538f3e0ce Mon Sep 17 00:00:00 2001 From: yanczi Date: Sun, 21 Dec 2025 09:29:07 +0100 Subject: [PATCH] =?UTF-8?q?Strumieniowe=20pakowanie=20z=20podzia=C5=82em?= =?UTF-8?q?=20na=20chunki?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ChunkManager.cpp | 4 ++ CreateCargo.cpp | 99 ++++++++++++++++++++++++++++++++++++++++++------ CreateCargo.h | 4 ++ DataStruct.h | 3 ++ ExtractCargo.cpp | 14 +++---- ExtractCargo.h | 1 - voidcmd.vcxproj | 2 +- 7 files changed, 107 insertions(+), 20 deletions(-) diff --git a/ChunkManager.cpp b/ChunkManager.cpp index cb3d664..aa7780c 100644 --- a/ChunkManager.cpp +++ b/ChunkManager.cpp @@ -105,6 +105,10 @@ std::vector ChunkManager::dechunked(const std::vector& zip, const bo const uint32_t chunkBeforeSize = getIntFromVector(zip, offset); const uint32_t chunkLastSize = getIntFromVector(zip, offset); + std::cout << "Q: " << chunkLen << std::endl; + std::cout << "C: " << chunkBeforeSize << std::endl; + std::cout << "L: " << chunkBeforeSize << std::endl; + std::vector chunksString; // Dekompresja bloków diff --git a/CreateCargo.cpp b/CreateCargo.cpp index 50dc8f9..b1b97a9 100644 --- a/CreateCargo.cpp +++ b/CreateCargo.cpp @@ -24,10 +24,12 @@ CreateCargo::CreateCargo() , extension(EXTENSION) , version(VERSION) , methodFlags(0) + , xxhState(XXH64_createState()) , offset(0) , hppKey(false) { // TODO Auto-generated constructor stub + XXH64_reset(xxhState, 0); } CreateCargo::~CreateCargo() { @@ -241,6 +243,8 @@ std::vector CreateCargo::ComputingHeadFiles() std::string path = PathToUnixLike(RemoveStartPath(file.path)); std::ifstream f(file.path, std::ios::binary | std::ios::ate); + std::cout << path << std::endl; + //Obliczanie rozmiaru pliku size_t size = f.tellg(); f.seekg(0, std::ios::beg); @@ -251,32 +255,105 @@ std::vector CreateCargo::ComputingHeadFiles() } else { + XXH64_reset(xxhState, 0); + //Wczytanie pliku do pamięci - std::vector buffer(size); - f.read(buffer.data(), size); + std::vector buffer(CHUNK_STREAM_256MB); + + uint64_t sizeFile = 0; + + const uint32_t chunkBlockSize = CHUNK_BLOCK_SIZE; + const uint32_t quantity = (size + chunkBlockSize) / chunkBlockSize; + const uint32_t lastChunkSize = size - (chunkBlockSize * (quantity - 1)); + + // Jeśli jest ustawiona flaga inna niż RAW + // Dodaj do kontenera konfigurację chunków + if (file.parameter != FILE_FLAG_RAW) + { + std::cout << "CHUNK PARAM" << std::endl; + cargo.write(reinterpret_cast(&quantity), sizeof(quantity)); + cargo.write(reinterpret_cast(&chunkBlockSize), sizeof(chunkBlockSize)); + cargo.write(reinterpret_cast(&lastChunkSize), sizeof(lastChunkSize)); + sizeFile = sizeof(quantity) + sizeof(chunkBlockSize) + sizeof(lastChunkSize); + } + + // Strumieniowanie danych + while (f.read(buffer.data(), CHUNK_STREAM_256MB) || f.gcount() > 0) + { + const int bufferSize = f.gcount(); + buffer.resize(bufferSize); + + // Aktualizacja XXH64 + XXH64_update(xxhState, buffer.data(), buffer.size()); + + if (file.parameter == FILE_FLAG_RAW) + { + // Zapisywanie strumienia do kontenera + cargo.write(reinterpret_cast(buffer.data()), buffer.size()); + sizeFile += bufferSize; + } + else + { + for (uint32_t ofs = 0; ofs < bufferSize; ofs += chunkBlockSize) + { + // Rozmiar chunka + const uint32_t chunkSize = std::min(chunkBlockSize, bufferSize - ofs); + + auto begin = buffer.begin() + ofs; + auto end = begin + chunkSize; + + // Skopiuj fragment danych do chunka + std::vector chunk(begin, end); + + std::vector outChunk; + + // Przetwórz chunki i przetwórz + if ((file.parameter & FILE_FLAG_COMPRESS) == FILE_FLAG_COMPRESS) + { + // Zaszyfruj i skompresuj lub tylko skompresuj + outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ? + eman.encrypt(cman.compress(chunk)) : cman.compress(chunk); + } + else + { + // Zaszyfruj lub skopiuj + outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ? + eman.encrypt(cman.compress(chunk)) : cman.compress(chunk); + } + + const uint32_t outSize = outChunk.size(); + + cargo.write(reinterpret_cast(&outSize), sizeof(outSize)); + sizeFile += sizeof(outSize); + + cargo.write(reinterpret_cast(outChunk.data()), outChunk.size()); + sizeFile += outSize; + } + std::cout << "SIZE: " << sizeFile << std::endl; + } + } + f.close(); //Tworzenie hashu CRC - const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0); + //const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0); //Kompresjia - std::vector pakBuffer; - computingBytes(file.parameter, buffer, pakBuffer); - - std::cout << static_cast(file.parameter) << std::endl; + //std::vector pakBuffer; + //computingBytes(file.parameter, buffer, pakBuffer); FilesTable ft; ft.nameFile = path; ft.nameLen = path.length(); ft.offset = offset; - ft.size = pakBuffer.size(); + ft.size = sizeFile; ft.flag = file.parameter; - ft.crc = crc; + ft.crc = XXH64_digest(xxhState); - cargo.write(reinterpret_cast(pakBuffer.data()), pakBuffer.size()); + //cargo.write(reinterpret_cast(pakBuffer.data()), pakBuffer.size()); filesTable.push_back(ft); - offset += pakBuffer.size(); + offset += sizeFile; } } diff --git a/CreateCargo.h b/CreateCargo.h index a870d6d..e31b3e6 100644 --- a/CreateCargo.h +++ b/CreateCargo.h @@ -35,6 +35,7 @@ #include "DataStruct.h" #include "ChunkManager.h" #include "EncryptionManager.h" +#include "CompressionManager.h" #define KEY_ZIP "compress" // Pliki do skompresowania @@ -65,6 +66,8 @@ private: uint8_t methodFlags; + XXH64_state_t* xxhState; + std::string catalogPath; std::string tempFile; @@ -73,6 +76,7 @@ private: std::vector filesList; EncryptionManager eman; + CompressionManager cman; bool hppKey; // listy wyjątków diff --git a/DataStruct.h b/DataStruct.h index c4113c6..5a17980 100644 --- a/DataStruct.h +++ b/DataStruct.h @@ -34,6 +34,9 @@ #define CHUNK_STREAM_16MB 16777216 // 16MB #define CHUNK_STREAM_256MB 268435456 // 256MB +// Rozmiar pojedynczego bloku +#define CHUNK_BLOCK_SIZE 131072 // 128KB + #define FILE_FLAG_RAW 0x00 #define FILE_FLAG_COMPRESS 0x0F #define FILE_FLAG_ENCRYPT 0xF0 diff --git a/ExtractCargo.cpp b/ExtractCargo.cpp index 6006138..c59d8d8 100644 --- a/ExtractCargo.cpp +++ b/ExtractCargo.cpp @@ -22,7 +22,6 @@ ExtractCargo::ExtractCargo() :filesLen(0) , tablePosition(0) - , filesHeadsOffset(0) , version(VERSION) , signature(SIGNATURE) { @@ -98,18 +97,15 @@ bool ExtractCargo::CheckCargoFile() } cargoFile.read(magic.data(), magic.size()); - - // Pobierz pozycję tablicy plików i jej rozmiar - cargoFile.read(reinterpret_cast(&tablePosition), sizeof(tablePosition)); - cargoFile.read(reinterpret_cast(&filesLen), sizeof(filesLen)); - if (std::string(magic.begin(), magic.end()) != signature) { std::cerr << "Error: Corrupted Cargo" << std::endl; return false; } - filesHeadsOffset = signature.length() + sizeof(cargoVer) + sizeof(filesLen); + // Pobierz pozycję tablicy plików i jej rozmiar + cargoFile.read(reinterpret_cast(&tablePosition), sizeof(tablePosition)); + cargoFile.read(reinterpret_cast(&filesLen), sizeof(filesLen)); return true; } @@ -179,6 +175,9 @@ void ExtractCargo::LoadFilesTable() cargoFile.read(reinterpret_cast(&fhTmp.crc), sizeof(fhTmp.crc)); cargoFile.read(reinterpret_cast(&fhTmp.flag), sizeof(fhTmp.flag)); + std::cout << tablePosition << std::endl; + std::cout << "Size: " << fhTmp.size << std::endl; + filesHeads.push_back(fhTmp); } } @@ -194,6 +193,7 @@ void ExtractCargo::ExtractingFilesFromCargo() CreateDirections(dir); std::ofstream file(dir, std::ios::binary); + std::cout << fh.size << std::endl; cargoFile.seekg(fh.offset); std::vector buffor(fh.size); diff --git a/ExtractCargo.h b/ExtractCargo.h index e5cc23c..9ee7e1f 100644 --- a/ExtractCargo.h +++ b/ExtractCargo.h @@ -47,7 +47,6 @@ private: uint32_t filesLen; uint64_t tablePosition; - int filesHeadsOffset; const int8_t version; const std::string signature; diff --git a/voidcmd.vcxproj b/voidcmd.vcxproj index dcc841e..49f706f 100644 --- a/voidcmd.vcxproj +++ b/voidcmd.vcxproj @@ -103,7 +103,7 @@ true _DEBUG;_CONSOLE;%(PreprocessorDefinitions);SODIUM_STATIC true - stdcpp17 + stdcpp20 3rd\libsodium\include;3rd\json\include;3rd\zstd\include;3rd\xxhash\include