Merge pull request 'Strumieniowe pakowanie z podziałem na chunki' (#13) from Wywołanie_i/o into master

Reviewed-on: #13

ChunkManager.cpp

@@ -82,6 +82,10 @@ std::vector<char> ChunkManager::dechunked(const std::vector<char>& zip, const bo
 	const uint32_t chunkBeforeSize = getIntFromVector<uint32_t>(zip, offset);
 	const uint32_t chunkLastSize = getIntFromVector<uint32_t>(zip, offset);
 
+	std::cout << "Q: " << chunkLen << std::endl;
+	std::cout << "C: " << chunkBeforeSize << std::endl;
+	std::cout << "L: " << chunkBeforeSize << std::endl;
+
 	std::vector<char> chunksString;
 
 	// Dekompresja bloków

CreateCargo.cpp

@@ -24,10 +24,12 @@ CreateCargo::CreateCargo()
 	, extension(EXTENSION)
 	, version(VERSION)
 	, methodFlags(0)
+	, xxhState(XXH64_createState())
 	, offset(0)
 	, hppKey(false)
 {
 	// TODO Auto-generated constructor stub
+	XXH64_reset(xxhState, 0);
 }
 
 CreateCargo::~CreateCargo() {
@@ -241,6 +243,8 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 		std::string path = PathToUnixLike(RemoveStartPath(file.path));
 		std::ifstream f(file.path, std::ios::binary | std::ios::ate);
 
+		std::cout << path << std::endl;
+
 		//Obliczanie rozmiaru pliku
 		size_t size = f.tellg();
 		f.seekg(0, std::ios::beg);
@@ -251,32 +255,105 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 		}
 		else
 		{
+			XXH64_reset(xxhState, 0);
+
 			//Wczytanie pliku do pamiêci
-			std::vector<char> buffer(size);
+			std::vector<char> buffer(CHUNK_STREAM_256MB);
-			f.read(buffer.data(), size);
+
+			uint64_t sizeFile = 0;
+
+			const uint32_t chunkBlockSize = CHUNK_BLOCK_SIZE;
+			const uint32_t quantity = (size + chunkBlockSize) / chunkBlockSize;
+			const uint32_t lastChunkSize = size - (chunkBlockSize * (quantity - 1));
+
+			// Jeœli jest ustawiona flaga inna ni¿ RAW
+			// Dodaj do kontenera konfiguracjê chunków
+			if (file.parameter != FILE_FLAG_RAW)
+			{
+				std::cout << "CHUNK PARAM" << std::endl;
+				cargo.write(reinterpret_cast<const char*>(&quantity), sizeof(quantity));
+				cargo.write(reinterpret_cast<const char*>(&chunkBlockSize), sizeof(chunkBlockSize));
+				cargo.write(reinterpret_cast<const char*>(&lastChunkSize), sizeof(lastChunkSize));
+				sizeFile = sizeof(quantity) + sizeof(chunkBlockSize) + sizeof(lastChunkSize);
+			}
+
+			// Strumieniowanie danych
+			while (f.read(buffer.data(), CHUNK_STREAM_256MB) || f.gcount() > 0)
+			{
+				const int bufferSize = f.gcount();
+				buffer.resize(bufferSize);
+
+				// Aktualizacja XXH64
+				XXH64_update(xxhState, buffer.data(), buffer.size());
+
+				if (file.parameter == FILE_FLAG_RAW)
+				{
+					// Zapisywanie strumienia do kontenera
+					cargo.write(reinterpret_cast<const char*>(buffer.data()), buffer.size());
+					sizeFile += bufferSize;
+				}
+				else
+				{
+					for (uint32_t ofs = 0; ofs < bufferSize; ofs += chunkBlockSize)
+					{
+						// Rozmiar chunka
+						const uint32_t chunkSize = std::min(chunkBlockSize, bufferSize - ofs);
+
+						auto begin = buffer.begin() + ofs;
+						auto end = begin + chunkSize;
+
+						// Skopiuj fragment danych do chunka
+						std::vector<char> chunk(begin, end);
+
+						std::vector<char> outChunk;
+
+						// Przetwórz chunki i przetwórz
+						if ((file.parameter & FILE_FLAG_COMPRESS) == FILE_FLAG_COMPRESS)
+						{
+							// Zaszyfruj i skompresuj lub tylko skompresuj
+							outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ? 
+								eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
+						}
+						else
+						{
+							// Zaszyfruj lub skopiuj
+							outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ? 
+								eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
+						}
+
+						const uint32_t outSize = outChunk.size();
+
+						cargo.write(reinterpret_cast<const char*>(&outSize), sizeof(outSize));
+						sizeFile += sizeof(outSize);
+
+						cargo.write(reinterpret_cast<const char*>(outChunk.data()), outChunk.size());
+						sizeFile += outSize;
+					}
+					std::cout << "SIZE: " << sizeFile << std::endl;
+				}
+			}
+
 			f.close();
 
 			//Tworzenie hashu CRC
-			const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
+			//const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
 
 			//Kompresjia
-			std::vector<char> pakBuffer;
+			//std::vector<char> pakBuffer;
-			computingBytes(file.parameter, buffer, pakBuffer);
+			//computingBytes(file.parameter, buffer, pakBuffer);
-
-			std::cout << static_cast<int>(file.parameter) << std::endl;
 
 			FilesTable ft;
 			ft.nameFile = path;
 			ft.nameLen = path.length();
 			ft.offset = offset;
-			ft.size = pakBuffer.size();
+			ft.size = sizeFile;
 			ft.flag = file.parameter;
-			ft.crc = crc;
+			ft.crc = XXH64_digest(xxhState);
 
-			cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
+			//cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
 
 			filesTable.push_back(ft);
-			offset += pakBuffer.size();
+			offset += sizeFile;
 		}
 	}
 

CreateCargo.h

@@ -35,6 +35,7 @@
 #include "DataStruct.h"
 #include "ChunkManager.h"
 #include "EncryptionManager.h"
+#include "CompressionManager.h"
 
 
 #define KEY_ZIP             "compress"      // Pliki do skompresowania
@@ -65,6 +66,8 @@ private:
 
 	uint8_t methodFlags;
 
+	XXH64_state_t* xxhState;
+
 
 	std::string catalogPath;
 	std::string tempFile;
@@ -73,6 +76,7 @@ private:
 	std::vector<std::string> filesList;
 
 	EncryptionManager eman;
+	CompressionManager cman;
 	bool hppKey;
 
 	// listy wyj¹tków

DataStruct.h

@@ -34,6 +34,9 @@
 #define CHUNK_STREAM_16MB  16777216 // 16MB
 #define CHUNK_STREAM_256MB 268435456 // 256MB
 
+// Rozmiar pojedynczego bloku
+#define CHUNK_BLOCK_SIZE   131072 // 128KB
+
 #define FILE_FLAG_RAW       0x00
 #define FILE_FLAG_COMPRESS  0x0F
 #define FILE_FLAG_ENCRYPT   0xF0

ExtractCargo.cpp

@@ -22,7 +22,6 @@
 ExtractCargo::ExtractCargo()
 	:filesLen(0)
 	, tablePosition(0)
-	, filesHeadsOffset(0)
 	, version(VERSION)
 	, signature(SIGNATURE)
 {
@@ -98,18 +97,15 @@ bool ExtractCargo::CheckCargoFile()
 	}
 
 	cargoFile.read(magic.data(), magic.size());
-
-	// Pobierz pozycjê tablicy plików i jej rozmiar
-	cargoFile.read(reinterpret_cast<char*>(&tablePosition), sizeof(tablePosition));
-	cargoFile.read(reinterpret_cast<char*>(&filesLen), sizeof(filesLen));
-
 	if (std::string(magic.begin(), magic.end()) != signature)
 	{
 		std::cerr << "Error: Corrupted Cargo" << std::endl;
 		return false;
 	}
 
-	filesHeadsOffset = signature.length() + sizeof(cargoVer) + sizeof(filesLen);
+	// Pobierz pozycjê tablicy plików i jej rozmiar
+	cargoFile.read(reinterpret_cast<char*>(&tablePosition), sizeof(tablePosition));
+	cargoFile.read(reinterpret_cast<char*>(&filesLen), sizeof(filesLen));
 
 	return true;
 }
@@ -179,6 +175,9 @@ void ExtractCargo::LoadFilesTable()
 		cargoFile.read(reinterpret_cast<char*>(&fhTmp.crc), sizeof(fhTmp.crc));
 		cargoFile.read(reinterpret_cast<char*>(&fhTmp.flag), sizeof(fhTmp.flag));
 
+		std::cout << tablePosition << std::endl;
+		std::cout << "Size: " << fhTmp.size << std::endl;
+
 		filesHeads.push_back(fhTmp);
 	}
 }
@@ -194,6 +193,7 @@ void ExtractCargo::ExtractingFilesFromCargo()
 		CreateDirections(dir);
 		std::ofstream file(dir, std::ios::binary);
 
+		std::cout << fh.size << std::endl;
 		cargoFile.seekg(fh.offset);
 		std::vector<char> buffor(fh.size);
 

ExtractCargo.h

@@ -47,7 +47,6 @@ private:
 
 	uint32_t filesLen;
 	uint64_t tablePosition;
-	int filesHeadsOffset;
 
 	const int8_t version;
 	const std::string signature;

voidcmd.vcxproj

@@ -103,7 +103,7 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);SODIUM_STATIC</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpp20</LanguageStandard>
       <AdditionalIncludeDirectories>3rd\libsodium\include;3rd\json\include;3rd\zstd\include;3rd\xxhash\include</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>