Merge pull request 'streaming' (#14) from streaming into master

Reviewed-on: #14

ChunkManager.cpp

@@ -0,0 +1,122 @@
+#include "CompressingManager.h"
+
+CompressingManager::CompressingManager()
+{ }
+
+ChunkManager::~ChunkManager()
+{ }
+
+//-----------------------------------------------------------------------------
+//	Kompresja blokowa
+//	
+// Dzielenie vectora na chunki dok³adnie po 128KB
+// Kompresowanie chunków bez nag³ówka
+//-----------------------------------------------------------------------------
+std::vector<char> ChunkManager::chunked(const std::vector<char>& raw, const bool& compress, const bool& encrypt)
+{
+	//std::vector<BlockSize> blockSizes;
+
+	// Maksymalny rozmiar chunka
+	const size_t maxBlockSize = BLOCK_SIZE;
+	const size_t rawSize = raw.size();
+
+	uint32_t blockLen = 0;
+	uint32_t lastChunkRawSize;
+	std::vector<char> compressedBlocks;
+	for (size_t offset = 0; offset < rawSize; offset += maxBlockSize)
+	{
+		// Rozmiar chunka
+		const size_t chunkSize = std::min(maxBlockSize, rawSize - offset);
+
+		auto begin = raw.begin() + offset;
+		auto end = begin + chunkSize;
+
+		// Skopiuj fragment danych do chunka
+		std::vector<char> chunk(begin, end);
+
+		std::vector<char> outChunk;
+
+		// Buffor wyjœciowy nadpisany skompresowanymi danymi
+		std::vector<char> zipChunk(maxZipChunkSize);
+
+		// Kompresja
+		int zipSize = LZ4_compress_default(chunk.data(), zipChunk.data(), chunkSize, maxZipChunkSize);
+
+		// Zmiana rozmiaru do faktycznego rozmiaru po kompresji
+		zipChunk.resize(zipSize);
+
+		uint32_t chs = static_cast<uint32_t>(chunk.size());
+		uint32_t zch = static_cast<uint32_t>(outChunk.size());
+
+		//addIntToVector<uint32_t>(compressedBlocks, chs);
+		lastChunkRawSize = chs;
+		addIntToVector<uint32_t>(compressedBlocks, zch);
+		compressedBlocks.insert(compressedBlocks.end(), outChunk.begin(), outChunk.end());
+
+		blockLen++;
+	}
+
+	std::vector<char> zip;
+	// Wstaw liczbê o iloœci bloków do vectora;
+	// Przekonpwertuj usigned int32 na ci¹g znkaów
+	// uint16_t blockLen = blockSizes .size();
+	addIntToVector<uint32_t>(zip, blockLen);
+	addIntToVector<uint32_t>(zip, maxBlockSize);
+	addIntToVector<uint32_t>(zip, lastChunkRawSize);
+
+	// Dodaj skompresowane dane
+	zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end());
+
+	return zip;
+}
+
+//////////////////////////////////////////////////////////////
+
+//-----------------------------------------------------------------------------
+//	Dekompresja blokowa
+//-----------------------------------------------------------------------------
+std::vector<char> ChunkManager::dechunked(const std::vector<char>& zip, const bool& compress, const bool& encrypt)
+{
+	size_t offset = 0;
+	const uint32_t chunkLen = getIntFromVector<uint32_t>(zip, offset);
+	const uint32_t chunkBeforeSize = getIntFromVector<uint32_t>(zip, offset);
+	const uint32_t chunkLastSize = getIntFromVector<uint32_t>(zip, offset);
+
+	std::cout << "Q: " << chunkLen << std::endl;
+	std::cout << "C: " << chunkBeforeSize << std::endl;
+	std::cout << "L: " << chunkBeforeSize << std::endl;
+
+	std::vector<char> chunksString;
+
+	// Dekompresja bloków
+	for (size_t i = 0; i < chunkLen; ++i)
+	{
+		// Pobierz rozmiar chunków przed i po skompresowaniem
+		uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize;
+		uint32_t chunkZipSize = getIntFromVector<uint32_t>(zip, offset);
+
+		// Pobierz blok chunka
+		std::vector<char> inChunk(chunkZipSize);
+		std::memcpy(inChunk.data(), zip.data() + offset, chunkZipSize);
+		offset += chunkZipSize;
+
+		// Zdeklarój pusty chunk
+		std::vector<char> chunk(chunkSize);
+
+		// Dekompresja chunka
+		int sizeData = LZ4_decompress_safe(zipChunk.data(), chunk.data(), static_cast<int>(chunkZipSize), static_cast<int>(chunkSize));
+
+		if (sizeData < 0)
+		{
+			throw std::runtime_error("LZ4 Decompressing Error");
+		}
+
+		// Dostosowanie rozmiaru vectora po skompresowaniu
+		chunk.resize(sizeData);
+
+		// Scal chunki
+		chunksString.insert(chunksString.end(), chunk.begin(), chunk.end());
+	}
+
+	return chunksString;
+}

ChunkManager.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <vector>
+#include <cstdint>
+#include <cstring>
+#include <stdexcept>
+#include <algorithm>
+#include <iostream>
+
+#include "EncryptionManager.h"
+#include "CompressionManager.h"
+
+#define BLOCK_SIZE          131072 // 128KB
+
+class ChunkManager
+{
+public:
+	ChunkManager(EncryptionManager& em);
+	~ChunkManager();
+
+	// Podzia³ na chunki
+	std::vector<char> chunked(const std::vector<char>&, const bool&, const bool&);
+
+	// Zcalanie chunków
+	std::vector<char> dechunked(const std::vector<char>&, const bool&, const bool&);
+
+private:
+	EncryptionManager eman;
+	CompressionManager cman;
+
+	// Przekonwertuj zmienn¹ na ci¹g na vector
+	template <typename T>
+	void addIntToVector(std::vector<char>& vec, const T& val)
+	{
+		size_t tmpSize = vec.size();
+		vec.resize(tmpSize + sizeof(val));
+		std::memcpy(vec.data() + tmpSize, &val, sizeof(val));
+	}
+
+	// Pobierz zmienn¹ z vectora
+	template <typename T>
+	T getIntFromVector(const std::vector<char>& vec, size_t& offset)
+	{
+		T tmp{};
+		std::memcpy(&tmp, vec.data() + offset, sizeof(tmp));
+		offset += sizeof(tmp);
+	
+		return tmp;
+	}
+};
+

CreateCargo.cpp

@@ -42,7 +42,7 @@ CreateCargo::~CreateCargo() {
 //-----------------------------------------------------------------------------
 bool CreateCargo::Create(const std::string& path, const uint8_t& flag)
 {
-	cargoFile = path + "." + extension;
+	cargoFile = path + "." + signature;
 	catalogPath = path;
 	methodFlags = flag;
 
@@ -241,6 +241,7 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 			// Dodaj do kontenera konfiguracjê chunków
 			if (file.parameter != flag::raw)
 			{
+				std::cout << "CHUNK PARAM" << std::endl;
 				cargo.write(reinterpret_cast<const char*>(&quantity), sizeof(quantity));
 				cargo.write(reinterpret_cast<const char*>(&chunkBlockSize), sizeof(chunkBlockSize));
 				cargo.write(reinterpret_cast<const char*>(&lastChunkSize), sizeof(lastChunkSize));
@@ -287,7 +288,8 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 						else
 						{
 							// Zaszyfruj lub skopiuj
-							outChunk = eman.encrypt(chunk);
+							outChunk = (file.parameter & flag::enc) == flag::enc ?
+								eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
 						}
 
 						const uint32_t outSize = outChunk.size();
@@ -298,11 +300,19 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 						cargo.write(reinterpret_cast<const char*>(outChunk.data()), outChunk.size());
 						sizeFile += outSize;
 					}
+					std::cout << "SIZE: " << sizeFile << std::endl;
 				}
 			}
 
 			f.close();
 
+			//Tworzenie hashu CRC
+			//const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
+
+			//Kompresjia
+			//std::vector<char> pakBuffer;
+			//computingBytes(file.parameter, buffer, pakBuffer);
+
 			FilesTable ft;
 			ft.nameFile = path;
 			ft.nameLen = path.length();
@@ -311,6 +321,8 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 			ft.flag = file.parameter;
 			ft.crc = XXH64_digest(xxhState);
 
+			//cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
+
 			filesTable.push_back(ft);
 			offset += sizeFile;
 		}

CreateCargo.h

@@ -33,6 +33,7 @@
 #include <nlohmann/json.hpp>
 
 #include "DataStruct.h"
+#include "ChunkManager.h"
 #include "EncryptionManager.h"
 #include "CompressionManager.h"
 

DataStruct.h

@@ -64,13 +64,13 @@ namespace fl {
 namespace ds
 {
 	// Chunki streamowania
-	inline constexpr uint32_t chunk_stream = 268435456; // 256MB
+	inline constexpr int chunk_stream = 268435456; // 256MB
 
 	// Blok chunków
-	inline constexpr uint32_t block_size   = 131072;    // 128KB
+	inline constexpr int block_size   = 131072;    // 128KB
 
 	// Maksymalny rozmiar pliku do spakowania
-	inline constexpr uint64_t maxFileSize = 8589934592; // 8GB
+	inline constexpr int maxFileSize = 8589934592; // 8GB
 }
 
 // Flagi

ExtractCargo.cpp

@@ -110,6 +110,51 @@ bool ExtractCargo::CheckCargoFile()
 	return true;
 }
 
+//-----------------------------------------------------------------------------
+//   Sprawdzanie sumy kontrolnej
+//-----------------------------------------------------------------------------
+bool ExtractCargo::HashValid(const std::vector<char>& data, const uint64_t& crc)
+{
+	uint64_t actualCrc = XXH64(data.data(), data.size(), 0);
+
+	if (actualCrc != crc)
+	{
+		return false;
+	}
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+//   Magiczna funkcja do dekompresji i deszyfracji danych
+//-----------------------------------------------------------------------------
+void ExtractCargo::computingBytes(const std::vector<char>& input, std::vector<char>& output, const uint8_t& flag)
+{
+	ChunkManager cm(eman);
+
+	std::cout << static_cast<int>(flag) << std::endl;
+	
+	switch (flag)
+	{
+	case FILE_FLAG_COMPRESS:
+		output = cm.dechunked(input, true, false);
+		break;
+	
+	case FILE_FLAG_ENCRYPT:
+		output = cm.dechunked(input, false, true);
+		break;
+	
+	case FILE_FLAG_ZIPENC:
+		output = cm.dechunked(input, true, true);
+		std::cout << "DENC" << std::endl;
+		break;
+	
+	default:
+		output = input;
+		break;
+	}
+}
+
 //-----------------------------------------------------------------------------
 //   Pobieranie nagłówków plików
 //-----------------------------------------------------------------------------
@@ -130,6 +175,9 @@ void ExtractCargo::LoadFilesTable()
 		cargoFile.read(reinterpret_cast<char*>(&fhTmp.crc), sizeof(fhTmp.crc));
 		cargoFile.read(reinterpret_cast<char*>(&fhTmp.flag), sizeof(fhTmp.flag));
 
+		std::cout << tablePosition << std::endl;
+		std::cout << "Size: " << fhTmp.size << std::endl;
+
 		filesHeads.push_back(fhTmp);
 	}
 }
@@ -143,76 +191,25 @@ void ExtractCargo::ExtractingFilesFromCargo()
 	{
 		std::filesystem::path dir = cargoFileName.stem() / fh.nameFile;
 		CreateDirections(dir);
-
-		std::cout << dir.string() << std::endl;
-
 		std::ofstream file(dir, std::ios::binary);
+
+		std::cout << fh.size << std::endl;
 		cargoFile.seekg(fh.offset);
+		std::vector<char> buffor(fh.size);
 
-		XXH64_reset(xxhState, 0);
+		cargoFile.read(buffor.data(), fh.size);
 
-		// Strumieñ wyci¹gaj¹cy
+		std::vector<char> rawBuffor;
-		if (fh.flag == flag::raw)
+		computingBytes(buffor, rawBuffor, fh.flag);
+
+		if (!HashValid(rawBuffor, fh.crc))
 		{
-			for (uint64_t sc = 0; sc < fh.size; sc += ds::chunk_stream)
+			std::cerr << fh.nameFile << "  Error: Corrupted data integration CRC" << std::endl;
-			{
-				const uint32_t streamChunk = std::min(ds::chunk_stream, static_cast<uint32_t>(fh.size - sc));
-
-				std::vector<char> buffer(streamChunk);
-				cargoFile.read(buffer.data(), streamChunk);
-				XXH64_update(xxhState, buffer.data(), buffer.size());
-				file.write(reinterpret_cast<const char*>(buffer.data()), streamChunk);
-			}
-		}
-		else
-		{
-			uint32_t chunkLen;
-			uint32_t chunkBeforeSize;
-			uint32_t chunkLastSize;
-
-			cargoFile.read(reinterpret_cast<char*>(&chunkLen), sizeof(chunkLen));
-			cargoFile.read(reinterpret_cast<char*>(&chunkBeforeSize), sizeof(chunkBeforeSize));
-			cargoFile.read(reinterpret_cast<char*>(&chunkLastSize), sizeof(chunkLastSize));
-
-			std::vector<char> chunksString;
-
-			// Dekompresja bloków
-			for (size_t i = 0; i < chunkLen; ++i)
-			{
-				// Pobierz rozmiar chunków przed i po skompresowaniem
-				uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize;
-
-				uint32_t chunkZipSize;
-				cargoFile.read(reinterpret_cast<char*>(&chunkZipSize), sizeof(chunkZipSize));
-
-				// Pobierz blok chunka
-				std::vector<char> buffer(chunkZipSize);
-				cargoFile.read(buffer.data(), chunkZipSize);
-
-				std::vector<char> rawBuffer(chunkSize);
-				if ((fh.flag & flag::zip) == flag::zip)
-				{
-					rawBuffer = (fh.flag & flag::enc) == flag::enc ?
-						cman.decompress(eman.decrypt(buffer), chunkSize) :
-						cman.decompress(buffer, chunkSize);
-				}
-				else
-				{
-					rawBuffer = eman.decrypt(buffer);
 		}
 
-				XXH64_update(xxhState, rawBuffer.data(), rawBuffer.size());
+		file.write(reinterpret_cast<const char*>(rawBuffor.data()), rawBuffor.size());
-				file.write(reinterpret_cast<const char*>(rawBuffer.data()), chunkSize);
-			}
-		}
 
 		file.close();
-
-		if (XXH64_digest(xxhState) != fh.crc)
-		{
-			std::cerr << dir.string() << "  Error: Corrupted data integration CRC" << std::endl;
-
-		}
 	}
 
 	std::cout << "Unpacking complete!" << std::endl;

ExtractCargo.h

@@ -28,12 +28,11 @@
 #include <sstream>
 #include <cstdint>
 #include <string>
-#include <utility>
 #include <xxhash.h>
 
 #include "DataStruct.h"
+#include "ChunkManager.h"
 #include "EncryptionManager.h"
-#include "CompressionManager.h"
 
 class ExtractCargo {
 public:
@@ -58,7 +57,7 @@ private:
 	std::ifstream cargoFile;
 
 	EncryptionManager eman;
-	CompressionManager cman;
+
 
 	// Sprawdzenie poprawnoœci archiwum
 	bool CheckCargoFile();
@@ -69,7 +68,13 @@ private:
 	// Pobieranie nag³ówków plików
 	void LoadFilesTable();
 
+	// Sprawdzanie sumy kontrolnej
+	bool HashValid(const std::vector<char>&, const uint64_t&);
+
 	// Utwórz katalog
 	void CreateDirections(std::filesystem::path);
 
+	// Magiczna funkcja do dekompresji i deszyfracji danych
+	void computingBytes(const std::vector<char>&, std::vector<char>&, const uint8_t&);
+
 };

README.md

@@ -16,6 +16,8 @@
 
 **Visual Studio 2022**
 
+**CMAKE**
+--soon--
 
 
 **=========================================================================================**
@@ -34,7 +36,13 @@
 
     \*\*-c  - Packing catalog width compressing all files\*\*
 
-    \*\*-p  - Packing catalog width out compressing\*\*
+    \*\*-r  - Raw files\*\*
+
+    \*\*-c  - Compressing\*\*
+
+    \*\*-e  - Encrypted\*\*
+
+    \*\*-s  - Compressing and encrypted\*\*
 
     \*\*-f  - Packing catalog width unique config for individual files (look on under section)\*\*
 
@@ -47,42 +55,6 @@
 
 
 
-**-f Parameters instruction**
-
-
-
-**Create regular txt file in same directory on catalog width same name.**
-
-
-
-**This is a config file width parameters**
-
-
-
-**{COMPRESS} - key for compress files list**
-
-
-
-**{COMPRESS}**
-
-**textures/texture.png**
-
-**\*.png**
-
-
-
-**{IGNORE} - key for ignoring files list**
-
-
-
-**{IGNORE}**
-
-**testfile.odt**
-
-**\*.odt**
-
-
-
 **=========================================================================================**
 
 ### **Components:**

voidcmd.vcxproj

@@ -132,6 +132,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="ChunkManager.cpp" />
     <ClCompile Include="CompressionManager.cpp" />
     <ClCompile Include="CreateCargo.cpp" />
     <ClCompile Include="EncryptionManager.cpp" />
@@ -140,6 +141,7 @@
     <ClCompile Include="voidcmd.cpp" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="ChunkManager.h" />
     <ClInclude Include="CompressionManager.h" />
     <ClInclude Include="CreateCargo.h" />
     <ClInclude Include="DataStruct.h" />

voidcmd.vcxproj.filters

@@ -27,6 +27,9 @@
     <ClCompile Include="ViewCargo.cpp">
       <Filter>Pliki źródłowe</Filter>
     </ClCompile>
+    <ClCompile Include="ChunkManager.cpp">
+      <Filter>Pliki źródłowe</Filter>
+    </ClCompile>
     <ClCompile Include="EncryptionManager.cpp">
       <Filter>Pliki źródłowe</Filter>
     </ClCompile>
@@ -47,6 +50,9 @@
     <ClInclude Include="DataStruct.h">
       <Filter>Pliki nagłówkowe</Filter>
     </ClInclude>
+    <ClInclude Include="ChunkManager.h">
+      <Filter>Pliki nagłówkowe</Filter>
+    </ClInclude>
     <ClInclude Include="EncryptionManager.h">
       <Filter>Pliki nagłówkowe</Filter>
     </ClInclude>