Merge branch 'streaming'

ChunkManager.cpp

@@ -1,122 +0,0 @@
-#include "CompressingManager.h"
-
-CompressingManager::CompressingManager()
-{ }
-
-ChunkManager::~ChunkManager()
-{ }
-
-//-----------------------------------------------------------------------------
-//	Kompresja blokowa
-//	
-// Dzielenie vectora na chunki dok³adnie po 128KB
-// Kompresowanie chunków bez nag³ówka
-//-----------------------------------------------------------------------------
-std::vector<char> ChunkManager::chunked(const std::vector<char>& raw, const bool& compress, const bool& encrypt)
-{
-	//std::vector<BlockSize> blockSizes;
-
-	// Maksymalny rozmiar chunka
-	const size_t maxBlockSize = BLOCK_SIZE;
-	const size_t rawSize = raw.size();
-
-	uint32_t blockLen = 0;
-	uint32_t lastChunkRawSize;
-	std::vector<char> compressedBlocks;
-	for (size_t offset = 0; offset < rawSize; offset += maxBlockSize)
-	{
-		// Rozmiar chunka
-		const size_t chunkSize = std::min(maxBlockSize, rawSize - offset);
-
-		auto begin = raw.begin() + offset;
-		auto end = begin + chunkSize;
-
-		// Skopiuj fragment danych do chunka
-		std::vector<char> chunk(begin, end);
-
-		std::vector<char> outChunk;
-
-		// Buffor wyjœciowy nadpisany skompresowanymi danymi
-		std::vector<char> zipChunk(maxZipChunkSize);
-
-		// Kompresja
-		int zipSize = LZ4_compress_default(chunk.data(), zipChunk.data(), chunkSize, maxZipChunkSize);
-
-		// Zmiana rozmiaru do faktycznego rozmiaru po kompresji
-		zipChunk.resize(zipSize);
-
-		uint32_t chs = static_cast<uint32_t>(chunk.size());
-		uint32_t zch = static_cast<uint32_t>(outChunk.size());
-
-		//addIntToVector<uint32_t>(compressedBlocks, chs);
-		lastChunkRawSize = chs;
-		addIntToVector<uint32_t>(compressedBlocks, zch);
-		compressedBlocks.insert(compressedBlocks.end(), outChunk.begin(), outChunk.end());
-
-		blockLen++;
-	}
-
-	std::vector<char> zip;
-	// Wstaw liczbê o iloœci bloków do vectora;
-	// Przekonpwertuj usigned int32 na ci¹g znkaów
-	// uint16_t blockLen = blockSizes .size();
-	addIntToVector<uint32_t>(zip, blockLen);
-	addIntToVector<uint32_t>(zip, maxBlockSize);
-	addIntToVector<uint32_t>(zip, lastChunkRawSize);
-
-	// Dodaj skompresowane dane
-	zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end());
-
-	return zip;
-}
-
-//////////////////////////////////////////////////////////////
-
-//-----------------------------------------------------------------------------
-//	Dekompresja blokowa
-//-----------------------------------------------------------------------------
-std::vector<char> ChunkManager::dechunked(const std::vector<char>& zip, const bool& compress, const bool& encrypt)
-{
-	size_t offset = 0;
-	const uint32_t chunkLen = getIntFromVector<uint32_t>(zip, offset);
-	const uint32_t chunkBeforeSize = getIntFromVector<uint32_t>(zip, offset);
-	const uint32_t chunkLastSize = getIntFromVector<uint32_t>(zip, offset);
-
-	std::cout << "Q: " << chunkLen << std::endl;
-	std::cout << "C: " << chunkBeforeSize << std::endl;
-	std::cout << "L: " << chunkBeforeSize << std::endl;
-
-	std::vector<char> chunksString;
-
-	// Dekompresja bloków
-	for (size_t i = 0; i < chunkLen; ++i)
-	{
-		// Pobierz rozmiar chunków przed i po skompresowaniem
-		uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize;
-		uint32_t chunkZipSize = getIntFromVector<uint32_t>(zip, offset);
-
-		// Pobierz blok chunka
-		std::vector<char> inChunk(chunkZipSize);
-		std::memcpy(inChunk.data(), zip.data() + offset, chunkZipSize);
-		offset += chunkZipSize;
-
-		// Zdeklarój pusty chunk
-		std::vector<char> chunk(chunkSize);
-
-		// Dekompresja chunka
-		int sizeData = LZ4_decompress_safe(zipChunk.data(), chunk.data(), static_cast<int>(chunkZipSize), static_cast<int>(chunkSize));
-
-		if (sizeData < 0)
-		{
-			throw std::runtime_error("LZ4 Decompressing Error");
-		}
-
-		// Dostosowanie rozmiaru vectora po skompresowaniu
-		chunk.resize(sizeData);
-
-		// Scal chunki
-		chunksString.insert(chunksString.end(), chunk.begin(), chunk.end());
-	}
-
-	return chunksString;
-}

ChunkManager.h

@@ -1,51 +0,0 @@
-#pragma once
-
-#include <vector>
-#include <cstdint>
-#include <cstring>
-#include <stdexcept>
-#include <algorithm>
-#include <iostream>
-
-#include "EncryptionManager.h"
-#include "CompressionManager.h"
-
-#define BLOCK_SIZE          131072 // 128KB
-
-class ChunkManager
-{
-public:
-	ChunkManager(EncryptionManager& em);
-	~ChunkManager();
-
-	// Podzia³ na chunki
-	std::vector<char> chunked(const std::vector<char>&, const bool&, const bool&);
-
-	// Zcalanie chunków
-	std::vector<char> dechunked(const std::vector<char>&, const bool&, const bool&);
-
-private:
-	EncryptionManager eman;
-	CompressionManager cman;
-
-	// Przekonwertuj zmienn¹ na ci¹g na vector
-	template <typename T>
-	void addIntToVector(std::vector<char>& vec, const T& val)
-	{
-		size_t tmpSize = vec.size();
-		vec.resize(tmpSize + sizeof(val));
-		std::memcpy(vec.data() + tmpSize, &val, sizeof(val));
-	}
-
-	// Pobierz zmienn¹ z vectora
-	template <typename T>
-	T getIntFromVector(const std::vector<char>& vec, size_t& offset)
-	{
-		T tmp{};
-		std::memcpy(&tmp, vec.data() + offset, sizeof(tmp));
-		offset += sizeof(tmp);
-	
-		return tmp;
-	}
-};
-

CreateCargo.cpp

@@ -42,7 +42,7 @@ CreateCargo::~CreateCargo() {
 //-----------------------------------------------------------------------------
 bool CreateCargo::Create(const std::string& path, const uint8_t& flag)
 {
-	cargoFile = path + "." + signature;
+	cargoFile = path + "." + extension;
 	catalogPath = path;
 	methodFlags = flag;
 
@@ -241,7 +241,6 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 			// Dodaj do kontenera konfiguracjê chunków
 			if (file.parameter != flag::raw)
 			{
-				std::cout << "CHUNK PARAM" << std::endl;
 				cargo.write(reinterpret_cast<const char*>(&quantity), sizeof(quantity));
 				cargo.write(reinterpret_cast<const char*>(&chunkBlockSize), sizeof(chunkBlockSize));
 				cargo.write(reinterpret_cast<const char*>(&lastChunkSize), sizeof(lastChunkSize));
@@ -288,8 +287,7 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 						else
 						{
 							// Zaszyfruj lub skopiuj
-							outChunk = (file.parameter & flag::enc) == flag::enc ?
+							outChunk = eman.encrypt(chunk);
-								eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
 						}
 
 						const uint32_t outSize = outChunk.size();
@@ -300,19 +298,11 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 						cargo.write(reinterpret_cast<const char*>(outChunk.data()), outChunk.size());
 						sizeFile += outSize;
 					}
-					std::cout << "SIZE: " << sizeFile << std::endl;
 				}
 			}
 
 			f.close();
 
-			//Tworzenie hashu CRC
-			//const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
-
-			//Kompresjia
-			//std::vector<char> pakBuffer;
-			//computingBytes(file.parameter, buffer, pakBuffer);
-
 			FilesTable ft;
 			ft.nameFile = path;
 			ft.nameLen = path.length();
@@ -321,8 +311,6 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
 			ft.flag = file.parameter;
 			ft.crc = XXH64_digest(xxhState);
 
-			//cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
-
 			filesTable.push_back(ft);
 			offset += sizeFile;
 		}

CreateCargo.h

@@ -33,7 +33,6 @@
 #include <nlohmann/json.hpp>
 
 #include "DataStruct.h"
-#include "ChunkManager.h"
 #include "EncryptionManager.h"
 #include "CompressionManager.h"
 

DataStruct.h

@@ -64,13 +64,13 @@ namespace fl {
 namespace ds
 {
 	// Chunki streamowania
-	inline constexpr int chunk_stream = 268435456; // 256MB
+	inline constexpr uint32_t chunk_stream = 268435456; // 256MB
 
 	// Blok chunków
-	inline constexpr int block_size   = 131072;    // 128KB
+	inline constexpr uint32_t block_size   = 131072;    // 128KB
 
 	// Maksymalny rozmiar pliku do spakowania
-	inline constexpr int maxFileSize = 8589934592; // 8GB
+	inline constexpr uint64_t maxFileSize = 8589934592; // 8GB
 }
 
 // Flagi

ExtractCargo.cpp

@@ -110,51 +110,6 @@ bool ExtractCargo::CheckCargoFile()
 	return true;
 }
 
-//-----------------------------------------------------------------------------
-//   Sprawdzanie sumy kontrolnej
-//-----------------------------------------------------------------------------
-bool ExtractCargo::HashValid(const std::vector<char>& data, const uint64_t& crc)
-{
-	uint64_t actualCrc = XXH64(data.data(), data.size(), 0);
-
-	if (actualCrc != crc)
-	{
-		return false;
-	}
-
-	return true;
-}
-
-//-----------------------------------------------------------------------------
-//   Magiczna funkcja do dekompresji i deszyfracji danych
-//-----------------------------------------------------------------------------
-void ExtractCargo::computingBytes(const std::vector<char>& input, std::vector<char>& output, const uint8_t& flag)
-{
-	ChunkManager cm(eman);
-
-	std::cout << static_cast<int>(flag) << std::endl;
-	
-	switch (flag)
-	{
-	case FILE_FLAG_COMPRESS:
-		output = cm.dechunked(input, true, false);
-		break;
-	
-	case FILE_FLAG_ENCRYPT:
-		output = cm.dechunked(input, false, true);
-		break;
-	
-	case FILE_FLAG_ZIPENC:
-		output = cm.dechunked(input, true, true);
-		std::cout << "DENC" << std::endl;
-		break;
-	
-	default:
-		output = input;
-		break;
-	}
-}
-
 //-----------------------------------------------------------------------------
 //   Pobieranie nagłówków plików
 //-----------------------------------------------------------------------------
@@ -175,9 +130,6 @@ void ExtractCargo::LoadFilesTable()
 		cargoFile.read(reinterpret_cast<char*>(&fhTmp.crc), sizeof(fhTmp.crc));
 		cargoFile.read(reinterpret_cast<char*>(&fhTmp.flag), sizeof(fhTmp.flag));
 
-		std::cout << tablePosition << std::endl;
-		std::cout << "Size: " << fhTmp.size << std::endl;
-
 		filesHeads.push_back(fhTmp);
 	}
 }
@@ -191,25 +143,76 @@ void ExtractCargo::ExtractingFilesFromCargo()
 	{
 		std::filesystem::path dir = cargoFileName.stem() / fh.nameFile;
 		CreateDirections(dir);
+
+		std::cout << dir.string() << std::endl;
+
 		std::ofstream file(dir, std::ios::binary);
-
-		std::cout << fh.size << std::endl;
 		cargoFile.seekg(fh.offset);
-		std::vector<char> buffor(fh.size);
 
-		cargoFile.read(buffor.data(), fh.size);
+		XXH64_reset(xxhState, 0);
 
-		std::vector<char> rawBuffor;
+		// Strumieñ wyci¹gaj¹cy
-		computingBytes(buffor, rawBuffor, fh.flag);
+		if (fh.flag == flag::raw)
-
-		if (!HashValid(rawBuffor, fh.crc))
 		{
-			std::cerr << fh.nameFile << "  Error: Corrupted data integration CRC" << std::endl;
+			for (uint64_t sc = 0; sc < fh.size; sc += ds::chunk_stream)
+			{
+				const uint32_t streamChunk = std::min(ds::chunk_stream, static_cast<uint32_t>(fh.size - sc));
+
+				std::vector<char> buffer(streamChunk);
+				cargoFile.read(buffer.data(), streamChunk);
+				XXH64_update(xxhState, buffer.data(), buffer.size());
+				file.write(reinterpret_cast<const char*>(buffer.data()), streamChunk);
+			}
+		}
+		else
+		{
+			uint32_t chunkLen;
+			uint32_t chunkBeforeSize;
+			uint32_t chunkLastSize;
+
+			cargoFile.read(reinterpret_cast<char*>(&chunkLen), sizeof(chunkLen));
+			cargoFile.read(reinterpret_cast<char*>(&chunkBeforeSize), sizeof(chunkBeforeSize));
+			cargoFile.read(reinterpret_cast<char*>(&chunkLastSize), sizeof(chunkLastSize));
+
+			std::vector<char> chunksString;
+
+			// Dekompresja bloków
+			for (size_t i = 0; i < chunkLen; ++i)
+			{
+				// Pobierz rozmiar chunków przed i po skompresowaniem
+				uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize;
+
+				uint32_t chunkZipSize;
+				cargoFile.read(reinterpret_cast<char*>(&chunkZipSize), sizeof(chunkZipSize));
+
+				// Pobierz blok chunka
+				std::vector<char> buffer(chunkZipSize);
+				cargoFile.read(buffer.data(), chunkZipSize);
+
+				std::vector<char> rawBuffer(chunkSize);
+				if ((fh.flag & flag::zip) == flag::zip)
+				{
+					rawBuffer = (fh.flag & flag::enc) == flag::enc ?
+						cman.decompress(eman.decrypt(buffer), chunkSize) :
+						cman.decompress(buffer, chunkSize);
+				}
+				else
+				{
+					rawBuffer = eman.decrypt(buffer);
+				}
+
+				XXH64_update(xxhState, rawBuffer.data(), rawBuffer.size());
+				file.write(reinterpret_cast<const char*>(rawBuffer.data()), chunkSize);
+			}
 		}
 
-		file.write(reinterpret_cast<const char*>(rawBuffor.data()), rawBuffor.size());
-
 		file.close();
+
+		if (XXH64_digest(xxhState) != fh.crc)
+		{
+			std::cerr << dir.string() << "  Error: Corrupted data integration CRC" << std::endl;
+
+		}
 	}
 
 	std::cout << "Unpacking complete!" << std::endl;

ExtractCargo.h

@@ -28,11 +28,12 @@
 #include <sstream>
 #include <cstdint>
 #include <string>
+#include <utility>
 #include <xxhash.h>
 
 #include "DataStruct.h"
-#include "ChunkManager.h"
 #include "EncryptionManager.h"
+#include "CompressionManager.h"
 
 class ExtractCargo {
 public:
@@ -57,7 +58,7 @@ private:
 	std::ifstream cargoFile;
 
 	EncryptionManager eman;
-
+	CompressionManager cman;
 
 	// Sprawdzenie poprawnoœci archiwum
 	bool CheckCargoFile();
@@ -68,13 +69,7 @@ private:
 	// Pobieranie nag³ówków plików
 	void LoadFilesTable();
 
-	// Sprawdzanie sumy kontrolnej
-	bool HashValid(const std::vector<char>&, const uint64_t&);
-
 	// Utwórz katalog
 	void CreateDirections(std::filesystem::path);
 
-	// Magiczna funkcja do dekompresji i deszyfracji danych
-	void computingBytes(const std::vector<char>&, std::vector<char>&, const uint8_t&);
-
 };

README.md

@@ -16,8 +16,6 @@
 
 **Visual Studio 2022**
 
-**CMAKE**
---soon--
 
 
 **=========================================================================================**
@@ -36,13 +34,7 @@
 
     \*\*-c  - Packing catalog width compressing all files\*\*
 
-    \*\*-r  - Raw files\*\*
+    \*\*-p  - Packing catalog width out compressing\*\*
-
-    \*\*-c  - Compressing\*\*
-
-    \*\*-e  - Encrypted\*\*
-
-    \*\*-s  - Compressing and encrypted\*\*
 
     \*\*-f  - Packing catalog width unique config for individual files (look on under section)\*\*
 
@@ -55,6 +47,42 @@
 
 
 
+**-f Parameters instruction**
+
+
+
+**Create regular txt file in same directory on catalog width same name.**
+
+
+
+**This is a config file width parameters**
+
+
+
+**{COMPRESS} - key for compress files list**
+
+
+
+**{COMPRESS}**
+
+**textures/texture.png**
+
+**\*.png**
+
+
+
+**{IGNORE} - key for ignoring files list**
+
+
+
+**{IGNORE}**
+
+**testfile.odt**
+
+**\*.odt**
+
+
+
 **=========================================================================================**
 
 ### **Components:**

voidcmd.vcxproj

@@ -132,7 +132,6 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="ChunkManager.cpp" />
     <ClCompile Include="CompressionManager.cpp" />
     <ClCompile Include="CreateCargo.cpp" />
     <ClCompile Include="EncryptionManager.cpp" />
@@ -141,7 +140,6 @@
     <ClCompile Include="voidcmd.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="ChunkManager.h" />
     <ClInclude Include="CompressionManager.h" />
     <ClInclude Include="CreateCargo.h" />
     <ClInclude Include="DataStruct.h" />

voidcmd.vcxproj.filters

@@ -27,9 +27,6 @@
     <ClCompile Include="ViewCargo.cpp">
       <Filter>Pliki źródłowe</Filter>
     </ClCompile>
-    <ClCompile Include="ChunkManager.cpp">
-      <Filter>Pliki źródłowe</Filter>
-    </ClCompile>
     <ClCompile Include="EncryptionManager.cpp">
       <Filter>Pliki źródłowe</Filter>
     </ClCompile>
@@ -50,9 +47,6 @@
     <ClInclude Include="DataStruct.h">
       <Filter>Pliki nagłówkowe</Filter>
     </ClInclude>
-    <ClInclude Include="ChunkManager.h">
-      <Filter>Pliki nagłówkowe</Filter>
-    </ClInclude>
     <ClInclude Include="EncryptionManager.h">
       <Filter>Pliki nagłówkowe</Filter>
     </ClInclude>