diff --git a/ChunkManager.cpp b/ChunkManager.cpp new file mode 100644 index 0000000..6da9cc4 --- /dev/null +++ b/ChunkManager.cpp @@ -0,0 +1,122 @@ +#include "CompressingManager.h" + +CompressingManager::CompressingManager() +{ } + +ChunkManager::~ChunkManager() +{ } + +//----------------------------------------------------------------------------- +// Kompresja blokowa +// +// Dzielenie vectora na chunki dokładnie po 128KB +// Kompresowanie chunków bez nagłówka +//----------------------------------------------------------------------------- +std::vector ChunkManager::chunked(const std::vector& raw, const bool& compress, const bool& encrypt) +{ + //std::vector blockSizes; + + // Maksymalny rozmiar chunka + const size_t maxBlockSize = BLOCK_SIZE; + const size_t rawSize = raw.size(); + + uint32_t blockLen = 0; + uint32_t lastChunkRawSize; + std::vector compressedBlocks; + for (size_t offset = 0; offset < rawSize; offset += maxBlockSize) + { + // Rozmiar chunka + const size_t chunkSize = std::min(maxBlockSize, rawSize - offset); + + auto begin = raw.begin() + offset; + auto end = begin + chunkSize; + + // Skopiuj fragment danych do chunka + std::vector chunk(begin, end); + + std::vector outChunk; + + // Buffor wyjściowy nadpisany skompresowanymi danymi + std::vector zipChunk(maxZipChunkSize); + + // Kompresja + int zipSize = LZ4_compress_default(chunk.data(), zipChunk.data(), chunkSize, maxZipChunkSize); + + // Zmiana rozmiaru do faktycznego rozmiaru po kompresji + zipChunk.resize(zipSize); + + uint32_t chs = static_cast(chunk.size()); + uint32_t zch = static_cast(outChunk.size()); + + //addIntToVector(compressedBlocks, chs); + lastChunkRawSize = chs; + addIntToVector(compressedBlocks, zch); + compressedBlocks.insert(compressedBlocks.end(), outChunk.begin(), outChunk.end()); + + blockLen++; + } + + std::vector zip; + // Wstaw liczbę o ilości bloków do vectora; + // Przekonpwertuj usigned int32 na ciąg znkaów + // uint16_t blockLen = blockSizes .size(); + addIntToVector(zip, blockLen); + addIntToVector(zip, maxBlockSize); + addIntToVector(zip, lastChunkRawSize); + + // Dodaj skompresowane dane + zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end()); + + return zip; +} + +////////////////////////////////////////////////////////////// + +//----------------------------------------------------------------------------- +// Dekompresja blokowa +//----------------------------------------------------------------------------- +std::vector ChunkManager::dechunked(const std::vector& zip, const bool& compress, const bool& encrypt) +{ + size_t offset = 0; + const uint32_t chunkLen = getIntFromVector(zip, offset); + const uint32_t chunkBeforeSize = getIntFromVector(zip, offset); + const uint32_t chunkLastSize = getIntFromVector(zip, offset); + + std::cout << "Q: " << chunkLen << std::endl; + std::cout << "C: " << chunkBeforeSize << std::endl; + std::cout << "L: " << chunkBeforeSize << std::endl; + + std::vector chunksString; + + // Dekompresja bloków + for (size_t i = 0; i < chunkLen; ++i) + { + // Pobierz rozmiar chunków przed i po skompresowaniem + uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; + uint32_t chunkZipSize = getIntFromVector(zip, offset); + + // Pobierz blok chunka + std::vector inChunk(chunkZipSize); + std::memcpy(inChunk.data(), zip.data() + offset, chunkZipSize); + offset += chunkZipSize; + + // Zdeklarój pusty chunk + std::vector chunk(chunkSize); + + // Dekompresja chunka + int sizeData = LZ4_decompress_safe(zipChunk.data(), chunk.data(), static_cast(chunkZipSize), static_cast(chunkSize)); + + if (sizeData < 0) + { + throw std::runtime_error("LZ4 Decompressing Error"); + } + + // Dostosowanie rozmiaru vectora po skompresowaniu + chunk.resize(sizeData); + + // Scal chunki + chunksString.insert(chunksString.end(), chunk.begin(), chunk.end()); + } + + return chunksString; +} \ No newline at end of file diff --git a/ChunkManager.h b/ChunkManager.h new file mode 100644 index 0000000..c4e1f30 --- /dev/null +++ b/ChunkManager.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "EncryptionManager.h" +#include "CompressionManager.h" + +#define BLOCK_SIZE 131072 // 128KB + +class ChunkManager +{ +public: + ChunkManager(EncryptionManager& em); + ~ChunkManager(); + + // Podział na chunki + std::vector chunked(const std::vector&, const bool&, const bool&); + + // Zcalanie chunków + std::vector dechunked(const std::vector&, const bool&, const bool&); + +private: + EncryptionManager eman; + CompressionManager cman; + + // Przekonwertuj zmienną na ciąg na vector + template + void addIntToVector(std::vector& vec, const T& val) + { + size_t tmpSize = vec.size(); + vec.resize(tmpSize + sizeof(val)); + std::memcpy(vec.data() + tmpSize, &val, sizeof(val)); + } + + // Pobierz zmienną z vectora + template + T getIntFromVector(const std::vector& vec, size_t& offset) + { + T tmp{}; + std::memcpy(&tmp, vec.data() + offset, sizeof(tmp)); + offset += sizeof(tmp); + + return tmp; + } +}; + diff --git a/CreateCargo.cpp b/CreateCargo.cpp index b24a880..adbddf5 100644 --- a/CreateCargo.cpp +++ b/CreateCargo.cpp @@ -42,7 +42,7 @@ CreateCargo::~CreateCargo() { //----------------------------------------------------------------------------- bool CreateCargo::Create(const std::string& path, const uint8_t& flag) { - cargoFile = path + "." + extension; + cargoFile = path + "." + signature; catalogPath = path; methodFlags = flag; @@ -241,6 +241,7 @@ std::vector CreateCargo::ComputingHeadFiles() // Dodaj do kontenera konfigurację chunków if (file.parameter != flag::raw) { + std::cout << "CHUNK PARAM" << std::endl; cargo.write(reinterpret_cast(&quantity), sizeof(quantity)); cargo.write(reinterpret_cast(&chunkBlockSize), sizeof(chunkBlockSize)); cargo.write(reinterpret_cast(&lastChunkSize), sizeof(lastChunkSize)); @@ -287,7 +288,8 @@ std::vector CreateCargo::ComputingHeadFiles() else { // Zaszyfruj lub skopiuj - outChunk = eman.encrypt(chunk); + outChunk = (file.parameter & flag::enc) == flag::enc ? + eman.encrypt(cman.compress(chunk)) : cman.compress(chunk); } const uint32_t outSize = outChunk.size(); @@ -298,11 +300,19 @@ std::vector CreateCargo::ComputingHeadFiles() cargo.write(reinterpret_cast(outChunk.data()), outChunk.size()); sizeFile += outSize; } + std::cout << "SIZE: " << sizeFile << std::endl; } } f.close(); + //Tworzenie hashu CRC + //const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0); + + //Kompresjia + //std::vector pakBuffer; + //computingBytes(file.parameter, buffer, pakBuffer); + FilesTable ft; ft.nameFile = path; ft.nameLen = path.length(); @@ -311,6 +321,8 @@ std::vector CreateCargo::ComputingHeadFiles() ft.flag = file.parameter; ft.crc = XXH64_digest(xxhState); + //cargo.write(reinterpret_cast(pakBuffer.data()), pakBuffer.size()); + filesTable.push_back(ft); offset += sizeFile; } diff --git a/CreateCargo.h b/CreateCargo.h index 958dcf2..3879779 100644 --- a/CreateCargo.h +++ b/CreateCargo.h @@ -33,6 +33,7 @@ #include #include "DataStruct.h" +#include "ChunkManager.h" #include "EncryptionManager.h" #include "CompressionManager.h" diff --git a/DataStruct.h b/DataStruct.h index 75f5a79..af4da80 100644 --- a/DataStruct.h +++ b/DataStruct.h @@ -64,13 +64,13 @@ namespace fl { namespace ds { // Chunki streamowania - inline constexpr uint32_t chunk_stream = 268435456; // 256MB + inline constexpr int chunk_stream = 268435456; // 256MB // Blok chunków - inline constexpr uint32_t block_size = 131072; // 128KB + inline constexpr int block_size = 131072; // 128KB // Maksymalny rozmiar pliku do spakowania - inline constexpr uint64_t maxFileSize = 8589934592; // 8GB + inline constexpr int maxFileSize = 8589934592; // 8GB } // Flagi diff --git a/ExtractCargo.cpp b/ExtractCargo.cpp index 3d1398a..83f53a5 100644 --- a/ExtractCargo.cpp +++ b/ExtractCargo.cpp @@ -110,6 +110,51 @@ bool ExtractCargo::CheckCargoFile() return true; } +//----------------------------------------------------------------------------- +// Sprawdzanie sumy kontrolnej +//----------------------------------------------------------------------------- +bool ExtractCargo::HashValid(const std::vector& data, const uint64_t& crc) +{ + uint64_t actualCrc = XXH64(data.data(), data.size(), 0); + + if (actualCrc != crc) + { + return false; + } + + return true; +} + +//----------------------------------------------------------------------------- +// Magiczna funkcja do dekompresji i deszyfracji danych +//----------------------------------------------------------------------------- +void ExtractCargo::computingBytes(const std::vector& input, std::vector& output, const uint8_t& flag) +{ + ChunkManager cm(eman); + + std::cout << static_cast(flag) << std::endl; + + switch (flag) + { + case FILE_FLAG_COMPRESS: + output = cm.dechunked(input, true, false); + break; + + case FILE_FLAG_ENCRYPT: + output = cm.dechunked(input, false, true); + break; + + case FILE_FLAG_ZIPENC: + output = cm.dechunked(input, true, true); + std::cout << "DENC" << std::endl; + break; + + default: + output = input; + break; + } +} + //----------------------------------------------------------------------------- // Pobieranie nagłówków plików //----------------------------------------------------------------------------- @@ -130,6 +175,9 @@ void ExtractCargo::LoadFilesTable() cargoFile.read(reinterpret_cast(&fhTmp.crc), sizeof(fhTmp.crc)); cargoFile.read(reinterpret_cast(&fhTmp.flag), sizeof(fhTmp.flag)); + std::cout << tablePosition << std::endl; + std::cout << "Size: " << fhTmp.size << std::endl; + filesHeads.push_back(fhTmp); } } @@ -143,76 +191,25 @@ void ExtractCargo::ExtractingFilesFromCargo() { std::filesystem::path dir = cargoFileName.stem() / fh.nameFile; CreateDirections(dir); - - std::cout << dir.string() << std::endl; - std::ofstream file(dir, std::ios::binary); + + std::cout << fh.size << std::endl; cargoFile.seekg(fh.offset); + std::vector buffor(fh.size); - XXH64_reset(xxhState, 0); + cargoFile.read(buffor.data(), fh.size); - // Strumień wyciągający - if (fh.flag == flag::raw) + std::vector rawBuffor; + computingBytes(buffor, rawBuffor, fh.flag); + + if (!HashValid(rawBuffor, fh.crc)) { - for (uint64_t sc = 0; sc < fh.size; sc += ds::chunk_stream) - { - const uint32_t streamChunk = std::min(ds::chunk_stream, static_cast(fh.size - sc)); - - std::vector buffer(streamChunk); - cargoFile.read(buffer.data(), streamChunk); - XXH64_update(xxhState, buffer.data(), buffer.size()); - file.write(reinterpret_cast(buffer.data()), streamChunk); - } + std::cerr << fh.nameFile << " Error: Corrupted data integration CRC" << std::endl; } - else - { - uint32_t chunkLen; - uint32_t chunkBeforeSize; - uint32_t chunkLastSize; - cargoFile.read(reinterpret_cast(&chunkLen), sizeof(chunkLen)); - cargoFile.read(reinterpret_cast(&chunkBeforeSize), sizeof(chunkBeforeSize)); - cargoFile.read(reinterpret_cast(&chunkLastSize), sizeof(chunkLastSize)); - - std::vector chunksString; - - // Dekompresja bloków - for (size_t i = 0; i < chunkLen; ++i) - { - // Pobierz rozmiar chunków przed i po skompresowaniem - uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; - - uint32_t chunkZipSize; - cargoFile.read(reinterpret_cast(&chunkZipSize), sizeof(chunkZipSize)); - - // Pobierz blok chunka - std::vector buffer(chunkZipSize); - cargoFile.read(buffer.data(), chunkZipSize); - - std::vector rawBuffer(chunkSize); - if ((fh.flag & flag::zip) == flag::zip) - { - rawBuffer = (fh.flag & flag::enc) == flag::enc ? - cman.decompress(eman.decrypt(buffer), chunkSize) : - cman.decompress(buffer, chunkSize); - } - else - { - rawBuffer = eman.decrypt(buffer); - } - - XXH64_update(xxhState, rawBuffer.data(), rawBuffer.size()); - file.write(reinterpret_cast(rawBuffer.data()), chunkSize); - } - } + file.write(reinterpret_cast(rawBuffor.data()), rawBuffor.size()); file.close(); - - if (XXH64_digest(xxhState) != fh.crc) - { - std::cerr << dir.string() << " Error: Corrupted data integration CRC" << std::endl; - - } } std::cout << "Unpacking complete!" << std::endl; diff --git a/ExtractCargo.h b/ExtractCargo.h index 9fce5f5..3cab45c 100644 --- a/ExtractCargo.h +++ b/ExtractCargo.h @@ -28,12 +28,11 @@ #include #include #include -#include #include #include "DataStruct.h" +#include "ChunkManager.h" #include "EncryptionManager.h" -#include "CompressionManager.h" class ExtractCargo { public: @@ -58,7 +57,7 @@ private: std::ifstream cargoFile; EncryptionManager eman; - CompressionManager cman; + // Sprawdzenie poprawności archiwum bool CheckCargoFile(); @@ -69,7 +68,13 @@ private: // Pobieranie nagłówków plików void LoadFilesTable(); + // Sprawdzanie sumy kontrolnej + bool HashValid(const std::vector&, const uint64_t&); + // Utwórz katalog void CreateDirections(std::filesystem::path); + // Magiczna funkcja do dekompresji i deszyfracji danych + void computingBytes(const std::vector&, std::vector&, const uint8_t&); + }; diff --git a/README.md b/README.md index 704bf0a..20f79de 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ **Visual Studio 2022** +**CMAKE** +--soon-- **=========================================================================================** @@ -34,7 +36,13 @@   \*\*-c - Packing catalog width compressing all files\*\* - \*\*-p - Packing catalog width out compressing\*\* + \*\*-r - Raw files\*\* + + \*\*-c - Compressing\*\* + + \*\*-e - Encrypted\*\* + + \*\*-s - Compressing and encrypted\*\* \*\*-f - Packing catalog width unique config for individual files (look on under section)\*\* @@ -47,42 +55,6 @@ -**-f Parameters instruction** - - - -**Create regular txt file in same directory on catalog width same name.** - - - -**This is a config file width parameters** - - - -**{COMPRESS} - key for compress files list** - - - -**{COMPRESS}** - -**textures/texture.png** - -**\*.png** - - - -**{IGNORE} - key for ignoring files list** - - - -**{IGNORE}** - -**testfile.odt** - -**\*.odt** - - - **=========================================================================================** ### **Components:** diff --git a/voidcmd.vcxproj b/voidcmd.vcxproj index ee82946..49f706f 100644 --- a/voidcmd.vcxproj +++ b/voidcmd.vcxproj @@ -132,6 +132,7 @@ + @@ -140,6 +141,7 @@ + diff --git a/voidcmd.vcxproj.filters b/voidcmd.vcxproj.filters index 4dec7bb..563ca03 100644 --- a/voidcmd.vcxproj.filters +++ b/voidcmd.vcxproj.filters @@ -27,6 +27,9 @@ Pliki ĹşrĂłdĹ‚owe + + Pliki ĹşrĂłdĹ‚owe + Pliki ĹşrĂłdĹ‚owe @@ -47,6 +50,9 @@ Pliki nagĹ‚Ăłwkowe + + Pliki nagĹ‚Ăłwkowe + Pliki nagĹ‚Ăłwkowe