diff --git a/ChunkManager.cpp b/ChunkManager.cpp deleted file mode 100644 index 6da9cc4..0000000 --- a/ChunkManager.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "CompressingManager.h" - -CompressingManager::CompressingManager() -{ } - -ChunkManager::~ChunkManager() -{ } - -//----------------------------------------------------------------------------- -// Kompresja blokowa -// -// Dzielenie vectora na chunki dokładnie po 128KB -// Kompresowanie chunków bez nagłówka -//----------------------------------------------------------------------------- -std::vector ChunkManager::chunked(const std::vector& raw, const bool& compress, const bool& encrypt) -{ - //std::vector blockSizes; - - // Maksymalny rozmiar chunka - const size_t maxBlockSize = BLOCK_SIZE; - const size_t rawSize = raw.size(); - - uint32_t blockLen = 0; - uint32_t lastChunkRawSize; - std::vector compressedBlocks; - for (size_t offset = 0; offset < rawSize; offset += maxBlockSize) - { - // Rozmiar chunka - const size_t chunkSize = std::min(maxBlockSize, rawSize - offset); - - auto begin = raw.begin() + offset; - auto end = begin + chunkSize; - - // Skopiuj fragment danych do chunka - std::vector chunk(begin, end); - - std::vector outChunk; - - // Buffor wyjściowy nadpisany skompresowanymi danymi - std::vector zipChunk(maxZipChunkSize); - - // Kompresja - int zipSize = LZ4_compress_default(chunk.data(), zipChunk.data(), chunkSize, maxZipChunkSize); - - // Zmiana rozmiaru do faktycznego rozmiaru po kompresji - zipChunk.resize(zipSize); - - uint32_t chs = static_cast(chunk.size()); - uint32_t zch = static_cast(outChunk.size()); - - //addIntToVector(compressedBlocks, chs); - lastChunkRawSize = chs; - addIntToVector(compressedBlocks, zch); - compressedBlocks.insert(compressedBlocks.end(), outChunk.begin(), outChunk.end()); - - blockLen++; - } - - std::vector zip; - // Wstaw liczbę o ilości bloków do vectora; - // Przekonpwertuj usigned int32 na ciąg znkaów - // uint16_t blockLen = blockSizes .size(); - addIntToVector(zip, blockLen); - addIntToVector(zip, maxBlockSize); - addIntToVector(zip, lastChunkRawSize); - - // Dodaj skompresowane dane - zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end()); - - return zip; -} - -////////////////////////////////////////////////////////////// - -//----------------------------------------------------------------------------- -// Dekompresja blokowa -//----------------------------------------------------------------------------- -std::vector ChunkManager::dechunked(const std::vector& zip, const bool& compress, const bool& encrypt) -{ - size_t offset = 0; - const uint32_t chunkLen = getIntFromVector(zip, offset); - const uint32_t chunkBeforeSize = getIntFromVector(zip, offset); - const uint32_t chunkLastSize = getIntFromVector(zip, offset); - - std::cout << "Q: " << chunkLen << std::endl; - std::cout << "C: " << chunkBeforeSize << std::endl; - std::cout << "L: " << chunkBeforeSize << std::endl; - - std::vector chunksString; - - // Dekompresja bloków - for (size_t i = 0; i < chunkLen; ++i) - { - // Pobierz rozmiar chunków przed i po skompresowaniem - uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; - uint32_t chunkZipSize = getIntFromVector(zip, offset); - - // Pobierz blok chunka - std::vector inChunk(chunkZipSize); - std::memcpy(inChunk.data(), zip.data() + offset, chunkZipSize); - offset += chunkZipSize; - - // Zdeklarój pusty chunk - std::vector chunk(chunkSize); - - // Dekompresja chunka - int sizeData = LZ4_decompress_safe(zipChunk.data(), chunk.data(), static_cast(chunkZipSize), static_cast(chunkSize)); - - if (sizeData < 0) - { - throw std::runtime_error("LZ4 Decompressing Error"); - } - - // Dostosowanie rozmiaru vectora po skompresowaniu - chunk.resize(sizeData); - - // Scal chunki - chunksString.insert(chunksString.end(), chunk.begin(), chunk.end()); - } - - return chunksString; -} \ No newline at end of file diff --git a/ChunkManager.h b/ChunkManager.h deleted file mode 100644 index c4e1f30..0000000 --- a/ChunkManager.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "EncryptionManager.h" -#include "CompressionManager.h" - -#define BLOCK_SIZE 131072 // 128KB - -class ChunkManager -{ -public: - ChunkManager(EncryptionManager& em); - ~ChunkManager(); - - // Podział na chunki - std::vector chunked(const std::vector&, const bool&, const bool&); - - // Zcalanie chunków - std::vector dechunked(const std::vector&, const bool&, const bool&); - -private: - EncryptionManager eman; - CompressionManager cman; - - // Przekonwertuj zmienną na ciąg na vector - template - void addIntToVector(std::vector& vec, const T& val) - { - size_t tmpSize = vec.size(); - vec.resize(tmpSize + sizeof(val)); - std::memcpy(vec.data() + tmpSize, &val, sizeof(val)); - } - - // Pobierz zmienną z vectora - template - T getIntFromVector(const std::vector& vec, size_t& offset) - { - T tmp{}; - std::memcpy(&tmp, vec.data() + offset, sizeof(tmp)); - offset += sizeof(tmp); - - return tmp; - } -}; - diff --git a/CreateCargo.cpp b/CreateCargo.cpp index adbddf5..b24a880 100644 --- a/CreateCargo.cpp +++ b/CreateCargo.cpp @@ -42,7 +42,7 @@ CreateCargo::~CreateCargo() { //----------------------------------------------------------------------------- bool CreateCargo::Create(const std::string& path, const uint8_t& flag) { - cargoFile = path + "." + signature; + cargoFile = path + "." + extension; catalogPath = path; methodFlags = flag; @@ -241,7 +241,6 @@ std::vector CreateCargo::ComputingHeadFiles() // Dodaj do kontenera konfigurację chunków if (file.parameter != flag::raw) { - std::cout << "CHUNK PARAM" << std::endl; cargo.write(reinterpret_cast(&quantity), sizeof(quantity)); cargo.write(reinterpret_cast(&chunkBlockSize), sizeof(chunkBlockSize)); cargo.write(reinterpret_cast(&lastChunkSize), sizeof(lastChunkSize)); @@ -288,8 +287,7 @@ std::vector CreateCargo::ComputingHeadFiles() else { // Zaszyfruj lub skopiuj - outChunk = (file.parameter & flag::enc) == flag::enc ? - eman.encrypt(cman.compress(chunk)) : cman.compress(chunk); + outChunk = eman.encrypt(chunk); } const uint32_t outSize = outChunk.size(); @@ -300,19 +298,11 @@ std::vector CreateCargo::ComputingHeadFiles() cargo.write(reinterpret_cast(outChunk.data()), outChunk.size()); sizeFile += outSize; } - std::cout << "SIZE: " << sizeFile << std::endl; } } f.close(); - //Tworzenie hashu CRC - //const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0); - - //Kompresjia - //std::vector pakBuffer; - //computingBytes(file.parameter, buffer, pakBuffer); - FilesTable ft; ft.nameFile = path; ft.nameLen = path.length(); @@ -321,8 +311,6 @@ std::vector CreateCargo::ComputingHeadFiles() ft.flag = file.parameter; ft.crc = XXH64_digest(xxhState); - //cargo.write(reinterpret_cast(pakBuffer.data()), pakBuffer.size()); - filesTable.push_back(ft); offset += sizeFile; } diff --git a/CreateCargo.h b/CreateCargo.h index 3879779..958dcf2 100644 --- a/CreateCargo.h +++ b/CreateCargo.h @@ -33,7 +33,6 @@ #include #include "DataStruct.h" -#include "ChunkManager.h" #include "EncryptionManager.h" #include "CompressionManager.h" diff --git a/DataStruct.h b/DataStruct.h index af4da80..75f5a79 100644 --- a/DataStruct.h +++ b/DataStruct.h @@ -64,13 +64,13 @@ namespace fl { namespace ds { // Chunki streamowania - inline constexpr int chunk_stream = 268435456; // 256MB + inline constexpr uint32_t chunk_stream = 268435456; // 256MB // Blok chunków - inline constexpr int block_size = 131072; // 128KB + inline constexpr uint32_t block_size = 131072; // 128KB // Maksymalny rozmiar pliku do spakowania - inline constexpr int maxFileSize = 8589934592; // 8GB + inline constexpr uint64_t maxFileSize = 8589934592; // 8GB } // Flagi diff --git a/ExtractCargo.cpp b/ExtractCargo.cpp index 83f53a5..3d1398a 100644 --- a/ExtractCargo.cpp +++ b/ExtractCargo.cpp @@ -110,51 +110,6 @@ bool ExtractCargo::CheckCargoFile() return true; } -//----------------------------------------------------------------------------- -// Sprawdzanie sumy kontrolnej -//----------------------------------------------------------------------------- -bool ExtractCargo::HashValid(const std::vector& data, const uint64_t& crc) -{ - uint64_t actualCrc = XXH64(data.data(), data.size(), 0); - - if (actualCrc != crc) - { - return false; - } - - return true; -} - -//----------------------------------------------------------------------------- -// Magiczna funkcja do dekompresji i deszyfracji danych -//----------------------------------------------------------------------------- -void ExtractCargo::computingBytes(const std::vector& input, std::vector& output, const uint8_t& flag) -{ - ChunkManager cm(eman); - - std::cout << static_cast(flag) << std::endl; - - switch (flag) - { - case FILE_FLAG_COMPRESS: - output = cm.dechunked(input, true, false); - break; - - case FILE_FLAG_ENCRYPT: - output = cm.dechunked(input, false, true); - break; - - case FILE_FLAG_ZIPENC: - output = cm.dechunked(input, true, true); - std::cout << "DENC" << std::endl; - break; - - default: - output = input; - break; - } -} - //----------------------------------------------------------------------------- // Pobieranie nagłówków plików //----------------------------------------------------------------------------- @@ -175,9 +130,6 @@ void ExtractCargo::LoadFilesTable() cargoFile.read(reinterpret_cast(&fhTmp.crc), sizeof(fhTmp.crc)); cargoFile.read(reinterpret_cast(&fhTmp.flag), sizeof(fhTmp.flag)); - std::cout << tablePosition << std::endl; - std::cout << "Size: " << fhTmp.size << std::endl; - filesHeads.push_back(fhTmp); } } @@ -191,25 +143,76 @@ void ExtractCargo::ExtractingFilesFromCargo() { std::filesystem::path dir = cargoFileName.stem() / fh.nameFile; CreateDirections(dir); + + std::cout << dir.string() << std::endl; + std::ofstream file(dir, std::ios::binary); - - std::cout << fh.size << std::endl; cargoFile.seekg(fh.offset); - std::vector buffor(fh.size); - cargoFile.read(buffor.data(), fh.size); + XXH64_reset(xxhState, 0); - std::vector rawBuffor; - computingBytes(buffor, rawBuffor, fh.flag); - - if (!HashValid(rawBuffor, fh.crc)) + // Strumień wyciągający + if (fh.flag == flag::raw) { - std::cerr << fh.nameFile << " Error: Corrupted data integration CRC" << std::endl; + for (uint64_t sc = 0; sc < fh.size; sc += ds::chunk_stream) + { + const uint32_t streamChunk = std::min(ds::chunk_stream, static_cast(fh.size - sc)); + + std::vector buffer(streamChunk); + cargoFile.read(buffer.data(), streamChunk); + XXH64_update(xxhState, buffer.data(), buffer.size()); + file.write(reinterpret_cast(buffer.data()), streamChunk); + } + } + else + { + uint32_t chunkLen; + uint32_t chunkBeforeSize; + uint32_t chunkLastSize; + + cargoFile.read(reinterpret_cast(&chunkLen), sizeof(chunkLen)); + cargoFile.read(reinterpret_cast(&chunkBeforeSize), sizeof(chunkBeforeSize)); + cargoFile.read(reinterpret_cast(&chunkLastSize), sizeof(chunkLastSize)); + + std::vector chunksString; + + // Dekompresja bloków + for (size_t i = 0; i < chunkLen; ++i) + { + // Pobierz rozmiar chunków przed i po skompresowaniem + uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; + + uint32_t chunkZipSize; + cargoFile.read(reinterpret_cast(&chunkZipSize), sizeof(chunkZipSize)); + + // Pobierz blok chunka + std::vector buffer(chunkZipSize); + cargoFile.read(buffer.data(), chunkZipSize); + + std::vector rawBuffer(chunkSize); + if ((fh.flag & flag::zip) == flag::zip) + { + rawBuffer = (fh.flag & flag::enc) == flag::enc ? + cman.decompress(eman.decrypt(buffer), chunkSize) : + cman.decompress(buffer, chunkSize); + } + else + { + rawBuffer = eman.decrypt(buffer); + } + + XXH64_update(xxhState, rawBuffer.data(), rawBuffer.size()); + file.write(reinterpret_cast(rawBuffer.data()), chunkSize); + } } - file.write(reinterpret_cast(rawBuffor.data()), rawBuffor.size()); - file.close(); + + if (XXH64_digest(xxhState) != fh.crc) + { + std::cerr << dir.string() << " Error: Corrupted data integration CRC" << std::endl; + + } } std::cout << "Unpacking complete!" << std::endl; diff --git a/ExtractCargo.h b/ExtractCargo.h index 3cab45c..9fce5f5 100644 --- a/ExtractCargo.h +++ b/ExtractCargo.h @@ -28,11 +28,12 @@ #include #include #include +#include #include #include "DataStruct.h" -#include "ChunkManager.h" #include "EncryptionManager.h" +#include "CompressionManager.h" class ExtractCargo { public: @@ -57,7 +58,7 @@ private: std::ifstream cargoFile; EncryptionManager eman; - + CompressionManager cman; // Sprawdzenie poprawności archiwum bool CheckCargoFile(); @@ -68,13 +69,7 @@ private: // Pobieranie nagłówków plików void LoadFilesTable(); - // Sprawdzanie sumy kontrolnej - bool HashValid(const std::vector&, const uint64_t&); - // Utwórz katalog void CreateDirections(std::filesystem::path); - // Magiczna funkcja do dekompresji i deszyfracji danych - void computingBytes(const std::vector&, std::vector&, const uint8_t&); - }; diff --git a/README.md b/README.md index 20f79de..704bf0a 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ **Visual Studio 2022** -**CMAKE** ---soon-- **=========================================================================================** @@ -36,13 +34,7 @@   \*\*-c - Packing catalog width compressing all files\*\* - \*\*-r - Raw files\*\* - - \*\*-c - Compressing\*\* - - \*\*-e - Encrypted\*\* - - \*\*-s - Compressing and encrypted\*\* + \*\*-p - Packing catalog width out compressing\*\* \*\*-f - Packing catalog width unique config for individual files (look on under section)\*\* @@ -55,6 +47,42 @@ +**-f Parameters instruction** + + + +**Create regular txt file in same directory on catalog width same name.** + + + +**This is a config file width parameters** + + + +**{COMPRESS} - key for compress files list** + + + +**{COMPRESS}** + +**textures/texture.png** + +**\*.png** + + + +**{IGNORE} - key for ignoring files list** + + + +**{IGNORE}** + +**testfile.odt** + +**\*.odt** + + + **=========================================================================================** ### **Components:** diff --git a/voidcmd.vcxproj b/voidcmd.vcxproj index 49f706f..ee82946 100644 --- a/voidcmd.vcxproj +++ b/voidcmd.vcxproj @@ -132,7 +132,6 @@ - @@ -141,7 +140,6 @@ - diff --git a/voidcmd.vcxproj.filters b/voidcmd.vcxproj.filters index 563ca03..4dec7bb 100644 --- a/voidcmd.vcxproj.filters +++ b/voidcmd.vcxproj.filters @@ -27,9 +27,6 @@ Pliki ĹşrĂłdĹ‚owe - - Pliki ĹşrĂłdĹ‚owe - Pliki ĹşrĂłdĹ‚owe @@ -50,9 +47,6 @@ Pliki nagĹ‚Ăłwkowe - - Pliki nagĹ‚Ăłwkowe - Pliki nagĹ‚Ăłwkowe