diff --git a/.gitignore b/.gitignore index 9491a2f..92b3625 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,11 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd \ No newline at end of file +FodyWeavers.xsd + +3rd/ +test/ +test2/ +*.pak +expak/ +x64/ \ No newline at end of file diff --git a/CompressingManager.cpp b/CompressingManager.cpp new file mode 100644 index 0000000..86f9886 --- /dev/null +++ b/CompressingManager.cpp @@ -0,0 +1,121 @@ +#include "CompressingManager.h" + +CompressingManager::CompressingManager() +{ } + +CompressingManager::~CompressingManager() +{ } + +//----------------------------------------------------------------------------- +// Kompresja blokowa +// +// Dzielenie vectora na chunki dokładnie po 128KB +// Kompresowanie chunków bez nagłówka +//----------------------------------------------------------------------------- +std::vector CompressingManager::compress(const std::vector& raw) +{ + //std::vector blockSizes; + + // Maksymalny rozmiar chunka + const size_t maxBlockSize = BLOCK_SIZE; + const size_t rawSize = raw.size(); + + uint16_t blockLen = 0; + uint32_t lastChunkRawSize; + std::vector compressedBlocks; + for (size_t offset = 0; offset < rawSize; offset += maxBlockSize) + { + // Rozmiar chunka + const size_t chunkSize = std::min(maxBlockSize, rawSize - offset); + + auto begin = raw.begin() + offset; + auto end = begin + chunkSize; + + // Skopiuj fragment danych do chunka + std::vector chunk(begin, end); + + // Obliczanie rozmiaru skompresowanego bloku + int maxZipChunkSize = LZ4_compressBound(chunkSize); + + // Buffor wyjściowy nadpisany skompresowanymi danymi + std::vector zipChunk(maxZipChunkSize); + + // Kompresja + int zipSize = LZ4_compress_default(chunk.data(), zipChunk.data(), chunkSize, maxZipChunkSize); + + // Zmiana rozmiaru do faktycznego rozmiaru po kompresji + zipChunk.resize(zipSize); + + uint32_t chs = chunk.size(); + uint32_t zch = zipChunk.size(); + + //addIntToVector(compressedBlocks, chs); + lastChunkRawSize = chs; + addIntToVector(compressedBlocks, zch); + compressedBlocks.insert(compressedBlocks.end(), zipChunk.begin(), zipChunk.end()); + + blockLen++; + } + + std::vector zip; + // Wstaw liczbę o ilości bloków do vectora; + // Przekonpwertuj usigned int32 na ciąg znkaów + //uint16_t blockLen = blockSizes .size(); + addIntToVector(zip, blockLen); + addIntToVector(zip, maxBlockSize); + addIntToVector(zip, lastChunkRawSize); + + // Dodaj skompresowane dane + zip.insert(zip.end(), compressedBlocks.begin(), compressedBlocks.end()); + + return zip; +} + +////////////////////////////////////////////////////////////// + +//----------------------------------------------------------------------------- +// Dekompresja blokowa +//----------------------------------------------------------------------------- +std::vector CompressingManager::decompress(const std::vector& zip) +{ + size_t offset = 0; + const uint16_t chunkLen = getIntFromVector(zip, offset); + const uint32_t chunkBeforeSize = getIntFromVector(zip, offset); + const uint32_t chunkLastSize = getIntFromVector(zip, offset); + + std::vector chunksString; + + // Dekompresja bloków + for (size_t i = 0; i < chunkLen; ++i) + { + // Pobierz rozmiar chunków przed i po skompresowaniem + uint32_t chunkSize = i < chunkLen - 1 ? chunkBeforeSize : chunkLastSize; + uint32_t chunkZipSize = getIntFromVector(zip, offset); + + // Pobierz blok chunka + std::vector zipChunk(chunkZipSize); + std::memcpy(zipChunk.data(), zip.data() + offset, chunkZipSize); + offset += chunkZipSize; + + // Zdeklarój pusty chunk + std::vector chunk(chunkSize); + + // Dekompresja chunka + int sizeData = LZ4_decompress_safe(zipChunk.data(), chunk.data(), static_cast(chunkZipSize), static_cast(chunkSize)); + + if (sizeData < 0) + { + throw std::runtime_error("LZ4 Decompressing Error"); + } + + // Dostosowanie rozmiaru vectora po skompresowaniu + chunk.resize(sizeData); + + // Scal chunki + chunksString.insert(chunksString.end(), chunk.begin(), chunk.end()); + } + + std::cout << "FULL: " << chunksString.size() << std::endl; + + return chunksString; +} \ No newline at end of file diff --git a/CompressingManager.h b/CompressingManager.h new file mode 100644 index 0000000..e3fd839 --- /dev/null +++ b/CompressingManager.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#define BLOCK_SIZE 131072 // 128KB + +struct BlockSize +{ + uint32_t raw; + uint32_t zip; +}; + +class CompressingManager +{ +public: + CompressingManager(); + ~CompressingManager(); + + // Kompresja danych + std::vector compress(const std::vector&); + + // Dekompresja + std::vector decompress(const std::vector&); + +private: + std::vector blockSizes; + + // Przekonwertuj zmienną na ciąg na vector + template + void addIntToVector(std::vector& vec, const T& val) + { + size_t tmpSize = vec.size(); + vec.resize(tmpSize + sizeof(val)); + std::memcpy(vec.data() + tmpSize, &val, sizeof(val)); + } + + // Pobierz zmienną z vectora + template + T getIntFromVector(const std::vector& vec, size_t& offset) + { + T tmp{}; + std::memcpy(&tmp, vec.data() + offset, sizeof(tmp)); + offset += sizeof(tmp); + + return tmp; + } +}; + diff --git a/CreateCargo.cpp b/CreateCargo.cpp index 66dd7b4..a505f5f 100644 --- a/CreateCargo.cpp +++ b/CreateCargo.cpp @@ -157,48 +157,38 @@ CargoHead CreateCargo::CreateCargoHead(const uint32_t& filesLen, const uint64_t& return ch; } -//----------------------------------------------------------------------------- -// Kompresowanie -//----------------------------------------------------------------------------- -std::vector CreateCargo::Compressing(const std::vector& raw) -{ - int maxZipSize = LZ4_compressBound(raw.size()); - std::vector zip(maxZipSize); - int zipSize = LZ4_compress_default(raw.data(), zip.data(), raw.size(), maxZipSize); - zip.resize(zipSize); - - return zip; -} - //----------------------------------------------------------------------------- // Sprawdza czy plik znajduje się na liście //----------------------------------------------------------------------------- -bool CreateCargo::CheckFileOnTheList(const std::string& path, std::vector& input, std::vector& output) +uint8_t CreateCargo::CheckFileOnTheList(const std::string& path, std::vector& input, std::vector& output) { //Flaga aktywna sprawdza czy plik jest na liście. Jeśli jest to zwraca surowedane //Przeciwnie kompresuje dane + CompressingManager cm; + if (filteringFlag) { if (FilteringData(path)) { - output = Compressing(input); - return true; + output = cm.compress(input); + return ZIP_FILE; } else { output = std::move(input); - return false; + return RAW_FILE; } } //Flaga aktywna kompresuje dane if (compressingFlag) { - output = Compressing(input); - return true; + + output = cm.compress(input); + return ZIP_FILE; } output = std::move(input); - return false; + return RAW_FILE; } //----------------------------------------------------------------------------- @@ -238,15 +228,15 @@ std::vector CreateCargo::ComputingHeadFiles() //Kompresjia std::vector zip; - bool isZip = CheckFileOnTheList(path, buffor, zip); + uint8_t method = CheckFileOnTheList(path, buffor, zip); FilesTable ft; ft.nameFile = path; ft.nameLen = path.length(); + ft.hashName = fnv32(path); ft.offset = offset; ft.size = zip.size(); - ft.rawSize = size; - ft.isZip = isZip; + ft.isZip = method; ft.crc = crc; cargo.write(reinterpret_cast(zip.data()), zip.size()); @@ -323,6 +313,24 @@ std::string CreateCargo::UpperString(std::string s) { return s; } +//----------------------------------------------------------------------------- +// Wygenerój FNV-1a HASH +//----------------------------------------------------------------------------- +uint32_t CreateCargo::fnv32(const std::string& data) +{ + const uint32_t fnvOffset = 2166136261u; + const uint32_t fnvPrime = 16777619u; + + uint32_t hash = fnvOffset; + for (unsigned char c : data) + { + hash ^= c; + hash *= fnvPrime; + } + + return hash; +} + //----------------------------------------------------------------------------- // Sprawdzanie czy plik znajduje się na liście //----------------------------------------------------------------------------- @@ -412,9 +420,9 @@ bool CreateCargo::WriteCargo() cargo.write(reinterpret_cast(&head.nameLen), sizeof(head.nameLen)); cargo.write(head.nameFile.data(), head.nameLen); + cargo.write(reinterpret_cast(&head.hashName), sizeof(head.hashName)); cargo.write(reinterpret_cast(&head.offset), sizeof(head.offset)); cargo.write(reinterpret_cast(&head.size), sizeof(head.size)); - cargo.write(reinterpret_cast(&head.rawSize), sizeof(head.rawSize)); cargo.write(reinterpret_cast(&head.crc), sizeof(head.crc)); cargo.write(reinterpret_cast(&head.isZip), sizeof(head.isZip)); } diff --git a/CreateCargo.h b/CreateCargo.h index ecb8259..a705d18 100644 --- a/CreateCargo.h +++ b/CreateCargo.h @@ -33,6 +33,7 @@ #include "DataStruct.h" #include "Txtpp.h" #include "xxhash.h" +#include "CompressingManager.h" @@ -98,9 +99,6 @@ private: // Tworzenie nagłowka pliku CargoHead CreateCargoHead(const uint32_t&, const uint64_t&); - // Kompresowanie - std::vector Compressing(const std::vector&); - // Przygotowanie nagłówków i plików std::vector ComputingHeadFiles(); @@ -111,7 +109,7 @@ private: void GetFilters(const std::string&); // Sprawdza czy plik znajduje się na liście - bool CheckFileOnTheList(const std::string&, std::vector&, std::vector&); + uint8_t CheckFileOnTheList(const std::string&, std::vector&, std::vector&); // Kasowanie z listy plików ignorow bool CheckIgnorePath(const std::string&); @@ -122,6 +120,9 @@ private: // Zamień cały ciąg na duże litery std::string UpperString(std::string); + // Wygenerój FNV-1a HASH + uint32_t fnv32(const std::string& data); + // Rozdzielanie paternu od ścieżki void ExtPatternAndPathDetection(const std::vector&, std::vector&, std::vector&); diff --git a/DataStruct.h b/DataStruct.h index ce2541b..b6c64dd 100644 --- a/DataStruct.h +++ b/DataStruct.h @@ -25,24 +25,32 @@ #define EXTENSION "pak" -#define SIGNATURE "VoidFS" +#define SIGNATURE "XPAK" -#define VERSION 11 +#define VERSION 100 //Prgoram title -#define PROGRAM_TITLE "Void Archive Tool" -#define PROGRAM_VERSION "v1.1" +#define PROGRAM_TITLE "eXtendet PAK" +#define PROGRAM_VERSION "v1.2" #define PROGRAM_AUTHOR "Yanczi" -#define PROGRAM_COMPILING "1 September 2025" +#define PROGRAM_COMPILING "28 September 2025" #define PROGRAM_LICENSE "GNU LGPL v3" +//Limity +#define MAX_FILE_SIZE 2147483648 // 2GB +#define MAX_PAK_SIZE 8796093022208 // 8TB +// Metody zapisania pliku +#define RAW_FILE 0 +#define ZIP_FILE 1 +#define CRYPT_FILE 2 +#define CRYPT_ZIP 3 struct CargoHead { std::string signature; - uint8_t version; + uint16_t version; uint32_t files; uint64_t table; }; @@ -51,9 +59,9 @@ struct FilesTable { uint8_t nameLen; std::string nameFile; + uint32_t hashName; uint64_t offset; uint32_t size; - uint32_t rawSize; uint64_t crc; - bool isZip; + uint8_t isZip; }; \ No newline at end of file diff --git a/ExtractCargo.cpp b/ExtractCargo.cpp index 1a55271..bf5e028 100644 --- a/ExtractCargo.cpp +++ b/ExtractCargo.cpp @@ -78,8 +78,8 @@ bool ExtractCargo::Extract(const std::string& cFile) //----------------------------------------------------------------------------- bool ExtractCargo::CheckCargoFile() { - std::vector magic(6); - uint8_t cargoVer = 0; + std::vector magic(signature.size()); + uint16_t cargoVer = 0; if (!cargoFile.is_open()) { @@ -92,6 +92,8 @@ bool ExtractCargo::CheckCargoFile() cargoFile.read(reinterpret_cast(&filesLen), sizeof(filesLen)); cargoFile.read(reinterpret_cast(&tablePosition), sizeof(tablePosition)); + std::cout << std::string(magic.begin(), magic.end()) << std::endl; + if (std::string(magic.begin(), magic.end()) != signature) { std::cerr << "Error: Corrupted Cargo" << std::endl; @@ -109,29 +111,6 @@ bool ExtractCargo::CheckCargoFile() return true; } -//----------------------------------------------------------------------------- -// Dekomprezja danych -//----------------------------------------------------------------------------- -std::vector ExtractCargo::DecompressingData(const std::vector& zip, const uint32_t& estimatedSize) -{ - std::vector unzip(estimatedSize); - - int sizeData = LZ4_decompress_safe( - zip.data(), - unzip.data(), - static_cast(zip.size()), - static_cast(estimatedSize) - ); - - if (sizeData < 0) - { - throw std::runtime_error("LZ4 Decompressing Error"); - } - unzip.resize(sizeData); - - return unzip; -} - //----------------------------------------------------------------------------- // Sprawdzanie sumy kontrolnej //----------------------------------------------------------------------------- @@ -163,9 +142,9 @@ void ExtractCargo::LoadFilesTable() cargoFile.read(nameBuffor.data(), fhTmp.nameLen); fhTmp.nameFile = std::string(nameBuffor.begin(), nameBuffor.end()); + cargoFile.read(reinterpret_cast(&fhTmp.hashName), sizeof(fhTmp.hashName)); cargoFile.read(reinterpret_cast(&fhTmp.offset), sizeof(fhTmp.offset)); cargoFile.read(reinterpret_cast(&fhTmp.size), sizeof(fhTmp.size)); - cargoFile.read(reinterpret_cast(&fhTmp.rawSize), sizeof(fhTmp.rawSize)); cargoFile.read(reinterpret_cast(&fhTmp.crc), sizeof(fhTmp.crc)); cargoFile.read(reinterpret_cast(&fhTmp.isZip), sizeof(fhTmp.isZip)); @@ -178,6 +157,8 @@ void ExtractCargo::LoadFilesTable() //----------------------------------------------------------------------------- void ExtractCargo::ExtractingFilesFromCargo() { + CompressingManager cm; + for (const auto& fh : filesHeads) { std::filesystem::path dir = cargoFileName.stem() / fh.nameFile; @@ -189,7 +170,7 @@ void ExtractCargo::ExtractingFilesFromCargo() cargoFile.read(buffor.data(), fh.size); - std::vector rawBuffor = fh.isZip ? DecompressingData(buffor, fh.rawSize) : buffor; + std::vector rawBuffor = fh.isZip ? cm.decompress(buffor) : buffor; if (!HashValid(rawBuffor, fh.crc)) { diff --git a/ExtractCargo.h b/ExtractCargo.h index 6dea90a..fbb28a6 100644 --- a/ExtractCargo.h +++ b/ExtractCargo.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -30,6 +29,7 @@ #include "DataStruct.h" #include "xxhash.h" +#include "CompressingManager.h" class ExtractCargo { public: @@ -65,9 +65,6 @@ private: // Pobieranie nagłówków plików void LoadFilesTable(); - // Dekomprezja danych - std::vector DecompressingData(const std::vector&, const uint32_t&); - // Sprawdzanie sumy kontrolnej bool HashValid(const std::vector&, const uint64_t&); diff --git a/Txtpp.h b/Txtpp.h index b95e986..70940bb 100644 --- a/Txtpp.h +++ b/Txtpp.h @@ -71,6 +71,32 @@ public: return tmp; } + template + T getValue(const std::string& key, const std::string& val) + { + std::vector tmp; + Parse(key, tmp); + + for (const auto& line : tmp) + { + std::string cleanLine = RemoveSpaces(line); + std::string t; + std::string v; + + bool tv = false; + + for (const char& c : cleanLine) + { + if (c != ":") {tv = true;} + + if (!tv) { t += c; } + else { v += c; } + } + } + + return tmp; + } + private: const char sectionStart = '{'; const char sectionEnd = '}'; diff --git a/ViewCargo.cpp b/ViewCargo.cpp index 87d5378..d2b4349 100644 --- a/ViewCargo.cpp +++ b/ViewCargo.cpp @@ -60,7 +60,7 @@ bool ViewCargo::View(const std::string& path) headElements.push_back(ftxui::text(" Zip ") | ftxui::bold); headElements.push_back(ftxui::text("Nazwa pliku") | ftxui::bold | ftxui::size(ftxui::WIDTH, ftxui::EQUAL, 56)); - headElements.push_back(ftxui::text("CRC") | ftxui::bold | ftxui::size(ftxui::WIDTH, ftxui::EQUAL, 20)); + headElements.push_back(ftxui::text("Hash Name") | ftxui::bold | ftxui::size(ftxui::WIDTH, ftxui::EQUAL, 20)); filesList.push_back(hbox(std::move(headElements))); @@ -80,7 +80,7 @@ bool ViewCargo::View(const std::string& path) bool ViewCargo::CheckCargoFile(const std::string& path) { std::vector magic(signature.length()); - uint8_t cargoVer = 0; + uint16_t cargoVer = 0; std::ifstream cargo(path, std::ios::binary); @@ -139,14 +139,14 @@ void ViewCargo::GetFileList(const std::string& path) cargo.read(nameBuffor.data(), fhTmp.nameLen); fhTmp.nameFile = std::string(nameBuffor.begin(), nameBuffor.end()); + cargo.read(reinterpret_cast(&fhTmp.hashName), sizeof(fhTmp.hashName)); cargo.read(reinterpret_cast(&fhTmp.offset), sizeof(fhTmp.offset)); cargo.read(reinterpret_cast(&fhTmp.size), sizeof(fhTmp.size)); - cargo.read(reinterpret_cast(&fhTmp.rawSize), sizeof(fhTmp.rawSize)); cargo.read(reinterpret_cast(&fhTmp.crc), sizeof(fhTmp.crc)); cargo.read(reinterpret_cast(&fhTmp.isZip), sizeof(fhTmp.isZip)); //Tworzenie wierszy tabeli - CreateTableRow(fhTmp.nameFile, fhTmp.isZip, fhTmp.crc); + CreateTableRow(fhTmp.nameFile, fhTmp.isZip, fhTmp.hashName); } cargo.close(); @@ -155,7 +155,7 @@ void ViewCargo::GetFileList(const std::string& path) //----------------------------------------------------------------------------- // Generowanie wierszy do tabeli //----------------------------------------------------------------------------- -void ViewCargo::CreateTableRow(std::string file, bool zip, uint64_t hash) +void ViewCargo::CreateTableRow(const std::string& file, const uint8_t& zip, const uint32_t& hash) { //Zamiania crc liczbowej na hex string std::stringstream ss; @@ -166,13 +166,13 @@ void ViewCargo::CreateTableRow(std::string file, bool zip, uint64_t hash) ftxui::Element eZip; //Dodawanie check boxa czy plik jest spakowany czy nie - if (zip) + if (zip == 1) { - eZip = ftxui::text(" [x] ") | ftxui::color(ftxui::Color::Green); + eZip = ftxui::text(" [x] ") | ftxui::color(ftxui::Color::Cyan); } else { - eZip = ftxui::text(" [ ] ") | ftxui::color(ftxui::Color::Red); + eZip = ftxui::text(" [ ] ") | ftxui::color(ftxui::Color::White); } //Dodawanie komurek diff --git a/ViewCargo.h b/ViewCargo.h index e00ef0e..23e487e 100644 --- a/ViewCargo.h +++ b/ViewCargo.h @@ -39,7 +39,7 @@ public: private: const std::string signature; - const uint8_t version; + const uint16_t version; uint32_t filesLen; uint64_t tablePos; @@ -48,7 +48,7 @@ private: bool CheckCargoFile(const std::string&); void GetFileList(const std::string&); void RenderList(); - void CreateTableRow(std::string, bool, uint64_t); + void CreateTableRow(const std::string&, const uint8_t&, const uint32_t&); }; diff --git a/voidcmd.sln b/expak.sln similarity index 100% rename from voidcmd.sln rename to expak.sln diff --git a/voidcmd.cpp b/voidcmd.cpp index 11d3f82..7031729 100644 --- a/voidcmd.cpp +++ b/voidcmd.cpp @@ -38,21 +38,30 @@ void RenderHelp() { const std::string HelpTitle = "< Manual >"; const std::string HelpInstruction = - "voidarchive \n" - " \n" - "COMPRESSION -c Pack and compress files from the specified directory \n" - "PACKING -p Pack files from the specified directory \n" - "FILTERING -f Pack the files according to the guidelines given in the .txt\n" - "EXTRACTING -x Extract files from the specified container \n" - "LISTING -ls List files stored in a container \n" - " \n" - " \n" - ".txt \n" - " \n" - "Keys: \n" - " \n" - " {compress} - Compressing files -> /path/data/file.txt *.txt *.* - All files \n" - " {ignore} - Ignoring concrete files -> /path/data/file.txt *.txt \n"; + "pakcmd \n" + " \n" + " -c Pack and compress with LZ4 \n" + " -p Pack files from the specified directory \n" + " -e Pack and encrypted from the specified directory \n" + " -f Pack the files according to the guidelines given in the .txt \n" + " \n" + "Extracting: \n" + " -x Extract files from the specified container \n" + " -ls List files stored in a container \n" + " \n" + " \n" + ".txt \n" + " \n" + "Keys: \n" + " \n" + " {compress} - Compressing files \n" + " {crypt} - Encrypted files with AES256 \n" + " {ignore} - Ignoring concrete files \n" + " \n" + " /path/to/file.ext - Concrete file \n" + " *.ext - All files with concrete extension \n" + " *.* - All files !NOT WORKING WITH {ignore} KEY! \n" + " \n"; Interface tui; tui.TextBorder(HelpTitle, HelpInstruction); diff --git a/voidcmd.vcxproj b/voidcmd.vcxproj index c321cd8..736b318 100644 --- a/voidcmd.vcxproj +++ b/voidcmd.vcxproj @@ -24,6 +24,7 @@ {fc5a6d25-b824-4f4f-86c6-c2cc11d4f02b} voidcmd 10.0 + expak @@ -103,10 +104,13 @@ _DEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp17 + 3rd\lz4\include Console true + 3rd\lz4\lib + liblz4_static.lib @@ -118,13 +122,17 @@ NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp17 + 3rd\lz4\include Console true + 3rd\lz4\lib + liblz4_static.lib + @@ -133,6 +141,7 @@ + diff --git a/voidcmd.vcxproj.filters b/voidcmd.vcxproj.filters index 035a3ea..b2edffa 100644 --- a/voidcmd.vcxproj.filters +++ b/voidcmd.vcxproj.filters @@ -33,6 +33,9 @@ Pliki ĹşrĂłdĹ‚owe + + Pliki ĹşrĂłdĹ‚owe + @@ -56,6 +59,9 @@ Pliki nagĹ‚Ăłwkowe + + Pliki nagĹ‚Ăłwkowe +