yanczi/VoidArchive
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'Strumieniowe pakowanie z podziałem na chunki' (#13) from Wywołanie_i/o into master

Browse source 
Reviewed-on: #13
This commit is contained in:
yanczi 2025-12-21 09:30:37 +01:00
commit 93296a1f6e
7 changed files with 107 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ChunkManager.cpp
View file

@ -82,6 +82,10 @@ std::vector<char> ChunkManager::dechunked(const std::vector<char>& zip, const bo
const uint32_t chunkBeforeSize = getIntFromVector<uint32_t>(zip, offset);
const uint32_t chunkLastSize = getIntFromVector<uint32_t>(zip, offset);
std::cout << "Q: " << chunkLen << std::endl;
std::cout << "C: " << chunkBeforeSize << std::endl;
std::cout << "L: " << chunkBeforeSize << std::endl;
std::vector<char> chunksString;
// Dekompresja bloków

99
CreateCargo.cpp
View file

@ -24,10 +24,12 @@ CreateCargo::CreateCargo()
, extension(EXTENSION)
, version(VERSION)
, methodFlags(0)
, xxhState(XXH64_createState())
, offset(0)
, hppKey(false)
{
// TODO Auto-generated constructor stub
XXH64_reset(xxhState, 0);
}
CreateCargo::~CreateCargo() {
@ -241,6 +243,8 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
std::string path = PathToUnixLike(RemoveStartPath(file.path));
std::ifstream f(file.path, std::ios::binary | std::ios::ate);
std::cout << path << std::endl;
//Obliczanie rozmiaru pliku
size_t size = f.tellg();
f.seekg(0, std::ios::beg);
@ -251,32 +255,105 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
}
else
{
XXH64_reset(xxhState, 0);
//Wczytanie pliku do pamiêci
std::vector<char> buffer(size);
f.read(buffer.data(), size);
std::vector<char> buffer(CHUNK_STREAM_256MB);
uint64_t sizeFile = 0;
const uint32_t chunkBlockSize = CHUNK_BLOCK_SIZE;
const uint32_t quantity = (size + chunkBlockSize) / chunkBlockSize;
const uint32_t lastChunkSize = size - (chunkBlockSize * (quantity - 1));
// Jeœli jest ustawiona flaga inna ni¿ RAW
// Dodaj do kontenera konfiguracjê chunków
if (file.parameter != FILE_FLAG_RAW)
{
std::cout << "CHUNK PARAM" << std::endl;
cargo.write(reinterpret_cast<const char*>(&quantity), sizeof(quantity));
cargo.write(reinterpret_cast<const char*>(&chunkBlockSize), sizeof(chunkBlockSize));
cargo.write(reinterpret_cast<const char*>(&lastChunkSize), sizeof(lastChunkSize));
sizeFile = sizeof(quantity) + sizeof(chunkBlockSize) + sizeof(lastChunkSize);
}
// Strumieniowanie danych
while (f.read(buffer.data(), CHUNK_STREAM_256MB) || f.gcount() > 0)
{
const int bufferSize = f.gcount();
buffer.resize(bufferSize);
// Aktualizacja XXH64
XXH64_update(xxhState, buffer.data(), buffer.size());
if (file.parameter == FILE_FLAG_RAW)
{
// Zapisywanie strumienia do kontenera
cargo.write(reinterpret_cast<const char*>(buffer.data()), buffer.size());
sizeFile += bufferSize;
}
else
{
for (uint32_t ofs = 0; ofs < bufferSize; ofs += chunkBlockSize)
{
// Rozmiar chunka
const uint32_t chunkSize = std::min(chunkBlockSize, bufferSize - ofs);
auto begin = buffer.begin() + ofs;
auto end = begin + chunkSize;
// Skopiuj fragment danych do chunka
std::vector<char> chunk(begin, end);
std::vector<char> outChunk;
// Przetwórz chunki i przetwórz
if ((file.parameter & FILE_FLAG_COMPRESS) == FILE_FLAG_COMPRESS)
{
// Zaszyfruj i skompresuj lub tylko skompresuj
outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ?
eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
}
else
{
// Zaszyfruj lub skopiuj
outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ?
eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
}
const uint32_t outSize = outChunk.size();
cargo.write(reinterpret_cast<const char*>(&outSize), sizeof(outSize));
sizeFile += sizeof(outSize);
cargo.write(reinterpret_cast<const char*>(outChunk.data()), outChunk.size());
sizeFile += outSize;
}
std::cout << "SIZE: " << sizeFile << std::endl;
}
}
f.close();
//Tworzenie hashu CRC
const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
//const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
//Kompresjia
std::vector<char> pakBuffer;
computingBytes(file.parameter, buffer, pakBuffer);
std::cout << static_cast<int>(file.parameter) << std::endl;
//std::vector<char> pakBuffer;
//computingBytes(file.parameter, buffer, pakBuffer);
FilesTable ft;
ft.nameFile = path;
ft.nameLen = path.length();
ft.offset = offset;
ft.size = pakBuffer.size();
ft.size = sizeFile;
ft.flag = file.parameter;
ft.crc = crc;
ft.crc = XXH64_digest(xxhState);
cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
//cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
filesTable.push_back(ft);
offset += pakBuffer.size();
offset += sizeFile;
}
}

4
CreateCargo.h
View file

@ -35,6 +35,7 @@
#include "DataStruct.h"
#include "ChunkManager.h"
#include "EncryptionManager.h"
#include "CompressionManager.h"
#define KEY_ZIP "compress" // Pliki do skompresowania
@ -65,6 +66,8 @@ private:
uint8_t methodFlags;
XXH64_state_t* xxhState;
std::string catalogPath;
std::string tempFile;
@ -73,6 +76,7 @@ private:
std::vector<std::string> filesList;
EncryptionManager eman;
CompressionManager cman;
bool hppKey;
// listy wyj¹tków

3
DataStruct.h
View file

@ -34,6 +34,9 @@
#define CHUNK_STREAM_16MB 16777216 // 16MB
#define CHUNK_STREAM_256MB 268435456 // 256MB
// Rozmiar pojedynczego bloku
#define CHUNK_BLOCK_SIZE 131072 // 128KB
#define FILE_FLAG_RAW 0x00
#define FILE_FLAG_COMPRESS 0x0F
#define FILE_FLAG_ENCRYPT 0xF0

14
ExtractCargo.cpp
View file

@ -22,7 +22,6 @@
ExtractCargo::ExtractCargo()
:filesLen(0)
, tablePosition(0)
, filesHeadsOffset(0)
, version(VERSION)
, signature(SIGNATURE)
{
@ -98,18 +97,15 @@ bool ExtractCargo::CheckCargoFile()
}
cargoFile.read(magic.data(), magic.size());
// Pobierz pozycjê tablicy plików i jej rozmiar
cargoFile.read(reinterpret_cast<char*>(&tablePosition), sizeof(tablePosition));
cargoFile.read(reinterpret_cast<char*>(&filesLen), sizeof(filesLen));
if (std::string(magic.begin(), magic.end()) != signature)
{
std::cerr << "Error: Corrupted Cargo" << std::endl;
return false;
}
filesHeadsOffset = signature.length() + sizeof(cargoVer) + sizeof(filesLen);
// Pobierz pozycjê tablicy plików i jej rozmiar
cargoFile.read(reinterpret_cast<char*>(&tablePosition), sizeof(tablePosition));
cargoFile.read(reinterpret_cast<char*>(&filesLen), sizeof(filesLen));
return true;
}
@ -179,6 +175,9 @@ void ExtractCargo::LoadFilesTable()
cargoFile.read(reinterpret_cast<char*>(&fhTmp.crc), sizeof(fhTmp.crc));
cargoFile.read(reinterpret_cast<char*>(&fhTmp.flag), sizeof(fhTmp.flag));
std::cout << tablePosition << std::endl;
std::cout << "Size: " << fhTmp.size << std::endl;
filesHeads.push_back(fhTmp);
}
}
@ -194,6 +193,7 @@ void ExtractCargo::ExtractingFilesFromCargo()
CreateDirections(dir);
std::ofstream file(dir, std::ios::binary);
std::cout << fh.size << std::endl;
cargoFile.seekg(fh.offset);
std::vector<char> buffor(fh.size);

1
ExtractCargo.h
View file

@ -47,7 +47,6 @@ private:
uint32_t filesLen;
uint64_t tablePosition;
int filesHeadsOffset;
const int8_t version;
const std::string signature;

2
voidcmd.vcxproj
View file

@ -103,7 +103,7 @@
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);SODIUM_STATIC</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpp17</LanguageStandard>
<LanguageStandard>stdcpp20</LanguageStandard>
<AdditionalIncludeDirectories>3rd\libsodium\include;3rd\json\include;3rd\zstd\include;3rd\xxhash\include</AdditionalIncludeDirectories>
</ClCompile>
<Link>