yanczi/VoidArchive
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Strumieniowe pakowanie z podziałem na chunki

Browse source
This commit is contained in:
yanczi 2025-12-21 09:29:07 +01:00
parent 91aaa279ec
commit 022bc0d069
7 changed files with 107 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

99
CreateCargo.cpp
View file

@ -24,10 +24,12 @@ CreateCargo::CreateCargo()
, extension(EXTENSION)
, version(VERSION)
, methodFlags(0)
, xxhState(XXH64_createState())
, offset(0)
, hppKey(false)
{
// TODO Auto-generated constructor stub
XXH64_reset(xxhState, 0);
}
CreateCargo::~CreateCargo() {
@ -241,6 +243,8 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
std::string path = PathToUnixLike(RemoveStartPath(file.path));
std::ifstream f(file.path, std::ios::binary | std::ios::ate);
std::cout << path << std::endl;
//Obliczanie rozmiaru pliku
size_t size = f.tellg();
f.seekg(0, std::ios::beg);
@ -251,32 +255,105 @@ std::vector<FilesTable> CreateCargo::ComputingHeadFiles()
}
else
{
XXH64_reset(xxhState, 0);
//Wczytanie pliku do pamiêci
std::vector<char> buffer(size);
f.read(buffer.data(), size);
std::vector<char> buffer(CHUNK_STREAM_256MB);
uint64_t sizeFile = 0;
const uint32_t chunkBlockSize = CHUNK_BLOCK_SIZE;
const uint32_t quantity = (size + chunkBlockSize) / chunkBlockSize;
const uint32_t lastChunkSize = size - (chunkBlockSize * (quantity - 1));
// Jeœli jest ustawiona flaga inna ni¿ RAW
// Dodaj do kontenera konfiguracjê chunków
if (file.parameter != FILE_FLAG_RAW)
{
std::cout << "CHUNK PARAM" << std::endl;
cargo.write(reinterpret_cast<const char*>(&quantity), sizeof(quantity));
cargo.write(reinterpret_cast<const char*>(&chunkBlockSize), sizeof(chunkBlockSize));
cargo.write(reinterpret_cast<const char*>(&lastChunkSize), sizeof(lastChunkSize));
sizeFile = sizeof(quantity) + sizeof(chunkBlockSize) + sizeof(lastChunkSize);
}
// Strumieniowanie danych
while (f.read(buffer.data(), CHUNK_STREAM_256MB) || f.gcount() > 0)
{
const int bufferSize = f.gcount();
buffer.resize(bufferSize);
// Aktualizacja XXH64
XXH64_update(xxhState, buffer.data(), buffer.size());
if (file.parameter == FILE_FLAG_RAW)
{
// Zapisywanie strumienia do kontenera
cargo.write(reinterpret_cast<const char*>(buffer.data()), buffer.size());
sizeFile += bufferSize;
}
else
{
for (uint32_t ofs = 0; ofs < bufferSize; ofs += chunkBlockSize)
{
// Rozmiar chunka
const uint32_t chunkSize = std::min(chunkBlockSize, bufferSize - ofs);
auto begin = buffer.begin() + ofs;
auto end = begin + chunkSize;
// Skopiuj fragment danych do chunka
std::vector<char> chunk(begin, end);
std::vector<char> outChunk;
// Przetwórz chunki i przetwórz
if ((file.parameter & FILE_FLAG_COMPRESS) == FILE_FLAG_COMPRESS)
{
// Zaszyfruj i skompresuj lub tylko skompresuj
outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ?
eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
}
else
{
// Zaszyfruj lub skopiuj
outChunk = (file.parameter & FILE_FLAG_ENCRYPT) == FILE_FLAG_ENCRYPT ?
eman.encrypt(cman.compress(chunk)) : cman.compress(chunk);
}
const uint32_t outSize = outChunk.size();
cargo.write(reinterpret_cast<const char*>(&outSize), sizeof(outSize));
sizeFile += sizeof(outSize);
cargo.write(reinterpret_cast<const char*>(outChunk.data()), outChunk.size());
sizeFile += outSize;
}
std::cout << "SIZE: " << sizeFile << std::endl;
}
}
f.close();
//Tworzenie hashu CRC
const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
//const uint64_t crc = XXH64(buffer.data(), buffer.size(), 0);
//Kompresjia
std::vector<char> pakBuffer;
computingBytes(file.parameter, buffer, pakBuffer);
std::cout << static_cast<int>(file.parameter) << std::endl;
//std::vector<char> pakBuffer;
//computingBytes(file.parameter, buffer, pakBuffer);
FilesTable ft;
ft.nameFile = path;
ft.nameLen = path.length();
ft.offset = offset;
ft.size = pakBuffer.size();
ft.size = sizeFile;
ft.flag = file.parameter;
ft.crc = crc;
ft.crc = XXH64_digest(xxhState);
cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
//cargo.write(reinterpret_cast<const char*>(pakBuffer.data()), pakBuffer.size());
filesTable.push_back(ft);
offset += pakBuffer.size();
offset += sizeFile;
}
}