Add ZoneWriting basis

This commit is contained in:
Jan
2021-03-15 22:36:07 +01:00
parent 301f6e3e7a
commit 9d26c9c927
66 changed files with 1344 additions and 111 deletions

View File

View File

@ -0,0 +1,6 @@
#pragma once
class AssetWriter
{
};

View File

@ -0,0 +1,29 @@
#pragma once
#include "Zone/Zone.h"
#include "Zone/Stream/IZoneOutputStream.h"
class ContentWriterBase
{
protected:
static constexpr void* PTR_FOLLOWING = reinterpret_cast<void*>(-1);
static constexpr void* PTR_INSERT = reinterpret_cast<void*>(-2);
const char** varXString;
Zone* m_zone;
IZoneOutputStream* m_stream;
ContentWriterBase();
ContentWriterBase(Zone* zone, IZoneOutputStream* stream);
void WriteXString(bool atStreamStart) const;
void WriteXStringArray(bool atStreamStart, size_t count);
public:
virtual ~ContentWriterBase() = default;
ContentWriterBase(const ContentWriterBase& other) = default;
ContentWriterBase(ContentWriterBase&& other) noexcept = default;
ContentWriterBase& operator=(const ContentWriterBase& other) = default;
ContentWriterBase& operator=(ContentWriterBase&& other) noexcept = default;
};

View File

@ -0,0 +1,17 @@
#pragma once
#include "Zone/Zone.h"
#include "Zone/Stream/IZoneOutputStream.h"
class IContentWritingEntryPoint
{
public:
IContentWritingEntryPoint() = default;
virtual ~IContentWritingEntryPoint() = default;
IContentWritingEntryPoint(const IContentWritingEntryPoint& other) = default;
IContentWritingEntryPoint(IContentWritingEntryPoint&& other) noexcept = default;
IContentWritingEntryPoint& operator=(const IContentWritingEntryPoint& other) = default;
IContentWritingEntryPoint& operator=(IContentWritingEntryPoint&& other) noexcept = default;
virtual void WriteContent(Zone* zone, IZoneOutputStream* stream) = 0;
};

View File

@ -0,0 +1,19 @@
#pragma once
#include "Writing/ZoneWriter.h"
#include "IWritingStream.h"
class ZoneWriter;
class IWritingStep
{
public:
IWritingStep() = default;
virtual ~IWritingStep() = default;
IWritingStep(const IWritingStep& other) = default;
IWritingStep(IWritingStep&& other) noexcept = default;
IWritingStep& operator=(const IWritingStep& other) = default;
IWritingStep& operator=(IWritingStep&& other) noexcept = default;
virtual void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) = 0;
};

View File

@ -0,0 +1,18 @@
#pragma once
#include <cstdint>
#include <cstddef>
class IWritingStream
{
public:
IWritingStream() = default;
virtual ~IWritingStream() = default;
IWritingStream(const IWritingStream& other) = default;
IWritingStream(IWritingStream&& other) noexcept = default;
IWritingStream& operator=(const IWritingStream& other) = default;
IWritingStream& operator=(IWritingStream&& other) noexcept = default;
virtual void Write(const void* buffer, size_t length) = 0;
virtual int64_t Pos() = 0;
};

View File

@ -0,0 +1,19 @@
#pragma once
#include "Utils/ClassUtils.h"
#include "ZoneWriter.h"
#include "Zone/Zone.h"
class IZoneWriterFactory
{
public:
IZoneWriterFactory() = default;
virtual ~IZoneWriterFactory() = default;
IZoneWriterFactory(const IZoneWriterFactory& other) = default;
IZoneWriterFactory(IZoneWriterFactory&& other) noexcept = default;
IZoneWriterFactory& operator=(const IZoneWriterFactory& other) = default;
IZoneWriterFactory& operator=(IZoneWriterFactory&& other) noexcept = default;
_NODISCARD virtual bool SupportsZone(Zone* zone) const = 0;
_NODISCARD virtual std::unique_ptr<ZoneWriter> CreateWriter(Zone* zone) const = 0;
};

View File

@ -0,0 +1,31 @@
#include "InMemoryZoneData.h"
#include <cassert>
InMemoryZoneData::InMemoryZoneData()
: m_total_size(0)
{
m_buffers.emplace_back(BUFFER_SIZE);
}
InMemoryZoneData::MemoryBuffer::MemoryBuffer(const size_t size)
: m_data(std::make_unique<char[]>(size)),
m_size(0)
{
}
void* InMemoryZoneData::GetBufferOfSize(const size_t size)
{
assert(size <= BUFFER_SIZE);
if(m_buffers.back().m_size + size > BUFFER_SIZE)
{
m_buffers.emplace_back(BUFFER_SIZE);
}
auto& backBuffer = m_buffers.back();
void* result = &backBuffer.m_data[backBuffer.m_size];
backBuffer.m_size += size;
m_total_size += size;
return result;
}

View File

@ -0,0 +1,25 @@
#pragma once
#include <vector>
#include <memory>
class InMemoryZoneData
{
static constexpr size_t BUFFER_SIZE = 0x10000;
public:
class MemoryBuffer
{
public:
std::unique_ptr<char[]> m_data;
size_t m_size;
explicit MemoryBuffer(size_t size);
};
int64_t m_total_size;
std::vector<MemoryBuffer> m_buffers;
InMemoryZoneData();
void* GetBufferOfSize(size_t size);
};

View File

@ -0,0 +1,11 @@
#include "OutputStreamProcessor.h"
OutputStreamProcessor::OutputStreamProcessor()
{
m_base_stream = nullptr;
}
void OutputStreamProcessor::SetBaseStream(IWritingStream* baseStream)
{
m_base_stream = baseStream;
}

View File

@ -0,0 +1,14 @@
#pragma once
#include "IWritingStream.h"
class OutputStreamProcessor : public IWritingStream
{
protected:
IWritingStream* m_base_stream;
public:
OutputStreamProcessor();
void SetBaseStream(IWritingStream* baseStream);
};

View File

@ -0,0 +1,46 @@
#include "OutputProcessorXChunks.h"
class OutputProcessorXChunks::Impl
{
};
OutputProcessorXChunks::OutputProcessorXChunks(int numStreams, size_t xChunkSize)
{
}
OutputProcessorXChunks::OutputProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize)
{
}
OutputProcessorXChunks::~OutputProcessorXChunks()
{
delete m_impl;
m_impl = nullptr;
}
OutputProcessorXChunks::OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept
: m_impl(other.m_impl)
{
other.m_impl = nullptr;
}
OutputProcessorXChunks& OutputProcessorXChunks::operator=(OutputProcessorXChunks&& other) noexcept
{
m_impl = other.m_impl;
other.m_impl = nullptr;
return *this;
}
void OutputProcessorXChunks::AddChunkProcessor(std::unique_ptr<IXChunkOutputProcessor> chunkProcessor) const
{
}
void OutputProcessorXChunks::Write(const void* buffer, size_t length)
{
}
int64_t OutputProcessorXChunks::Pos()
{
return 0;
}

View File

@ -0,0 +1,26 @@
#pragma once
#include <memory>
#include "Writing/OutputStreamProcessor.h"
#include "XChunks/IXChunkOutputProcessor.h"
class OutputProcessorXChunks final : public OutputStreamProcessor
{
class Impl;
Impl* m_impl;
public:
OutputProcessorXChunks(int numStreams, size_t xChunkSize);
OutputProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize);
~OutputProcessorXChunks() override;
OutputProcessorXChunks(const OutputProcessorXChunks& other) = delete;
OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept;
OutputProcessorXChunks& operator=(const OutputProcessorXChunks& other) = delete;
OutputProcessorXChunks& operator=(OutputProcessorXChunks&& other) noexcept;
void AddChunkProcessor(std::unique_ptr<IXChunkOutputProcessor> chunkProcessor) const;
void Write(const void* buffer, size_t length) override;
int64_t Pos() override;
};

View File

@ -0,0 +1,17 @@
#pragma once
#include <cstdint>
#include <cstddef>
class IXChunkOutputProcessor
{
public:
IXChunkOutputProcessor() = default;
virtual ~IXChunkOutputProcessor() = default;
IXChunkOutputProcessor(const IXChunkOutputProcessor& other) = default;
IXChunkOutputProcessor(IXChunkOutputProcessor&& other) noexcept = default;
IXChunkOutputProcessor& operator=(const IXChunkOutputProcessor& other) = default;
IXChunkOutputProcessor& operator=(IXChunkOutputProcessor&& other) noexcept = default;
virtual size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) = 0;
};

View File

@ -0,0 +1,8 @@
#pragma once
#include "IXChunkOutputProcessor.h"
class XChunkOutputProcessorDeflate final : public IXChunkOutputProcessor
{
public:
size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
};

View File

@ -0,0 +1,20 @@
#pragma once
#include "IXChunkOutputProcessor.h"
#include "Utils/ICapturedDataProvider.h"
class XChunkOutputProcessorSalsa20 final : public IXChunkOutputProcessor, public ICapturedDataProvider
{
class Impl;
Impl* m_impl;
public:
XChunkOutputProcessorSalsa20(int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
~XChunkOutputProcessorSalsa20() override;
XChunkOutputProcessorSalsa20(const XChunkOutputProcessorSalsa20& other) = delete;
XChunkOutputProcessorSalsa20(XChunkOutputProcessorSalsa20&& other) noexcept = default;
XChunkOutputProcessorSalsa20& operator=(const XChunkOutputProcessorSalsa20& other) = delete;
XChunkOutputProcessorSalsa20& operator=(XChunkOutputProcessorSalsa20&& other) noexcept = default;
size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
void GetCapturedData(const uint8_t** pCapturedData, size_t* pSize) override;
};

View File

@ -0,0 +1,16 @@
#include "StepAddOutputProcessor.h"
#include <cassert>
StepAddOutputProcessor::StepAddOutputProcessor(std::unique_ptr<OutputStreamProcessor> streamProcessor)
: m_stream_processor(std::move(streamProcessor))
{
}
void StepAddOutputProcessor::PerformStep(ZoneWriter* zoneLoader, IWritingStream* stream)
{
assert(zoneLoader != nullptr);
assert(m_stream_processor != nullptr);
zoneLoader->AddStreamProcessor(std::move(m_stream_processor));
m_stream_processor = nullptr;
}

View File

@ -0,0 +1,15 @@
#pragma once
#include <memory>
#include "Writing/IWritingStep.h"
class StepAddOutputProcessor final : public IWritingStep
{
std::unique_ptr<OutputStreamProcessor> m_stream_processor;
public:
explicit StepAddOutputProcessor(std::unique_ptr<OutputStreamProcessor> streamProcessor);
void PerformStep(ZoneWriter* zoneLoader, IWritingStream* stream) override;
};

View File

@ -0,0 +1,10 @@
#include "StepWriteXBlockSizes.h"
StepWriteXBlockSizes::StepWriteXBlockSizes(Zone* zone)
: m_zone(zone)
{
}
void StepWriteXBlockSizes::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
{
}

View File

@ -0,0 +1,13 @@
#pragma once
#include "Writing/IWritingStep.h"
#include "Zone/Zone.h"
class StepWriteXBlockSizes final : public IWritingStep
{
Zone* m_zone;
public:
explicit StepWriteXBlockSizes(Zone* zone);
void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) override;
};

View File

@ -0,0 +1,20 @@
#include "StepWriteZero.h"
#include <algorithm>
StepWriteZero::StepWriteZero(const size_t count)
: m_count(count)
{
}
void StepWriteZero::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
{
const uint64_t num = 0;
size_t toWrite;
for(auto i = 0u; i < m_count; i += toWrite)
{
toWrite = std::min(sizeof(uint64_t), m_count - i);
stream->Write(&num, toWrite);
}
}

View File

@ -0,0 +1,12 @@
#pragma once
#include "Writing/IWritingStep.h"
class StepWriteZero final : public IWritingStep
{
size_t m_count;
public:
explicit StepWriteZero(size_t count);
void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) override;
};

View File

@ -0,0 +1,10 @@
#include "StepWriteZoneContentToFile.h"
StepWriteZoneContentToFile::StepWriteZoneContentToFile(StepWriteZoneContentToMemory* zoneMemory)
: m_memory(zoneMemory)
{
}
void StepWriteZoneContentToFile::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
{
}

View File

@ -0,0 +1,13 @@
#pragma once
#include "StepWriteZoneContentToMemory.h"
#include "Writing/IWritingStep.h"
class StepWriteZoneContentToFile final : public IWritingStep
{
StepWriteZoneContentToMemory* m_memory;
public:
explicit StepWriteZoneContentToFile(StepWriteZoneContentToMemory* zoneMemory);
void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) override;
};

View File

@ -0,0 +1,27 @@
#include "StepWriteZoneContentToMemory.h"
#include "Zone/Stream/Impl/InMemoryZoneOutputStream.h"
StepWriteZoneContentToMemory::StepWriteZoneContentToMemory(std::unique_ptr<IContentWritingEntryPoint> entryPoint, Zone* zone, int offsetBlockBitCount, block_t insertBlock)
: m_content_loader(std::move(entryPoint)),
m_zone_data(std::make_unique<InMemoryZoneData>()),
m_zone(zone),
m_offset_block_bit_count(offsetBlockBitCount),
m_insert_block(insertBlock)
{
}
void StepWriteZoneContentToMemory::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
{
std::vector<XBlock*> blocks;
for (const auto& block : zoneWriter->m_blocks)
blocks.push_back(block.get());
const auto zoneOutputStream = std::make_unique<InMemoryZoneOutputStream>(m_zone_data.get(), std::move(blocks), m_offset_block_bit_count, m_insert_block);
m_content_loader->WriteContent(m_zone, zoneOutputStream.get());
}
InMemoryZoneData* StepWriteZoneContentToMemory::GetData() const
{
return m_zone_data.get();
}

View File

@ -0,0 +1,23 @@
#pragma once
#include <memory>
#include "Utils/ClassUtils.h"
#include "Writing/IContentWritingEntryPoint.h"
#include "Writing/InMemoryZoneData.h"
#include "Writing/IWritingStep.h"
class StepWriteZoneContentToMemory final : public IWritingStep
{
std::unique_ptr<IContentWritingEntryPoint> m_content_loader;
std::unique_ptr<InMemoryZoneData> m_zone_data;
Zone* m_zone;
int m_offset_block_bit_count;
block_t m_insert_block;
public:
StepWriteZoneContentToMemory(std::unique_ptr<IContentWritingEntryPoint> entryPoint, Zone* zone, int offsetBlockBitCount, block_t insertBlock);
void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) override;
_NODISCARD InMemoryZoneData* GetData() const;
};

View File

@ -0,0 +1,11 @@
#include "StepWriteZoneHeader.h"
StepWriteZoneHeader::StepWriteZoneHeader(const ZoneHeader header)
: m_header(header)
{
}
void StepWriteZoneHeader::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
{
stream->Write(&m_header, sizeof(m_header));
}

View File

@ -0,0 +1,12 @@
#pragma once
#include "Writing/IWritingStep.h"
class StepWriteZoneHeader final : public IWritingStep
{
ZoneHeader m_header;
public:
explicit StepWriteZoneHeader(ZoneHeader header);
void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) override;
};

View File

@ -0,0 +1,15 @@
#include "StepWriteZoneSizes.h"
StepWriteZoneSizes::StepWriteZoneSizes(StepWriteZoneContentToMemory* memory)
: m_memory(memory)
{
}
void StepWriteZoneSizes::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
{
auto totalSize = static_cast<size_t>(m_memory->GetData()->m_total_size);
size_t externalSize = 0;
stream->Write(&totalSize, sizeof(totalSize));
stream->Write(&externalSize, sizeof(externalSize));
}

View File

@ -0,0 +1,13 @@
#pragma once
#include "StepWriteZoneContentToMemory.h"
#include "Writing/IWritingStep.h"
class StepWriteZoneSizes final : public IWritingStep
{
StepWriteZoneContentToMemory* m_memory;
public:
explicit StepWriteZoneSizes(StepWriteZoneContentToMemory* memory);
void PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) override;
};

View File

@ -0,0 +1,16 @@
#include "WritingException.h"
WritingException::WritingException(std::string message)
: m_error_message(std::move(message))
{
}
char const* WritingException::what() const
{
return m_error_message.c_str();
}
const std::string& WritingException::Message() const
{
return m_error_message;
}

View File

@ -0,0 +1,16 @@
#pragma once
#include <string>
#include <exception>
#include "Utils/ClassUtils.h"
class WritingException final : public std::exception
{
std::string m_error_message;
public:
explicit WritingException(std::string message);
_NODISCARD char const* what() const override;
_NODISCARD const std::string& Message() const;
};

View File

@ -0,0 +1,16 @@
#include "WritingFileStream.h"
WritingFileStream::WritingFileStream(std::ostream& stream)
: m_stream(stream)
{
}
void WritingFileStream::Write(const void* buffer, const size_t length)
{
m_stream.write(static_cast<const char*>(buffer), length);
}
int64_t WritingFileStream::Pos()
{
return m_stream.tellp();
}

View File

@ -0,0 +1,15 @@
#pragma once
#include <ostream>
#include "IWritingStream.h"
class WritingFileStream final : public IWritingStream
{
std::ostream& m_stream;
public:
explicit WritingFileStream(std::ostream& stream);
void Write(const void* buffer, size_t length) override;
int64_t Pos() override;
};

View File

@ -0,0 +1,79 @@
#include "ZoneWriter.h"
#include <iostream>
#include "WritingException.h"
#include "WritingFileStream.h"
ZoneWriter::ZoneWriter()
: m_processor_chain_dirty(false)
{
}
IWritingStream* ZoneWriter::BuildWritingChain(IWritingStream* rootStream)
{
auto* currentStream = rootStream;
for (const auto& processor : m_processors)
{
processor->SetBaseStream(currentStream);
currentStream = processor.get();
}
m_processor_chain_dirty = false;
return currentStream;
}
void ZoneWriter::AddXBlock(std::unique_ptr<XBlock> block)
{
m_blocks.emplace_back(std::move(block));
}
void ZoneWriter::AddWritingStep(std::unique_ptr<IWritingStep> step)
{
m_steps.emplace_back(std::move(step));
}
void ZoneWriter::AddStreamProcessor(std::unique_ptr<OutputStreamProcessor> streamProcessor)
{
m_processors.push_back(std::move(streamProcessor));
m_processor_chain_dirty = true;
}
void ZoneWriter::RemoveStreamProcessor(OutputStreamProcessor* streamProcessor)
{
for (auto i = m_processors.begin(); i < m_processors.end(); ++i)
{
if (i->get() == streamProcessor)
{
m_processors.erase(i);
m_processor_chain_dirty = true;
break;
}
}
}
bool ZoneWriter::WriteZone(std::ostream& stream)
{
WritingFileStream fileStream(stream);
auto* endStream = BuildWritingChain(&fileStream);
try
{
for (const auto& step : m_steps)
{
step->PerformStep(this, endStream);
if (m_processor_chain_dirty)
endStream = BuildWritingChain(&fileStream);
}
}
catch (WritingException& e)
{
std::cout << "Writing fastfile failed: " << e.Message() << std::endl;
return false;
}
return true;
}

View File

@ -0,0 +1,33 @@
#pragma once
#include <vector>
#include <memory>
#include "IWritingStep.h"
#include "OutputStreamProcessor.h"
#include "Zone/Zone.h"
class IWritingStep;
class ZoneWriter
{
std::vector<std::unique_ptr<IWritingStep>> m_steps;
std::vector<std::unique_ptr<OutputStreamProcessor>> m_processors;
bool m_processor_chain_dirty;
IWritingStream* BuildWritingChain(IWritingStream* rootStream);
public:
std::vector<std::unique_ptr<XBlock>> m_blocks;
ZoneWriter();
void AddXBlock(std::unique_ptr<XBlock> block);
void AddWritingStep(std::unique_ptr<IWritingStep> step);
void AddStreamProcessor(std::unique_ptr<OutputStreamProcessor> streamProcessor);
void RemoveStreamProcessor(OutputStreamProcessor* streamProcessor);
bool WriteZone(std::ostream& stream);
};