Rename ZoneLoader and ZoneWriter components to ZoneLoading and ZoneWriting to make a difference between the executive class and the component class

src/ZoneLoading/Game/T6/ContentLoaderT6.cpp

@@ -0,0 +1,136 @@
+#include "ContentLoaderT6.h"
+#include "Game/T6/T6.h"
+#include "Loading/Exception/UnsupportedAssetTypeException.h"
+
+#include <cassert>
+
+#include "Game/T6/XAssets/rawfile/rawfile_load_db.h"
+
+using namespace T6;
+
+ContentLoaderT6::ContentLoaderT6()
+{
+    // Script String 0 is always empty string
+    m_script_strings.emplace_back("");
+}
+
+void ContentLoaderT6::LoadScriptStringList(ScriptStringList* scriptStringList)
+{
+    m_stream->PushBlock(XFILE_BLOCK_VIRTUAL);
+
+    if(scriptStringList->strings != nullptr)
+    {
+        assert(scriptStringList->strings == PTR_FOLLOWING);
+
+        scriptStringList->strings = m_stream->Alloc<const char*>();
+        LoadXStringArray(scriptStringList->strings, scriptStringList->count, true);
+
+        for(int i = 0; i < scriptStringList->count; i++)
+        {
+            m_script_strings.emplace_back(scriptStringList->strings[i]);
+        }
+    }
+
+    m_stream->PopBlock();
+}
+
+void ContentLoaderT6::LoadXAsset(XAsset* pXAsset, const bool atStreamStart)
+{
+    assert(pXAsset != nullptr);
+
+    if(atStreamStart)
+        m_stream->Load<XAsset>();
+
+    switch(pXAsset->type)
+    {
+    case ASSET_TYPE_RAWFILE:
+        {
+            RawFileLoader rawFileLoader(this, m_zone, m_stream);
+
+            rawFileLoader.LoadRawFilePtr(&pXAsset->header.rawfile);
+            break;
+        }
+
+    default:
+        {
+            throw UnsupportedAssetTypeException(pXAsset->type);
+        }
+    }
+}
+
+void ContentLoaderT6::LoadXAssetArray(XAsset* pArray, const size_t count, const bool atStreamStart)
+{
+    assert(pArray != nullptr);
+
+    if(atStreamStart)
+        m_stream->Load<XAsset>(count);
+
+    size_t assetCounts[ASSET_TYPE_COUNT]{0};
+
+    for(size_t index = 0; index < count; index++)
+    {
+        assert(pArray[index].type >= 0 && pArray[index].type < ASSET_TYPE_COUNT);
+
+        if(pArray[index].type >= 0 && pArray[index].type < ASSET_TYPE_COUNT)
+        {
+            assetCounts[pArray[index].type]++;
+        }
+    }
+
+    // Special case: CLIPMAP and CLIPMAP_PVS are the same struct and therefore share the same asset pool
+    assetCounts[ASSET_TYPE_CLIPMAP_PVS] += assetCounts[ASSET_TYPE_CLIPMAP];
+    assetCounts[ASSET_TYPE_CLIPMAP] = 0;
+
+    for(asset_type_t assetType = 0; assetType < ASSET_TYPE_COUNT; assetType++)
+    {
+        m_zone->GetPools()->InitPoolStatic(assetType, assetCounts[assetType]);
+    }
+
+    for(size_t index = 0; index < count; index++)
+    {
+        LoadXAsset(&pArray[index], false);
+    }
+}
+
+void ContentLoaderT6::Load(Zone* zone, IZoneInputStream* stream)
+{
+    m_zone = zone;
+    m_stream = stream;
+
+    m_stream->PushBlock(XFILE_BLOCK_VIRTUAL);
+
+    auto* assetList = m_stream->Alloc<XAssetList>();
+    m_stream->Load<XAssetList>();
+
+    LoadScriptStringList(&assetList->stringList);
+
+    if(assetList->depends != nullptr)
+    {
+        assert(assetList->depends == PTR_FOLLOWING);
+
+        assetList->depends = m_stream->Alloc<const char*>();
+        LoadXStringArray(assetList->depends, assetList->dependCount, true);
+    }
+
+    if(assetList->assets != nullptr)
+    {
+        assert(assetList->assets == PTR_FOLLOWING);
+
+        assetList->assets = m_stream->Alloc<XAsset>();
+        LoadXAssetArray(assetList->assets, assetList->assetCount, true);
+    }
+
+    m_stream->PopBlock();
+}
+
+std::string& ContentLoaderT6::GetZoneScriptString(const scr_string_t scrString)
+{
+    assert(scrString >= 0 && scrString < m_script_strings.size());
+
+    if(scrString < 0 || scrString >= m_script_strings.size())
+    {
+        return m_script_strings[0];
+    }
+
+    return m_script_strings[scrString];
+}

src/ZoneLoading/Game/T6/ContentLoaderT6.h

@@ -0,0 +1,21 @@
+#pragma once
+#include "Loading/ContentLoader.h"
+#include "Loading/IContentLoadingEntryPoint.h"
+#include "Game/T6/T6.h"
+#include "Loading/IZoneScriptStringProvider.h"
+
+class ContentLoaderT6 final : public ContentLoader, public IContentLoadingEntryPoint, public IZoneScriptStringProvider
+{
+    std::vector<std::string> m_script_strings;
+
+    void LoadScriptStringList(T6::ScriptStringList* scriptStringList);
+
+    void LoadXAsset(T6::XAsset* pXAsset, bool atStreamStart);
+    void LoadXAssetArray(T6::XAsset* pArray, size_t count, bool atStreamStart);
+
+public:
+    ContentLoaderT6();
+
+    void Load(Zone* zone, IZoneInputStream* stream) override;
+    std::string& GetZoneScriptString(scr_string_t scrString) override;
+};

src/ZoneLoading/Game/T6/XAssets/rawfile/rawfile_load_db.cpp

@@ -0,0 +1,70 @@
+#include "rawfile_load_db.h"
+#include <cassert>
+
+using namespace T6;
+
+RawFileLoader::RawFileLoader(IZoneScriptStringProvider* scriptStringProvider, Zone* zone, IZoneInputStream* stream) 
+    : AssetLoader(ASSET_TYPE_RAWFILE, scriptStringProvider, zone, stream){}
+
+void RawFileLoader::LoadRawFile(RawFile* pRawFile, const bool atStreamStart)
+{
+    assert(pRawFile != nullptr);
+
+    if(atStreamStart)
+        m_stream->Load<RawFile>();
+
+    m_stream->PushBlock(XFILE_BLOCK_VIRTUAL);
+
+    LoadXString(&pRawFile->name);
+
+    if(pRawFile->buffer != nullptr)
+    {
+        assert(pRawFile->buffer == PTR_FOLLOWING);
+
+        pRawFile->buffer = m_stream->Alloc<const char>(16);
+        m_stream->Load<const char>(pRawFile->len + 1);
+    }
+
+    m_stream->PopBlock();
+}
+
+void RawFileLoader::LoadRawFileAsset(RawFile** pPtr)
+{
+    assert(pPtr != nullptr);
+    *pPtr = static_cast<RawFile*>(LinkAsset(GetRawFileName(*pPtr), *pPtr));
+}
+
+void RawFileLoader::LoadRawFilePtr(RawFile** pPtr)
+{
+    assert(pPtr != nullptr);
+    
+    m_stream->PushBlock(XFILE_BLOCK_TEMP);
+
+    if(*pPtr != nullptr)
+    {
+        if(*pPtr == PTR_FOLLOWING || *pPtr == PTR_INSERT)
+        {
+            RawFile** toInsert = nullptr;
+            if(*pPtr == PTR_INSERT)
+                toInsert = m_stream->InsertPointer<RawFile>();
+
+            *pPtr = m_stream->Alloc<RawFile>();
+            LoadRawFile(*pPtr, true);
+            LoadRawFileAsset(pPtr);
+
+            if(toInsert != nullptr)
+                *toInsert = *pPtr;
+        }
+        else
+        {
+            *pPtr = m_stream->ConvertOffsetToAlias(*pPtr);
+        }
+    }
+
+    m_stream->PopBlock();
+}
+
+std::string RawFileLoader::GetRawFileName(RawFile* pRawFile)
+{
+    return pRawFile->name;
+}

src/ZoneLoading/Game/T6/XAssets/rawfile/rawfile_load_db.h

@@ -0,0 +1,18 @@
+#pragma once
+#include "Loading/AssetLoader.h"
+#include "Game/T6/T6.h"
+
+namespace T6
+{
+    class RawFileLoader final : public AssetLoader
+    {
+        void LoadRawFile(RawFile* pRawFile, bool atStreamStart);
+        void LoadRawFileAsset(RawFile** pPtr);
+
+    public:
+        RawFileLoader(IZoneScriptStringProvider* scriptStringProvider, Zone* zone, IZoneInputStream* stream);
+
+		void LoadRawFilePtr(RawFile** pPtr);
+        static std::string GetRawFileName(RawFile* pRawFile);
+    };
+}

src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp

@@ -0,0 +1,240 @@
+#include "ZoneLoaderFactoryT6.h"
+#include "Game/T6/T6.h"
+
+#include "Loading/Steps/StepVerifyMagic.h"
+#include "Loading/Steps/StepSkipBytes.h"
+#include "Loading/Steps/StepVerifyFileName.h"
+#include "Loading/Steps/StepLoadSignature.h"
+#include "Loading/Steps/StepVerifySignature.h"
+#include "Loading/Steps/StepAddProcessor.h"
+#include "Loading/Steps/StepAllocXBlocks.h"
+
+#include "Loading/Processor/ProcessorXChunks.h"
+#include "Loading/Processor/XChunks/ChunkProcessorSalsa20.h"
+#include "Loading/Processor/XChunks/ChunkProcessorInflate.h"
+
+#include "Utils/ClassUtils.h"
+#include <cassert>
+#include "Loading/Steps/StepLoadZoneContent.h"
+#include "ContentLoaderT6.h"
+#include "Game/T6/GameAssetPoolT6.h"
+#include "Game/T6/GameT6.h"
+
+const std::string ZoneLoaderFactoryT6::MAGIC_SIGNED_TREYARCH = "TAff0100";
+const std::string ZoneLoaderFactoryT6::MAGIC_SIGNED_ASSET_BUILDER = "ABff0100";
+const std::string ZoneLoaderFactoryT6::MAGIC_UNSIGNED = "TAffu100";
+const int ZoneLoaderFactoryT6::VERSION = 147;
+
+const int ZoneLoaderFactoryT6::STREAM_COUNT = 4;
+const int ZoneLoaderFactoryT6::XCHUNK_SIZE = 0x8000;
+const int ZoneLoaderFactoryT6::OFFSET_BLOCK_BIT_COUNT = 3;
+const block_t ZoneLoaderFactoryT6::INSERT_BLOCK = T6::XFILE_BLOCK_VIRTUAL;
+
+const std::string ZoneLoaderFactoryT6::MAGIC_AUTH_HEADER = "PHEEBs71";
+const uint8_t ZoneLoaderFactoryT6::SALSA20_KEY_TREYARCH[]
+{
+	0x64, 0x1D, 0x8A, 0x2F,
+	0xE3, 0x1D, 0x3A, 0xA6,
+	0x36, 0x22, 0xBB, 0xC9,
+	0xCE, 0x85, 0x87, 0x22,
+	0x9D, 0x42, 0xB0, 0xF8,
+	0xED, 0x9B, 0x92, 0x41,
+	0x30, 0xBF, 0x88, 0xB6,
+	0x5E, 0xDC, 0x50, 0xBE
+};
+
+const uint8_t ZoneLoaderFactoryT6::RSA_PUBLIC_KEY_TREYARCH[]
+{
+    0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01,
+    0x00, 0xc7, 0x9d, 0x33, 0xe0, 0x75, 0xaf, 0xef,
+    0x08, 0x08, 0x2b, 0x89, 0xd9, 0x3b, 0xf3, 0xd5,
+    0x9a, 0x65, 0xa6, 0xde, 0x3b, 0x1e, 0x20, 0xde,
+    0x59, 0x19, 0x43, 0x88, 0x1a, 0x8b, 0x39, 0x13,
+    0x60, 0x12, 0xd3, 0xb2, 0x77, 0x6d, 0xe1, 0x99,
+    0x75, 0x24, 0xb4, 0x0d, 0x8c, 0xb7, 0x84, 0xf2,
+    0x48, 0x8f, 0xd5, 0x4c, 0xb7, 0x64, 0x44, 0xa3,
+    0xa8, 0x4a, 0xac, 0x2d, 0x54, 0x15, 0x2b, 0x1f,
+    0xb3, 0xf4, 0x4c, 0x16, 0xa0, 0x92, 0x8e, 0xd2,
+    0xfa, 0xcc, 0x11, 0x6a, 0x74, 0x6a, 0x70, 0xb8,
+    0xd3, 0x34, 0x6b, 0x39, 0xc6, 0x2a, 0x69, 0xde,
+    0x31, 0x34, 0xdf, 0xe7, 0x8b, 0x7e, 0x17, 0xa3,
+    0x17, 0xd9, 0x5e, 0x88, 0x39, 0x21, 0xf8, 0x7d,
+    0x3c, 0x29, 0x21, 0x6c, 0x0e, 0xf1, 0xb4, 0x09,
+    0x54, 0xe8, 0x20, 0x34, 0x90, 0x2e, 0xb4, 0x1a,
+    0x95, 0x95, 0x90, 0xe5, 0xfb, 0xce, 0xfe, 0x8a,
+    0xbf, 0xea, 0xaf, 0x09, 0x0c, 0x0b, 0x87, 0x22,
+    0xe1, 0xfe, 0x82, 0x6e, 0x91, 0xe8, 0xd1, 0xb6,
+    0x35, 0x03, 0x4f, 0xdb, 0xc1, 0x31, 0xe2, 0xba,
+    0xa0, 0x13, 0xf6, 0xdb, 0x07, 0x9b, 0xcb, 0x99,
+    0xce, 0x9f, 0x49, 0xc4, 0x51, 0x8e, 0xf1, 0x04,
+    0x9b, 0x30, 0xc3, 0x02, 0xff, 0x7b, 0x94, 0xca,
+    0x12, 0x69, 0x1e, 0xdb, 0x2d, 0x3e, 0xbd, 0x48,
+    0x16, 0xe1, 0x72, 0x37, 0xb8, 0x5f, 0x61, 0xfa,
+    0x24, 0x16, 0x3a, 0xde, 0xbf, 0x6a, 0x71, 0x62,
+    0x32, 0xf3, 0xaa, 0x7f, 0x28, 0x3a, 0x0c, 0x27,
+    0xeb, 0xa9, 0x0a, 0x4c, 0x79, 0x88, 0x84, 0xb3,
+    0xe2, 0x52, 0xb9, 0x68, 0x1e, 0x82, 0xcf, 0x67,
+    0x43, 0xf3, 0x68, 0xf7, 0x26, 0x19, 0xaa, 0xdd,
+    0x3f, 0x1e, 0xc6, 0x46, 0x11, 0x9f, 0x24, 0x23,
+    0xa7, 0xb0, 0x1b, 0x79, 0xa7, 0x0c, 0x5a, 0xfe,
+    0x96, 0xf7, 0xe7, 0x88, 0x09, 0xa6, 0x69, 0xe3,
+    0x8b, 0x02, 0x03, 0x01, 0x00, 0x01
+};
+
+class ZoneLoaderFactoryT6::ZoneLoaderFactoryT6Impl
+{
+    static bool CanLoad(ZoneHeader& header, bool* isSecure, bool* isOfficial)
+    {
+        assert(isSecure != nullptr);
+        assert(isOfficial != nullptr);
+
+        if(header.m_version != VERSION)
+        {
+            return false;
+        }
+
+        if(!memcmp(header.m_magic, MAGIC_SIGNED_TREYARCH.c_str(), 8))
+        {
+            *isSecure = true;
+            *isOfficial = true;
+            return true;
+        }
+        
+        if(!memcmp(header.m_magic, MAGIC_SIGNED_ASSET_BUILDER.c_str(), 8))
+        {
+            *isSecure = true;
+            *isOfficial = false;
+            return true;
+        }
+
+        if(!memcmp(header.m_magic, MAGIC_UNSIGNED.c_str(), 8))
+        {
+            *isSecure = false;
+            *isOfficial = true;
+            return true;
+        }
+
+        return false;
+    }
+
+    static void SetupBlock(ZoneLoader* zoneLoader)
+    {
+#define XBLOCK_DEF(name, type) new XBlock(STR(name), name, type)
+
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_TEMP, XBlock::BLOCK_TYPE_TEMP));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_RUNTIME_VIRTUAL, XBlock::BLOCK_TYPE_RUNTIME));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_RUNTIME_PHYSICAL, XBlock::BLOCK_TYPE_RUNTIME));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_DELAY_VIRTUAL, XBlock::BLOCK_TYPE_DELAY));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_DELAY_PHYSICAL, XBlock::BLOCK_TYPE_DELAY));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_VIRTUAL, XBlock::BLOCK_TYPE_NORMAL));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_PHYSICAL, XBlock::BLOCK_TYPE_NORMAL));
+        zoneLoader->AddXBlock(XBLOCK_DEF(T6::XFILE_BLOCK_STREAMER_RESERVE, XBlock::BLOCK_TYPE_NORMAL));
+
+#undef XBLOCK_DEF
+    }
+
+    static IPublicKeyAlgorithm* SetupRSA(const bool isOfficial)
+    {
+        if(isOfficial)
+        {
+            auto* rsa = Crypto::CreateRSA(IPublicKeyAlgorithm::RSA_HASH_SHA256, Crypto::RSA_PADDING_PSS);
+
+            if(!rsa->SetKey(RSA_PUBLIC_KEY_TREYARCH, sizeof(RSA_PUBLIC_KEY_TREYARCH)))
+            {
+                printf("Invalid public key for signature checking\n");
+
+                delete rsa;
+                return nullptr;
+            }
+
+            return rsa;
+        }
+        else
+        {
+            assert(false);
+
+            // TODO: Load custom RSA key here
+            return nullptr;
+        }
+    }
+
+    static ISignatureProvider* AddAuthHeaderSteps(const bool isSecure, ZoneLoader* zoneLoader, std::string& fileName)
+    {
+        // Unsigned zones do not have an auth header
+        if(!isSecure)
+            return nullptr;
+        
+        zoneLoader->AddLoadingStep(new StepVerifyMagic(MAGIC_AUTH_HEADER.c_str()));
+        zoneLoader->AddLoadingStep(new StepSkipBytes(4)); // Loading Flags which are always zero
+        zoneLoader->AddLoadingStep(new StepVerifyFileName(fileName, 32));
+
+        auto* signatureLoadStep = new StepLoadSignature(256);
+        zoneLoader->AddLoadingStep(signatureLoadStep);
+
+        return signatureLoadStep;
+    }
+
+    static ISignatureDataProvider* AddXChunkProcessor(ZoneLoader* zoneLoader, std::string& fileName)
+    {
+        auto* xChunkProcessor = new ProcessorXChunks(STREAM_COUNT, XCHUNK_SIZE);
+
+        // First decrypt the chunks with Salsa20
+        auto* chunkProcessorSalsa20 = new ChunkProcessorSalsa20(STREAM_COUNT, fileName, SALSA20_KEY_TREYARCH, sizeof(SALSA20_KEY_TREYARCH));
+        xChunkProcessor->AddChunkProcessor(chunkProcessorSalsa20);
+
+        // Then decompress the chunks using zlib
+        xChunkProcessor->AddChunkProcessor(new ChunkProcessorInflate());
+        zoneLoader->AddLoadingStep(new StepAddProcessor(xChunkProcessor));
+
+        // The signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm
+        return chunkProcessorSalsa20;
+    }
+
+public:
+    static ZoneLoader* CreateLoaderForHeader(ZoneHeader& header, std::string& fileName)
+    {
+        bool isSecure;
+        bool isOfficial;
+
+        // Check if this file is a supported T6 zone.
+        if(!CanLoad(header, &isSecure, &isOfficial))
+            return nullptr;
+
+        // Create new zone
+        auto* zone = new Zone(fileName, 0, new GameAssetPoolT6(0), &game_t6);
+        
+        // File is supported. Now setup all required steps for loading this file.
+        auto* zoneLoader = new ZoneLoader(zone);
+
+        SetupBlock(zoneLoader);
+
+        // If file is signed setup a RSA instance.
+        IPublicKeyAlgorithm* rsa = isSecure ? SetupRSA(isOfficial) : nullptr;
+
+        // Add steps for loading the auth header which also contain the signature of the zone if it is signed.
+        ISignatureProvider* signatureProvider = AddAuthHeaderSteps(isSecure, zoneLoader, fileName);
+
+        // Setup loading XChunks from the zone from this point on.
+        ISignatureDataProvider* signatureDataProvider = AddXChunkProcessor(zoneLoader, fileName);
+
+        // Start of the XFile struct
+        zoneLoader->AddLoadingStep(new StepSkipBytes(8)); // Skip size and externalSize fields since they are not interesting for us
+        zoneLoader->AddLoadingStep(new StepAllocXBlocks());
+
+        // Start of the zone content
+        zoneLoader->AddLoadingStep(new StepLoadZoneContent(new ContentLoaderT6(), zone, OFFSET_BLOCK_BIT_COUNT, INSERT_BLOCK));
+
+        if(isSecure)
+        {
+            zoneLoader->AddLoadingStep(new StepVerifySignature(rsa, signatureProvider, signatureDataProvider));
+        }
+
+        // Return the fully setup zoneloader
+        return zoneLoader;
+    }
+};
+
+ZoneLoader* ZoneLoaderFactoryT6::CreateLoaderForHeader(ZoneHeader& header, std::string& fileName)
+{
+    return ZoneLoaderFactoryT6Impl::CreateLoaderForHeader(header, fileName);
+}

src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "Loading/IZoneLoaderFactory.h"
+#include <string>
+
+class ZoneLoaderFactoryT6 final : public IZoneLoaderFactory
+{
+    static const std::string MAGIC_SIGNED_TREYARCH;
+    static const std::string MAGIC_SIGNED_ASSET_BUILDER;
+    static const std::string MAGIC_UNSIGNED;
+    static const int VERSION;
+
+    static const int STREAM_COUNT;
+    static const int XCHUNK_SIZE;
+    static const int OFFSET_BLOCK_BIT_COUNT;
+    static const block_t INSERT_BLOCK;
+
+    static const std::string MAGIC_AUTH_HEADER;
+    static const uint8_t SALSA20_KEY_TREYARCH[];
+    static const uint8_t RSA_PUBLIC_KEY_TREYARCH[];
+
+    class ZoneLoaderFactoryT6Impl;
+
+public:
+    ZoneLoader* CreateLoaderForHeader(ZoneHeader& header, std::string& fileName) override;
+};