wip: change redirect pointer to lookup

src/ZoneCodeGeneratorLib/Domain/Computations/StructureComputations.cpp

@@ -61,3 +61,8 @@ std::vector<MemberInformation*> StructureComputations::GetUsedMembers() const
 
     return members;
 }
+
+bool StructureComputations::IsInTempBlock() const
+{
+    return m_info->m_block != nullptr && m_info->m_block->m_type == FastFileBlockType::TEMP;
+}

src/ZoneCodeGeneratorLib/Domain/Computations/StructureComputations.h

@@ -11,6 +11,7 @@ public:
     [[nodiscard]] MemberInformation* GetDynamicMember() const;
     [[nodiscard]] bool HasNonDynamicMember() const;
     [[nodiscard]] std::vector<MemberInformation*> GetUsedMembers() const;
+    [[nodiscard]] bool IsInTempBlock() const;
 
 private:
     const StructureInformation* m_info;

src/ZoneCodeGeneratorLib/Generating/Templates/ZoneLoadTemplate.cpp

@@ -565,9 +565,10 @@ namespace
                       MakeMemberAccess(&structInfo, &memberInfo, modifier),
                       OffsetForMemberModifier(memberInfo, modifier, nestedBaseOffset))
 
-                if (memberInfo.m_is_reusable || (memberInfo.m_type && StructureComputations(memberInfo.m_type).IsAsset()))
+                if (!StructureComputations(&structInfo).IsInTempBlock()
+                    && (memberInfo.m_is_reusable || (memberInfo.m_type && StructureComputations(memberInfo.m_type).IsAsset())))
                 {
-                    LINEF("fillAccessor.InsertPointerRedirect(m_stream.AllocRedirectEntry(&{0}), {1});",
+                    LINEF("m_stream.AddPointerLookup(&{0}, fillAccessor.BlockBuffer({1}));",
                           MakeMemberAccess(&structInfo, &memberInfo, modifier),
                           OffsetForMemberModifier(memberInfo, modifier, nestedBaseOffset))
                 }
@@ -684,6 +685,12 @@ namespace
 
         void PrintFillStruct_Struct(const StructureInformation& info)
         {
+            if (info.m_reusable_reference_exists)
+            {
+                LINEF("m_stream.AddPointerLookup({0}, fillAccessor.BlockBuffer(0));", MakeTypeVarName(info.m_definition))
+                LINE("")
+            }
+
             const auto* dynamicMember = StructureComputations(&info).GetDynamicMember();
 
             if (dynamicMember)
@@ -976,7 +983,7 @@ namespace
 
                     if (info && !info->m_has_matching_cross_platform_structure)
                     {
-                        LINEF("*{0} = m_stream.ConvertOffsetToPointerRedirect(*{0});", MakeTypePtrVarName(def))
+                        LINEF("*{0} = m_stream.ConvertOffsetToPointerLookup(*{0});", MakeTypePtrVarName(def))
                     }
                     else
                     {
@@ -1019,9 +1026,7 @@ namespace
 
                 if (reusable || (info && StructureComputations(info).IsAsset()))
                 {
-                    LINEF("ptrArrayFill.InsertPointerRedirect(m_stream.AllocRedirectEntry(&{0}[index]), {1} * index);",
+                    LINEF("m_stream.AddPointerLookup(&{0}[index], ptrArrayFill.BlockBuffer({1} * index));", MakeTypePtrVarName(def), m_env.m_pointer_size)
-                          MakeTypePtrVarName(def),
-                          m_env.m_pointer_size)
                 }
 
                 m_intendation--;
@@ -1644,7 +1649,7 @@ namespace
 
                 if (ShouldAllocOutOfBlock(*member, loadType))
                 {
-                    LINEF("{0} = m_stream.ConvertOffsetToPointerRedirect({0});", MakeMemberAccess(info, member, modifier))
+                    LINEF("{0} = m_stream.ConvertOffsetToPointerLookup({0});", MakeMemberAccess(info, member, modifier))
                 }
                 else
                 {

src/ZoneLoading/Game/IW4/ContentLoaderIW4.cpp

@@ -155,7 +155,7 @@ void ContentLoader::LoadXAssetArray(const bool atStreamStart, const size_t count
         {
             fill.Fill(varXAsset[index].type, 8u * index);
             fill.FillPtr(varXAsset[index].header.data, 8u * index + 4u);
-            fill.InsertPointerRedirect(m_stream.AllocRedirectEntry(&varXAsset[index].header.data), 8u * index + 4u);
+            m_stream.AddPointerLookup(&varXAsset[index].header.data, fill.BlockBuffer(8u * index + 4u));
         }
 #endif
     }

src/ZoneLoading/Loading/ContentLoaderBase.cpp

@@ -50,7 +50,10 @@ void ContentLoaderBase::LoadXStringArray(const bool atStreamStart, const size_t
         const auto fill = m_stream.LoadWithFill(4u * count);
 
         for (size_t index = 0; index < count; index++)
+        {
             fill.FillPtr(varXString[index], 4u * index);
+            m_stream.AddPointerLookup(&varXString[index], fill.BlockBuffer(4u * index));
+        }
     }
 
     for (size_t index = 0; index < count; index++)

src/ZoneLoading/Loading/Exception/InvalidAliasLookupException.cpp

@@ -1,19 +0,0 @@
-#include "InvalidAliasLookupException.h"
-
-#include <format>
-
-InvalidAliasLookupException::InvalidAliasLookupException(const size_t lookupIndex, const size_t lookupCount)
-    : m_lookup_index(lookupIndex),
-      m_lookup_count(lookupCount)
-{
-}
-
-std::string InvalidAliasLookupException::DetailedMessage()
-{
-    return std::format("Tried to resolve zone alias lookup {} when there are only {} entries in the lookup", m_lookup_index, m_lookup_count);
-}
-
-char const* InvalidAliasLookupException::what() const noexcept
-{
-    return "Tried to resolve invalid zone alias lookup";
-}

src/ZoneLoading/Loading/Exception/InvalidAliasLookupException.h

@@ -1,16 +0,0 @@
-#pragma once
-#include "LoadingException.h"
-#include "Zone/XBlock.h"
-
-class InvalidAliasLookupException final : public LoadingException
-{
-public:
-    InvalidAliasLookupException(size_t lookupIndex, size_t lookupCount);
-
-    std::string DetailedMessage() override;
-    [[nodiscard]] char const* what() const noexcept override;
-
-private:
-    size_t m_lookup_index;
-    size_t m_lookup_count;
-};

src/ZoneLoading/Zone/Stream/ZoneInputStream.cpp

@@ -1,7 +1,6 @@
 #include "ZoneInputStream.h"
 
 #include "Loading/Exception/BlockOverflowException.h"
-#include "Loading/Exception/InvalidAliasLookupException.h"
 #include "Loading/Exception/InvalidOffsetBlockException.h"
 #include "Loading/Exception/InvalidOffsetBlockOffsetException.h"
 #include "Loading/Exception/OutOfBlockBoundsException.h"
@@ -12,11 +11,10 @@
 #include <iostream>
 #include <sstream>
 #include <stack>
+#include <unordered_map>
 
-ZoneStreamFillReadAccessor::ZoneStreamFillReadAccessor(
+ZoneStreamFillReadAccessor::ZoneStreamFillReadAccessor(void* blockBuffer, const size_t bufferSize, const unsigned pointerByteCount, const size_t offset)
-    const void* dataBuffer, void* blockBuffer, const size_t bufferSize, const unsigned pointerByteCount, const size_t offset)
+    : m_block_buffer(blockBuffer),
-    : m_data_buffer(dataBuffer),
-      m_block_buffer(blockBuffer),
       m_buffer_size(bufferSize),
       m_pointer_byte_count(pointerByteCount),
       m_offset(offset)
@@ -28,11 +26,7 @@ ZoneStreamFillReadAccessor::ZoneStreamFillReadAccessor(
 ZoneStreamFillReadAccessor ZoneStreamFillReadAccessor::AtOffset(const size_t offset) const
 {
     assert(offset <= m_buffer_size);
-    return ZoneStreamFillReadAccessor(static_cast<const char*>(m_data_buffer) + offset,
+    return ZoneStreamFillReadAccessor(static_cast<char*>(m_block_buffer) + offset, m_buffer_size - offset, m_pointer_byte_count, m_offset + offset);
-                                      static_cast<char*>(m_block_buffer) + offset,
-                                      m_buffer_size - offset,
-                                      m_pointer_byte_count,
-                                      m_offset + offset);
 }
 
 size_t ZoneStreamFillReadAccessor::Offset() const
@@ -40,16 +34,9 @@ size_t ZoneStreamFillReadAccessor::Offset() const
     return m_offset;
 }
 
-void ZoneStreamFillReadAccessor::InsertPointerRedirect(const uintptr_t aliasValue, const size_t offset) const
+void* ZoneStreamFillReadAccessor::BlockBuffer(const size_t offset) const
 {
-    // Memory should be zero by default
+    return static_cast<uint8_t*>(m_block_buffer) + offset;
-    if (aliasValue == 0)
-        return;
-
-    assert(offset < m_buffer_size);
-    assert(m_block_buffer);
-
-    std::memcpy(static_cast<char*>(m_block_buffer) + offset, &aliasValue, m_pointer_byte_count);
 }
 
 namespace
@@ -70,7 +57,7 @@ namespace
               m_block_mask((std::numeric_limits<uintptr_t>::max() >> (sizeof(uintptr_t) * 8 - blockBitCount)) << (pointerBitCount - blockBitCount)),
               m_block_shift(pointerBitCount - blockBitCount),
               m_offset_mask(std::numeric_limits<uintptr_t>::max() >> (sizeof(uintptr_t) * 8 - (pointerBitCount - blockBitCount))),
-              m_alias_mask(1uz << (pointerBitCount - 1uz))
+              m_last_fill_size(0)
         {
             assert(pointerBitCount % 8u == 0u);
             assert(insertBlock < static_cast<block_t>(blocks.size()));
@@ -215,8 +202,7 @@ namespace
 
         ZoneStreamFillReadAccessor LoadWithFill(const size_t size) override
         {
-            m_fill_buffer.resize(size);
+            m_last_fill_size = size;
-            auto* dst = m_fill_buffer.data();
 
             // If no block has been pushed, load raw
             if (!m_block_stack.empty())
@@ -224,35 +210,34 @@ namespace
                 const auto* block = m_block_stack.top();
                 auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]];
 
-                LoadDataFromBlock(*block, dst, size);
+                LoadDataFromBlock(*block, blockBufferForFill, size);
 
-                return ZoneStreamFillReadAccessor(dst, blockBufferForFill, size, m_pointer_byte_count, 0);
+                return ZoneStreamFillReadAccessor(blockBufferForFill, size, m_pointer_byte_count, 0);
             }
 
-            m_stream.Load(dst, size);
+            m_fill_buffer.resize(size);
-            return ZoneStreamFillReadAccessor(dst, nullptr, size, m_pointer_byte_count, 0);
+            m_stream.Load(m_fill_buffer.data(), size);
+            return ZoneStreamFillReadAccessor(m_fill_buffer.data(), size, m_pointer_byte_count, 0);
         }
 
         ZoneStreamFillReadAccessor AppendToFill(const size_t appendSize) override
         {
-            const auto appendOffset = m_fill_buffer.size();
+            const auto appendOffset = m_last_fill_size;
-            m_fill_buffer.resize(appendOffset + appendSize);
+            m_last_fill_size += appendSize;
-            auto* dst = m_fill_buffer.data() + appendOffset;
 
-            const auto newTotalSize = appendOffset + appendSize;
             // If no block has been pushed, load raw
             if (!m_block_stack.empty())
             {
                 const auto* block = m_block_stack.top();
                 auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]] - appendOffset;
+                LoadDataFromBlock(*block, &block->m_buffer[m_block_offsets[block->m_index]], appendSize);
 
-                LoadDataFromBlock(*block, dst, appendSize);
+                return ZoneStreamFillReadAccessor(blockBufferForFill, m_last_fill_size, m_pointer_byte_count, 0);
-
-                return ZoneStreamFillReadAccessor(m_fill_buffer.data(), blockBufferForFill, newTotalSize, m_pointer_byte_count, 0);
             }
 
-            m_stream.Load(dst, appendSize);
+            m_fill_buffer.resize(appendOffset + appendSize);
-            return ZoneStreamFillReadAccessor(m_fill_buffer.data(), nullptr, newTotalSize, m_pointer_byte_count, 0);
+            m_stream.Load(m_fill_buffer.data() + appendOffset, appendSize);
+            return ZoneStreamFillReadAccessor(m_fill_buffer.data(), m_last_fill_size, m_pointer_byte_count, 0);
         }
 
         ZoneStreamFillReadAccessor GetLastFill() override
@@ -262,12 +247,13 @@ namespace
             if (!m_block_stack.empty())
             {
                 const auto* block = m_block_stack.top();
-                auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]] - m_fill_buffer.size();
+                auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]] - m_last_fill_size;
 
-                return ZoneStreamFillReadAccessor(m_fill_buffer.data(), blockBufferForFill, m_fill_buffer.size(), m_pointer_byte_count, 0);
+                return ZoneStreamFillReadAccessor(blockBufferForFill, m_last_fill_size, m_pointer_byte_count, 0);
             }
 
-            return ZoneStreamFillReadAccessor(m_fill_buffer.data(), nullptr, m_fill_buffer.size(), m_pointer_byte_count, 0);
+            assert(m_fill_buffer.size() == m_last_fill_size);
+            return ZoneStreamFillReadAccessor(m_fill_buffer.data(), m_last_fill_size, m_pointer_byte_count, 0);
         }
 
         void* InsertPointerNative() override
@@ -293,27 +279,19 @@ namespace
             if (m_block_offsets[m_insert_block->m_index] + m_pointer_byte_count > m_insert_block->m_buffer_size)
                 throw BlockOverflowException(m_insert_block);
 
-            auto* ptr = static_cast<void*>(&m_insert_block->m_buffer[m_block_offsets[m_insert_block->m_index]]);
+            const auto blockNum = m_insert_block->m_index;
+            const auto blockOffset = static_cast<uintptr_t>(m_block_offsets[m_insert_block->m_index]);
+            const auto zonePtr = (static_cast<uintptr_t>(blockNum) << m_block_shift) | (blockOffset & m_offset_mask);
 
             IncBlockPos(*m_insert_block, m_pointer_byte_count);
 
-            const auto newLookupIndex = static_cast<uintptr_t>(m_alias_lookup.size()) | m_alias_mask;
+            return zonePtr;
-            m_alias_lookup.emplace_back(nullptr);
-
-            std::memcpy(ptr, &newLookupIndex, m_pointer_byte_count);
-
-            return newLookupIndex;
         }
 
-        void SetInsertedPointerAliasLookup(const uintptr_t lookupEntry, void* value) override
+        void SetInsertedPointerAliasLookup(const uintptr_t zonePtr, void* value) override
         {
-            assert(lookupEntry & m_alias_mask);
+            assert((static_cast<block_t>((zonePtr & m_block_mask) >> m_block_shift)) < m_blocks.size());
-
+            m_alias_redirect_lookup.emplace(zonePtr, value);
-            const auto aliasIndex = lookupEntry & ~m_alias_mask;
-
-            assert(aliasIndex < m_alias_lookup.size());
-
-            m_alias_lookup[aliasIndex] = value;
         }
 
         void* ConvertOffsetToPointerNative(const void* offset) override
@@ -355,21 +333,23 @@ namespace
             return *reinterpret_cast<void**>(&block->m_buffer[blockOffset]);
         }
 
-        uintptr_t AllocRedirectEntry(void* alias) override
+        void AddPointerLookup(void* alias, const void* blockPtr) override
         {
-            // nullptr is always lookup alias 0
+            assert(!m_block_stack.empty());
-            assert(alias);
+            const auto* block = m_block_stack.top();
-            if (alias == nullptr)
+            assert(blockPtr >= block->m_buffer.get() && blockPtr < block->m_buffer.get() + block->m_buffer_size);
-                return 0;
 
-            const auto newIndex = m_pointer_redirect_lookup.size();
+            // Non-normal blocks cannot be referenced via zone pointer anyway
+            if (block->m_type != XBlockType::BLOCK_TYPE_NORMAL)
+                return;
 
-            m_pointer_redirect_lookup.emplace_back(alias);
+            const auto blockNum = block->m_index;
-
+            const auto blockOffset = static_cast<uintptr_t>(static_cast<const uint8_t*>(blockPtr) - block->m_buffer.get());
-            return static_cast<uintptr_t>(newIndex + 1);
+            const auto zonePtr = (static_cast<uintptr_t>(blockNum) << m_block_shift) | (blockOffset & m_offset_mask);
+            m_pointer_redirect_lookup.emplace(zonePtr, alias);
         }
 
-        void* ConvertOffsetToPointerRedirect(const void* offset) override
+        void* ConvertOffsetToPointerLookup(const void* offset) override
         {
             // For details see ConvertOffsetToPointer
             const auto offsetInt = reinterpret_cast<uintptr_t>(offset) - 1u;
@@ -385,16 +365,12 @@ namespace
             if (block->m_buffer_size <= blockOffset + sizeof(void*))
                 throw InvalidOffsetBlockOffsetException(block, blockOffset);
 
-            uintptr_t lookupEntry = 0u;
+            const auto foundPointerLookup = m_pointer_redirect_lookup.find(offsetInt);
-            std::memcpy(&lookupEntry, &block->m_buffer[blockOffset], m_pointer_byte_count);
+            if (foundPointerLookup != m_pointer_redirect_lookup.end())
+                return foundPointerLookup->second;
 
-            assert(lookupEntry != 0);
+            assert(false);
-            if (lookupEntry == 0)
+            return &block->m_buffer[blockOffset];
-                return nullptr;
-            if (lookupEntry > m_pointer_redirect_lookup.size())
-                throw InvalidAliasLookupException(lookupEntry - 1, m_pointer_redirect_lookup.size());
-
-            return m_pointer_redirect_lookup[lookupEntry - 1];
         }
 
         void* ConvertOffsetToAliasLookup(const void* offset) override
@@ -410,31 +386,19 @@ namespace
 
             auto* block = m_blocks[blockNum];
 
-            if (block->m_buffer_size <= blockOffset + sizeof(void*))
+            if (block->m_buffer_size <= blockOffset + sizeof(uintptr_t))
                 throw InvalidOffsetBlockOffsetException(block, blockOffset);
 
-            uintptr_t lookupEntry = 0u;
+            const auto foundAliasLookup = m_alias_redirect_lookup.find(offsetInt);
-            std::memcpy(&lookupEntry, &block->m_buffer[blockOffset], m_pointer_byte_count);
+            if (foundAliasLookup != m_alias_redirect_lookup.end())
+                return foundAliasLookup->second;
 
-            if (lookupEntry == 0)
+            const auto foundPointerLookup = m_pointer_redirect_lookup.find(offsetInt);
-                return nullptr;
+            if (foundPointerLookup != m_pointer_redirect_lookup.end())
+                return *static_cast<void**>(foundPointerLookup->second);
 
-            if (lookupEntry & m_alias_mask)
+            assert(false);
-            {
+            throw InvalidOffsetBlockOffsetException(block, blockOffset);
-                const auto aliasIndex = lookupEntry & ~m_alias_mask;
-
-                if (aliasIndex >= m_alias_lookup.size())
-                    throw InvalidAliasLookupException(aliasIndex, m_alias_lookup.size());
-
-                return m_alias_lookup[aliasIndex];
-            }
-
-            const auto redirectIndex = lookupEntry - 1;
-
-            if (redirectIndex >= m_pointer_redirect_lookup.size())
-                throw InvalidAliasLookupException(redirectIndex, m_pointer_redirect_lookup.size());
-
-            return *static_cast<void**>(m_pointer_redirect_lookup[redirectIndex]);
         }
 
 #ifdef DEBUG_OFFSETS
@@ -508,9 +472,10 @@ namespace
         XBlock* m_insert_block;
 
         std::vector<uint8_t> m_fill_buffer;
-        std::vector<void*> m_pointer_redirect_lookup;
+        size_t m_last_fill_size;
-        std::vector<void*> m_alias_lookup;
+        // These lookups map a block offset to a pointer in case of a platform mismatch
-        size_t m_alias_mask;
+        std::unordered_map<uintptr_t, void*> m_pointer_redirect_lookup;
+        std::unordered_map<uintptr_t, void*> m_alias_redirect_lookup;
     };
 } // namespace
 

src/ZoneLoading/Zone/Stream/ZoneInputStream.h

@@ -16,23 +16,24 @@
 class ZoneStreamFillReadAccessor
 {
 public:
-    ZoneStreamFillReadAccessor(const void* dataBuffer, void* blockBuffer, size_t bufferSize, unsigned pointerByteCount, size_t offset);
+    ZoneStreamFillReadAccessor(void* blockBuffer, size_t bufferSize, unsigned pointerByteCount, size_t offset);
 
     [[nodiscard]] ZoneStreamFillReadAccessor AtOffset(size_t offset) const;
     [[nodiscard]] size_t Offset() const;
+    [[nodiscard]] void* BlockBuffer(size_t offset) const;
 
     template<typename T> void Fill(T& value, const size_t offset) const
     {
         assert(offset + sizeof(T) <= m_buffer_size);
 
-        value = *reinterpret_cast<const T*>(static_cast<const char*>(m_data_buffer) + offset);
+        value = *reinterpret_cast<const T*>(static_cast<const char*>(m_block_buffer) + offset);
     }
 
     template<typename T, size_t S> void FillArray(T (&value)[S], const size_t offset) const
     {
         assert(offset + sizeof(T) * S <= m_buffer_size);
 
-        std::memcpy(value, static_cast<const char*>(m_data_buffer) + offset, sizeof(T) * S);
+        std::memcpy(value, static_cast<const char*>(m_block_buffer) + offset, sizeof(T) * S);
     }
 
     template<typename T> void FillPtr(T*& value, const size_t offset) const
@@ -41,13 +42,10 @@ public:
         assert(m_pointer_byte_count <= sizeof(uintptr_t));
 
         value = nullptr;
-        std::memcpy(&value, static_cast<const char*>(m_data_buffer) + offset, m_pointer_byte_count);
+        std::memcpy(&value, static_cast<const char*>(m_block_buffer) + offset, m_pointer_byte_count);
     }
 
-    void InsertPointerRedirect(uintptr_t aliasValue, size_t offset) const;
-
 private:
-    const void* m_data_buffer;
     void* m_block_buffer;
     size_t m_buffer_size;
     unsigned m_pointer_byte_count;
@@ -134,7 +132,6 @@ public:
     }
 
     virtual uintptr_t InsertPointerAliasLookup() = 0;
-
     virtual void SetInsertedPointerAliasLookup(uintptr_t lookupEntry, void* value) = 0;
 
     virtual void* ConvertOffsetToPointerNative(const void* offset) = 0;
@@ -151,13 +148,18 @@ public:
         return static_cast<T*>(ConvertOffsetToAliasNative(static_cast<const void*>(offset)));
     }
 
-    virtual uintptr_t AllocRedirectEntry(void* alias) = 0;
+    /**
+     * \brief Adds a lookup from a block pointer to out of block data
+     * \param redirectTo A pointer to the out of block data to redirect to
+     * \param redirectFrom A pointer to the block data to redirect from
+     */
+    virtual void AddPointerLookup(void* redirectTo, const void* redirectFrom) = 0;
 
-    virtual void* ConvertOffsetToPointerRedirect(const void* offset) = 0;
+    virtual void* ConvertOffsetToPointerLookup(const void* offset) = 0;
 
-    template<typename T> T* ConvertOffsetToPointerRedirect(T* offset)
+    template<typename T> T* ConvertOffsetToPointerLookup(T* offset)
     {
-        return static_cast<T*>(ConvertOffsetToPointerRedirect(static_cast<const void*>(offset)));
+        return static_cast<T*>(ConvertOffsetToPointerLookup(static_cast<const void*>(offset)));
     }
 
     virtual void* ConvertOffsetToAliasLookup(const void* offset) = 0;