INSANEMODE/OpenAssetTools
1
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-06-16 01:37:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

wip: change redirect pointer to lookup

Browse Source
This commit is contained in:
Jan
2025-06-13 20:25:33 +01:00
parent de2ad3d9b0
commit 53f46ccc23
9 changed files with 97 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/ZoneCodeGeneratorLib/Domain/Computations/StructureComputations.cpp
View File

@ -61,3 +61,8 @@ std::vector<MemberInformation*> StructureComputations::GetUsedMembers() const
return members;
}
bool StructureComputations::IsInTempBlock() const
{
return m_info->m_block != nullptr && m_info->m_block->m_type == FastFileBlockType::TEMP;
}

1
src/ZoneCodeGeneratorLib/Domain/Computations/StructureComputations.h
View File

@ -11,6 +11,7 @@ public:
[[nodiscard]] MemberInformation* GetDynamicMember() const;
[[nodiscard]] bool HasNonDynamicMember() const;
[[nodiscard]] std::vector<MemberInformation*> GetUsedMembers() const;
[[nodiscard]] bool IsInTempBlock() const;
private:
const StructureInformation* m_info;

19
src/ZoneCodeGeneratorLib/Generating/Templates/ZoneLoadTemplate.cpp
View File

@ -565,9 +565,10 @@ namespace
MakeMemberAccess(&structInfo, &memberInfo, modifier),
OffsetForMemberModifier(memberInfo, modifier, nestedBaseOffset))
if (memberInfo.m_is_reusable || (memberInfo.m_type && StructureComputations(memberInfo.m_type).IsAsset()))
if (!StructureComputations(&structInfo).IsInTempBlock()
&& (memberInfo.m_is_reusable || (memberInfo.m_type && StructureComputations(memberInfo.m_type).IsAsset())))
{
LINEF("fillAccessor.InsertPointerRedirect(m_stream.AllocRedirectEntry(&{0}), {1});",
LINEF("m_stream.AddPointerLookup(&{0}, fillAccessor.BlockBuffer({1}));",
MakeMemberAccess(&structInfo, &memberInfo, modifier),
OffsetForMemberModifier(memberInfo, modifier, nestedBaseOffset))
}
@ -684,6 +685,12 @@ namespace
void PrintFillStruct_Struct(const StructureInformation& info)
{
if (info.m_reusable_reference_exists)
{
LINEF("m_stream.AddPointerLookup({0}, fillAccessor.BlockBuffer(0));", MakeTypeVarName(info.m_definition))
LINE("")
}
const auto* dynamicMember = StructureComputations(&info).GetDynamicMember();
if (dynamicMember)
@ -976,7 +983,7 @@ namespace
if (info && !info->m_has_matching_cross_platform_structure)
{
LINEF("*{0} = m_stream.ConvertOffsetToPointerRedirect(*{0});", MakeTypePtrVarName(def))
LINEF("*{0} = m_stream.ConvertOffsetToPointerLookup(*{0});", MakeTypePtrVarName(def))
}
else
{
@ -1019,9 +1026,7 @@ namespace
if (reusable || (info && StructureComputations(info).IsAsset()))
{
LINEF("ptrArrayFill.InsertPointerRedirect(m_stream.AllocRedirectEntry(&{0}[index]), {1} * index);",
MakeTypePtrVarName(def),
m_env.m_pointer_size)
LINEF("m_stream.AddPointerLookup(&{0}[index], ptrArrayFill.BlockBuffer({1} * index));", MakeTypePtrVarName(def), m_env.m_pointer_size)
}
m_intendation--;
@ -1644,7 +1649,7 @@ namespace
if (ShouldAllocOutOfBlock(*member, loadType))
{
LINEF("{0} = m_stream.ConvertOffsetToPointerRedirect({0});", MakeMemberAccess(info, member, modifier))
LINEF("{0} = m_stream.ConvertOffsetToPointerLookup({0});", MakeMemberAccess(info, member, modifier))
}
else
{

2
src/ZoneLoading/Game/IW4/ContentLoaderIW4.cpp
View File

@ -155,7 +155,7 @@ void ContentLoader::LoadXAssetArray(const bool atStreamStart, const size_t count
{
fill.Fill(varXAsset[index].type, 8u * index);
fill.FillPtr(varXAsset[index].header.data, 8u * index + 4u);
fill.InsertPointerRedirect(m_stream.AllocRedirectEntry(&varXAsset[index].header.data), 8u * index + 4u);
m_stream.AddPointerLookup(&varXAsset[index].header.data, fill.BlockBuffer(8u * index + 4u));
}
#endif
}

3
src/ZoneLoading/Loading/ContentLoaderBase.cpp
View File

@ -50,7 +50,10 @@ void ContentLoaderBase::LoadXStringArray(const bool atStreamStart, const size_t
const auto fill = m_stream.LoadWithFill(4u * count);
for (size_t index = 0; index < count; index++)
{
fill.FillPtr(varXString[index], 4u * index);
m_stream.AddPointerLookup(&varXString[index], fill.BlockBuffer(4u * index));
}
}
for (size_t index = 0; index < count; index++)

19
src/ZoneLoading/Loading/Exception/InvalidAliasLookupException.cpp
View File

@ -1,19 +0,0 @@
#include "InvalidAliasLookupException.h"
#include <format>
InvalidAliasLookupException::InvalidAliasLookupException(const size_t lookupIndex, const size_t lookupCount)
: m_lookup_index(lookupIndex),
m_lookup_count(lookupCount)
{
}
std::string InvalidAliasLookupException::DetailedMessage()
{
return std::format("Tried to resolve zone alias lookup {} when there are only {} entries in the lookup", m_lookup_index, m_lookup_count);
}
char const* InvalidAliasLookupException::what() const noexcept
{
return "Tried to resolve invalid zone alias lookup";
}

16
src/ZoneLoading/Loading/Exception/InvalidAliasLookupException.h
View File

@ -1,16 +0,0 @@
#pragma once
#include "LoadingException.h"
#include "Zone/XBlock.h"
class InvalidAliasLookupException final : public LoadingException
{
public:
InvalidAliasLookupException(size_t lookupIndex, size_t lookupCount);
std::string DetailedMessage() override;
[[nodiscard]] char const* what() const noexcept override;
private:
size_t m_lookup_index;
size_t m_lookup_count;
};

157
src/ZoneLoading/Zone/Stream/ZoneInputStream.cpp
View File

@ -1,7 +1,6 @@
#include "ZoneInputStream.h"
#include "Loading/Exception/BlockOverflowException.h"
#include "Loading/Exception/InvalidAliasLookupException.h"
#include "Loading/Exception/InvalidOffsetBlockException.h"
#include "Loading/Exception/InvalidOffsetBlockOffsetException.h"
#include "Loading/Exception/OutOfBlockBoundsException.h"
@ -12,11 +11,10 @@
#include <iostream>
#include <sstream>
#include <stack>
#include <unordered_map>
ZoneStreamFillReadAccessor::ZoneStreamFillReadAccessor(
const void* dataBuffer, void* blockBuffer, const size_t bufferSize, const unsigned pointerByteCount, const size_t offset)
: m_data_buffer(dataBuffer),
m_block_buffer(blockBuffer),
ZoneStreamFillReadAccessor::ZoneStreamFillReadAccessor(void* blockBuffer, const size_t bufferSize, const unsigned pointerByteCount, const size_t offset)
: m_block_buffer(blockBuffer),
m_buffer_size(bufferSize),
m_pointer_byte_count(pointerByteCount),
m_offset(offset)
@ -28,11 +26,7 @@ ZoneStreamFillReadAccessor::ZoneStreamFillReadAccessor(
ZoneStreamFillReadAccessor ZoneStreamFillReadAccessor::AtOffset(const size_t offset) const
{
assert(offset <= m_buffer_size);
return ZoneStreamFillReadAccessor(static_cast<const char*>(m_data_buffer) + offset,
static_cast<char*>(m_block_buffer) + offset,
m_buffer_size - offset,
m_pointer_byte_count,
m_offset + offset);
return ZoneStreamFillReadAccessor(static_cast<char*>(m_block_buffer) + offset, m_buffer_size - offset, m_pointer_byte_count, m_offset + offset);
}
size_t ZoneStreamFillReadAccessor::Offset() const
@ -40,16 +34,9 @@ size_t ZoneStreamFillReadAccessor::Offset() const
return m_offset;
}
void ZoneStreamFillReadAccessor::InsertPointerRedirect(const uintptr_t aliasValue, const size_t offset) const
void* ZoneStreamFillReadAccessor::BlockBuffer(const size_t offset) const
{
// Memory should be zero by default
if (aliasValue == 0)
return;
assert(offset < m_buffer_size);
assert(m_block_buffer);
std::memcpy(static_cast<char*>(m_block_buffer) + offset, &aliasValue, m_pointer_byte_count);
return static_cast<uint8_t*>(m_block_buffer) + offset;
}
namespace
@ -70,7 +57,7 @@ namespace
m_block_mask((std::numeric_limits<uintptr_t>::max() >> (sizeof(uintptr_t) * 8 - blockBitCount)) << (pointerBitCount - blockBitCount)),
m_block_shift(pointerBitCount - blockBitCount),
m_offset_mask(std::numeric_limits<uintptr_t>::max() >> (sizeof(uintptr_t) * 8 - (pointerBitCount - blockBitCount))),
m_alias_mask(1uz << (pointerBitCount - 1uz))
m_last_fill_size(0)
{
assert(pointerBitCount % 8u == 0u);
assert(insertBlock < static_cast<block_t>(blocks.size()));
@ -215,8 +202,7 @@ namespace
ZoneStreamFillReadAccessor LoadWithFill(const size_t size) override
{
m_fill_buffer.resize(size);
auto* dst = m_fill_buffer.data();
m_last_fill_size = size;
// If no block has been pushed, load raw
if (!m_block_stack.empty())
@ -224,35 +210,34 @@ namespace
const auto* block = m_block_stack.top();
auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]];
LoadDataFromBlock(*block, dst, size);
LoadDataFromBlock(*block, blockBufferForFill, size);
return ZoneStreamFillReadAccessor(dst, blockBufferForFill, size, m_pointer_byte_count, 0);
return ZoneStreamFillReadAccessor(blockBufferForFill, size, m_pointer_byte_count, 0);
}
m_stream.Load(dst, size);
return ZoneStreamFillReadAccessor(dst, nullptr, size, m_pointer_byte_count, 0);
m_fill_buffer.resize(size);
m_stream.Load(m_fill_buffer.data(), size);
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), size, m_pointer_byte_count, 0);
}
ZoneStreamFillReadAccessor AppendToFill(const size_t appendSize) override
{
const auto appendOffset = m_fill_buffer.size();
m_fill_buffer.resize(appendOffset + appendSize);
auto* dst = m_fill_buffer.data() + appendOffset;
const auto appendOffset = m_last_fill_size;
m_last_fill_size += appendSize;
const auto newTotalSize = appendOffset + appendSize;
// If no block has been pushed, load raw
if (!m_block_stack.empty())
{
const auto* block = m_block_stack.top();
auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]] - appendOffset;
LoadDataFromBlock(*block, &block->m_buffer[m_block_offsets[block->m_index]], appendSize);
LoadDataFromBlock(*block, dst, appendSize);
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), blockBufferForFill, newTotalSize, m_pointer_byte_count, 0);
return ZoneStreamFillReadAccessor(blockBufferForFill, m_last_fill_size, m_pointer_byte_count, 0);
}
m_stream.Load(dst, appendSize);
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), nullptr, newTotalSize, m_pointer_byte_count, 0);
m_fill_buffer.resize(appendOffset + appendSize);
m_stream.Load(m_fill_buffer.data() + appendOffset, appendSize);
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), m_last_fill_size, m_pointer_byte_count, 0);
}
ZoneStreamFillReadAccessor GetLastFill() override
@ -262,12 +247,13 @@ namespace
if (!m_block_stack.empty())
{
const auto* block = m_block_stack.top();
auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]] - m_fill_buffer.size();
auto* blockBufferForFill = &block->m_buffer[m_block_offsets[block->m_index]] - m_last_fill_size;
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), blockBufferForFill, m_fill_buffer.size(), m_pointer_byte_count, 0);
return ZoneStreamFillReadAccessor(blockBufferForFill, m_last_fill_size, m_pointer_byte_count, 0);
}
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), nullptr, m_fill_buffer.size(), m_pointer_byte_count, 0);
assert(m_fill_buffer.size() == m_last_fill_size);
return ZoneStreamFillReadAccessor(m_fill_buffer.data(), m_last_fill_size, m_pointer_byte_count, 0);
}
void* InsertPointerNative() override
@ -293,27 +279,19 @@ namespace
if (m_block_offsets[m_insert_block->m_index] + m_pointer_byte_count > m_insert_block->m_buffer_size)
throw BlockOverflowException(m_insert_block);
auto* ptr = static_cast<void*>(&m_insert_block->m_buffer[m_block_offsets[m_insert_block->m_index]]);
const auto blockNum = m_insert_block->m_index;
const auto blockOffset = static_cast<uintptr_t>(m_block_offsets[m_insert_block->m_index]);
const auto zonePtr = (static_cast<uintptr_t>(blockNum) << m_block_shift) | (blockOffset & m_offset_mask);
IncBlockPos(*m_insert_block, m_pointer_byte_count);
const auto newLookupIndex = static_cast<uintptr_t>(m_alias_lookup.size()) | m_alias_mask;
m_alias_lookup.emplace_back(nullptr);
std::memcpy(ptr, &newLookupIndex, m_pointer_byte_count);
return newLookupIndex;
return zonePtr;
}
void SetInsertedPointerAliasLookup(const uintptr_t lookupEntry, void* value) override
void SetInsertedPointerAliasLookup(const uintptr_t zonePtr, void* value) override
{
assert(lookupEntry & m_alias_mask);
const auto aliasIndex = lookupEntry & ~m_alias_mask;
assert(aliasIndex < m_alias_lookup.size());
m_alias_lookup[aliasIndex] = value;
assert((static_cast<block_t>((zonePtr & m_block_mask) >> m_block_shift)) < m_blocks.size());
m_alias_redirect_lookup.emplace(zonePtr, value);
}
void* ConvertOffsetToPointerNative(const void* offset) override
@ -355,21 +333,23 @@ namespace
return *reinterpret_cast<void**>(&block->m_buffer[blockOffset]);
}
uintptr_t AllocRedirectEntry(void* alias) override
void AddPointerLookup(void* alias, const void* blockPtr) override
{
// nullptr is always lookup alias 0
assert(alias);
if (alias == nullptr)
return 0;
assert(!m_block_stack.empty());
const auto* block = m_block_stack.top();
assert(blockPtr >= block->m_buffer.get() && blockPtr < block->m_buffer.get() + block->m_buffer_size);
const auto newIndex = m_pointer_redirect_lookup.size();
// Non-normal blocks cannot be referenced via zone pointer anyway
if (block->m_type != XBlockType::BLOCK_TYPE_NORMAL)
return;
m_pointer_redirect_lookup.emplace_back(alias);
return static_cast<uintptr_t>(newIndex + 1);
const auto blockNum = block->m_index;
const auto blockOffset = static_cast<uintptr_t>(static_cast<const uint8_t*>(blockPtr) - block->m_buffer.get());
const auto zonePtr = (static_cast<uintptr_t>(blockNum) << m_block_shift) | (blockOffset & m_offset_mask);
m_pointer_redirect_lookup.emplace(zonePtr, alias);
}
void* ConvertOffsetToPointerRedirect(const void* offset) override
void* ConvertOffsetToPointerLookup(const void* offset) override
{
// For details see ConvertOffsetToPointer
const auto offsetInt = reinterpret_cast<uintptr_t>(offset) - 1u;
@ -385,16 +365,12 @@ namespace
if (block->m_buffer_size <= blockOffset + sizeof(void*))
throw InvalidOffsetBlockOffsetException(block, blockOffset);
uintptr_t lookupEntry = 0u;
std::memcpy(&lookupEntry, &block->m_buffer[blockOffset], m_pointer_byte_count);
const auto foundPointerLookup = m_pointer_redirect_lookup.find(offsetInt);
if (foundPointerLookup != m_pointer_redirect_lookup.end())
return foundPointerLookup->second;
assert(lookupEntry != 0);
if (lookupEntry == 0)
return nullptr;
if (lookupEntry > m_pointer_redirect_lookup.size())
throw InvalidAliasLookupException(lookupEntry - 1, m_pointer_redirect_lookup.size());
return m_pointer_redirect_lookup[lookupEntry - 1];
assert(false);
return &block->m_buffer[blockOffset];
}
void* ConvertOffsetToAliasLookup(const void* offset) override
@ -410,31 +386,19 @@ namespace
auto* block = m_blocks[blockNum];
if (block->m_buffer_size <= blockOffset + sizeof(void*))
if (block->m_buffer_size <= blockOffset + sizeof(uintptr_t))
throw InvalidOffsetBlockOffsetException(block, blockOffset);
uintptr_t lookupEntry = 0u;
std::memcpy(&lookupEntry, &block->m_buffer[blockOffset], m_pointer_byte_count);
const auto foundAliasLookup = m_alias_redirect_lookup.find(offsetInt);
if (foundAliasLookup != m_alias_redirect_lookup.end())
return foundAliasLookup->second;
if (lookupEntry == 0)
return nullptr;
const auto foundPointerLookup = m_pointer_redirect_lookup.find(offsetInt);
if (foundPointerLookup != m_pointer_redirect_lookup.end())
return *static_cast<void**>(foundPointerLookup->second);
if (lookupEntry & m_alias_mask)
{
const auto aliasIndex = lookupEntry & ~m_alias_mask;
if (aliasIndex >= m_alias_lookup.size())
throw InvalidAliasLookupException(aliasIndex, m_alias_lookup.size());
return m_alias_lookup[aliasIndex];
}
const auto redirectIndex = lookupEntry - 1;
if (redirectIndex >= m_pointer_redirect_lookup.size())
throw InvalidAliasLookupException(redirectIndex, m_pointer_redirect_lookup.size());
return *static_cast<void**>(m_pointer_redirect_lookup[redirectIndex]);
assert(false);
throw InvalidOffsetBlockOffsetException(block, blockOffset);
}
#ifdef DEBUG_OFFSETS
@ -508,9 +472,10 @@ namespace
XBlock* m_insert_block;
std::vector<uint8_t> m_fill_buffer;
std::vector<void*> m_pointer_redirect_lookup;
std::vector<void*> m_alias_lookup;
size_t m_alias_mask;
size_t m_last_fill_size;
// These lookups map a block offset to a pointer in case of a platform mismatch
std::unordered_map<uintptr_t, void*> m_pointer_redirect_lookup;
std::unordered_map<uintptr_t, void*> m_alias_redirect_lookup;
};
} // namespace

26
src/ZoneLoading/Zone/Stream/ZoneInputStream.h
View File

@ -16,23 +16,24 @@
class ZoneStreamFillReadAccessor
{
public:
ZoneStreamFillReadAccessor(const void* dataBuffer, void* blockBuffer, size_t bufferSize, unsigned pointerByteCount, size_t offset);
ZoneStreamFillReadAccessor(void* blockBuffer, size_t bufferSize, unsigned pointerByteCount, size_t offset);
[[nodiscard]] ZoneStreamFillReadAccessor AtOffset(size_t offset) const;
[[nodiscard]] size_t Offset() const;
[[nodiscard]] void* BlockBuffer(size_t offset) const;
template<typename T> void Fill(T& value, const size_t offset) const
{
assert(offset + sizeof(T) <= m_buffer_size);
value = *reinterpret_cast<const T*>(static_cast<const char*>(m_data_buffer) + offset);
value = *reinterpret_cast<const T*>(static_cast<const char*>(m_block_buffer) + offset);
}
template<typename T, size_t S> void FillArray(T (&value)[S], const size_t offset) const
{
assert(offset + sizeof(T) * S <= m_buffer_size);
std::memcpy(value, static_cast<const char*>(m_data_buffer) + offset, sizeof(T) * S);
std::memcpy(value, static_cast<const char*>(m_block_buffer) + offset, sizeof(T) * S);
}
template<typename T> void FillPtr(T*& value, const size_t offset) const
@ -41,13 +42,10 @@ public:
assert(m_pointer_byte_count <= sizeof(uintptr_t));
value = nullptr;
std::memcpy(&value, static_cast<const char*>(m_data_buffer) + offset, m_pointer_byte_count);
std::memcpy(&value, static_cast<const char*>(m_block_buffer) + offset, m_pointer_byte_count);
}
void InsertPointerRedirect(uintptr_t aliasValue, size_t offset) const;
private:
const void* m_data_buffer;
void* m_block_buffer;
size_t m_buffer_size;
unsigned m_pointer_byte_count;
@ -134,7 +132,6 @@ public:
}
virtual uintptr_t InsertPointerAliasLookup() = 0;
virtual void SetInsertedPointerAliasLookup(uintptr_t lookupEntry, void* value) = 0;
virtual void* ConvertOffsetToPointerNative(const void* offset) = 0;
@ -151,13 +148,18 @@ public:
return static_cast<T*>(ConvertOffsetToAliasNative(static_cast<const void*>(offset)));
}
virtual uintptr_t AllocRedirectEntry(void* alias) = 0;
/**
* \brief Adds a lookup from a block pointer to out of block data
* \param redirectTo A pointer to the out of block data to redirect to
* \param redirectFrom A pointer to the block data to redirect from
*/
virtual void AddPointerLookup(void* redirectTo, const void* redirectFrom) = 0;
virtual void* ConvertOffsetToPointerRedirect(const void* offset) = 0;
virtual void* ConvertOffsetToPointerLookup(const void* offset) = 0;
template<typename T> T* ConvertOffsetToPointerRedirect(T* offset)
template<typename T> T* ConvertOffsetToPointerLookup(T* offset)
{
return static_cast<T*>(ConvertOffsetToPointerRedirect(static_cast<const void*>(offset)));
return static_cast<T*>(ConvertOffsetToPointerLookup(static_cast<const void*>(offset)));
}
virtual void* ConvertOffsetToAliasLookup(const void* offset) = 0;