From 8ac276f3f63332bb0ac7eac5845ae6480368f683 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Wed, 9 Apr 2025 17:45:37 +0200 Subject: [PATCH 01/12] Added XMODEL_BIN Export --- .gitmodules | 3 + docs/SupportedAssetTypes.md | 10 +- premake5.lua | 2 + src/ObjWriting.lua | 3 +- .../IW3/AssetDumpers/AssetDumperXModel.cpp | 17 + .../IW4/AssetDumpers/AssetDumperXModel.cpp | 18 + src/ObjWriting/ObjWriting.h | 1 + .../XModel/Export/XModelBinWriter.cpp | 414 ++++++++++++++++++ .../XModel/Export/XModelBinWriter.h | 11 + .../XModel/XModelDumper.cpp.template | 19 + src/Unlinker/UnlinkerArgs.cpp | 8 +- src/Utils/Utils/MemoryWriter.cpp | 73 +++ src/Utils/Utils/MemoryWriter.h | 33 ++ thirdparty/README.md | 6 +- thirdparty/lz4 | 1 + thirdparty/lz4.lua | 48 ++ 16 files changed, 659 insertions(+), 8 deletions(-) create mode 100644 src/Utils/Utils/MemoryWriter.cpp create mode 100644 src/Utils/Utils/MemoryWriter.h create mode 160000 thirdparty/lz4 create mode 100644 thirdparty/lz4.lua diff --git a/.gitmodules b/.gitmodules index 7bfede4e..da2074a0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "thirdparty/eigen"] path = thirdparty/eigen url = https://gitlab.com/libeigen/eigen.git +[submodule "thirdparty/lz4"] + path = thirdparty/lz4 + url = git@github.com:lz4/lz4.git diff --git a/docs/SupportedAssetTypes.md b/docs/SupportedAssetTypes.md index a9c9e375..5b0d8f05 100644 --- a/docs/SupportedAssetTypes.md +++ b/docs/SupportedAssetTypes.md @@ -13,7 +13,7 @@ The following section specify which assets are supported to be dumped to disk (u | -------------------- | --------------- | --------------- | ----------------------------------------------------------------- | | PhysPreset | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialTechniqueSet | ❌ | ❌ | | | GfxImage | ✅ | ✅ | | @@ -44,7 +44,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysPreset | ✅ | ✅ | | | PhysCollmap | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialPixelShader | ✅ | ✅ | Shaders are compiled. Only dumps/loads shader bytecode. | | MaterialVertexShader | ✅ | ✅ | Shaders are compiled. Only dumps/loads shader bytecode. | @@ -85,7 +85,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysCollmap | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | | XModelSurfs | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialPixelShader | ❌ | ❌ | | | MaterialVertexShader | ❌ | ❌ | | @@ -130,7 +130,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysConstraints | ❌ | ❌ | | | DestructibleDef | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialTechniqueSet | ❌ | ❌ | | | GfxImage | ✅ | ❌ | A few special image encodings are not yet supported. | @@ -167,7 +167,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysConstraints | ✅ | ✅ | | | DestructibleDef | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ✅ | ✅ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ✅ | ✅ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ⁉️ | ⁉️ | Dumping/Loading is currently possible for materials in their compiled form. There is currently no material pipeline. | | MaterialTechniqueSet | ⁉️ | ❌ | Only dumps compiled shaders. | | GfxImage | ✅ | ✅ | A few special image encodings are not yet supported. | diff --git a/premake5.lua b/premake5.lua index 7a20bd50..cfa67e29 100644 --- a/premake5.lua +++ b/premake5.lua @@ -97,6 +97,7 @@ include "thirdparty/minilzo.lua" include "thirdparty/minizip.lua" include "thirdparty/salsa20.lua" include "thirdparty/zlib.lua" +include "thirdparty/lz4.lua" -- ThirdParty group: All projects that are external dependencies group "ThirdParty" @@ -109,6 +110,7 @@ group "ThirdParty" minizip:project() salsa20:project() zlib:project() + lz4:project() group "" -- ======================== diff --git a/src/ObjWriting.lua b/src/ObjWriting.lua index 186fa9d1..1f715ade 100644 --- a/src/ObjWriting.lua +++ b/src/ObjWriting.lua @@ -21,6 +21,7 @@ function ObjWriting:link(links) links:linkto(ZoneCommon) links:linkto(minizip) links:linkto(libtomcrypt) + links:linkto(lz4) end function ObjWriting:use() @@ -60,5 +61,5 @@ function ObjWriting:project() eigen:include(includes) json:include(includes) libtomcrypt:include(includes) - + lz4:include(includes) end diff --git a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp index cd6789e1..24b4246a 100644 --- a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp @@ -5,6 +5,7 @@ #include "Utils/DistinctMapper.h" #include "Utils/HalfFloat.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -458,6 +459,18 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".XMODEL_EXPORT")); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -495,6 +508,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; diff --git a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp index 19d5ec79..71474a51 100644 --- a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp @@ -5,6 +5,7 @@ #include "Utils/DistinctMapper.h" #include "Utils/HalfFloat.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -446,6 +447,19 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto* modelSurfs = model->lodInfo[lod].modelSurfs; + const auto assetFile = context.OpenAssetFile(std::format("model_export/{}.XMODEL_BIN", modelSurfs->name)); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -484,6 +498,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; diff --git a/src/ObjWriting/ObjWriting.h b/src/ObjWriting/ObjWriting.h index b27beb25..0fe1ec93 100644 --- a/src/ObjWriting/ObjWriting.h +++ b/src/ObjWriting/ObjWriting.h @@ -20,6 +20,7 @@ public: enum class ModelOutputFormat_e { XMODEL_EXPORT, + XMODEL_BIN, OBJ, GLTF, GLB diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index e69de29b..6081923d 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -0,0 +1,414 @@ +#include "XModelBinWriter.h" + +#include "Utils/MemoryWriter.h" + +#pragma warning(push, 0) +#include +#pragma warning(pop) + +#include +#include +#include +#include + +class XModelBinWriterBase : public XModelWriter +{ +protected: + void PrepareVertexMerger(const XModelCommon& xmodel) + { + m_vertex_merger = VertexMerger(xmodel.m_vertices.size()); + + auto vertexOffset = 0u; + for (const auto& vertex : xmodel.m_vertices) + { + XModelVertexBoneWeights weights{0, 0}; + + if (vertexOffset < xmodel.m_vertex_bone_weights.size()) + weights = xmodel.m_vertex_bone_weights[vertexOffset]; + + m_vertex_merger.Add(VertexMergerPos{vertex.coordinates[0], + vertex.coordinates[1], + vertex.coordinates[2], + xmodel.m_bone_weight_data.weights.empty() ? nullptr : &xmodel.m_bone_weight_data.weights[weights.weightOffset], + weights.weightCount}); + + vertexOffset++; + } + } + + void WriteAlignedString(const std::string& string) + { + auto paddingSize = ((string.size() + 1 + 0x3) & 0xFFFFFFFFFFFFFC) - (string.size() + 1); + auto padding = std::make_unique(paddingSize); + + m_writer.WriteNullTerminatedString(string); + + m_writer.Write(reinterpret_cast(padding.get()), paddingSize); + } + + void WriteComment(const std::string& comment) + { + m_writer.Write(0xC355); + WriteAlignedString(comment); + } + + void WriteInt16(const int16_t hash, const int16_t value) + { + m_writer.Write(hash); + m_writer.Write(value); + } + + void WriteUInt16(const int16_t hash, const uint16_t value) + { + m_writer.Write(hash); + m_writer.Write(value); + } + + int16_t ClampFloatToShort(const float value) const + { + return std::clamp(static_cast(32767 * std::clamp(value, -1.0f, 1.0f)), static_cast(-32767), 32767i16); + } + + uint8_t ClampFloatToUByte(const float value) const + { + return std::clamp(static_cast(255 * std::clamp(value, -1.0f, 1.0f)), 0ui8, 255ui8); + } + + void WriteHeader(int16_t version) + { + WriteComment("OpenAssetTools XMODEL_BIN File"); + WriteComment(std::format("Game Origin: {}", m_game_name)); + WriteComment(std::format("Zone Origin: {}", m_zone_name)); + m_writer.Write(0x46C8); // Model Hash + WriteInt16(0x24D1, version); + } + + void WriteBones(const XModelCommon& xmodel) + { + WriteInt16(0x76BA, xmodel.m_bones.size()); + + auto boneNum = 0; + for (const auto& bone : xmodel.m_bones) + { + m_writer.Write(0xF099); // Bone Hash + m_writer.Write(boneNum); + if (bone.parentIndex) + m_writer.Write(static_cast(*bone.parentIndex)); + else + m_writer.Write(-1); + + WriteAlignedString(bone.name); + boneNum++; + } + + boneNum = 0; + for (const auto& bone : xmodel.m_bones) + { + WriteInt16(0xDD9A, boneNum); + + m_writer.Write(0x9383); // Bone Offset Hash + m_writer.Write(bone.globalOffset[0]); // X + m_writer.Write(bone.globalOffset[1]); // Y + m_writer.Write(bone.globalOffset[2]); // Z + + m_writer.Write(0x1C56); // Bone Scale Hash + m_writer.Write(bone.scale[0]); // X + m_writer.Write(bone.scale[1]); // Y + m_writer.Write(bone.scale[2]); // Z + + const auto mat = Eigen::Quaternionf(bone.globalRotation.w, bone.globalRotation.x, bone.globalRotation.y, bone.globalRotation.z).matrix(); + + m_writer.Write(0xDCFDi16); // Bone Matrix X Hash + m_writer.Write(ClampFloatToShort(mat(0, 0))); + m_writer.Write(ClampFloatToShort(mat(0, 1))); + m_writer.Write(ClampFloatToShort(mat(0, 2))); + + m_writer.Write(0xCCDCi16); // Bone Matrix Y Hash + m_writer.Write(ClampFloatToShort(mat(1, 0))); + m_writer.Write(ClampFloatToShort(mat(1, 1))); + m_writer.Write(ClampFloatToShort(mat(1, 2))); + + m_writer.Write(0xFCBFi16); // Bone Matrix Z Hash + m_writer.Write(ClampFloatToShort(mat(2, 0))); + m_writer.Write(ClampFloatToShort(mat(2, 1))); + m_writer.Write(ClampFloatToShort(mat(2, 2))); + + boneNum++; + } + } + + XModelBinWriterBase(std::ostream& stream, std::string gameName, std::string zoneName) + : m_writer(0x800000), // 8MB + m_stream(stream), + m_game_name(std::move(gameName)), + m_zone_name(std::move(zoneName)) + { + } + + MemoryWriter m_writer; + std::ostream& m_stream; + std::string m_game_name; + std::string m_zone_name; + VertexMerger m_vertex_merger; +}; + +class XModelBinWriter7 final : public XModelBinWriterBase +{ + void WriteVertices(const XModelCommon& xmodel) + { + const auto& distinctVertexValues = m_vertex_merger.GetDistinctValues(); + + if (distinctVertexValues.size() > UINT16_MAX) + { + // Use 32 bit + m_writer.Write(0x2AEC); // 32 bit Hash + m_writer.Write(static_cast(distinctVertexValues.size())); + } + else + { + // Use 16 bit + WriteUInt16(0x950D, distinctVertexValues.size()); + } + + size_t vertexNum = 0u; + for (const auto& vertexPos : distinctVertexValues) + { + if (vertexNum > UINT16_MAX) + { + // Use 32 bit + m_writer.Write(0xB097); // 32 bit Hash + m_writer.Write(static_cast(vertexNum)); + } + else + { + // Use 16 bit + WriteUInt16(0x8F03, vertexNum); + } + + m_writer.Write(0x9383); // Offset Hash + m_writer.Write(vertexPos.x); + m_writer.Write(vertexPos.y); + m_writer.Write(vertexPos.z); + + WriteInt16(0xEA46, vertexPos.weightCount); + + for (auto weightIndex = 0u; weightIndex < vertexPos.weightCount; weightIndex++) + { + const auto& weight = vertexPos.weights[weightIndex]; + + WriteInt16(0xF1AB, weight.boneIndex); + m_writer.Write(weight.weight); + } + vertexNum++; + } + } + + void WriteFaceVertex(const XModelVertex& vertex) + { + m_writer.Write(0x89ECi16); // Normal Hash + m_writer.Write(ClampFloatToShort(vertex.normal[0])); // X + m_writer.Write(ClampFloatToShort(vertex.normal[1])); // Y + m_writer.Write(ClampFloatToShort(vertex.normal[2])); // Z + + m_writer.Write(0x6DD8); // Color Hash + m_writer.Write(ClampFloatToUByte(vertex.color[0])); // R + m_writer.Write(ClampFloatToUByte(vertex.color[1])); // G + m_writer.Write(ClampFloatToUByte(vertex.color[2])); // B + m_writer.Write(ClampFloatToUByte(vertex.color[3])); // A + + m_writer.Write(0x1AD4i16); // UV Hash + m_writer.Write(1ui16); // Layer + m_writer.Write(vertex.uv[0]); + m_writer.Write(vertex.uv[1]); + } + + void WriteFaces(const XModelCommon& xmodel) + { + auto totalFaceCount = 0u; + for (const auto& object : xmodel.m_objects) + totalFaceCount += object.m_faces.size(); + + m_writer.Write(0xBE92); // Face Count Hash + m_writer.Write(totalFaceCount); + + auto objectIndex = 0u; + for (const auto& object : xmodel.m_objects) + { + for (const auto& face : object.m_faces) + { + const size_t distinctPositions[3]{ + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[0]), + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[1]), + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[2]), + }; + + const XModelVertex& v0 = xmodel.m_vertices[face.vertexIndex[0]]; + const XModelVertex& v1 = xmodel.m_vertices[face.vertexIndex[1]]; + const XModelVertex& v2 = xmodel.m_vertices[face.vertexIndex[2]]; + + m_writer.Write(0x562Fi16); // Face Info Hash + m_writer.Write(static_cast(objectIndex)); + m_writer.Write(static_cast(object.materialIndex)); + + if (m_vertex_merger.GetDistinctValues().size() > UINT16_MAX) + { + m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(static_cast(distinctPositions[0])); + WriteFaceVertex(v0); + + m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(static_cast(distinctPositions[1])); + WriteFaceVertex(v1); + + m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(static_cast(distinctPositions[2])); + WriteFaceVertex(v2); + } + else + { + WriteUInt16(0x8F03, distinctPositions[0]); + WriteFaceVertex(v0); + + WriteUInt16(0x8F03, distinctPositions[1]); + WriteFaceVertex(v1); + + WriteUInt16(0x8F03, distinctPositions[2]); + WriteFaceVertex(v2); + } + } + + objectIndex++; + } + } + + void WriteObjects(const XModelCommon& xmodel) + { + WriteInt16(0x62AF, xmodel.m_objects.size()); + + size_t objectNum = 0; + for (const auto& object : xmodel.m_objects) + { + m_writer.Write(0x87D4i16); // Object Info Hash + m_writer.Write(static_cast(objectNum)); + WriteAlignedString(object.name); + + objectNum++; + } + } + + void WriteMaterials(const XModelCommon& xmodel) + { + WriteInt16(0xA1B2, xmodel.m_materials.size()); + + size_t materialNum = 0u; + for (const auto& material : xmodel.m_materials) + { + const auto colorMapPath = "../images/" + material.colorMapName + ".dds"; + + WriteInt16(0xA700, materialNum); + WriteAlignedString(material.name); + WriteAlignedString(material.materialTypeName); + WriteAlignedString(colorMapPath); + + m_writer.Write(0x6DD8); // Vertex Color Hash + m_writer.Write(ClampFloatToUByte(material.color[0])); // R + m_writer.Write(ClampFloatToUByte(material.color[1])); // G + m_writer.Write(ClampFloatToUByte(material.color[2])); // B + m_writer.Write(ClampFloatToUByte(material.color[3])); // A + + m_writer.Write(0x6DAB); // Transparancy Hash + m_writer.Write(material.transparency[0]); + m_writer.Write(material.transparency[1]); + m_writer.Write(material.transparency[2]); + m_writer.Write(material.transparency[3]); + + m_writer.Write(0x37FF); // Ambient Color Hash + m_writer.Write(material.ambientColor[0]); // R + m_writer.Write(material.ambientColor[1]); // G + m_writer.Write(material.ambientColor[2]); // B + m_writer.Write(material.ambientColor[3]); // A + + m_writer.Write(0x4265); // Incandescence Hash + m_writer.Write(material.incandescence[0]); + m_writer.Write(material.incandescence[1]); + m_writer.Write(material.incandescence[2]); + m_writer.Write(material.incandescence[3]); + + m_writer.Write(0xC835); // Coeffs Hash + m_writer.Write(material.coeffs[0]); + m_writer.Write(material.coeffs[1]); + + m_writer.Write(0xFE0C); // Glow Hash + m_writer.Write(material.glow.x); + m_writer.Write(material.glow.y); + + m_writer.Write(0x7E24); // Refractive Hash + m_writer.Write(material.refractive.x); + m_writer.Write(material.refractive.y); + + m_writer.Write(0x317C); // Specular Color Hash + m_writer.Write(material.specularColor[0]); // R + m_writer.Write(material.specularColor[1]); // G + m_writer.Write(material.specularColor[2]); // B + m_writer.Write(material.specularColor[3]); // A + + m_writer.Write(0xE593); // Reflective Color Hash + m_writer.Write(material.reflectiveColor[0]); // R + m_writer.Write(material.reflectiveColor[1]); // G + m_writer.Write(material.reflectiveColor[2]); // B + m_writer.Write(material.reflectiveColor[3]); // A + + m_writer.Write(0x7D76); // Reflective Hash + m_writer.Write(material.reflective.x); + m_writer.Write(material.reflective.y); + + m_writer.Write(0x83C7); // Blinn Hash + m_writer.Write(material.blinn[0]); + m_writer.Write(material.blinn[1]); + + m_writer.Write(0x5CD2); // Phong Hash + m_writer.Write(material.phong); + + materialNum++; + } + } + +public: + XModelBinWriter7(std::ostream& stream, std::string gameName, std::string zoneName) + : XModelBinWriterBase(stream, std::move(gameName), std::move(zoneName)) + { + } + + void Write(const XModelCommon& xmodel) override + { + PrepareVertexMerger(xmodel); + WriteHeader(7); + WriteBones(xmodel); + WriteVertices(xmodel); + WriteFaces(xmodel); + WriteObjects(xmodel); + WriteMaterials(xmodel); + + auto estimatedCompressedFileSize = LZ4_compressBound(m_writer.GetPosition()); + auto compressedBuffer = new char[estimatedCompressedFileSize]; + auto actualCompressedFileSize = + LZ4_compress_default(reinterpret_cast(m_writer.Data()), compressedBuffer, m_writer.GetPosition(), estimatedCompressedFileSize); + + uint64_t uncompressedSize = m_writer.GetPosition(); + char uncompressedSizeChar[4]; + std::memcpy(uncompressedSizeChar, &uncompressedSize, 4); + + const char magic[5] = {0x2A, 0x4C, 0x5A, 0x34, 0x2A}; + m_stream.write(magic, 5); + m_stream.write(uncompressedSizeChar, 4); + m_stream.write(compressedBuffer, actualCompressedFileSize); + } +}; + +namespace xmodel_bin +{ + std::unique_ptr CreateWriterForVersion7(std::ostream& stream, std::string gameName, std::string zoneName) + { + return std::make_unique(stream, std::move(gameName), std::move(zoneName)); + } +} // namespace xmodel_bin diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.h b/src/ObjWriting/XModel/Export/XModelBinWriter.h index e69de29b..4dba0767 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.h +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.h @@ -0,0 +1,11 @@ +#pragma once + +#include "XModel/XModelWriter.h" + +#include +#include + +namespace xmodel_bin +{ + std::unique_ptr CreateWriterForVersion7(std::ostream& stream, std::string gameName, std::string zoneName); +} diff --git a/src/ObjWriting/XModel/XModelDumper.cpp.template b/src/ObjWriting/XModel/XModelDumper.cpp.template index f0e1cb26..2e6e00f5 100644 --- a/src/ObjWriting/XModel/XModelDumper.cpp.template +++ b/src/ObjWriting/XModel/XModelDumper.cpp.template @@ -22,6 +22,7 @@ #include "ObjWriting.h" #include "Utils/DistinctMapper.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -579,6 +580,18 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".xmodel_bin")); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -616,6 +629,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; @@ -666,6 +683,8 @@ namespace { case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_EXPORT: return ".xmodel_export"; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + return ".xmodel_bin"; case ObjWriting::Configuration_t::ModelOutputFormat_e::OBJ: return ".obj"; case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: diff --git a/src/Unlinker/UnlinkerArgs.cpp b/src/Unlinker/UnlinkerArgs.cpp index 3d492667..18e66f32 100644 --- a/src/Unlinker/UnlinkerArgs.cpp +++ b/src/Unlinker/UnlinkerArgs.cpp @@ -80,7 +80,7 @@ const CommandLineOption* const OPTION_IMAGE_FORMAT = const CommandLineOption* const OPTION_MODEL_FORMAT = CommandLineOption::Builder::Create() .WithLongName("model-format") - .WithDescription("Specifies the format of dumped model files. Valid values are: XMODEL_EXPORT, OBJ, GLTF, GLB") + .WithDescription("Specifies the format of dumped model files. Valid values are: XMODEL_EXPORT, XMODEL_BIN, OBJ, GLTF, GLB") .WithParameter("modelFormatValue") .Build(); @@ -210,6 +210,12 @@ bool UnlinkerArgs::SetModelDumpingMode() const return true; } + if (specifiedValue == "xmodel_bin") + { + ObjWriting::Configuration.ModelOutputFormat = ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN; + return true; + } + if (specifiedValue == "obj") { ObjWriting::Configuration.ModelOutputFormat = ObjWriting::Configuration_t::ModelOutputFormat_e::OBJ; diff --git a/src/Utils/Utils/MemoryWriter.cpp b/src/Utils/Utils/MemoryWriter.cpp new file mode 100644 index 00000000..0ad24c98 --- /dev/null +++ b/src/Utils/Utils/MemoryWriter.cpp @@ -0,0 +1,73 @@ +#include "MemoryWriter.h" + +#include + +MemoryWriter::MemoryWriter(uint32_t capacity) +{ + m_dataPointer = new int8_t[capacity]; + m_dataLength = capacity; + m_currentPosition = 0; +} + +MemoryWriter::~MemoryWriter() +{ + Close(); +} + +uint64_t MemoryWriter::GetLength() const +{ + return m_dataLength; +} + +uint64_t MemoryWriter::GetPosition() const +{ + return m_currentPosition; +} + +int8_t* MemoryWriter::Data() const +{ + return m_dataPointer; +} + +void MemoryWriter::Close() +{ + if (m_dataPointer != nullptr) + { + delete[] m_dataPointer; + m_dataPointer = nullptr; + } +} + +void MemoryWriter::Write(const uint8_t* buffer, uint32_t size) +{ + ValidateCapacity(m_currentPosition + size); + + std::memcpy(m_dataPointer + m_currentPosition, buffer, size); + m_currentPosition += size; +} + +void MemoryWriter::WriteNullTerminatedString(const std::string& string) +{ + Write(reinterpret_cast(string.c_str()), string.size() + 1); +} + +void MemoryWriter::Reallocate(uint64_t capacity) +{ + auto tempPtr = static_cast(std::realloc(m_dataPointer, capacity)); + + if (tempPtr != nullptr) + { + m_dataPointer = tempPtr; + m_dataLength = capacity; + } +} + +void MemoryWriter::ValidateCapacity(uint64_t newSize) +{ + if (newSize > m_dataLength) + { + auto newCapacity = m_dataLength * 2; + assert(newCapacity <= UINT64_MAX); + Reallocate(newCapacity); + } +} diff --git a/src/Utils/Utils/MemoryWriter.h b/src/Utils/Utils/MemoryWriter.h new file mode 100644 index 00000000..960bbdd4 --- /dev/null +++ b/src/Utils/Utils/MemoryWriter.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +class MemoryWriter +{ +public: + MemoryWriter(uint32_t capacity); + ~MemoryWriter(); + + uint64_t GetLength() const; + uint64_t GetPosition() const; + int8_t* Data() const; + + void Close(); + void Write(const uint8_t* buffer, uint32_t size); + void WriteNullTerminatedString(const std::string& string); + + template void Write(const T data) + { + Write((uint8_t*)&data, sizeof(T)); + } + +private: + void Reallocate(uint64_t capacity); + void ValidateCapacity(uint64_t size); + +private: + int8_t* m_dataPointer; + uint64_t m_dataLength; + uint64_t m_currentPosition; +}; diff --git a/thirdparty/README.md b/thirdparty/README.md index c3686e9e..6b9f9c3b 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -22,4 +22,8 @@ ## zlib - https://www.zlib.net/ -- Version 1.2.11 \ No newline at end of file +- Version 1.2.11 + +## lz4 +- https://lz4.org/ +- Version 1.10.0 \ No newline at end of file diff --git a/thirdparty/lz4 b/thirdparty/lz4 new file mode 160000 index 00000000..fa1634e2 --- /dev/null +++ b/thirdparty/lz4 @@ -0,0 +1 @@ +Subproject commit fa1634e2ccd41ac09c087ab65e96bcbbd003fd20 diff --git a/thirdparty/lz4.lua b/thirdparty/lz4.lua new file mode 100644 index 00000000..5554e196 --- /dev/null +++ b/thirdparty/lz4.lua @@ -0,0 +1,48 @@ +lz4 = {} + +function lz4:include(includes) + if includes:handle(self:name()) then + includedirs { + path.join(ThirdPartyFolder(), "lz4") + } + end +end + +function lz4:link(links) + links:add(self:name()) +end + +function lz4:use() + +end + +function lz4:name() + return "lz4" +end + +function lz4:project() + local folder = ThirdPartyFolder() + local includes = Includes:create() + + project(self:name()) + targetdir(TargetDirectoryLib) + location "%{wks.location}/thirdparty/%{prj.name}" + kind "StaticLib" + language "C" + + files { + path.join(folder, "lz4/lib/*.h"), + path.join(folder, "lz4/lib/*.c") + } + + defines { + "_CRT_SECURE_NO_WARNINGS", + "_CRT_NONSTDC_NO_DEPRECATE" + } + + + self:include(includes) + + -- Disable warnings. They do not have any value to us since it is not our code. + warnings "off" +end From eff80a0318db100ded215677d489a5563ac75043 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Fri, 11 Apr 2025 19:03:41 +0200 Subject: [PATCH 02/12] Improved XModelBin Hash Readability --- .../XModel/Export/XModelBinWriter.cpp | 137 ++++++++++++------ 1 file changed, 93 insertions(+), 44 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 6081923d..d9a7ea0a 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -13,6 +13,55 @@ class XModelBinWriterBase : public XModelWriter { +protected: + enum XModelBinHash : int32_t + { + COMMENT = 0xC355, + MODEL = 0x46C8, + VERSION = 0x24D1, + + BONE_COUNT = 0x76BA, + BONE = 0xF099, + BONE_INDEX = 0xDD9A, + OFFSET = 0x9383, // Bone/Vertex Offset + BONE_SCALE = 0x1C56, + BONE_MATRIX_X = 0xDCFD, + BONE_MATRIX_Y = 0xCCDC, + BONE_MATRIX_Z = 0xFCBF, + + VERT16_COUNT = 0x950D, + VERT32_COUNT = 0x2AEC, + VERT16 = 0x8F03, + VERT32 = 0xB097, + VERT_WEIGHT_COUNT = 0xEA46, + VERT_WEIGHT = 0xF1AB, + + FACE_COUNT = 0xBE92, + TRIANGLE16 = 0x6711, + TRIANGLE32 = 0x562F, + + NORMAL = 0x89EC, + COLOR = 0x6DD8, + UV = 0x1AD4, + + OBJECT_COUNT = 0x62AF, + OBJECT = 0x87D4, + + MATERIAL_COUNT = 0xA1B2, + MATERIAL = 0xA700, + MATERIAL_TRANSPARENCY = 0x6DAB, + MATERIAL_AMBIENT_COLOR = 0x37FF, + MATERIAL_INCANDESCENCE = 0x4265, + MATERIAL_COEFFS = 0xC835, + MATERIAL_GLOW = 0xFE0C, + MATERIAL_REFRACTIVE = 0x7E24, + MATERIAL_SPECULAR_COLOR = 0x317C, + MATERIAL_REFLECTIVE_COLOR = 0xE593, + MATERIAL_REFLECTIVE = 0x7D76, + MATERIAL_BLINN = 0x83C7, + MATERIAL_PHONG = 0x5CD2 + }; + protected: void PrepareVertexMerger(const XModelCommon& xmodel) { @@ -48,7 +97,7 @@ protected: void WriteComment(const std::string& comment) { - m_writer.Write(0xC355); + m_writer.Write(XModelBinHash::COMMENT); WriteAlignedString(comment); } @@ -79,18 +128,18 @@ protected: WriteComment("OpenAssetTools XMODEL_BIN File"); WriteComment(std::format("Game Origin: {}", m_game_name)); WriteComment(std::format("Zone Origin: {}", m_zone_name)); - m_writer.Write(0x46C8); // Model Hash - WriteInt16(0x24D1, version); + m_writer.Write(XModelBinHash::MODEL); + WriteInt16(XModelBinHash::VERSION, version); } void WriteBones(const XModelCommon& xmodel) { - WriteInt16(0x76BA, xmodel.m_bones.size()); + WriteInt16(XModelBinHash::BONE_COUNT, xmodel.m_bones.size()); auto boneNum = 0; for (const auto& bone : xmodel.m_bones) { - m_writer.Write(0xF099); // Bone Hash + m_writer.Write(XModelBinHash::BONE); m_writer.Write(boneNum); if (bone.parentIndex) m_writer.Write(static_cast(*bone.parentIndex)); @@ -104,31 +153,31 @@ protected: boneNum = 0; for (const auto& bone : xmodel.m_bones) { - WriteInt16(0xDD9A, boneNum); + WriteInt16(XModelBinHash::BONE_INDEX, boneNum); - m_writer.Write(0x9383); // Bone Offset Hash + m_writer.Write(XModelBinHash::OFFSET); m_writer.Write(bone.globalOffset[0]); // X m_writer.Write(bone.globalOffset[1]); // Y m_writer.Write(bone.globalOffset[2]); // Z - m_writer.Write(0x1C56); // Bone Scale Hash + m_writer.Write(XModelBinHash::BONE_SCALE); m_writer.Write(bone.scale[0]); // X m_writer.Write(bone.scale[1]); // Y m_writer.Write(bone.scale[2]); // Z const auto mat = Eigen::Quaternionf(bone.globalRotation.w, bone.globalRotation.x, bone.globalRotation.y, bone.globalRotation.z).matrix(); - m_writer.Write(0xDCFDi16); // Bone Matrix X Hash + m_writer.Write(static_cast(XModelBinHash::BONE_MATRIX_X)); m_writer.Write(ClampFloatToShort(mat(0, 0))); m_writer.Write(ClampFloatToShort(mat(0, 1))); m_writer.Write(ClampFloatToShort(mat(0, 2))); - m_writer.Write(0xCCDCi16); // Bone Matrix Y Hash + m_writer.Write(static_cast(XModelBinHash::BONE_MATRIX_Y)); m_writer.Write(ClampFloatToShort(mat(1, 0))); m_writer.Write(ClampFloatToShort(mat(1, 1))); m_writer.Write(ClampFloatToShort(mat(1, 2))); - m_writer.Write(0xFCBFi16); // Bone Matrix Z Hash + m_writer.Write(static_cast(XModelBinHash::BONE_MATRIX_Z)); m_writer.Write(ClampFloatToShort(mat(2, 0))); m_writer.Write(ClampFloatToShort(mat(2, 1))); m_writer.Write(ClampFloatToShort(mat(2, 2))); @@ -161,13 +210,13 @@ class XModelBinWriter7 final : public XModelBinWriterBase if (distinctVertexValues.size() > UINT16_MAX) { // Use 32 bit - m_writer.Write(0x2AEC); // 32 bit Hash + m_writer.Write(XModelBinHash::VERT32_COUNT); m_writer.Write(static_cast(distinctVertexValues.size())); } else { // Use 16 bit - WriteUInt16(0x950D, distinctVertexValues.size()); + WriteUInt16(XModelBinHash::VERT16_COUNT, distinctVertexValues.size()); } size_t vertexNum = 0u; @@ -176,16 +225,16 @@ class XModelBinWriter7 final : public XModelBinWriterBase if (vertexNum > UINT16_MAX) { // Use 32 bit - m_writer.Write(0xB097); // 32 bit Hash + m_writer.Write(XModelBinHash::VERT32); m_writer.Write(static_cast(vertexNum)); } else { // Use 16 bit - WriteUInt16(0x8F03, vertexNum); + WriteUInt16(XModelBinHash::VERT16, vertexNum); } - m_writer.Write(0x9383); // Offset Hash + m_writer.Write(XModelBinHash::OFFSET); m_writer.Write(vertexPos.x); m_writer.Write(vertexPos.y); m_writer.Write(vertexPos.z); @@ -205,19 +254,19 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteFaceVertex(const XModelVertex& vertex) { - m_writer.Write(0x89ECi16); // Normal Hash + m_writer.Write(static_cast(XModelBinHash::NORMAL)); m_writer.Write(ClampFloatToShort(vertex.normal[0])); // X m_writer.Write(ClampFloatToShort(vertex.normal[1])); // Y m_writer.Write(ClampFloatToShort(vertex.normal[2])); // Z - m_writer.Write(0x6DD8); // Color Hash + m_writer.Write(XModelBinHash::COLOR); m_writer.Write(ClampFloatToUByte(vertex.color[0])); // R m_writer.Write(ClampFloatToUByte(vertex.color[1])); // G m_writer.Write(ClampFloatToUByte(vertex.color[2])); // B m_writer.Write(ClampFloatToUByte(vertex.color[3])); // A - m_writer.Write(0x1AD4i16); // UV Hash - m_writer.Write(1ui16); // Layer + m_writer.Write(static_cast(XModelBinHash::UV)); + m_writer.Write(1ui16); // Layer m_writer.Write(vertex.uv[0]); m_writer.Write(vertex.uv[1]); } @@ -228,7 +277,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase for (const auto& object : xmodel.m_objects) totalFaceCount += object.m_faces.size(); - m_writer.Write(0xBE92); // Face Count Hash + m_writer.Write(XModelBinHash::FACE_COUNT); m_writer.Write(totalFaceCount); auto objectIndex = 0u; @@ -246,33 +295,33 @@ class XModelBinWriter7 final : public XModelBinWriterBase const XModelVertex& v1 = xmodel.m_vertices[face.vertexIndex[1]]; const XModelVertex& v2 = xmodel.m_vertices[face.vertexIndex[2]]; - m_writer.Write(0x562Fi16); // Face Info Hash + m_writer.Write(static_cast(XModelBinHash::TRIANGLE32)); m_writer.Write(static_cast(objectIndex)); m_writer.Write(static_cast(object.materialIndex)); if (m_vertex_merger.GetDistinctValues().size() > UINT16_MAX) { - m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(XModelBinHash::VERT32); m_writer.Write(static_cast(distinctPositions[0])); WriteFaceVertex(v0); - m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(XModelBinHash::VERT32); m_writer.Write(static_cast(distinctPositions[1])); WriteFaceVertex(v1); - m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(XModelBinHash::VERT32); m_writer.Write(static_cast(distinctPositions[2])); WriteFaceVertex(v2); } else { - WriteUInt16(0x8F03, distinctPositions[0]); + WriteUInt16(XModelBinHash::VERT16, distinctPositions[0]); WriteFaceVertex(v0); - WriteUInt16(0x8F03, distinctPositions[1]); + WriteUInt16(XModelBinHash::VERT16, distinctPositions[1]); WriteFaceVertex(v1); - WriteUInt16(0x8F03, distinctPositions[2]); + WriteUInt16(XModelBinHash::VERT16, distinctPositions[2]); WriteFaceVertex(v2); } } @@ -283,12 +332,12 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteObjects(const XModelCommon& xmodel) { - WriteInt16(0x62AF, xmodel.m_objects.size()); + WriteInt16(XModelBinHash::OBJECT_COUNT, xmodel.m_objects.size()); size_t objectNum = 0; for (const auto& object : xmodel.m_objects) { - m_writer.Write(0x87D4i16); // Object Info Hash + m_writer.Write(static_cast(XModelBinHash::OBJECT)); m_writer.Write(static_cast(objectNum)); WriteAlignedString(object.name); @@ -298,75 +347,75 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteMaterials(const XModelCommon& xmodel) { - WriteInt16(0xA1B2, xmodel.m_materials.size()); + WriteInt16(XModelBinHash::MATERIAL_COUNT, xmodel.m_materials.size()); size_t materialNum = 0u; for (const auto& material : xmodel.m_materials) { const auto colorMapPath = "../images/" + material.colorMapName + ".dds"; - WriteInt16(0xA700, materialNum); + WriteInt16(XModelBinHash::MATERIAL, materialNum); WriteAlignedString(material.name); WriteAlignedString(material.materialTypeName); WriteAlignedString(colorMapPath); - m_writer.Write(0x6DD8); // Vertex Color Hash + m_writer.Write(XModelBinHash::COLOR); m_writer.Write(ClampFloatToUByte(material.color[0])); // R m_writer.Write(ClampFloatToUByte(material.color[1])); // G m_writer.Write(ClampFloatToUByte(material.color[2])); // B m_writer.Write(ClampFloatToUByte(material.color[3])); // A - m_writer.Write(0x6DAB); // Transparancy Hash + m_writer.Write(XModelBinHash::MATERIAL_TRANSPARENCY); m_writer.Write(material.transparency[0]); m_writer.Write(material.transparency[1]); m_writer.Write(material.transparency[2]); m_writer.Write(material.transparency[3]); - m_writer.Write(0x37FF); // Ambient Color Hash + m_writer.Write(XModelBinHash::MATERIAL_AMBIENT_COLOR); m_writer.Write(material.ambientColor[0]); // R m_writer.Write(material.ambientColor[1]); // G m_writer.Write(material.ambientColor[2]); // B m_writer.Write(material.ambientColor[3]); // A - m_writer.Write(0x4265); // Incandescence Hash + m_writer.Write(XModelBinHash::MATERIAL_INCANDESCENCE); m_writer.Write(material.incandescence[0]); m_writer.Write(material.incandescence[1]); m_writer.Write(material.incandescence[2]); m_writer.Write(material.incandescence[3]); - m_writer.Write(0xC835); // Coeffs Hash + m_writer.Write(XModelBinHash::MATERIAL_COEFFS); m_writer.Write(material.coeffs[0]); m_writer.Write(material.coeffs[1]); - m_writer.Write(0xFE0C); // Glow Hash + m_writer.Write(XModelBinHash::MATERIAL_GLOW); m_writer.Write(material.glow.x); m_writer.Write(material.glow.y); - m_writer.Write(0x7E24); // Refractive Hash + m_writer.Write(XModelBinHash::MATERIAL_REFRACTIVE); m_writer.Write(material.refractive.x); m_writer.Write(material.refractive.y); - m_writer.Write(0x317C); // Specular Color Hash + m_writer.Write(XModelBinHash::MATERIAL_SPECULAR_COLOR); m_writer.Write(material.specularColor[0]); // R m_writer.Write(material.specularColor[1]); // G m_writer.Write(material.specularColor[2]); // B m_writer.Write(material.specularColor[3]); // A - m_writer.Write(0xE593); // Reflective Color Hash + m_writer.Write(XModelBinHash::MATERIAL_REFLECTIVE_COLOR); m_writer.Write(material.reflectiveColor[0]); // R m_writer.Write(material.reflectiveColor[1]); // G m_writer.Write(material.reflectiveColor[2]); // B m_writer.Write(material.reflectiveColor[3]); // A - m_writer.Write(0x7D76); // Reflective Hash + m_writer.Write(XModelBinHash::MATERIAL_REFLECTIVE); m_writer.Write(material.reflective.x); m_writer.Write(material.reflective.y); - m_writer.Write(0x83C7); // Blinn Hash + m_writer.Write(XModelBinHash::MATERIAL_BLINN); m_writer.Write(material.blinn[0]); m_writer.Write(material.blinn[1]); - m_writer.Write(0x5CD2); // Phong Hash + m_writer.Write(XModelBinHash::MATERIAL_PHONG); m_writer.Write(material.phong); materialNum++; From f2501fdc0b0a166d0ab4e50e6cfd5fab79649fd7 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Fri, 11 Apr 2025 19:17:46 +0200 Subject: [PATCH 03/12] Fixed Compilation Issue on Linux --- src/Utils/Utils/MemoryWriter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Utils/Utils/MemoryWriter.cpp b/src/Utils/Utils/MemoryWriter.cpp index 0ad24c98..68b2e326 100644 --- a/src/Utils/Utils/MemoryWriter.cpp +++ b/src/Utils/Utils/MemoryWriter.cpp @@ -1,6 +1,7 @@ #include "MemoryWriter.h" #include +#include MemoryWriter::MemoryWriter(uint32_t capacity) { From 812f7e1f916051ca19db0708c85ab1ccf34791f7 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Fri, 11 Apr 2025 22:57:09 +0200 Subject: [PATCH 04/12] Fixed Precision Issue --- src/ObjWriting/XModel/Export/XModelBinWriter.cpp | 6 +++--- src/Utils/Utils/MemoryWriter.cpp | 8 ++++---- src/Utils/Utils/MemoryWriter.h | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index d9a7ea0a..6bb87ea3 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -445,11 +445,11 @@ public: uint64_t uncompressedSize = m_writer.GetPosition(); char uncompressedSizeChar[4]; - std::memcpy(uncompressedSizeChar, &uncompressedSize, 4); + std::memcpy(uncompressedSizeChar, &uncompressedSize, sizeof(uncompressedSizeChar)); const char magic[5] = {0x2A, 0x4C, 0x5A, 0x34, 0x2A}; - m_stream.write(magic, 5); - m_stream.write(uncompressedSizeChar, 4); + m_stream.write(magic, sizeof(magic)); + m_stream.write(uncompressedSizeChar, sizeof(uncompressedSizeChar)); m_stream.write(compressedBuffer, actualCompressedFileSize); } }; diff --git a/src/Utils/Utils/MemoryWriter.cpp b/src/Utils/Utils/MemoryWriter.cpp index 68b2e326..efdb833a 100644 --- a/src/Utils/Utils/MemoryWriter.cpp +++ b/src/Utils/Utils/MemoryWriter.cpp @@ -15,12 +15,12 @@ MemoryWriter::~MemoryWriter() Close(); } -uint64_t MemoryWriter::GetLength() const +size_t MemoryWriter::GetLength() const { return m_dataLength; } -uint64_t MemoryWriter::GetPosition() const +size_t MemoryWriter::GetPosition() const { return m_currentPosition; } @@ -52,7 +52,7 @@ void MemoryWriter::WriteNullTerminatedString(const std::string& string) Write(reinterpret_cast(string.c_str()), string.size() + 1); } -void MemoryWriter::Reallocate(uint64_t capacity) +void MemoryWriter::Reallocate(size_t capacity) { auto tempPtr = static_cast(std::realloc(m_dataPointer, capacity)); @@ -63,7 +63,7 @@ void MemoryWriter::Reallocate(uint64_t capacity) } } -void MemoryWriter::ValidateCapacity(uint64_t newSize) +void MemoryWriter::ValidateCapacity(size_t newSize) { if (newSize > m_dataLength) { diff --git a/src/Utils/Utils/MemoryWriter.h b/src/Utils/Utils/MemoryWriter.h index 960bbdd4..979c6920 100644 --- a/src/Utils/Utils/MemoryWriter.h +++ b/src/Utils/Utils/MemoryWriter.h @@ -9,8 +9,8 @@ public: MemoryWriter(uint32_t capacity); ~MemoryWriter(); - uint64_t GetLength() const; - uint64_t GetPosition() const; + size_t GetLength() const; + size_t GetPosition() const; int8_t* Data() const; void Close(); @@ -23,11 +23,11 @@ public: } private: - void Reallocate(uint64_t capacity); - void ValidateCapacity(uint64_t size); + void Reallocate(size_t capacity); + void ValidateCapacity(size_t size); private: int8_t* m_dataPointer; - uint64_t m_dataLength; - uint64_t m_currentPosition; + size_t m_dataLength; + size_t m_currentPosition; }; From ef79504e716d01ccb00c803e8a363fe6db0cb3b1 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Fri, 11 Apr 2025 23:22:54 +0200 Subject: [PATCH 05/12] Fixed Int Literals Not Working On Linux --- .../XModel/Export/XModelBinWriter.cpp | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 6bb87ea3..c516a736 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -87,12 +87,11 @@ protected: void WriteAlignedString(const std::string& string) { - auto paddingSize = ((string.size() + 1 + 0x3) & 0xFFFFFFFFFFFFFC) - (string.size() + 1); - auto padding = std::make_unique(paddingSize); + const auto paddingSize = ((string.size() + 1 + 0x3) & 0xFFFFFFFFFFFFFC) - (string.size() + 1); + const auto padding = std::make_unique(static_cast(paddingSize)); m_writer.WriteNullTerminatedString(string); - - m_writer.Write(reinterpret_cast(padding.get()), paddingSize); + m_writer.Write(reinterpret_cast(padding.get()), static_cast(paddingSize)); } void WriteComment(const std::string& comment) @@ -129,12 +128,12 @@ protected: WriteComment(std::format("Game Origin: {}", m_game_name)); WriteComment(std::format("Zone Origin: {}", m_zone_name)); m_writer.Write(XModelBinHash::MODEL); - WriteInt16(XModelBinHash::VERSION, version); + WriteInt16(static_cast(XModelBinHash::VERSION), version); } void WriteBones(const XModelCommon& xmodel) { - WriteInt16(XModelBinHash::BONE_COUNT, xmodel.m_bones.size()); + WriteInt16(static_cast(XModelBinHash::BONE_COUNT), static_cast(xmodel.m_bones.size())); auto boneNum = 0; for (const auto& bone : xmodel.m_bones) @@ -153,7 +152,7 @@ protected: boneNum = 0; for (const auto& bone : xmodel.m_bones) { - WriteInt16(XModelBinHash::BONE_INDEX, boneNum); + WriteInt16(static_cast(XModelBinHash::BONE_INDEX), boneNum); m_writer.Write(XModelBinHash::OFFSET); m_writer.Write(bone.globalOffset[0]); // X @@ -216,7 +215,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase else { // Use 16 bit - WriteUInt16(XModelBinHash::VERT16_COUNT, distinctVertexValues.size()); + WriteUInt16(static_cast(XModelBinHash::VERT16_COUNT), static_cast(distinctVertexValues.size())); } size_t vertexNum = 0u; @@ -231,7 +230,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase else { // Use 16 bit - WriteUInt16(XModelBinHash::VERT16, vertexNum); + WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(vertexNum)); } m_writer.Write(XModelBinHash::OFFSET); @@ -239,13 +238,13 @@ class XModelBinWriter7 final : public XModelBinWriterBase m_writer.Write(vertexPos.y); m_writer.Write(vertexPos.z); - WriteInt16(0xEA46, vertexPos.weightCount); + WriteInt16(static_cast(XModelBinHash::VERT_WEIGHT_COUNT), static_cast(vertexPos.weightCount)); for (auto weightIndex = 0u; weightIndex < vertexPos.weightCount; weightIndex++) { const auto& weight = vertexPos.weights[weightIndex]; - WriteInt16(0xF1AB, weight.boneIndex); + WriteInt16(static_cast(XModelBinHash::VERT_WEIGHT), weight.boneIndex); m_writer.Write(weight.weight); } vertexNum++; @@ -266,7 +265,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase m_writer.Write(ClampFloatToUByte(vertex.color[3])); // A m_writer.Write(static_cast(XModelBinHash::UV)); - m_writer.Write(1ui16); // Layer + m_writer.Write(static_cast(1)); // Layer m_writer.Write(vertex.uv[0]); m_writer.Write(vertex.uv[1]); } @@ -315,13 +314,13 @@ class XModelBinWriter7 final : public XModelBinWriterBase } else { - WriteUInt16(XModelBinHash::VERT16, distinctPositions[0]); + WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(distinctPositions[0])); WriteFaceVertex(v0); - WriteUInt16(XModelBinHash::VERT16, distinctPositions[1]); + WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(distinctPositions[1])); WriteFaceVertex(v1); - WriteUInt16(XModelBinHash::VERT16, distinctPositions[2]); + WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(distinctPositions[2])); WriteFaceVertex(v2); } } @@ -332,7 +331,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteObjects(const XModelCommon& xmodel) { - WriteInt16(XModelBinHash::OBJECT_COUNT, xmodel.m_objects.size()); + WriteInt16(XModelBinHash::OBJECT_COUNT, static_cast(xmodel.m_objects.size())); size_t objectNum = 0; for (const auto& object : xmodel.m_objects) @@ -347,14 +346,14 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteMaterials(const XModelCommon& xmodel) { - WriteInt16(XModelBinHash::MATERIAL_COUNT, xmodel.m_materials.size()); + WriteInt16(static_cast(XModelBinHash::MATERIAL_COUNT), static_cast(xmodel.m_materials.size())); size_t materialNum = 0u; for (const auto& material : xmodel.m_materials) { const auto colorMapPath = "../images/" + material.colorMapName + ".dds"; - WriteInt16(XModelBinHash::MATERIAL, materialNum); + WriteInt16(static_cast(XModelBinHash::MATERIAL), static_cast(materialNum)); WriteAlignedString(material.name); WriteAlignedString(material.materialTypeName); WriteAlignedString(colorMapPath); From 7aebe085b338145d334fd8566dcc54c330e8b0b9 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Fri, 11 Apr 2025 23:39:39 +0200 Subject: [PATCH 06/12] Fixed More Int Literals Not Working On Linux --- src/ObjWriting/XModel/Export/XModelBinWriter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index c516a736..0b3b0e37 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -114,12 +114,12 @@ protected: int16_t ClampFloatToShort(const float value) const { - return std::clamp(static_cast(32767 * std::clamp(value, -1.0f, 1.0f)), static_cast(-32767), 32767i16); + return std::clamp(static_cast(32767 * std::clamp(value, -1.0f, 1.0f)), static_cast(-32767), static_cast(32767)); } uint8_t ClampFloatToUByte(const float value) const { - return std::clamp(static_cast(255 * std::clamp(value, -1.0f, 1.0f)), 0ui8, 255ui8); + return std::clamp(static_cast(255 * std::clamp(value, -1.0f, 1.0f)), static_cast(0), static_cast(255)); } void WriteHeader(int16_t version) From 2a45357d5ca7faa7934a7328fdea33ae22fa3568 Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Sun, 13 Apr 2025 14:50:11 +0200 Subject: [PATCH 07/12] Added Comments --- src/ObjWriting/XModel/Export/XModelBinWriter.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 0b3b0e37..7588d4a0 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -14,6 +14,18 @@ class XModelBinWriterBase : public XModelWriter { protected: + /* + * A XModelBin Files consists of blocks of data, each block is indentified by a hash and is then followed by the actual data + * e.g.: + * 0x62AF = Number of objects hash + * 2 = Number of objects + * 0x87D4 = Start of object hash + * 0 = Object index + * Test_Obj = Object name + * 0x87D4 = Start of object hash + * 1 = Object index + * Test_Obj2 = Object name + */ enum XModelBinHash : int32_t { COMMENT = 0xC355, @@ -446,7 +458,7 @@ public: char uncompressedSizeChar[4]; std::memcpy(uncompressedSizeChar, &uncompressedSize, sizeof(uncompressedSizeChar)); - const char magic[5] = {0x2A, 0x4C, 0x5A, 0x34, 0x2A}; + const char magic[5] = {0x2A, 0x4C, 0x5A, 0x34, 0x2A}; // *LZ4* m_stream.write(magic, sizeof(magic)); m_stream.write(uncompressedSizeChar, sizeof(uncompressedSizeChar)); m_stream.write(compressedBuffer, actualCompressedFileSize); From 2aeeccd492f20ed9019e0fcc601f5266fad797ec Mon Sep 17 00:00:00 2001 From: GoastcraftHD Date: Sun, 13 Apr 2025 17:48:52 +0200 Subject: [PATCH 08/12] Update .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index da2074a0..b283a8c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,4 +18,4 @@ url = https://gitlab.com/libeigen/eigen.git [submodule "thirdparty/lz4"] path = thirdparty/lz4 - url = git@github.com:lz4/lz4.git + url = https://github.com/lz4/lz4.git From 8946c28eab7888eed81312758f13731e0fd2abfe Mon Sep 17 00:00:00 2001 From: Simon Ickler Date: Sat, 19 Apr 2025 15:58:37 +0200 Subject: [PATCH 09/12] Fixed Code Style --- .../IW3/AssetDumpers/AssetDumperXModel.cpp | 2 +- .../IW4/AssetDumpers/AssetDumperXModel.cpp | 2 +- .../XModel/Export/XModelBinWriter.cpp | 274 +++++++++--------- src/Utils/Utils/MemoryWriter.cpp | 74 ----- src/Utils/Utils/MemoryWriter.h | 33 --- 5 files changed, 144 insertions(+), 241 deletions(-) delete mode 100644 src/Utils/Utils/MemoryWriter.cpp delete mode 100644 src/Utils/Utils/MemoryWriter.h diff --git a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp index 24b4246a..0382088e 100644 --- a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp @@ -462,7 +462,7 @@ namespace void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) { const auto* model = asset->Asset(); - const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".XMODEL_EXPORT")); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".xmodel_bin")); if (!assetFile) return; diff --git a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp index 71474a51..3b9987e6 100644 --- a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp @@ -451,7 +451,7 @@ namespace { const auto* model = asset->Asset(); const auto* modelSurfs = model->lodInfo[lod].modelSurfs; - const auto assetFile = context.OpenAssetFile(std::format("model_export/{}.XMODEL_BIN", modelSurfs->name)); + const auto assetFile = context.OpenAssetFile(std::format("model_export/{}.xmodel_bin", modelSurfs->name)); if (!assetFile) return; diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 7588d4a0..649c1e45 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -1,7 +1,5 @@ #include "XModelBinWriter.h" -#include "Utils/MemoryWriter.h" - #pragma warning(push, 0) #include #pragma warning(pop) @@ -10,6 +8,7 @@ #include #include #include +#include class XModelBinWriterBase : public XModelWriter { @@ -26,7 +25,7 @@ protected: * 1 = Object index * Test_Obj2 = Object name */ - enum XModelBinHash : int32_t + enum XModelBinHash : uint32_t { COMMENT = 0xC355, MODEL = 0x46C8, @@ -74,7 +73,6 @@ protected: MATERIAL_PHONG = 0x5CD2 }; -protected: void PrepareVertexMerger(const XModelCommon& xmodel) { m_vertex_merger = VertexMerger(xmodel.m_vertices.size()); @@ -97,41 +95,55 @@ protected: } } + template void Write(const T& data) + { + m_writer.write(reinterpret_cast(&data), sizeof(T)); + } + + void WriteNullTerminatedString(std::string string) + { + m_writer.write(string.c_str(), string.size() + 1); + } + void WriteAlignedString(const std::string& string) { - const auto paddingSize = ((string.size() + 1 + 0x3) & 0xFFFFFFFFFFFFFC) - (string.size() + 1); - const auto padding = std::make_unique(static_cast(paddingSize)); + static constexpr uint8_t PADDING[4] = {0u, 0u, 0u, 0u}; + const auto paddingSize = (4u - (string.size() + 1u) % 4u) % 4u; - m_writer.WriteNullTerminatedString(string); - m_writer.Write(reinterpret_cast(padding.get()), static_cast(paddingSize)); + WriteNullTerminatedString(string); + m_writer.write(reinterpret_cast(PADDING), paddingSize); } void WriteComment(const std::string& comment) { - m_writer.Write(XModelBinHash::COMMENT); + Write(XModelBinHash::COMMENT); WriteAlignedString(comment); } void WriteInt16(const int16_t hash, const int16_t value) { - m_writer.Write(hash); - m_writer.Write(value); + Write(hash); + Write(value); } void WriteUInt16(const int16_t hash, const uint16_t value) { - m_writer.Write(hash); - m_writer.Write(value); + Write(hash); + Write(value); } int16_t ClampFloatToShort(const float value) const { - return std::clamp(static_cast(32767 * std::clamp(value, -1.0f, 1.0f)), static_cast(-32767), static_cast(32767)); + return std::clamp(static_cast(std::numeric_limits::max() * std::clamp(value, -1.0f, 1.0f)), + std::numeric_limits::min(), + std::numeric_limits::max()); } uint8_t ClampFloatToUByte(const float value) const { - return std::clamp(static_cast(255 * std::clamp(value, -1.0f, 1.0f)), static_cast(0), static_cast(255)); + return std::clamp(static_cast(std::numeric_limits::max() * std::clamp(value, -1.0f, 1.0f)), + std::numeric_limits::min(), + std::numeric_limits::max()); } void WriteHeader(int16_t version) @@ -139,7 +151,7 @@ protected: WriteComment("OpenAssetTools XMODEL_BIN File"); WriteComment(std::format("Game Origin: {}", m_game_name)); WriteComment(std::format("Zone Origin: {}", m_zone_name)); - m_writer.Write(XModelBinHash::MODEL); + Write(XModelBinHash::MODEL); WriteInt16(static_cast(XModelBinHash::VERSION), version); } @@ -150,12 +162,12 @@ protected: auto boneNum = 0; for (const auto& bone : xmodel.m_bones) { - m_writer.Write(XModelBinHash::BONE); - m_writer.Write(boneNum); + Write(XModelBinHash::BONE); + Write(boneNum); if (bone.parentIndex) - m_writer.Write(static_cast(*bone.parentIndex)); + Write(static_cast(*bone.parentIndex)); else - m_writer.Write(-1); + Write(-1); WriteAlignedString(bone.name); boneNum++; @@ -166,46 +178,45 @@ protected: { WriteInt16(static_cast(XModelBinHash::BONE_INDEX), boneNum); - m_writer.Write(XModelBinHash::OFFSET); - m_writer.Write(bone.globalOffset[0]); // X - m_writer.Write(bone.globalOffset[1]); // Y - m_writer.Write(bone.globalOffset[2]); // Z + Write(XModelBinHash::OFFSET); + Write(bone.globalOffset[0]); // X + Write(bone.globalOffset[1]); // Y + Write(bone.globalOffset[2]); // Z - m_writer.Write(XModelBinHash::BONE_SCALE); - m_writer.Write(bone.scale[0]); // X - m_writer.Write(bone.scale[1]); // Y - m_writer.Write(bone.scale[2]); // Z + Write(XModelBinHash::BONE_SCALE); + Write(bone.scale[0]); // X + Write(bone.scale[1]); // Y + Write(bone.scale[2]); // Z const auto mat = Eigen::Quaternionf(bone.globalRotation.w, bone.globalRotation.x, bone.globalRotation.y, bone.globalRotation.z).matrix(); - m_writer.Write(static_cast(XModelBinHash::BONE_MATRIX_X)); - m_writer.Write(ClampFloatToShort(mat(0, 0))); - m_writer.Write(ClampFloatToShort(mat(0, 1))); - m_writer.Write(ClampFloatToShort(mat(0, 2))); + Write(static_cast(XModelBinHash::BONE_MATRIX_X)); + Write(ClampFloatToShort(mat(0, 0))); + Write(ClampFloatToShort(mat(0, 1))); + Write(ClampFloatToShort(mat(0, 2))); - m_writer.Write(static_cast(XModelBinHash::BONE_MATRIX_Y)); - m_writer.Write(ClampFloatToShort(mat(1, 0))); - m_writer.Write(ClampFloatToShort(mat(1, 1))); - m_writer.Write(ClampFloatToShort(mat(1, 2))); + Write(static_cast(XModelBinHash::BONE_MATRIX_Y)); + Write(ClampFloatToShort(mat(1, 0))); + Write(ClampFloatToShort(mat(1, 1))); + Write(ClampFloatToShort(mat(1, 2))); - m_writer.Write(static_cast(XModelBinHash::BONE_MATRIX_Z)); - m_writer.Write(ClampFloatToShort(mat(2, 0))); - m_writer.Write(ClampFloatToShort(mat(2, 1))); - m_writer.Write(ClampFloatToShort(mat(2, 2))); + Write(static_cast(XModelBinHash::BONE_MATRIX_Z)); + Write(ClampFloatToShort(mat(2, 0))); + Write(ClampFloatToShort(mat(2, 1))); + Write(ClampFloatToShort(mat(2, 2))); boneNum++; } } XModelBinWriterBase(std::ostream& stream, std::string gameName, std::string zoneName) - : m_writer(0x800000), // 8MB - m_stream(stream), + : m_stream(stream), m_game_name(std::move(gameName)), m_zone_name(std::move(zoneName)) { } - MemoryWriter m_writer; + std::ostringstream m_writer; std::ostream& m_stream; std::string m_game_name; std::string m_zone_name; @@ -221,8 +232,8 @@ class XModelBinWriter7 final : public XModelBinWriterBase if (distinctVertexValues.size() > UINT16_MAX) { // Use 32 bit - m_writer.Write(XModelBinHash::VERT32_COUNT); - m_writer.Write(static_cast(distinctVertexValues.size())); + XModelBinWriterBase::Write(XModelBinHash::VERT32_COUNT); + XModelBinWriterBase::Write(static_cast(distinctVertexValues.size())); } else { @@ -236,8 +247,8 @@ class XModelBinWriter7 final : public XModelBinWriterBase if (vertexNum > UINT16_MAX) { // Use 32 bit - m_writer.Write(XModelBinHash::VERT32); - m_writer.Write(static_cast(vertexNum)); + XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(vertexNum)); } else { @@ -245,10 +256,10 @@ class XModelBinWriter7 final : public XModelBinWriterBase WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(vertexNum)); } - m_writer.Write(XModelBinHash::OFFSET); - m_writer.Write(vertexPos.x); - m_writer.Write(vertexPos.y); - m_writer.Write(vertexPos.z); + XModelBinWriterBase::Write(XModelBinHash::OFFSET); + XModelBinWriterBase::Write(vertexPos.x); + XModelBinWriterBase::Write(vertexPos.y); + XModelBinWriterBase::Write(vertexPos.z); WriteInt16(static_cast(XModelBinHash::VERT_WEIGHT_COUNT), static_cast(vertexPos.weightCount)); @@ -257,7 +268,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase const auto& weight = vertexPos.weights[weightIndex]; WriteInt16(static_cast(XModelBinHash::VERT_WEIGHT), weight.boneIndex); - m_writer.Write(weight.weight); + XModelBinWriterBase::Write(weight.weight); } vertexNum++; } @@ -265,21 +276,21 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteFaceVertex(const XModelVertex& vertex) { - m_writer.Write(static_cast(XModelBinHash::NORMAL)); - m_writer.Write(ClampFloatToShort(vertex.normal[0])); // X - m_writer.Write(ClampFloatToShort(vertex.normal[1])); // Y - m_writer.Write(ClampFloatToShort(vertex.normal[2])); // Z + XModelBinWriterBase::Write(static_cast(XModelBinHash::NORMAL)); + XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[0])); // X + XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[1])); // Y + XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[2])); // Z - m_writer.Write(XModelBinHash::COLOR); - m_writer.Write(ClampFloatToUByte(vertex.color[0])); // R - m_writer.Write(ClampFloatToUByte(vertex.color[1])); // G - m_writer.Write(ClampFloatToUByte(vertex.color[2])); // B - m_writer.Write(ClampFloatToUByte(vertex.color[3])); // A + XModelBinWriterBase::Write(XModelBinHash::COLOR); + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[0])); // R + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[1])); // G + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[2])); // B + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[3])); // A - m_writer.Write(static_cast(XModelBinHash::UV)); - m_writer.Write(static_cast(1)); // Layer - m_writer.Write(vertex.uv[0]); - m_writer.Write(vertex.uv[1]); + XModelBinWriterBase::Write(static_cast(XModelBinHash::UV)); + XModelBinWriterBase::Write(static_cast(1)); // Layer + XModelBinWriterBase::Write(vertex.uv[0]); + XModelBinWriterBase::Write(vertex.uv[1]); } void WriteFaces(const XModelCommon& xmodel) @@ -288,8 +299,8 @@ class XModelBinWriter7 final : public XModelBinWriterBase for (const auto& object : xmodel.m_objects) totalFaceCount += object.m_faces.size(); - m_writer.Write(XModelBinHash::FACE_COUNT); - m_writer.Write(totalFaceCount); + XModelBinWriterBase::Write(XModelBinHash::FACE_COUNT); + XModelBinWriterBase::Write(totalFaceCount); auto objectIndex = 0u; for (const auto& object : xmodel.m_objects) @@ -306,22 +317,22 @@ class XModelBinWriter7 final : public XModelBinWriterBase const XModelVertex& v1 = xmodel.m_vertices[face.vertexIndex[1]]; const XModelVertex& v2 = xmodel.m_vertices[face.vertexIndex[2]]; - m_writer.Write(static_cast(XModelBinHash::TRIANGLE32)); - m_writer.Write(static_cast(objectIndex)); - m_writer.Write(static_cast(object.materialIndex)); + XModelBinWriterBase::Write(static_cast(XModelBinHash::TRIANGLE32)); + XModelBinWriterBase::Write(static_cast(objectIndex)); + XModelBinWriterBase::Write(static_cast(object.materialIndex)); if (m_vertex_merger.GetDistinctValues().size() > UINT16_MAX) { - m_writer.Write(XModelBinHash::VERT32); - m_writer.Write(static_cast(distinctPositions[0])); + XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(distinctPositions[0])); WriteFaceVertex(v0); - m_writer.Write(XModelBinHash::VERT32); - m_writer.Write(static_cast(distinctPositions[1])); + XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(distinctPositions[1])); WriteFaceVertex(v1); - m_writer.Write(XModelBinHash::VERT32); - m_writer.Write(static_cast(distinctPositions[2])); + XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(distinctPositions[2])); WriteFaceVertex(v2); } else @@ -348,8 +359,8 @@ class XModelBinWriter7 final : public XModelBinWriterBase size_t objectNum = 0; for (const auto& object : xmodel.m_objects) { - m_writer.Write(static_cast(XModelBinHash::OBJECT)); - m_writer.Write(static_cast(objectNum)); + XModelBinWriterBase::Write(static_cast(XModelBinHash::OBJECT)); + XModelBinWriterBase::Write(static_cast(objectNum)); WriteAlignedString(object.name); objectNum++; @@ -363,71 +374,71 @@ class XModelBinWriter7 final : public XModelBinWriterBase size_t materialNum = 0u; for (const auto& material : xmodel.m_materials) { - const auto colorMapPath = "../images/" + material.colorMapName + ".dds"; + const auto colorMapPath = std::format("../images/{}.dds", material.colorMapName); WriteInt16(static_cast(XModelBinHash::MATERIAL), static_cast(materialNum)); WriteAlignedString(material.name); WriteAlignedString(material.materialTypeName); WriteAlignedString(colorMapPath); - m_writer.Write(XModelBinHash::COLOR); - m_writer.Write(ClampFloatToUByte(material.color[0])); // R - m_writer.Write(ClampFloatToUByte(material.color[1])); // G - m_writer.Write(ClampFloatToUByte(material.color[2])); // B - m_writer.Write(ClampFloatToUByte(material.color[3])); // A + XModelBinWriterBase::Write(XModelBinHash::COLOR); + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[0])); // R + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[1])); // G + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[2])); // B + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[3])); // A - m_writer.Write(XModelBinHash::MATERIAL_TRANSPARENCY); - m_writer.Write(material.transparency[0]); - m_writer.Write(material.transparency[1]); - m_writer.Write(material.transparency[2]); - m_writer.Write(material.transparency[3]); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_TRANSPARENCY); + XModelBinWriterBase::Write(material.transparency[0]); + XModelBinWriterBase::Write(material.transparency[1]); + XModelBinWriterBase::Write(material.transparency[2]); + XModelBinWriterBase::Write(material.transparency[3]); - m_writer.Write(XModelBinHash::MATERIAL_AMBIENT_COLOR); - m_writer.Write(material.ambientColor[0]); // R - m_writer.Write(material.ambientColor[1]); // G - m_writer.Write(material.ambientColor[2]); // B - m_writer.Write(material.ambientColor[3]); // A + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_AMBIENT_COLOR); + XModelBinWriterBase::Write(material.ambientColor[0]); // R + XModelBinWriterBase::Write(material.ambientColor[1]); // G + XModelBinWriterBase::Write(material.ambientColor[2]); // B + XModelBinWriterBase::Write(material.ambientColor[3]); // A - m_writer.Write(XModelBinHash::MATERIAL_INCANDESCENCE); - m_writer.Write(material.incandescence[0]); - m_writer.Write(material.incandescence[1]); - m_writer.Write(material.incandescence[2]); - m_writer.Write(material.incandescence[3]); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_INCANDESCENCE); + XModelBinWriterBase::Write(material.incandescence[0]); + XModelBinWriterBase::Write(material.incandescence[1]); + XModelBinWriterBase::Write(material.incandescence[2]); + XModelBinWriterBase::Write(material.incandescence[3]); - m_writer.Write(XModelBinHash::MATERIAL_COEFFS); - m_writer.Write(material.coeffs[0]); - m_writer.Write(material.coeffs[1]); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_COEFFS); + XModelBinWriterBase::Write(material.coeffs[0]); + XModelBinWriterBase::Write(material.coeffs[1]); - m_writer.Write(XModelBinHash::MATERIAL_GLOW); - m_writer.Write(material.glow.x); - m_writer.Write(material.glow.y); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_GLOW); + XModelBinWriterBase::Write(material.glow.x); + XModelBinWriterBase::Write(material.glow.y); - m_writer.Write(XModelBinHash::MATERIAL_REFRACTIVE); - m_writer.Write(material.refractive.x); - m_writer.Write(material.refractive.y); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_REFRACTIVE); + XModelBinWriterBase::Write(material.refractive.x); + XModelBinWriterBase::Write(material.refractive.y); - m_writer.Write(XModelBinHash::MATERIAL_SPECULAR_COLOR); - m_writer.Write(material.specularColor[0]); // R - m_writer.Write(material.specularColor[1]); // G - m_writer.Write(material.specularColor[2]); // B - m_writer.Write(material.specularColor[3]); // A + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_SPECULAR_COLOR); + XModelBinWriterBase::Write(material.specularColor[0]); // R + XModelBinWriterBase::Write(material.specularColor[1]); // G + XModelBinWriterBase::Write(material.specularColor[2]); // B + XModelBinWriterBase::Write(material.specularColor[3]); // A - m_writer.Write(XModelBinHash::MATERIAL_REFLECTIVE_COLOR); - m_writer.Write(material.reflectiveColor[0]); // R - m_writer.Write(material.reflectiveColor[1]); // G - m_writer.Write(material.reflectiveColor[2]); // B - m_writer.Write(material.reflectiveColor[3]); // A + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_REFLECTIVE_COLOR); + XModelBinWriterBase::Write(material.reflectiveColor[0]); // R + XModelBinWriterBase::Write(material.reflectiveColor[1]); // G + XModelBinWriterBase::Write(material.reflectiveColor[2]); // B + XModelBinWriterBase::Write(material.reflectiveColor[3]); // A - m_writer.Write(XModelBinHash::MATERIAL_REFLECTIVE); - m_writer.Write(material.reflective.x); - m_writer.Write(material.reflective.y); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_REFLECTIVE); + XModelBinWriterBase::Write(material.reflective.x); + XModelBinWriterBase::Write(material.reflective.y); - m_writer.Write(XModelBinHash::MATERIAL_BLINN); - m_writer.Write(material.blinn[0]); - m_writer.Write(material.blinn[1]); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_BLINN); + XModelBinWriterBase::Write(material.blinn[0]); + XModelBinWriterBase::Write(material.blinn[1]); - m_writer.Write(XModelBinHash::MATERIAL_PHONG); - m_writer.Write(material.phong); + XModelBinWriterBase::Write(XModelBinHash::MATERIAL_PHONG); + XModelBinWriterBase::Write(material.phong); materialNum++; } @@ -449,17 +460,16 @@ public: WriteObjects(xmodel); WriteMaterials(xmodel); - auto estimatedCompressedFileSize = LZ4_compressBound(m_writer.GetPosition()); + auto estimatedCompressedFileSize = LZ4_compressBound(m_writer.str().size()); auto compressedBuffer = new char[estimatedCompressedFileSize]; - auto actualCompressedFileSize = - LZ4_compress_default(reinterpret_cast(m_writer.Data()), compressedBuffer, m_writer.GetPosition(), estimatedCompressedFileSize); + auto actualCompressedFileSize = LZ4_compress_default(m_writer.str().c_str(), compressedBuffer, m_writer.str().size(), estimatedCompressedFileSize); - uint64_t uncompressedSize = m_writer.GetPosition(); + auto uncompressedSize = m_writer.str().size(); char uncompressedSizeChar[4]; std::memcpy(uncompressedSizeChar, &uncompressedSize, sizeof(uncompressedSizeChar)); - const char magic[5] = {0x2A, 0x4C, 0x5A, 0x34, 0x2A}; // *LZ4* - m_stream.write(magic, sizeof(magic)); + static constexpr char MAGIC[5] = {'*', 'L', 'Z', '4', '*'}; + m_stream.write(MAGIC, sizeof(MAGIC)); m_stream.write(uncompressedSizeChar, sizeof(uncompressedSizeChar)); m_stream.write(compressedBuffer, actualCompressedFileSize); } diff --git a/src/Utils/Utils/MemoryWriter.cpp b/src/Utils/Utils/MemoryWriter.cpp deleted file mode 100644 index efdb833a..00000000 --- a/src/Utils/Utils/MemoryWriter.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include "MemoryWriter.h" - -#include -#include - -MemoryWriter::MemoryWriter(uint32_t capacity) -{ - m_dataPointer = new int8_t[capacity]; - m_dataLength = capacity; - m_currentPosition = 0; -} - -MemoryWriter::~MemoryWriter() -{ - Close(); -} - -size_t MemoryWriter::GetLength() const -{ - return m_dataLength; -} - -size_t MemoryWriter::GetPosition() const -{ - return m_currentPosition; -} - -int8_t* MemoryWriter::Data() const -{ - return m_dataPointer; -} - -void MemoryWriter::Close() -{ - if (m_dataPointer != nullptr) - { - delete[] m_dataPointer; - m_dataPointer = nullptr; - } -} - -void MemoryWriter::Write(const uint8_t* buffer, uint32_t size) -{ - ValidateCapacity(m_currentPosition + size); - - std::memcpy(m_dataPointer + m_currentPosition, buffer, size); - m_currentPosition += size; -} - -void MemoryWriter::WriteNullTerminatedString(const std::string& string) -{ - Write(reinterpret_cast(string.c_str()), string.size() + 1); -} - -void MemoryWriter::Reallocate(size_t capacity) -{ - auto tempPtr = static_cast(std::realloc(m_dataPointer, capacity)); - - if (tempPtr != nullptr) - { - m_dataPointer = tempPtr; - m_dataLength = capacity; - } -} - -void MemoryWriter::ValidateCapacity(size_t newSize) -{ - if (newSize > m_dataLength) - { - auto newCapacity = m_dataLength * 2; - assert(newCapacity <= UINT64_MAX); - Reallocate(newCapacity); - } -} diff --git a/src/Utils/Utils/MemoryWriter.h b/src/Utils/Utils/MemoryWriter.h deleted file mode 100644 index 979c6920..00000000 --- a/src/Utils/Utils/MemoryWriter.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include -#include - -class MemoryWriter -{ -public: - MemoryWriter(uint32_t capacity); - ~MemoryWriter(); - - size_t GetLength() const; - size_t GetPosition() const; - int8_t* Data() const; - - void Close(); - void Write(const uint8_t* buffer, uint32_t size); - void WriteNullTerminatedString(const std::string& string); - - template void Write(const T data) - { - Write((uint8_t*)&data, sizeof(T)); - } - -private: - void Reallocate(size_t capacity); - void ValidateCapacity(size_t size); - -private: - int8_t* m_dataPointer; - size_t m_dataLength; - size_t m_currentPosition; -}; From 101487d8691060c803ba22de70391e89010127e5 Mon Sep 17 00:00:00 2001 From: Simon Ickler Date: Tue, 22 Apr 2025 11:31:40 +0200 Subject: [PATCH 10/12] Fixed Memory leak + other small fixes --- .../XModel/Export/XModelBinWriter.cpp | 141 +++++++++--------- 1 file changed, 73 insertions(+), 68 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 649c1e45..739642e2 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -1,14 +1,20 @@ #include "XModelBinWriter.h" +#include "GitVersion.h" + #pragma warning(push, 0) #include #pragma warning(pop) #include +#include #include #include #include +#include #include +#include +#include class XModelBinWriterBase : public XModelWriter { @@ -25,7 +31,7 @@ protected: * 1 = Object index * Test_Obj2 = Object name */ - enum XModelBinHash : uint32_t + enum XModelBinHash : uint16_t { COMMENT = 0xC355, MODEL = 0x46C8, @@ -116,17 +122,17 @@ protected: void WriteComment(const std::string& comment) { - Write(XModelBinHash::COMMENT); + Write(static_cast(XModelBinHash::COMMENT)); WriteAlignedString(comment); } - void WriteInt16(const int16_t hash, const int16_t value) + void WriteInt16(const uint16_t hash, const int16_t value) { Write(hash); Write(value); } - void WriteUInt16(const int16_t hash, const uint16_t value) + void WriteUInt16(const uint16_t hash, const uint16_t value) { Write(hash); Write(value); @@ -146,23 +152,23 @@ protected: std::numeric_limits::max()); } - void WriteHeader(int16_t version) + void WriteHeader(uint16_t version) { - WriteComment("OpenAssetTools XMODEL_BIN File"); + WriteComment("OpenAssetTools " GIT_VERSION " XMODEL_BIN File"); WriteComment(std::format("Game Origin: {}", m_game_name)); WriteComment(std::format("Zone Origin: {}", m_zone_name)); - Write(XModelBinHash::MODEL); - WriteInt16(static_cast(XModelBinHash::VERSION), version); + Write(static_cast(XModelBinHash::MODEL)); + WriteUInt16(XModelBinHash::VERSION, version); } void WriteBones(const XModelCommon& xmodel) { - WriteInt16(static_cast(XModelBinHash::BONE_COUNT), static_cast(xmodel.m_bones.size())); + WriteUInt16(XModelBinHash::BONE_COUNT, static_cast(xmodel.m_bones.size())); - auto boneNum = 0; + auto boneNum = 0U; for (const auto& bone : xmodel.m_bones) { - Write(XModelBinHash::BONE); + Write(static_cast(XModelBinHash::BONE)); Write(boneNum); if (bone.parentIndex) Write(static_cast(*bone.parentIndex)); @@ -176,31 +182,31 @@ protected: boneNum = 0; for (const auto& bone : xmodel.m_bones) { - WriteInt16(static_cast(XModelBinHash::BONE_INDEX), boneNum); + WriteUInt16(XModelBinHash::BONE_INDEX, static_cast(boneNum)); - Write(XModelBinHash::OFFSET); + Write(static_cast(XModelBinHash::OFFSET)); Write(bone.globalOffset[0]); // X Write(bone.globalOffset[1]); // Y Write(bone.globalOffset[2]); // Z - Write(XModelBinHash::BONE_SCALE); + Write(static_cast(XModelBinHash::BONE_SCALE)); Write(bone.scale[0]); // X Write(bone.scale[1]); // Y Write(bone.scale[2]); // Z const auto mat = Eigen::Quaternionf(bone.globalRotation.w, bone.globalRotation.x, bone.globalRotation.y, bone.globalRotation.z).matrix(); - Write(static_cast(XModelBinHash::BONE_MATRIX_X)); + Write(XModelBinHash::BONE_MATRIX_X); Write(ClampFloatToShort(mat(0, 0))); Write(ClampFloatToShort(mat(0, 1))); Write(ClampFloatToShort(mat(0, 2))); - Write(static_cast(XModelBinHash::BONE_MATRIX_Y)); + Write(XModelBinHash::BONE_MATRIX_Y); Write(ClampFloatToShort(mat(1, 0))); Write(ClampFloatToShort(mat(1, 1))); Write(ClampFloatToShort(mat(1, 2))); - Write(static_cast(XModelBinHash::BONE_MATRIX_Z)); + Write(XModelBinHash::BONE_MATRIX_Z); Write(ClampFloatToShort(mat(2, 0))); Write(ClampFloatToShort(mat(2, 1))); Write(ClampFloatToShort(mat(2, 2))); @@ -209,15 +215,13 @@ protected: } } - XModelBinWriterBase(std::ostream& stream, std::string gameName, std::string zoneName) - : m_stream(stream), - m_game_name(std::move(gameName)), + XModelBinWriterBase(std::string gameName, std::string zoneName) + : m_game_name(std::move(gameName)), m_zone_name(std::move(zoneName)) { } std::ostringstream m_writer; - std::ostream& m_stream; std::string m_game_name; std::string m_zone_name; VertexMerger m_vertex_merger; @@ -229,45 +233,45 @@ class XModelBinWriter7 final : public XModelBinWriterBase { const auto& distinctVertexValues = m_vertex_merger.GetDistinctValues(); - if (distinctVertexValues.size() > UINT16_MAX) + if (distinctVertexValues.size() > std::numeric_limits::max()) { // Use 32 bit - XModelBinWriterBase::Write(XModelBinHash::VERT32_COUNT); + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32_COUNT)); XModelBinWriterBase::Write(static_cast(distinctVertexValues.size())); } else { // Use 16 bit - WriteUInt16(static_cast(XModelBinHash::VERT16_COUNT), static_cast(distinctVertexValues.size())); + WriteUInt16(XModelBinHash::VERT16_COUNT, static_cast(distinctVertexValues.size())); } size_t vertexNum = 0u; for (const auto& vertexPos : distinctVertexValues) { - if (vertexNum > UINT16_MAX) + if (vertexNum > std::numeric_limits::max()) { // Use 32 bit - XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); XModelBinWriterBase::Write(static_cast(vertexNum)); } else { // Use 16 bit - WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(vertexNum)); + WriteUInt16(XModelBinHash::VERT16, static_cast(vertexNum)); } - XModelBinWriterBase::Write(XModelBinHash::OFFSET); + XModelBinWriterBase::Write(static_cast(XModelBinHash::OFFSET)); XModelBinWriterBase::Write(vertexPos.x); XModelBinWriterBase::Write(vertexPos.y); XModelBinWriterBase::Write(vertexPos.z); - WriteInt16(static_cast(XModelBinHash::VERT_WEIGHT_COUNT), static_cast(vertexPos.weightCount)); + WriteUInt16(XModelBinHash::VERT_WEIGHT_COUNT, static_cast(vertexPos.weightCount)); for (auto weightIndex = 0u; weightIndex < vertexPos.weightCount; weightIndex++) { const auto& weight = vertexPos.weights[weightIndex]; - WriteInt16(static_cast(XModelBinHash::VERT_WEIGHT), weight.boneIndex); + WriteInt16(XModelBinHash::VERT_WEIGHT, weight.boneIndex); XModelBinWriterBase::Write(weight.weight); } vertexNum++; @@ -276,18 +280,18 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteFaceVertex(const XModelVertex& vertex) { - XModelBinWriterBase::Write(static_cast(XModelBinHash::NORMAL)); + XModelBinWriterBase::Write(XModelBinHash::NORMAL); XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[0])); // X XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[1])); // Y XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[2])); // Z - XModelBinWriterBase::Write(XModelBinHash::COLOR); + XModelBinWriterBase::Write(static_cast(XModelBinHash::COLOR)); XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[0])); // R XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[1])); // G XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[2])); // B XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[3])); // A - XModelBinWriterBase::Write(static_cast(XModelBinHash::UV)); + XModelBinWriterBase::Write(XModelBinHash::UV); XModelBinWriterBase::Write(static_cast(1)); // Layer XModelBinWriterBase::Write(vertex.uv[0]); XModelBinWriterBase::Write(vertex.uv[1]); @@ -299,7 +303,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase for (const auto& object : xmodel.m_objects) totalFaceCount += object.m_faces.size(); - XModelBinWriterBase::Write(XModelBinHash::FACE_COUNT); + XModelBinWriterBase::Write(static_cast(XModelBinHash::FACE_COUNT)); XModelBinWriterBase::Write(totalFaceCount); auto objectIndex = 0u; @@ -317,33 +321,33 @@ class XModelBinWriter7 final : public XModelBinWriterBase const XModelVertex& v1 = xmodel.m_vertices[face.vertexIndex[1]]; const XModelVertex& v2 = xmodel.m_vertices[face.vertexIndex[2]]; - XModelBinWriterBase::Write(static_cast(XModelBinHash::TRIANGLE32)); + XModelBinWriterBase::Write(XModelBinHash::TRIANGLE32); XModelBinWriterBase::Write(static_cast(objectIndex)); XModelBinWriterBase::Write(static_cast(object.materialIndex)); - if (m_vertex_merger.GetDistinctValues().size() > UINT16_MAX) + if (m_vertex_merger.GetDistinctValues().size() > std::numeric_limits::max()) { - XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); XModelBinWriterBase::Write(static_cast(distinctPositions[0])); WriteFaceVertex(v0); - XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); XModelBinWriterBase::Write(static_cast(distinctPositions[1])); WriteFaceVertex(v1); - XModelBinWriterBase::Write(XModelBinHash::VERT32); + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); XModelBinWriterBase::Write(static_cast(distinctPositions[2])); WriteFaceVertex(v2); } else { - WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(distinctPositions[0])); + WriteUInt16(XModelBinHash::VERT16, static_cast(distinctPositions[0])); WriteFaceVertex(v0); - WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(distinctPositions[1])); + WriteUInt16(XModelBinHash::VERT16, static_cast(distinctPositions[1])); WriteFaceVertex(v1); - WriteUInt16(static_cast(XModelBinHash::VERT16), static_cast(distinctPositions[2])); + WriteUInt16(XModelBinHash::VERT16, static_cast(distinctPositions[2])); WriteFaceVertex(v2); } } @@ -354,13 +358,13 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteObjects(const XModelCommon& xmodel) { - WriteInt16(XModelBinHash::OBJECT_COUNT, static_cast(xmodel.m_objects.size())); + WriteUInt16(XModelBinHash::OBJECT_COUNT, static_cast(xmodel.m_objects.size())); size_t objectNum = 0; for (const auto& object : xmodel.m_objects) { - XModelBinWriterBase::Write(static_cast(XModelBinHash::OBJECT)); - XModelBinWriterBase::Write(static_cast(objectNum)); + XModelBinWriterBase::Write(XModelBinHash::OBJECT); + XModelBinWriterBase::Write(static_cast(objectNum)); WriteAlignedString(object.name); objectNum++; @@ -369,84 +373,87 @@ class XModelBinWriter7 final : public XModelBinWriterBase void WriteMaterials(const XModelCommon& xmodel) { - WriteInt16(static_cast(XModelBinHash::MATERIAL_COUNT), static_cast(xmodel.m_materials.size())); + WriteUInt16(XModelBinHash::MATERIAL_COUNT, static_cast(xmodel.m_materials.size())); size_t materialNum = 0u; for (const auto& material : xmodel.m_materials) { const auto colorMapPath = std::format("../images/{}.dds", material.colorMapName); - WriteInt16(static_cast(XModelBinHash::MATERIAL), static_cast(materialNum)); + WriteUInt16(XModelBinHash::MATERIAL, static_cast(materialNum)); WriteAlignedString(material.name); WriteAlignedString(material.materialTypeName); WriteAlignedString(colorMapPath); - XModelBinWriterBase::Write(XModelBinHash::COLOR); + XModelBinWriterBase::Write(static_cast(XModelBinHash::COLOR)); XModelBinWriterBase::Write(ClampFloatToUByte(material.color[0])); // R XModelBinWriterBase::Write(ClampFloatToUByte(material.color[1])); // G XModelBinWriterBase::Write(ClampFloatToUByte(material.color[2])); // B XModelBinWriterBase::Write(ClampFloatToUByte(material.color[3])); // A - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_TRANSPARENCY); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_TRANSPARENCY)); XModelBinWriterBase::Write(material.transparency[0]); XModelBinWriterBase::Write(material.transparency[1]); XModelBinWriterBase::Write(material.transparency[2]); XModelBinWriterBase::Write(material.transparency[3]); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_AMBIENT_COLOR); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_AMBIENT_COLOR)); XModelBinWriterBase::Write(material.ambientColor[0]); // R XModelBinWriterBase::Write(material.ambientColor[1]); // G XModelBinWriterBase::Write(material.ambientColor[2]); // B XModelBinWriterBase::Write(material.ambientColor[3]); // A - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_INCANDESCENCE); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_INCANDESCENCE)); XModelBinWriterBase::Write(material.incandescence[0]); XModelBinWriterBase::Write(material.incandescence[1]); XModelBinWriterBase::Write(material.incandescence[2]); XModelBinWriterBase::Write(material.incandescence[3]); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_COEFFS); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_COEFFS)); XModelBinWriterBase::Write(material.coeffs[0]); XModelBinWriterBase::Write(material.coeffs[1]); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_GLOW); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_GLOW)); XModelBinWriterBase::Write(material.glow.x); XModelBinWriterBase::Write(material.glow.y); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_REFRACTIVE); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_REFRACTIVE)); XModelBinWriterBase::Write(material.refractive.x); XModelBinWriterBase::Write(material.refractive.y); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_SPECULAR_COLOR); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_SPECULAR_COLOR)); XModelBinWriterBase::Write(material.specularColor[0]); // R XModelBinWriterBase::Write(material.specularColor[1]); // G XModelBinWriterBase::Write(material.specularColor[2]); // B XModelBinWriterBase::Write(material.specularColor[3]); // A - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_REFLECTIVE_COLOR); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_REFLECTIVE_COLOR)); XModelBinWriterBase::Write(material.reflectiveColor[0]); // R XModelBinWriterBase::Write(material.reflectiveColor[1]); // G XModelBinWriterBase::Write(material.reflectiveColor[2]); // B XModelBinWriterBase::Write(material.reflectiveColor[3]); // A - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_REFLECTIVE); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_REFLECTIVE)); XModelBinWriterBase::Write(material.reflective.x); XModelBinWriterBase::Write(material.reflective.y); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_BLINN); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_BLINN)); XModelBinWriterBase::Write(material.blinn[0]); XModelBinWriterBase::Write(material.blinn[1]); - XModelBinWriterBase::Write(XModelBinHash::MATERIAL_PHONG); + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_PHONG)); XModelBinWriterBase::Write(material.phong); materialNum++; } } + std::ostream& m_stream; + public: XModelBinWriter7(std::ostream& stream, std::string gameName, std::string zoneName) - : XModelBinWriterBase(stream, std::move(gameName), std::move(zoneName)) + : m_stream(stream), + XModelBinWriterBase(std::move(gameName), std::move(zoneName)) { } @@ -460,18 +467,16 @@ public: WriteObjects(xmodel); WriteMaterials(xmodel); + auto uncompressedSize = static_cast(m_writer.str().size()); auto estimatedCompressedFileSize = LZ4_compressBound(m_writer.str().size()); - auto compressedBuffer = new char[estimatedCompressedFileSize]; - auto actualCompressedFileSize = LZ4_compress_default(m_writer.str().c_str(), compressedBuffer, m_writer.str().size(), estimatedCompressedFileSize); - - auto uncompressedSize = m_writer.str().size(); - char uncompressedSizeChar[4]; - std::memcpy(uncompressedSizeChar, &uncompressedSize, sizeof(uncompressedSizeChar)); + const auto compressedBuffer = std::make_unique(estimatedCompressedFileSize); + auto actualCompressedFileSize = + LZ4_compress_default(m_writer.str().c_str(), compressedBuffer.get(), m_writer.str().size(), estimatedCompressedFileSize); static constexpr char MAGIC[5] = {'*', 'L', 'Z', '4', '*'}; m_stream.write(MAGIC, sizeof(MAGIC)); - m_stream.write(uncompressedSizeChar, sizeof(uncompressedSizeChar)); - m_stream.write(compressedBuffer, actualCompressedFileSize); + m_stream.write(reinterpret_cast(&uncompressedSize), sizeof(uncompressedSize)); + m_stream.write(compressedBuffer.get(), actualCompressedFileSize); } }; From dc282170bc9e4361c4ad7f3b37345050a292c490 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 23 Apr 2025 16:52:04 +0100 Subject: [PATCH 11/12] chore: small code style adjustments for XModelBinWriter --- .../XModel/Export/XModelBinWriter.cpp | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 739642e2..92e66266 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -86,16 +86,17 @@ protected: auto vertexOffset = 0u; for (const auto& vertex : xmodel.m_vertices) { - XModelVertexBoneWeights weights{0, 0}; + XModelVertexBoneWeights weights{.weightOffset = 0, .weightCount = 0}; if (vertexOffset < xmodel.m_vertex_bone_weights.size()) weights = xmodel.m_vertex_bone_weights[vertexOffset]; - m_vertex_merger.Add(VertexMergerPos{vertex.coordinates[0], - vertex.coordinates[1], - vertex.coordinates[2], - xmodel.m_bone_weight_data.weights.empty() ? nullptr : &xmodel.m_bone_weight_data.weights[weights.weightOffset], - weights.weightCount}); + m_vertex_merger.Add( + VertexMergerPos{.x = vertex.coordinates[0], + .y = vertex.coordinates[1], + .z = vertex.coordinates[2], + .weights = xmodel.m_bone_weight_data.weights.empty() ? nullptr : &xmodel.m_bone_weight_data.weights[weights.weightOffset], + .weightCount = weights.weightCount}); vertexOffset++; } @@ -106,7 +107,7 @@ protected: m_writer.write(reinterpret_cast(&data), sizeof(T)); } - void WriteNullTerminatedString(std::string string) + void WriteNullTerminatedString(const std::string& string) { m_writer.write(string.c_str(), string.size() + 1); } @@ -138,21 +139,21 @@ protected: Write(value); } - int16_t ClampFloatToShort(const float value) const + [[nodiscard]] static int16_t ClampFloatToShort(const float value) { return std::clamp(static_cast(std::numeric_limits::max() * std::clamp(value, -1.0f, 1.0f)), std::numeric_limits::min(), std::numeric_limits::max()); } - uint8_t ClampFloatToUByte(const float value) const + [[nodiscard]] static uint8_t ClampFloatToUByte(const float value) { return std::clamp(static_cast(std::numeric_limits::max() * std::clamp(value, -1.0f, 1.0f)), std::numeric_limits::min(), std::numeric_limits::max()); } - void WriteHeader(uint16_t version) + void WriteHeader(const uint16_t version) { WriteComment("OpenAssetTools " GIT_VERSION " XMODEL_BIN File"); WriteComment(std::format("Game Origin: {}", m_game_name)); @@ -182,7 +183,7 @@ protected: boneNum = 0; for (const auto& bone : xmodel.m_bones) { - WriteUInt16(XModelBinHash::BONE_INDEX, static_cast(boneNum)); + WriteUInt16(XModelBinHash::BONE_INDEX, static_cast(boneNum)); Write(static_cast(XModelBinHash::OFFSET)); Write(bone.globalOffset[0]); // X @@ -271,7 +272,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase { const auto& weight = vertexPos.weights[weightIndex]; - WriteInt16(XModelBinHash::VERT_WEIGHT, weight.boneIndex); + WriteUInt16(XModelBinHash::VERT_WEIGHT, static_cast(weight.boneIndex)); XModelBinWriterBase::Write(weight.weight); } vertexNum++; @@ -292,7 +293,7 @@ class XModelBinWriter7 final : public XModelBinWriterBase XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[3])); // A XModelBinWriterBase::Write(XModelBinHash::UV); - XModelBinWriterBase::Write(static_cast(1)); // Layer + XModelBinWriterBase::Write(static_cast(1u)); // Layer XModelBinWriterBase::Write(vertex.uv[0]); XModelBinWriterBase::Write(vertex.uv[1]); } @@ -452,8 +453,8 @@ class XModelBinWriter7 final : public XModelBinWriterBase public: XModelBinWriter7(std::ostream& stream, std::string gameName, std::string zoneName) - : m_stream(stream), - XModelBinWriterBase(std::move(gameName), std::move(zoneName)) + : XModelBinWriterBase(std::move(gameName), std::move(zoneName)), + m_stream(stream) { } @@ -468,10 +469,10 @@ public: WriteMaterials(xmodel); auto uncompressedSize = static_cast(m_writer.str().size()); - auto estimatedCompressedFileSize = LZ4_compressBound(m_writer.str().size()); + const auto estimatedCompressedFileSize = LZ4_compressBound(static_cast(m_writer.str().size())); const auto compressedBuffer = std::make_unique(estimatedCompressedFileSize); - auto actualCompressedFileSize = - LZ4_compress_default(m_writer.str().c_str(), compressedBuffer.get(), m_writer.str().size(), estimatedCompressedFileSize); + const auto actualCompressedFileSize = + LZ4_compress_default(m_writer.str().c_str(), compressedBuffer.get(), static_cast(m_writer.str().size()), estimatedCompressedFileSize); static constexpr char MAGIC[5] = {'*', 'L', 'Z', '4', '*'}; m_stream.write(MAGIC, sizeof(MAGIC)); From 76c22ffce2cfdfd273ab372c5e0a7bbec4d5f4f3 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 23 Apr 2025 17:00:17 +0100 Subject: [PATCH 12/12] chore: adjust include path for lz4 dependency --- src/ObjWriting/XModel/Export/XModelBinWriter.cpp | 5 +---- thirdparty/lz4.lua | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index 92e66266..2387f937 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -8,11 +8,8 @@ #include #include -#include -#include -#include #include -#include +#include #include #include diff --git a/thirdparty/lz4.lua b/thirdparty/lz4.lua index 5554e196..ef315d74 100644 --- a/thirdparty/lz4.lua +++ b/thirdparty/lz4.lua @@ -3,7 +3,7 @@ lz4 = {} function lz4:include(includes) if includes:handle(self:name()) then includedirs { - path.join(ThirdPartyFolder(), "lz4") + path.join(ThirdPartyFolder(), "lz4/lib") } end end