diff --git a/.gitmodules b/.gitmodules index 7bfede4e..da2074a0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "thirdparty/eigen"] path = thirdparty/eigen url = https://gitlab.com/libeigen/eigen.git +[submodule "thirdparty/lz4"] + path = thirdparty/lz4 + url = git@github.com:lz4/lz4.git diff --git a/docs/SupportedAssetTypes.md b/docs/SupportedAssetTypes.md index a9c9e375..5b0d8f05 100644 --- a/docs/SupportedAssetTypes.md +++ b/docs/SupportedAssetTypes.md @@ -13,7 +13,7 @@ The following section specify which assets are supported to be dumped to disk (u | -------------------- | --------------- | --------------- | ----------------------------------------------------------------- | | PhysPreset | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialTechniqueSet | ❌ | ❌ | | | GfxImage | ✅ | ✅ | | @@ -44,7 +44,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysPreset | ✅ | ✅ | | | PhysCollmap | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialPixelShader | ✅ | ✅ | Shaders are compiled. Only dumps/loads shader bytecode. | | MaterialVertexShader | ✅ | ✅ | Shaders are compiled. Only dumps/loads shader bytecode. | @@ -85,7 +85,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysCollmap | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | | XModelSurfs | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialPixelShader | ❌ | ❌ | | | MaterialVertexShader | ❌ | ❌ | | @@ -130,7 +130,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysConstraints | ❌ | ❌ | | | DestructibleDef | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialTechniqueSet | ❌ | ❌ | | | GfxImage | ✅ | ❌ | A few special image encodings are not yet supported. | @@ -167,7 +167,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysConstraints | ✅ | ✅ | | | DestructibleDef | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ✅ | ✅ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ✅ | ✅ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ⁉️ | ⁉️ | Dumping/Loading is currently possible for materials in their compiled form. There is currently no material pipeline. | | MaterialTechniqueSet | ⁉️ | ❌ | Only dumps compiled shaders. | | GfxImage | ✅ | ✅ | A few special image encodings are not yet supported. | diff --git a/premake5.lua b/premake5.lua index 7a20bd50..cfa67e29 100644 --- a/premake5.lua +++ b/premake5.lua @@ -97,6 +97,7 @@ include "thirdparty/minilzo.lua" include "thirdparty/minizip.lua" include "thirdparty/salsa20.lua" include "thirdparty/zlib.lua" +include "thirdparty/lz4.lua" -- ThirdParty group: All projects that are external dependencies group "ThirdParty" @@ -109,6 +110,7 @@ group "ThirdParty" minizip:project() salsa20:project() zlib:project() + lz4:project() group "" -- ======================== diff --git a/src/ObjWriting.lua b/src/ObjWriting.lua index 186fa9d1..1f715ade 100644 --- a/src/ObjWriting.lua +++ b/src/ObjWriting.lua @@ -21,6 +21,7 @@ function ObjWriting:link(links) links:linkto(ZoneCommon) links:linkto(minizip) links:linkto(libtomcrypt) + links:linkto(lz4) end function ObjWriting:use() @@ -60,5 +61,5 @@ function ObjWriting:project() eigen:include(includes) json:include(includes) libtomcrypt:include(includes) - + lz4:include(includes) end diff --git a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp index cd6789e1..24b4246a 100644 --- a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp @@ -5,6 +5,7 @@ #include "Utils/DistinctMapper.h" #include "Utils/HalfFloat.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -458,6 +459,18 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".XMODEL_EXPORT")); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -495,6 +508,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; diff --git a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp index 19d5ec79..71474a51 100644 --- a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp @@ -5,6 +5,7 @@ #include "Utils/DistinctMapper.h" #include "Utils/HalfFloat.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -446,6 +447,19 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto* modelSurfs = model->lodInfo[lod].modelSurfs; + const auto assetFile = context.OpenAssetFile(std::format("model_export/{}.XMODEL_BIN", modelSurfs->name)); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -484,6 +498,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; diff --git a/src/ObjWriting/ObjWriting.h b/src/ObjWriting/ObjWriting.h index b27beb25..0fe1ec93 100644 --- a/src/ObjWriting/ObjWriting.h +++ b/src/ObjWriting/ObjWriting.h @@ -20,6 +20,7 @@ public: enum class ModelOutputFormat_e { XMODEL_EXPORT, + XMODEL_BIN, OBJ, GLTF, GLB diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index e69de29b..6081923d 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -0,0 +1,414 @@ +#include "XModelBinWriter.h" + +#include "Utils/MemoryWriter.h" + +#pragma warning(push, 0) +#include +#pragma warning(pop) + +#include +#include +#include +#include + +class XModelBinWriterBase : public XModelWriter +{ +protected: + void PrepareVertexMerger(const XModelCommon& xmodel) + { + m_vertex_merger = VertexMerger(xmodel.m_vertices.size()); + + auto vertexOffset = 0u; + for (const auto& vertex : xmodel.m_vertices) + { + XModelVertexBoneWeights weights{0, 0}; + + if (vertexOffset < xmodel.m_vertex_bone_weights.size()) + weights = xmodel.m_vertex_bone_weights[vertexOffset]; + + m_vertex_merger.Add(VertexMergerPos{vertex.coordinates[0], + vertex.coordinates[1], + vertex.coordinates[2], + xmodel.m_bone_weight_data.weights.empty() ? nullptr : &xmodel.m_bone_weight_data.weights[weights.weightOffset], + weights.weightCount}); + + vertexOffset++; + } + } + + void WriteAlignedString(const std::string& string) + { + auto paddingSize = ((string.size() + 1 + 0x3) & 0xFFFFFFFFFFFFFC) - (string.size() + 1); + auto padding = std::make_unique(paddingSize); + + m_writer.WriteNullTerminatedString(string); + + m_writer.Write(reinterpret_cast(padding.get()), paddingSize); + } + + void WriteComment(const std::string& comment) + { + m_writer.Write(0xC355); + WriteAlignedString(comment); + } + + void WriteInt16(const int16_t hash, const int16_t value) + { + m_writer.Write(hash); + m_writer.Write(value); + } + + void WriteUInt16(const int16_t hash, const uint16_t value) + { + m_writer.Write(hash); + m_writer.Write(value); + } + + int16_t ClampFloatToShort(const float value) const + { + return std::clamp(static_cast(32767 * std::clamp(value, -1.0f, 1.0f)), static_cast(-32767), 32767i16); + } + + uint8_t ClampFloatToUByte(const float value) const + { + return std::clamp(static_cast(255 * std::clamp(value, -1.0f, 1.0f)), 0ui8, 255ui8); + } + + void WriteHeader(int16_t version) + { + WriteComment("OpenAssetTools XMODEL_BIN File"); + WriteComment(std::format("Game Origin: {}", m_game_name)); + WriteComment(std::format("Zone Origin: {}", m_zone_name)); + m_writer.Write(0x46C8); // Model Hash + WriteInt16(0x24D1, version); + } + + void WriteBones(const XModelCommon& xmodel) + { + WriteInt16(0x76BA, xmodel.m_bones.size()); + + auto boneNum = 0; + for (const auto& bone : xmodel.m_bones) + { + m_writer.Write(0xF099); // Bone Hash + m_writer.Write(boneNum); + if (bone.parentIndex) + m_writer.Write(static_cast(*bone.parentIndex)); + else + m_writer.Write(-1); + + WriteAlignedString(bone.name); + boneNum++; + } + + boneNum = 0; + for (const auto& bone : xmodel.m_bones) + { + WriteInt16(0xDD9A, boneNum); + + m_writer.Write(0x9383); // Bone Offset Hash + m_writer.Write(bone.globalOffset[0]); // X + m_writer.Write(bone.globalOffset[1]); // Y + m_writer.Write(bone.globalOffset[2]); // Z + + m_writer.Write(0x1C56); // Bone Scale Hash + m_writer.Write(bone.scale[0]); // X + m_writer.Write(bone.scale[1]); // Y + m_writer.Write(bone.scale[2]); // Z + + const auto mat = Eigen::Quaternionf(bone.globalRotation.w, bone.globalRotation.x, bone.globalRotation.y, bone.globalRotation.z).matrix(); + + m_writer.Write(0xDCFDi16); // Bone Matrix X Hash + m_writer.Write(ClampFloatToShort(mat(0, 0))); + m_writer.Write(ClampFloatToShort(mat(0, 1))); + m_writer.Write(ClampFloatToShort(mat(0, 2))); + + m_writer.Write(0xCCDCi16); // Bone Matrix Y Hash + m_writer.Write(ClampFloatToShort(mat(1, 0))); + m_writer.Write(ClampFloatToShort(mat(1, 1))); + m_writer.Write(ClampFloatToShort(mat(1, 2))); + + m_writer.Write(0xFCBFi16); // Bone Matrix Z Hash + m_writer.Write(ClampFloatToShort(mat(2, 0))); + m_writer.Write(ClampFloatToShort(mat(2, 1))); + m_writer.Write(ClampFloatToShort(mat(2, 2))); + + boneNum++; + } + } + + XModelBinWriterBase(std::ostream& stream, std::string gameName, std::string zoneName) + : m_writer(0x800000), // 8MB + m_stream(stream), + m_game_name(std::move(gameName)), + m_zone_name(std::move(zoneName)) + { + } + + MemoryWriter m_writer; + std::ostream& m_stream; + std::string m_game_name; + std::string m_zone_name; + VertexMerger m_vertex_merger; +}; + +class XModelBinWriter7 final : public XModelBinWriterBase +{ + void WriteVertices(const XModelCommon& xmodel) + { + const auto& distinctVertexValues = m_vertex_merger.GetDistinctValues(); + + if (distinctVertexValues.size() > UINT16_MAX) + { + // Use 32 bit + m_writer.Write(0x2AEC); // 32 bit Hash + m_writer.Write(static_cast(distinctVertexValues.size())); + } + else + { + // Use 16 bit + WriteUInt16(0x950D, distinctVertexValues.size()); + } + + size_t vertexNum = 0u; + for (const auto& vertexPos : distinctVertexValues) + { + if (vertexNum > UINT16_MAX) + { + // Use 32 bit + m_writer.Write(0xB097); // 32 bit Hash + m_writer.Write(static_cast(vertexNum)); + } + else + { + // Use 16 bit + WriteUInt16(0x8F03, vertexNum); + } + + m_writer.Write(0x9383); // Offset Hash + m_writer.Write(vertexPos.x); + m_writer.Write(vertexPos.y); + m_writer.Write(vertexPos.z); + + WriteInt16(0xEA46, vertexPos.weightCount); + + for (auto weightIndex = 0u; weightIndex < vertexPos.weightCount; weightIndex++) + { + const auto& weight = vertexPos.weights[weightIndex]; + + WriteInt16(0xF1AB, weight.boneIndex); + m_writer.Write(weight.weight); + } + vertexNum++; + } + } + + void WriteFaceVertex(const XModelVertex& vertex) + { + m_writer.Write(0x89ECi16); // Normal Hash + m_writer.Write(ClampFloatToShort(vertex.normal[0])); // X + m_writer.Write(ClampFloatToShort(vertex.normal[1])); // Y + m_writer.Write(ClampFloatToShort(vertex.normal[2])); // Z + + m_writer.Write(0x6DD8); // Color Hash + m_writer.Write(ClampFloatToUByte(vertex.color[0])); // R + m_writer.Write(ClampFloatToUByte(vertex.color[1])); // G + m_writer.Write(ClampFloatToUByte(vertex.color[2])); // B + m_writer.Write(ClampFloatToUByte(vertex.color[3])); // A + + m_writer.Write(0x1AD4i16); // UV Hash + m_writer.Write(1ui16); // Layer + m_writer.Write(vertex.uv[0]); + m_writer.Write(vertex.uv[1]); + } + + void WriteFaces(const XModelCommon& xmodel) + { + auto totalFaceCount = 0u; + for (const auto& object : xmodel.m_objects) + totalFaceCount += object.m_faces.size(); + + m_writer.Write(0xBE92); // Face Count Hash + m_writer.Write(totalFaceCount); + + auto objectIndex = 0u; + for (const auto& object : xmodel.m_objects) + { + for (const auto& face : object.m_faces) + { + const size_t distinctPositions[3]{ + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[0]), + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[1]), + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[2]), + }; + + const XModelVertex& v0 = xmodel.m_vertices[face.vertexIndex[0]]; + const XModelVertex& v1 = xmodel.m_vertices[face.vertexIndex[1]]; + const XModelVertex& v2 = xmodel.m_vertices[face.vertexIndex[2]]; + + m_writer.Write(0x562Fi16); // Face Info Hash + m_writer.Write(static_cast(objectIndex)); + m_writer.Write(static_cast(object.materialIndex)); + + if (m_vertex_merger.GetDistinctValues().size() > UINT16_MAX) + { + m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(static_cast(distinctPositions[0])); + WriteFaceVertex(v0); + + m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(static_cast(distinctPositions[1])); + WriteFaceVertex(v1); + + m_writer.Write(0xB097); // 32 Bit Vertex Count Hash + m_writer.Write(static_cast(distinctPositions[2])); + WriteFaceVertex(v2); + } + else + { + WriteUInt16(0x8F03, distinctPositions[0]); + WriteFaceVertex(v0); + + WriteUInt16(0x8F03, distinctPositions[1]); + WriteFaceVertex(v1); + + WriteUInt16(0x8F03, distinctPositions[2]); + WriteFaceVertex(v2); + } + } + + objectIndex++; + } + } + + void WriteObjects(const XModelCommon& xmodel) + { + WriteInt16(0x62AF, xmodel.m_objects.size()); + + size_t objectNum = 0; + for (const auto& object : xmodel.m_objects) + { + m_writer.Write(0x87D4i16); // Object Info Hash + m_writer.Write(static_cast(objectNum)); + WriteAlignedString(object.name); + + objectNum++; + } + } + + void WriteMaterials(const XModelCommon& xmodel) + { + WriteInt16(0xA1B2, xmodel.m_materials.size()); + + size_t materialNum = 0u; + for (const auto& material : xmodel.m_materials) + { + const auto colorMapPath = "../images/" + material.colorMapName + ".dds"; + + WriteInt16(0xA700, materialNum); + WriteAlignedString(material.name); + WriteAlignedString(material.materialTypeName); + WriteAlignedString(colorMapPath); + + m_writer.Write(0x6DD8); // Vertex Color Hash + m_writer.Write(ClampFloatToUByte(material.color[0])); // R + m_writer.Write(ClampFloatToUByte(material.color[1])); // G + m_writer.Write(ClampFloatToUByte(material.color[2])); // B + m_writer.Write(ClampFloatToUByte(material.color[3])); // A + + m_writer.Write(0x6DAB); // Transparancy Hash + m_writer.Write(material.transparency[0]); + m_writer.Write(material.transparency[1]); + m_writer.Write(material.transparency[2]); + m_writer.Write(material.transparency[3]); + + m_writer.Write(0x37FF); // Ambient Color Hash + m_writer.Write(material.ambientColor[0]); // R + m_writer.Write(material.ambientColor[1]); // G + m_writer.Write(material.ambientColor[2]); // B + m_writer.Write(material.ambientColor[3]); // A + + m_writer.Write(0x4265); // Incandescence Hash + m_writer.Write(material.incandescence[0]); + m_writer.Write(material.incandescence[1]); + m_writer.Write(material.incandescence[2]); + m_writer.Write(material.incandescence[3]); + + m_writer.Write(0xC835); // Coeffs Hash + m_writer.Write(material.coeffs[0]); + m_writer.Write(material.coeffs[1]); + + m_writer.Write(0xFE0C); // Glow Hash + m_writer.Write(material.glow.x); + m_writer.Write(material.glow.y); + + m_writer.Write(0x7E24); // Refractive Hash + m_writer.Write(material.refractive.x); + m_writer.Write(material.refractive.y); + + m_writer.Write(0x317C); // Specular Color Hash + m_writer.Write(material.specularColor[0]); // R + m_writer.Write(material.specularColor[1]); // G + m_writer.Write(material.specularColor[2]); // B + m_writer.Write(material.specularColor[3]); // A + + m_writer.Write(0xE593); // Reflective Color Hash + m_writer.Write(material.reflectiveColor[0]); // R + m_writer.Write(material.reflectiveColor[1]); // G + m_writer.Write(material.reflectiveColor[2]); // B + m_writer.Write(material.reflectiveColor[3]); // A + + m_writer.Write(0x7D76); // Reflective Hash + m_writer.Write(material.reflective.x); + m_writer.Write(material.reflective.y); + + m_writer.Write(0x83C7); // Blinn Hash + m_writer.Write(material.blinn[0]); + m_writer.Write(material.blinn[1]); + + m_writer.Write(0x5CD2); // Phong Hash + m_writer.Write(material.phong); + + materialNum++; + } + } + +public: + XModelBinWriter7(std::ostream& stream, std::string gameName, std::string zoneName) + : XModelBinWriterBase(stream, std::move(gameName), std::move(zoneName)) + { + } + + void Write(const XModelCommon& xmodel) override + { + PrepareVertexMerger(xmodel); + WriteHeader(7); + WriteBones(xmodel); + WriteVertices(xmodel); + WriteFaces(xmodel); + WriteObjects(xmodel); + WriteMaterials(xmodel); + + auto estimatedCompressedFileSize = LZ4_compressBound(m_writer.GetPosition()); + auto compressedBuffer = new char[estimatedCompressedFileSize]; + auto actualCompressedFileSize = + LZ4_compress_default(reinterpret_cast(m_writer.Data()), compressedBuffer, m_writer.GetPosition(), estimatedCompressedFileSize); + + uint64_t uncompressedSize = m_writer.GetPosition(); + char uncompressedSizeChar[4]; + std::memcpy(uncompressedSizeChar, &uncompressedSize, 4); + + const char magic[5] = {0x2A, 0x4C, 0x5A, 0x34, 0x2A}; + m_stream.write(magic, 5); + m_stream.write(uncompressedSizeChar, 4); + m_stream.write(compressedBuffer, actualCompressedFileSize); + } +}; + +namespace xmodel_bin +{ + std::unique_ptr CreateWriterForVersion7(std::ostream& stream, std::string gameName, std::string zoneName) + { + return std::make_unique(stream, std::move(gameName), std::move(zoneName)); + } +} // namespace xmodel_bin diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.h b/src/ObjWriting/XModel/Export/XModelBinWriter.h index e69de29b..4dba0767 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.h +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.h @@ -0,0 +1,11 @@ +#pragma once + +#include "XModel/XModelWriter.h" + +#include +#include + +namespace xmodel_bin +{ + std::unique_ptr CreateWriterForVersion7(std::ostream& stream, std::string gameName, std::string zoneName); +} diff --git a/src/ObjWriting/XModel/XModelDumper.cpp.template b/src/ObjWriting/XModel/XModelDumper.cpp.template index f0e1cb26..2e6e00f5 100644 --- a/src/ObjWriting/XModel/XModelDumper.cpp.template +++ b/src/ObjWriting/XModel/XModelDumper.cpp.template @@ -22,6 +22,7 @@ #include "ObjWriting.h" #include "Utils/DistinctMapper.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -579,6 +580,18 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".xmodel_bin")); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -616,6 +629,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; @@ -666,6 +683,8 @@ namespace { case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_EXPORT: return ".xmodel_export"; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + return ".xmodel_bin"; case ObjWriting::Configuration_t::ModelOutputFormat_e::OBJ: return ".obj"; case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: diff --git a/src/Unlinker/UnlinkerArgs.cpp b/src/Unlinker/UnlinkerArgs.cpp index 3d492667..18e66f32 100644 --- a/src/Unlinker/UnlinkerArgs.cpp +++ b/src/Unlinker/UnlinkerArgs.cpp @@ -80,7 +80,7 @@ const CommandLineOption* const OPTION_IMAGE_FORMAT = const CommandLineOption* const OPTION_MODEL_FORMAT = CommandLineOption::Builder::Create() .WithLongName("model-format") - .WithDescription("Specifies the format of dumped model files. Valid values are: XMODEL_EXPORT, OBJ, GLTF, GLB") + .WithDescription("Specifies the format of dumped model files. Valid values are: XMODEL_EXPORT, XMODEL_BIN, OBJ, GLTF, GLB") .WithParameter("modelFormatValue") .Build(); @@ -210,6 +210,12 @@ bool UnlinkerArgs::SetModelDumpingMode() const return true; } + if (specifiedValue == "xmodel_bin") + { + ObjWriting::Configuration.ModelOutputFormat = ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN; + return true; + } + if (specifiedValue == "obj") { ObjWriting::Configuration.ModelOutputFormat = ObjWriting::Configuration_t::ModelOutputFormat_e::OBJ; diff --git a/src/Utils/Utils/MemoryWriter.cpp b/src/Utils/Utils/MemoryWriter.cpp new file mode 100644 index 00000000..0ad24c98 --- /dev/null +++ b/src/Utils/Utils/MemoryWriter.cpp @@ -0,0 +1,73 @@ +#include "MemoryWriter.h" + +#include + +MemoryWriter::MemoryWriter(uint32_t capacity) +{ + m_dataPointer = new int8_t[capacity]; + m_dataLength = capacity; + m_currentPosition = 0; +} + +MemoryWriter::~MemoryWriter() +{ + Close(); +} + +uint64_t MemoryWriter::GetLength() const +{ + return m_dataLength; +} + +uint64_t MemoryWriter::GetPosition() const +{ + return m_currentPosition; +} + +int8_t* MemoryWriter::Data() const +{ + return m_dataPointer; +} + +void MemoryWriter::Close() +{ + if (m_dataPointer != nullptr) + { + delete[] m_dataPointer; + m_dataPointer = nullptr; + } +} + +void MemoryWriter::Write(const uint8_t* buffer, uint32_t size) +{ + ValidateCapacity(m_currentPosition + size); + + std::memcpy(m_dataPointer + m_currentPosition, buffer, size); + m_currentPosition += size; +} + +void MemoryWriter::WriteNullTerminatedString(const std::string& string) +{ + Write(reinterpret_cast(string.c_str()), string.size() + 1); +} + +void MemoryWriter::Reallocate(uint64_t capacity) +{ + auto tempPtr = static_cast(std::realloc(m_dataPointer, capacity)); + + if (tempPtr != nullptr) + { + m_dataPointer = tempPtr; + m_dataLength = capacity; + } +} + +void MemoryWriter::ValidateCapacity(uint64_t newSize) +{ + if (newSize > m_dataLength) + { + auto newCapacity = m_dataLength * 2; + assert(newCapacity <= UINT64_MAX); + Reallocate(newCapacity); + } +} diff --git a/src/Utils/Utils/MemoryWriter.h b/src/Utils/Utils/MemoryWriter.h new file mode 100644 index 00000000..960bbdd4 --- /dev/null +++ b/src/Utils/Utils/MemoryWriter.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +class MemoryWriter +{ +public: + MemoryWriter(uint32_t capacity); + ~MemoryWriter(); + + uint64_t GetLength() const; + uint64_t GetPosition() const; + int8_t* Data() const; + + void Close(); + void Write(const uint8_t* buffer, uint32_t size); + void WriteNullTerminatedString(const std::string& string); + + template void Write(const T data) + { + Write((uint8_t*)&data, sizeof(T)); + } + +private: + void Reallocate(uint64_t capacity); + void ValidateCapacity(uint64_t size); + +private: + int8_t* m_dataPointer; + uint64_t m_dataLength; + uint64_t m_currentPosition; +}; diff --git a/thirdparty/README.md b/thirdparty/README.md index c3686e9e..6b9f9c3b 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -22,4 +22,8 @@ ## zlib - https://www.zlib.net/ -- Version 1.2.11 \ No newline at end of file +- Version 1.2.11 + +## lz4 +- https://lz4.org/ +- Version 1.10.0 \ No newline at end of file diff --git a/thirdparty/lz4 b/thirdparty/lz4 new file mode 160000 index 00000000..fa1634e2 --- /dev/null +++ b/thirdparty/lz4 @@ -0,0 +1 @@ +Subproject commit fa1634e2ccd41ac09c087ab65e96bcbbd003fd20 diff --git a/thirdparty/lz4.lua b/thirdparty/lz4.lua new file mode 100644 index 00000000..5554e196 --- /dev/null +++ b/thirdparty/lz4.lua @@ -0,0 +1,48 @@ +lz4 = {} + +function lz4:include(includes) + if includes:handle(self:name()) then + includedirs { + path.join(ThirdPartyFolder(), "lz4") + } + end +end + +function lz4:link(links) + links:add(self:name()) +end + +function lz4:use() + +end + +function lz4:name() + return "lz4" +end + +function lz4:project() + local folder = ThirdPartyFolder() + local includes = Includes:create() + + project(self:name()) + targetdir(TargetDirectoryLib) + location "%{wks.location}/thirdparty/%{prj.name}" + kind "StaticLib" + language "C" + + files { + path.join(folder, "lz4/lib/*.h"), + path.join(folder, "lz4/lib/*.c") + } + + defines { + "_CRT_SECURE_NO_WARNINGS", + "_CRT_NONSTDC_NO_DEPRECATE" + } + + + self:include(includes) + + -- Disable warnings. They do not have any value to us since it is not our code. + warnings "off" +end