diff --git a/.gitmodules b/.gitmodules index 7bfede4e..b283a8c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "thirdparty/eigen"] path = thirdparty/eigen url = https://gitlab.com/libeigen/eigen.git +[submodule "thirdparty/lz4"] + path = thirdparty/lz4 + url = https://github.com/lz4/lz4.git diff --git a/docs/SupportedAssetTypes.md b/docs/SupportedAssetTypes.md index a9c9e375..5b0d8f05 100644 --- a/docs/SupportedAssetTypes.md +++ b/docs/SupportedAssetTypes.md @@ -13,7 +13,7 @@ The following section specify which assets are supported to be dumped to disk (u | -------------------- | --------------- | --------------- | ----------------------------------------------------------------- | | PhysPreset | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialTechniqueSet | ❌ | ❌ | | | GfxImage | ✅ | ✅ | | @@ -44,7 +44,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysPreset | ✅ | ✅ | | | PhysCollmap | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialPixelShader | ✅ | ✅ | Shaders are compiled. Only dumps/loads shader bytecode. | | MaterialVertexShader | ✅ | ✅ | Shaders are compiled. Only dumps/loads shader bytecode. | @@ -85,7 +85,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysCollmap | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | | XModelSurfs | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialPixelShader | ❌ | ❌ | | | MaterialVertexShader | ❌ | ❌ | | @@ -130,7 +130,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysConstraints | ❌ | ❌ | | | DestructibleDef | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ⁉️ | ❌ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ❌ | ❌ | | | MaterialTechniqueSet | ❌ | ❌ | | | GfxImage | ✅ | ❌ | A few special image encodings are not yet supported. | @@ -167,7 +167,7 @@ The following section specify which assets are supported to be dumped to disk (u | PhysConstraints | ✅ | ✅ | | | DestructibleDef | ❌ | ❌ | | | XAnimParts | ❌ | ❌ | | -| XModel | ✅ | ✅ | Model data can be exported to `XMODEL_EXPORT`, `OBJ`, `GLB/GLTF`. | +| XModel | ✅ | ✅ | Model data can be exported to `XMODEL_EXPORT/XMODEL_BIN`, `OBJ`, `GLB/GLTF`. | | Material | ⁉️ | ⁉️ | Dumping/Loading is currently possible for materials in their compiled form. There is currently no material pipeline. | | MaterialTechniqueSet | ⁉️ | ❌ | Only dumps compiled shaders. | | GfxImage | ✅ | ✅ | A few special image encodings are not yet supported. | diff --git a/premake5.lua b/premake5.lua index 3d873b34..e0be302a 100644 --- a/premake5.lua +++ b/premake5.lua @@ -97,6 +97,7 @@ include "thirdparty/minilzo.lua" include "thirdparty/minizip.lua" include "thirdparty/salsa20.lua" include "thirdparty/zlib.lua" +include "thirdparty/lz4.lua" -- ThirdParty group: All projects that are external dependencies group "ThirdParty" @@ -109,6 +110,7 @@ group "ThirdParty" minizip:project() salsa20:project() zlib:project() + lz4:project() group "" -- ======================== diff --git a/src/ObjWriting.lua b/src/ObjWriting.lua index 186fa9d1..1f715ade 100644 --- a/src/ObjWriting.lua +++ b/src/ObjWriting.lua @@ -21,6 +21,7 @@ function ObjWriting:link(links) links:linkto(ZoneCommon) links:linkto(minizip) links:linkto(libtomcrypt) + links:linkto(lz4) end function ObjWriting:use() @@ -60,5 +61,5 @@ function ObjWriting:project() eigen:include(includes) json:include(includes) libtomcrypt:include(includes) - + lz4:include(includes) end diff --git a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp index cd6789e1..0382088e 100644 --- a/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW3/AssetDumpers/AssetDumperXModel.cpp @@ -5,6 +5,7 @@ #include "Utils/DistinctMapper.h" #include "Utils/HalfFloat.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -458,6 +459,18 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".xmodel_bin")); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -495,6 +508,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; diff --git a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp index 19d5ec79..3b9987e6 100644 --- a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp +++ b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperXModel.cpp @@ -5,6 +5,7 @@ #include "Utils/DistinctMapper.h" #include "Utils/HalfFloat.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -446,6 +447,19 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto* modelSurfs = model->lodInfo[lod].modelSurfs; + const auto assetFile = context.OpenAssetFile(std::format("model_export/{}.xmodel_bin", modelSurfs->name)); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -484,6 +498,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; diff --git a/src/ObjWriting/ObjWriting.h b/src/ObjWriting/ObjWriting.h index b27beb25..0fe1ec93 100644 --- a/src/ObjWriting/ObjWriting.h +++ b/src/ObjWriting/ObjWriting.h @@ -20,6 +20,7 @@ public: enum class ModelOutputFormat_e { XMODEL_EXPORT, + XMODEL_BIN, OBJ, GLTF, GLB diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp index e69de29b..2387f937 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.cpp +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.cpp @@ -0,0 +1,487 @@ +#include "XModelBinWriter.h" + +#include "GitVersion.h" + +#pragma warning(push, 0) +#include +#pragma warning(pop) + +#include +#include +#include +#include +#include +#include + +class XModelBinWriterBase : public XModelWriter +{ +protected: + /* + * A XModelBin Files consists of blocks of data, each block is indentified by a hash and is then followed by the actual data + * e.g.: + * 0x62AF = Number of objects hash + * 2 = Number of objects + * 0x87D4 = Start of object hash + * 0 = Object index + * Test_Obj = Object name + * 0x87D4 = Start of object hash + * 1 = Object index + * Test_Obj2 = Object name + */ + enum XModelBinHash : uint16_t + { + COMMENT = 0xC355, + MODEL = 0x46C8, + VERSION = 0x24D1, + + BONE_COUNT = 0x76BA, + BONE = 0xF099, + BONE_INDEX = 0xDD9A, + OFFSET = 0x9383, // Bone/Vertex Offset + BONE_SCALE = 0x1C56, + BONE_MATRIX_X = 0xDCFD, + BONE_MATRIX_Y = 0xCCDC, + BONE_MATRIX_Z = 0xFCBF, + + VERT16_COUNT = 0x950D, + VERT32_COUNT = 0x2AEC, + VERT16 = 0x8F03, + VERT32 = 0xB097, + VERT_WEIGHT_COUNT = 0xEA46, + VERT_WEIGHT = 0xF1AB, + + FACE_COUNT = 0xBE92, + TRIANGLE16 = 0x6711, + TRIANGLE32 = 0x562F, + + NORMAL = 0x89EC, + COLOR = 0x6DD8, + UV = 0x1AD4, + + OBJECT_COUNT = 0x62AF, + OBJECT = 0x87D4, + + MATERIAL_COUNT = 0xA1B2, + MATERIAL = 0xA700, + MATERIAL_TRANSPARENCY = 0x6DAB, + MATERIAL_AMBIENT_COLOR = 0x37FF, + MATERIAL_INCANDESCENCE = 0x4265, + MATERIAL_COEFFS = 0xC835, + MATERIAL_GLOW = 0xFE0C, + MATERIAL_REFRACTIVE = 0x7E24, + MATERIAL_SPECULAR_COLOR = 0x317C, + MATERIAL_REFLECTIVE_COLOR = 0xE593, + MATERIAL_REFLECTIVE = 0x7D76, + MATERIAL_BLINN = 0x83C7, + MATERIAL_PHONG = 0x5CD2 + }; + + void PrepareVertexMerger(const XModelCommon& xmodel) + { + m_vertex_merger = VertexMerger(xmodel.m_vertices.size()); + + auto vertexOffset = 0u; + for (const auto& vertex : xmodel.m_vertices) + { + XModelVertexBoneWeights weights{.weightOffset = 0, .weightCount = 0}; + + if (vertexOffset < xmodel.m_vertex_bone_weights.size()) + weights = xmodel.m_vertex_bone_weights[vertexOffset]; + + m_vertex_merger.Add( + VertexMergerPos{.x = vertex.coordinates[0], + .y = vertex.coordinates[1], + .z = vertex.coordinates[2], + .weights = xmodel.m_bone_weight_data.weights.empty() ? nullptr : &xmodel.m_bone_weight_data.weights[weights.weightOffset], + .weightCount = weights.weightCount}); + + vertexOffset++; + } + } + + template void Write(const T& data) + { + m_writer.write(reinterpret_cast(&data), sizeof(T)); + } + + void WriteNullTerminatedString(const std::string& string) + { + m_writer.write(string.c_str(), string.size() + 1); + } + + void WriteAlignedString(const std::string& string) + { + static constexpr uint8_t PADDING[4] = {0u, 0u, 0u, 0u}; + const auto paddingSize = (4u - (string.size() + 1u) % 4u) % 4u; + + WriteNullTerminatedString(string); + m_writer.write(reinterpret_cast(PADDING), paddingSize); + } + + void WriteComment(const std::string& comment) + { + Write(static_cast(XModelBinHash::COMMENT)); + WriteAlignedString(comment); + } + + void WriteInt16(const uint16_t hash, const int16_t value) + { + Write(hash); + Write(value); + } + + void WriteUInt16(const uint16_t hash, const uint16_t value) + { + Write(hash); + Write(value); + } + + [[nodiscard]] static int16_t ClampFloatToShort(const float value) + { + return std::clamp(static_cast(std::numeric_limits::max() * std::clamp(value, -1.0f, 1.0f)), + std::numeric_limits::min(), + std::numeric_limits::max()); + } + + [[nodiscard]] static uint8_t ClampFloatToUByte(const float value) + { + return std::clamp(static_cast(std::numeric_limits::max() * std::clamp(value, -1.0f, 1.0f)), + std::numeric_limits::min(), + std::numeric_limits::max()); + } + + void WriteHeader(const uint16_t version) + { + WriteComment("OpenAssetTools " GIT_VERSION " XMODEL_BIN File"); + WriteComment(std::format("Game Origin: {}", m_game_name)); + WriteComment(std::format("Zone Origin: {}", m_zone_name)); + Write(static_cast(XModelBinHash::MODEL)); + WriteUInt16(XModelBinHash::VERSION, version); + } + + void WriteBones(const XModelCommon& xmodel) + { + WriteUInt16(XModelBinHash::BONE_COUNT, static_cast(xmodel.m_bones.size())); + + auto boneNum = 0U; + for (const auto& bone : xmodel.m_bones) + { + Write(static_cast(XModelBinHash::BONE)); + Write(boneNum); + if (bone.parentIndex) + Write(static_cast(*bone.parentIndex)); + else + Write(-1); + + WriteAlignedString(bone.name); + boneNum++; + } + + boneNum = 0; + for (const auto& bone : xmodel.m_bones) + { + WriteUInt16(XModelBinHash::BONE_INDEX, static_cast(boneNum)); + + Write(static_cast(XModelBinHash::OFFSET)); + Write(bone.globalOffset[0]); // X + Write(bone.globalOffset[1]); // Y + Write(bone.globalOffset[2]); // Z + + Write(static_cast(XModelBinHash::BONE_SCALE)); + Write(bone.scale[0]); // X + Write(bone.scale[1]); // Y + Write(bone.scale[2]); // Z + + const auto mat = Eigen::Quaternionf(bone.globalRotation.w, bone.globalRotation.x, bone.globalRotation.y, bone.globalRotation.z).matrix(); + + Write(XModelBinHash::BONE_MATRIX_X); + Write(ClampFloatToShort(mat(0, 0))); + Write(ClampFloatToShort(mat(0, 1))); + Write(ClampFloatToShort(mat(0, 2))); + + Write(XModelBinHash::BONE_MATRIX_Y); + Write(ClampFloatToShort(mat(1, 0))); + Write(ClampFloatToShort(mat(1, 1))); + Write(ClampFloatToShort(mat(1, 2))); + + Write(XModelBinHash::BONE_MATRIX_Z); + Write(ClampFloatToShort(mat(2, 0))); + Write(ClampFloatToShort(mat(2, 1))); + Write(ClampFloatToShort(mat(2, 2))); + + boneNum++; + } + } + + XModelBinWriterBase(std::string gameName, std::string zoneName) + : m_game_name(std::move(gameName)), + m_zone_name(std::move(zoneName)) + { + } + + std::ostringstream m_writer; + std::string m_game_name; + std::string m_zone_name; + VertexMerger m_vertex_merger; +}; + +class XModelBinWriter7 final : public XModelBinWriterBase +{ + void WriteVertices(const XModelCommon& xmodel) + { + const auto& distinctVertexValues = m_vertex_merger.GetDistinctValues(); + + if (distinctVertexValues.size() > std::numeric_limits::max()) + { + // Use 32 bit + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32_COUNT)); + XModelBinWriterBase::Write(static_cast(distinctVertexValues.size())); + } + else + { + // Use 16 bit + WriteUInt16(XModelBinHash::VERT16_COUNT, static_cast(distinctVertexValues.size())); + } + + size_t vertexNum = 0u; + for (const auto& vertexPos : distinctVertexValues) + { + if (vertexNum > std::numeric_limits::max()) + { + // Use 32 bit + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); + XModelBinWriterBase::Write(static_cast(vertexNum)); + } + else + { + // Use 16 bit + WriteUInt16(XModelBinHash::VERT16, static_cast(vertexNum)); + } + + XModelBinWriterBase::Write(static_cast(XModelBinHash::OFFSET)); + XModelBinWriterBase::Write(vertexPos.x); + XModelBinWriterBase::Write(vertexPos.y); + XModelBinWriterBase::Write(vertexPos.z); + + WriteUInt16(XModelBinHash::VERT_WEIGHT_COUNT, static_cast(vertexPos.weightCount)); + + for (auto weightIndex = 0u; weightIndex < vertexPos.weightCount; weightIndex++) + { + const auto& weight = vertexPos.weights[weightIndex]; + + WriteUInt16(XModelBinHash::VERT_WEIGHT, static_cast(weight.boneIndex)); + XModelBinWriterBase::Write(weight.weight); + } + vertexNum++; + } + } + + void WriteFaceVertex(const XModelVertex& vertex) + { + XModelBinWriterBase::Write(XModelBinHash::NORMAL); + XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[0])); // X + XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[1])); // Y + XModelBinWriterBase::Write(ClampFloatToShort(vertex.normal[2])); // Z + + XModelBinWriterBase::Write(static_cast(XModelBinHash::COLOR)); + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[0])); // R + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[1])); // G + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[2])); // B + XModelBinWriterBase::Write(ClampFloatToUByte(vertex.color[3])); // A + + XModelBinWriterBase::Write(XModelBinHash::UV); + XModelBinWriterBase::Write(static_cast(1u)); // Layer + XModelBinWriterBase::Write(vertex.uv[0]); + XModelBinWriterBase::Write(vertex.uv[1]); + } + + void WriteFaces(const XModelCommon& xmodel) + { + auto totalFaceCount = 0u; + for (const auto& object : xmodel.m_objects) + totalFaceCount += object.m_faces.size(); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::FACE_COUNT)); + XModelBinWriterBase::Write(totalFaceCount); + + auto objectIndex = 0u; + for (const auto& object : xmodel.m_objects) + { + for (const auto& face : object.m_faces) + { + const size_t distinctPositions[3]{ + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[0]), + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[1]), + m_vertex_merger.GetDistinctPositionByInputPosition(face.vertexIndex[2]), + }; + + const XModelVertex& v0 = xmodel.m_vertices[face.vertexIndex[0]]; + const XModelVertex& v1 = xmodel.m_vertices[face.vertexIndex[1]]; + const XModelVertex& v2 = xmodel.m_vertices[face.vertexIndex[2]]; + + XModelBinWriterBase::Write(XModelBinHash::TRIANGLE32); + XModelBinWriterBase::Write(static_cast(objectIndex)); + XModelBinWriterBase::Write(static_cast(object.materialIndex)); + + if (m_vertex_merger.GetDistinctValues().size() > std::numeric_limits::max()) + { + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); + XModelBinWriterBase::Write(static_cast(distinctPositions[0])); + WriteFaceVertex(v0); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); + XModelBinWriterBase::Write(static_cast(distinctPositions[1])); + WriteFaceVertex(v1); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::VERT32)); + XModelBinWriterBase::Write(static_cast(distinctPositions[2])); + WriteFaceVertex(v2); + } + else + { + WriteUInt16(XModelBinHash::VERT16, static_cast(distinctPositions[0])); + WriteFaceVertex(v0); + + WriteUInt16(XModelBinHash::VERT16, static_cast(distinctPositions[1])); + WriteFaceVertex(v1); + + WriteUInt16(XModelBinHash::VERT16, static_cast(distinctPositions[2])); + WriteFaceVertex(v2); + } + } + + objectIndex++; + } + } + + void WriteObjects(const XModelCommon& xmodel) + { + WriteUInt16(XModelBinHash::OBJECT_COUNT, static_cast(xmodel.m_objects.size())); + + size_t objectNum = 0; + for (const auto& object : xmodel.m_objects) + { + XModelBinWriterBase::Write(XModelBinHash::OBJECT); + XModelBinWriterBase::Write(static_cast(objectNum)); + WriteAlignedString(object.name); + + objectNum++; + } + } + + void WriteMaterials(const XModelCommon& xmodel) + { + WriteUInt16(XModelBinHash::MATERIAL_COUNT, static_cast(xmodel.m_materials.size())); + + size_t materialNum = 0u; + for (const auto& material : xmodel.m_materials) + { + const auto colorMapPath = std::format("../images/{}.dds", material.colorMapName); + + WriteUInt16(XModelBinHash::MATERIAL, static_cast(materialNum)); + WriteAlignedString(material.name); + WriteAlignedString(material.materialTypeName); + WriteAlignedString(colorMapPath); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::COLOR)); + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[0])); // R + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[1])); // G + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[2])); // B + XModelBinWriterBase::Write(ClampFloatToUByte(material.color[3])); // A + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_TRANSPARENCY)); + XModelBinWriterBase::Write(material.transparency[0]); + XModelBinWriterBase::Write(material.transparency[1]); + XModelBinWriterBase::Write(material.transparency[2]); + XModelBinWriterBase::Write(material.transparency[3]); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_AMBIENT_COLOR)); + XModelBinWriterBase::Write(material.ambientColor[0]); // R + XModelBinWriterBase::Write(material.ambientColor[1]); // G + XModelBinWriterBase::Write(material.ambientColor[2]); // B + XModelBinWriterBase::Write(material.ambientColor[3]); // A + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_INCANDESCENCE)); + XModelBinWriterBase::Write(material.incandescence[0]); + XModelBinWriterBase::Write(material.incandescence[1]); + XModelBinWriterBase::Write(material.incandescence[2]); + XModelBinWriterBase::Write(material.incandescence[3]); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_COEFFS)); + XModelBinWriterBase::Write(material.coeffs[0]); + XModelBinWriterBase::Write(material.coeffs[1]); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_GLOW)); + XModelBinWriterBase::Write(material.glow.x); + XModelBinWriterBase::Write(material.glow.y); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_REFRACTIVE)); + XModelBinWriterBase::Write(material.refractive.x); + XModelBinWriterBase::Write(material.refractive.y); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_SPECULAR_COLOR)); + XModelBinWriterBase::Write(material.specularColor[0]); // R + XModelBinWriterBase::Write(material.specularColor[1]); // G + XModelBinWriterBase::Write(material.specularColor[2]); // B + XModelBinWriterBase::Write(material.specularColor[3]); // A + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_REFLECTIVE_COLOR)); + XModelBinWriterBase::Write(material.reflectiveColor[0]); // R + XModelBinWriterBase::Write(material.reflectiveColor[1]); // G + XModelBinWriterBase::Write(material.reflectiveColor[2]); // B + XModelBinWriterBase::Write(material.reflectiveColor[3]); // A + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_REFLECTIVE)); + XModelBinWriterBase::Write(material.reflective.x); + XModelBinWriterBase::Write(material.reflective.y); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_BLINN)); + XModelBinWriterBase::Write(material.blinn[0]); + XModelBinWriterBase::Write(material.blinn[1]); + + XModelBinWriterBase::Write(static_cast(XModelBinHash::MATERIAL_PHONG)); + XModelBinWriterBase::Write(material.phong); + + materialNum++; + } + } + + std::ostream& m_stream; + +public: + XModelBinWriter7(std::ostream& stream, std::string gameName, std::string zoneName) + : XModelBinWriterBase(std::move(gameName), std::move(zoneName)), + m_stream(stream) + { + } + + void Write(const XModelCommon& xmodel) override + { + PrepareVertexMerger(xmodel); + WriteHeader(7); + WriteBones(xmodel); + WriteVertices(xmodel); + WriteFaces(xmodel); + WriteObjects(xmodel); + WriteMaterials(xmodel); + + auto uncompressedSize = static_cast(m_writer.str().size()); + const auto estimatedCompressedFileSize = LZ4_compressBound(static_cast(m_writer.str().size())); + const auto compressedBuffer = std::make_unique(estimatedCompressedFileSize); + const auto actualCompressedFileSize = + LZ4_compress_default(m_writer.str().c_str(), compressedBuffer.get(), static_cast(m_writer.str().size()), estimatedCompressedFileSize); + + static constexpr char MAGIC[5] = {'*', 'L', 'Z', '4', '*'}; + m_stream.write(MAGIC, sizeof(MAGIC)); + m_stream.write(reinterpret_cast(&uncompressedSize), sizeof(uncompressedSize)); + m_stream.write(compressedBuffer.get(), actualCompressedFileSize); + } +}; + +namespace xmodel_bin +{ + std::unique_ptr CreateWriterForVersion7(std::ostream& stream, std::string gameName, std::string zoneName) + { + return std::make_unique(stream, std::move(gameName), std::move(zoneName)); + } +} // namespace xmodel_bin diff --git a/src/ObjWriting/XModel/Export/XModelBinWriter.h b/src/ObjWriting/XModel/Export/XModelBinWriter.h index e69de29b..4dba0767 100644 --- a/src/ObjWriting/XModel/Export/XModelBinWriter.h +++ b/src/ObjWriting/XModel/Export/XModelBinWriter.h @@ -0,0 +1,11 @@ +#pragma once + +#include "XModel/XModelWriter.h" + +#include +#include + +namespace xmodel_bin +{ + std::unique_ptr CreateWriterForVersion7(std::ostream& stream, std::string gameName, std::string zoneName); +} diff --git a/src/ObjWriting/XModel/XModelDumper.cpp.template b/src/ObjWriting/XModel/XModelDumper.cpp.template index f0e1cb26..2e6e00f5 100644 --- a/src/ObjWriting/XModel/XModelDumper.cpp.template +++ b/src/ObjWriting/XModel/XModelDumper.cpp.template @@ -22,6 +22,7 @@ #include "ObjWriting.h" #include "Utils/DistinctMapper.h" #include "Utils/QuatInt16.h" +#include "XModel/Export/XModelBinWriter.h" #include "XModel/Export/XModelExportWriter.h" #include "XModel/Gltf/GltfBinOutput.h" #include "XModel/Gltf/GltfTextOutput.h" @@ -579,6 +580,18 @@ namespace writer->Write(common); } + void DumpXModelBinLod(const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod) + { + const auto* model = asset->Asset(); + const auto assetFile = context.OpenAssetFile(GetFileNameForLod(model->name, lod, ".xmodel_bin")); + + if (!assetFile) + return; + + const auto writer = xmodel_bin::CreateWriterForVersion7(*assetFile, context.m_zone.m_game->GetShortName(), context.m_zone.m_name); + writer->Write(common); + } + template void DumpGltfLod( const XModelCommon& common, const AssetDumpingContext& context, const XAssetInfo* asset, const unsigned lod, const std::string& extension) @@ -616,6 +629,10 @@ namespace DumpXModelExportLod(common, context, asset, currentLod); break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + DumpXModelBinLod(common, context, asset, currentLod); + break; + case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: DumpGltfLod(common, context, asset, currentLod, ".gltf"); break; @@ -666,6 +683,8 @@ namespace { case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_EXPORT: return ".xmodel_export"; + case ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN: + return ".xmodel_bin"; case ObjWriting::Configuration_t::ModelOutputFormat_e::OBJ: return ".obj"; case ObjWriting::Configuration_t::ModelOutputFormat_e::GLTF: diff --git a/src/Unlinker/UnlinkerArgs.cpp b/src/Unlinker/UnlinkerArgs.cpp index 3d492667..18e66f32 100644 --- a/src/Unlinker/UnlinkerArgs.cpp +++ b/src/Unlinker/UnlinkerArgs.cpp @@ -80,7 +80,7 @@ const CommandLineOption* const OPTION_IMAGE_FORMAT = const CommandLineOption* const OPTION_MODEL_FORMAT = CommandLineOption::Builder::Create() .WithLongName("model-format") - .WithDescription("Specifies the format of dumped model files. Valid values are: XMODEL_EXPORT, OBJ, GLTF, GLB") + .WithDescription("Specifies the format of dumped model files. Valid values are: XMODEL_EXPORT, XMODEL_BIN, OBJ, GLTF, GLB") .WithParameter("modelFormatValue") .Build(); @@ -210,6 +210,12 @@ bool UnlinkerArgs::SetModelDumpingMode() const return true; } + if (specifiedValue == "xmodel_bin") + { + ObjWriting::Configuration.ModelOutputFormat = ObjWriting::Configuration_t::ModelOutputFormat_e::XMODEL_BIN; + return true; + } + if (specifiedValue == "obj") { ObjWriting::Configuration.ModelOutputFormat = ObjWriting::Configuration_t::ModelOutputFormat_e::OBJ; diff --git a/thirdparty/README.md b/thirdparty/README.md index c3686e9e..6b9f9c3b 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -22,4 +22,8 @@ ## zlib - https://www.zlib.net/ -- Version 1.2.11 \ No newline at end of file +- Version 1.2.11 + +## lz4 +- https://lz4.org/ +- Version 1.10.0 \ No newline at end of file diff --git a/thirdparty/lz4 b/thirdparty/lz4 new file mode 160000 index 00000000..fa1634e2 --- /dev/null +++ b/thirdparty/lz4 @@ -0,0 +1 @@ +Subproject commit fa1634e2ccd41ac09c087ab65e96bcbbd003fd20 diff --git a/thirdparty/lz4.lua b/thirdparty/lz4.lua new file mode 100644 index 00000000..ef315d74 --- /dev/null +++ b/thirdparty/lz4.lua @@ -0,0 +1,48 @@ +lz4 = {} + +function lz4:include(includes) + if includes:handle(self:name()) then + includedirs { + path.join(ThirdPartyFolder(), "lz4/lib") + } + end +end + +function lz4:link(links) + links:add(self:name()) +end + +function lz4:use() + +end + +function lz4:name() + return "lz4" +end + +function lz4:project() + local folder = ThirdPartyFolder() + local includes = Includes:create() + + project(self:name()) + targetdir(TargetDirectoryLib) + location "%{wks.location}/thirdparty/%{prj.name}" + kind "StaticLib" + language "C" + + files { + path.join(folder, "lz4/lib/*.h"), + path.join(folder, "lz4/lib/*.c") + } + + defines { + "_CRT_SECURE_NO_WARNINGS", + "_CRT_NONSTDC_NO_DEPRECATE" + } + + + self:include(includes) + + -- Disable warnings. They do not have any value to us since it is not our code. + warnings "off" +end