Merge pull request #12982 from FearlessTobi/fs-rewrite-part0

fs: Add FileSystemAccessor and use cmif serialization

src/core/file_sys/fs_filesystem.h

@@ -23,6 +23,8 @@ enum class OpenDirectoryMode : u64 {
     File = (1 << 1),
 
     All = (Directory | File),
+
+    NotRequireFileSize = (1ULL << 31),
 };
 DECLARE_ENUM_FLAG_OPERATORS(OpenDirectoryMode)
 
@@ -36,4 +38,29 @@ enum class CreateOption : u8 {
     BigFile = (1 << 0),
 };
 
+struct FileSystemAttribute {
+    u8 dir_entry_name_length_max_defined;
+    u8 file_entry_name_length_max_defined;
+    u8 dir_path_name_length_max_defined;
+    u8 file_path_name_length_max_defined;
+    INSERT_PADDING_BYTES_NOINIT(0x5);
+    u8 utf16_dir_entry_name_length_max_defined;
+    u8 utf16_file_entry_name_length_max_defined;
+    u8 utf16_dir_path_name_length_max_defined;
+    u8 utf16_file_path_name_length_max_defined;
+    INSERT_PADDING_BYTES_NOINIT(0x18);
+    s32 dir_entry_name_length_max;
+    s32 file_entry_name_length_max;
+    s32 dir_path_name_length_max;
+    s32 file_path_name_length_max;
+    INSERT_PADDING_WORDS_NOINIT(0x5);
+    s32 utf16_dir_entry_name_length_max;
+    s32 utf16_file_entry_name_length_max;
+    s32 utf16_dir_path_name_length_max;
+    s32 utf16_file_path_name_length_max;
+    INSERT_PADDING_WORDS_NOINIT(0x18);
+    INSERT_PADDING_WORDS_NOINIT(0x1);
+};
+static_assert(sizeof(FileSystemAttribute) == 0xC0, "FileSystemAttribute has incorrect size");
+
 } // namespace FileSys

src/core/file_sys/fs_memory_management.h

@@ -10,7 +10,7 @@ namespace FileSys {
 
 constexpr size_t RequiredAlignment = alignof(u64);
 
-void* AllocateUnsafe(size_t size) {
+inline void* AllocateUnsafe(size_t size) {
     // Allocate
     void* const ptr = ::operator new(size, std::align_val_t{RequiredAlignment});
 
@@ -21,16 +21,16 @@ void* AllocateUnsafe(size_t size) {
     return ptr;
 }
 
-void DeallocateUnsafe(void* ptr, size_t size) {
+inline void DeallocateUnsafe(void* ptr, size_t size) {
     // Deallocate the pointer
     ::operator delete(ptr, std::align_val_t{RequiredAlignment});
 }
 
-void* Allocate(size_t size) {
+inline void* Allocate(size_t size) {
     return AllocateUnsafe(size);
 }
 
-void Deallocate(void* ptr, size_t size) {
+inline void Deallocate(void* ptr, size_t size) {
     // If the pointer is non-null, deallocate it
     if (ptr != nullptr) {
         DeallocateUnsafe(ptr, size);

src/core/file_sys/fs_path.h

@@ -381,7 +381,7 @@ public:
 
         // Check that it's possible for us to remove a child
         auto* p = m_write_buffer.Get();
-        s32 len = std::strlen(p);
+        s32 len = static_cast<s32>(std::strlen(p));
         R_UNLESS(len != 1 || (p[0] != '/' && p[0] != '.'), ResultNotImplemented);
 
         // Handle a trailing separator

src/core/file_sys/fs_path_utility.h

@@ -426,9 +426,10 @@ public:
         R_SUCCEED();
     }
 
-    static Result Normalize(char* dst, size_t* out_len, const char* path, size_t max_out_size,
-                            bool is_windows_path, bool is_drive_relative_path,
-                            bool allow_all_characters = false) {
+    static constexpr Result Normalize(char* dst, size_t* out_len, const char* path,
+                                      size_t max_out_size, bool is_windows_path,
+                                      bool is_drive_relative_path,
+                                      bool allow_all_characters = false) {
         // Use StringTraits names for remainder of scope
         using namespace StringTraits;
 

src/core/file_sys/fs_string_util.h

@@ -19,6 +19,11 @@ constexpr int Strlen(const T* str) {
     return length;
 }
 
+template <typename T>
+constexpr int Strnlen(const T* str, std::size_t count) {
+    return Strnlen(str, static_cast<int>(count));
+}
+
 template <typename T>
 constexpr int Strnlen(const T* str, int count) {
     ASSERT(str != nullptr);
@@ -32,6 +37,11 @@ constexpr int Strnlen(const T* str, int count) {
     return length;
 }
 
+template <typename T>
+constexpr int Strncmp(const T* lhs, const T* rhs, std::size_t count) {
+    return Strncmp(lhs, rhs, static_cast<int>(count));
+}
+
 template <typename T>
 constexpr int Strncmp(const T* lhs, const T* rhs, int count) {
     ASSERT(lhs != nullptr);
@@ -51,6 +61,11 @@ constexpr int Strncmp(const T* lhs, const T* rhs, int count) {
     return l - r;
 }
 
+template <typename T>
+static constexpr int Strlcpy(T* dst, const T* src, std::size_t count) {
+    return Strlcpy<T>(dst, src, static_cast<int>(count));
+}
+
 template <typename T>
 static constexpr int Strlcpy(T* dst, const T* src, int count) {
     ASSERT(dst != nullptr);

src/core/file_sys/fsa/fs_i_directory.h

@@ -0,0 +1,91 @@
+// SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "common/common_types.h"
+#include "core/file_sys/errors.h"
+#include "core/file_sys/fs_directory.h"
+#include "core/file_sys/fs_file.h"
+#include "core/file_sys/fs_filesystem.h"
+#include "core/file_sys/savedata_factory.h"
+#include "core/file_sys/vfs/vfs.h"
+#include "core/hle/result.h"
+
+namespace FileSys::Fsa {
+
+class IDirectory {
+public:
+    explicit IDirectory(VirtualDir backend_, OpenDirectoryMode mode)
+        : backend(std::move(backend_)) {
+        // TODO(DarkLordZach): Verify that this is the correct behavior.
+        // Build entry index now to save time later.
+        if (True(mode & OpenDirectoryMode::Directory)) {
+            BuildEntryIndex(backend->GetSubdirectories(), DirectoryEntryType::Directory);
+        }
+        if (True(mode & OpenDirectoryMode::File)) {
+            BuildEntryIndex(backend->GetFiles(), DirectoryEntryType::File);
+        }
+    }
+    virtual ~IDirectory() {}
+
+    Result Read(s64* out_count, DirectoryEntry* out_entries, s64 max_entries) {
+        R_UNLESS(out_count != nullptr, ResultNullptrArgument);
+        if (max_entries == 0) {
+            *out_count = 0;
+            R_SUCCEED();
+        }
+        R_UNLESS(out_entries != nullptr, ResultNullptrArgument);
+        R_UNLESS(max_entries > 0, ResultInvalidArgument);
+        R_RETURN(this->DoRead(out_count, out_entries, max_entries));
+    }
+
+    Result GetEntryCount(s64* out) {
+        R_UNLESS(out != nullptr, ResultNullptrArgument);
+        R_RETURN(this->DoGetEntryCount(out));
+    }
+
+private:
+    Result DoRead(s64* out_count, DirectoryEntry* out_entries, s64 max_entries) {
+        const u64 actual_entries =
+            std::min(static_cast<u64>(max_entries), entries.size() - next_entry_index);
+        const auto* begin = reinterpret_cast<u8*>(entries.data() + next_entry_index);
+        const auto* end = reinterpret_cast<u8*>(entries.data() + next_entry_index + actual_entries);
+        const auto range_size = static_cast<std::size_t>(std::distance(begin, end));
+
+        next_entry_index += actual_entries;
+        *out_count = actual_entries;
+
+        std::memcpy(out_entries, begin, range_size);
+
+        R_SUCCEED();
+    }
+
+    Result DoGetEntryCount(s64* out) {
+        *out = entries.size() - next_entry_index;
+        R_SUCCEED();
+    }
+
+    // TODO: Remove this when VFS is gone
+    template <typename T>
+    void BuildEntryIndex(const std::vector<T>& new_data, DirectoryEntryType type) {
+        entries.reserve(entries.size() + new_data.size());
+
+        for (const auto& new_entry : new_data) {
+            auto name = new_entry->GetName();
+
+            if (type == DirectoryEntryType::File && name == GetSaveDataSizeFileName()) {
+                continue;
+            }
+
+            entries.emplace_back(name, static_cast<s8>(type),
+                                 type == DirectoryEntryType::Directory ? 0 : new_entry->GetSize());
+        }
+    }
+
+    VirtualDir backend;
+    std::vector<DirectoryEntry> entries;
+    u64 next_entry_index = 0;
+};
+
+} // namespace FileSys::Fsa

src/core/file_sys/fsa/fs_i_file.h

@@ -0,0 +1,167 @@
+// SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "common/overflow.h"
+#include "core/file_sys/errors.h"
+#include "core/file_sys/fs_file.h"
+#include "core/file_sys/fs_filesystem.h"
+#include "core/file_sys/fs_operate_range.h"
+#include "core/file_sys/vfs/vfs.h"
+#include "core/file_sys/vfs/vfs_types.h"
+#include "core/hle/result.h"
+
+namespace FileSys::Fsa {
+
+class IFile {
+public:
+    explicit IFile(VirtualFile backend_) : backend(std::move(backend_)) {}
+    virtual ~IFile() {}
+
+    Result Read(size_t* out, s64 offset, void* buffer, size_t size, const ReadOption& option) {
+        // Check that we have an output pointer
+        R_UNLESS(out != nullptr, ResultNullptrArgument);
+
+        // If we have nothing to read, just succeed
+        if (size == 0) {
+            *out = 0;
+            R_SUCCEED();
+        }
+
+        // Check that the read is valid
+        R_UNLESS(buffer != nullptr, ResultNullptrArgument);
+        R_UNLESS(offset >= 0, ResultOutOfRange);
+        R_UNLESS(Common::CanAddWithoutOverflow<s64>(offset, size), ResultOutOfRange);
+
+        // Do the read
+        R_RETURN(this->DoRead(out, offset, buffer, size, option));
+    }
+
+    Result Read(size_t* out, s64 offset, void* buffer, size_t size) {
+        R_RETURN(this->Read(out, offset, buffer, size, ReadOption::None));
+    }
+
+    Result GetSize(s64* out) {
+        R_UNLESS(out != nullptr, ResultNullptrArgument);
+        R_RETURN(this->DoGetSize(out));
+    }
+
+    Result Flush() {
+        R_RETURN(this->DoFlush());
+    }
+
+    Result Write(s64 offset, const void* buffer, size_t size, const WriteOption& option) {
+        // Handle the zero-size case
+        if (size == 0) {
+            if (option.HasFlushFlag()) {
+                R_TRY(this->Flush());
+            }
+            R_SUCCEED();
+        }
+
+        // Check the write is valid
+        R_UNLESS(buffer != nullptr, ResultNullptrArgument);
+        R_UNLESS(offset >= 0, ResultOutOfRange);
+        R_UNLESS(Common::CanAddWithoutOverflow<s64>(offset, size), ResultOutOfRange);
+
+        R_RETURN(this->DoWrite(offset, buffer, size, option));
+    }
+
+    Result SetSize(s64 size) {
+        R_UNLESS(size >= 0, ResultOutOfRange);
+        R_RETURN(this->DoSetSize(size));
+    }
+
+    Result OperateRange(void* dst, size_t dst_size, OperationId op_id, s64 offset, s64 size,
+                        const void* src, size_t src_size) {
+        R_RETURN(this->DoOperateRange(dst, dst_size, op_id, offset, size, src, src_size));
+    }
+
+    Result OperateRange(OperationId op_id, s64 offset, s64 size) {
+        R_RETURN(this->DoOperateRange(nullptr, 0, op_id, offset, size, nullptr, 0));
+    }
+
+protected:
+    Result DryRead(size_t* out, s64 offset, size_t size, const ReadOption& option,
+                   OpenMode open_mode) {
+        // Check that we can read
+        R_UNLESS(static_cast<u32>(open_mode & OpenMode::Read) != 0, ResultReadNotPermitted);
+
+        // Get the file size, and validate our offset
+        s64 file_size = 0;
+        R_TRY(this->DoGetSize(std::addressof(file_size)));
+        R_UNLESS(offset <= file_size, ResultOutOfRange);
+
+        *out = static_cast<size_t>(std::min(file_size - offset, static_cast<s64>(size)));
+        R_SUCCEED();
+    }
+
+    Result DrySetSize(s64 size, OpenMode open_mode) {
+        // Check that we can write
+        R_UNLESS(static_cast<u32>(open_mode & OpenMode::Write) != 0, ResultWriteNotPermitted);
+        R_SUCCEED();
+    }
+
+    Result DryWrite(bool* out_append, s64 offset, size_t size, const WriteOption& option,
+                    OpenMode open_mode) {
+        // Check that we can write
+        R_UNLESS(static_cast<u32>(open_mode & OpenMode::Write) != 0, ResultWriteNotPermitted);
+
+        // Get the file size
+        s64 file_size = 0;
+        R_TRY(this->DoGetSize(&file_size));
+
+        // Determine if we need to append
+        *out_append = false;
+        if (file_size < offset + static_cast<s64>(size)) {
+            R_UNLESS(static_cast<u32>(open_mode & OpenMode::AllowAppend) != 0,
+                     ResultFileExtensionWithoutOpenModeAllowAppend);
+            *out_append = true;
+        }
+
+        R_SUCCEED();
+    }
+
+private:
+    Result DoRead(size_t* out, s64 offset, void* buffer, size_t size, const ReadOption& option) {
+        const auto read_size = backend->Read(static_cast<u8*>(buffer), size, offset);
+        *out = read_size;
+
+        R_SUCCEED();
+    }
+
+    Result DoGetSize(s64* out) {
+        *out = backend->GetSize();
+        R_SUCCEED();
+    }
+
+    Result DoFlush() {
+        // Exists for SDK compatibiltity -- No need to flush file.
+        R_SUCCEED();
+    }
+
+    Result DoWrite(s64 offset, const void* buffer, size_t size, const WriteOption& option) {
+        const std::size_t written = backend->Write(static_cast<const u8*>(buffer), size, offset);
+
+        ASSERT_MSG(written == size,
+                   "Could not write all bytes to file (requested={:016X}, actual={:016X}).", size,
+                   written);
+
+        R_SUCCEED();
+    }
+
+    Result DoSetSize(s64 size) {
+        backend->Resize(size);
+        R_SUCCEED();
+    }
+
+    Result DoOperateRange(void* dst, size_t dst_size, OperationId op_id, s64 offset, s64 size,
+                          const void* src, size_t src_size) {
+        R_THROW(ResultNotImplemented);
+    }
+
+    VirtualFile backend;
+};
+
+} // namespace FileSys::Fsa

src/core/file_sys/fsa/fs_i_filesystem.h

@@ -0,0 +1,206 @@
+// SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "core/file_sys/errors.h"
+#include "core/file_sys/fs_filesystem.h"
+#include "core/file_sys/fs_path.h"
+#include "core/file_sys/vfs/vfs_types.h"
+#include "core/hle/result.h"
+#include "core/hle/service/filesystem/filesystem.h"
+
+namespace FileSys::Fsa {
+
+class IFile;
+class IDirectory;
+
+enum class QueryId : u32 {
+    SetConcatenationFileAttribute = 0,
+    UpdateMac = 1,
+    IsSignedSystemPartitionOnSdCardValid = 2,
+    QueryUnpreparedFileInformation = 3,
+};
+
+class IFileSystem {
+public:
+    explicit IFileSystem(VirtualDir backend_) : backend{std::move(backend_)} {}
+    virtual ~IFileSystem() {}
+
+    Result CreateFile(const Path& path, s64 size, CreateOption option) {
+        R_UNLESS(size >= 0, ResultOutOfRange);
+        R_RETURN(this->DoCreateFile(path, size, static_cast<int>(option)));
+    }
+
+    Result CreateFile(const Path& path, s64 size) {
+        R_RETURN(this->CreateFile(path, size, CreateOption::None));
+    }
+
+    Result DeleteFile(const Path& path) {
+        R_RETURN(this->DoDeleteFile(path));
+    }
+
+    Result CreateDirectory(const Path& path) {
+        R_RETURN(this->DoCreateDirectory(path));
+    }
+
+    Result DeleteDirectory(const Path& path) {
+        R_RETURN(this->DoDeleteDirectory(path));
+    }
+
+    Result DeleteDirectoryRecursively(const Path& path) {
+        R_RETURN(this->DoDeleteDirectoryRecursively(path));
+    }
+
+    Result RenameFile(const Path& old_path, const Path& new_path) {
+        R_RETURN(this->DoRenameFile(old_path, new_path));
+    }
+
+    Result RenameDirectory(const Path& old_path, const Path& new_path) {
+        R_RETURN(this->DoRenameDirectory(old_path, new_path));
+    }
+
+    Result GetEntryType(DirectoryEntryType* out, const Path& path) {
+        R_RETURN(this->DoGetEntryType(out, path));
+    }
+
+    Result OpenFile(VirtualFile* out_file, const Path& path, OpenMode mode) {
+        R_UNLESS(out_file != nullptr, ResultNullptrArgument);
+        R_UNLESS(static_cast<u32>(mode & OpenMode::ReadWrite) != 0, ResultInvalidOpenMode);
+        R_UNLESS(static_cast<u32>(mode & ~OpenMode::All) == 0, ResultInvalidOpenMode);
+        R_RETURN(this->DoOpenFile(out_file, path, mode));
+    }
+
+    Result OpenDirectory(VirtualDir* out_dir, const Path& path, OpenDirectoryMode mode) {
+        R_UNLESS(out_dir != nullptr, ResultNullptrArgument);
+        R_UNLESS(static_cast<u64>(mode & OpenDirectoryMode::All) != 0, ResultInvalidOpenMode);
+        R_UNLESS(static_cast<u64>(
+                     mode & ~(OpenDirectoryMode::All | OpenDirectoryMode::NotRequireFileSize)) == 0,
+                 ResultInvalidOpenMode);
+        R_RETURN(this->DoOpenDirectory(out_dir, path, mode));
+    }
+
+    Result Commit() {
+        R_RETURN(this->DoCommit());
+    }
+
+    Result GetFreeSpaceSize(s64* out, const Path& path) {
+        R_UNLESS(out != nullptr, ResultNullptrArgument);
+        R_RETURN(this->DoGetFreeSpaceSize(out, path));
+    }
+
+    Result GetTotalSpaceSize(s64* out, const Path& path) {
+        R_UNLESS(out != nullptr, ResultNullptrArgument);
+        R_RETURN(this->DoGetTotalSpaceSize(out, path));
+    }
+
+    Result CleanDirectoryRecursively(const Path& path) {
+        R_RETURN(this->DoCleanDirectoryRecursively(path));
+    }
+
+    Result GetFileTimeStampRaw(FileTimeStampRaw* out, const Path& path) {
+        R_UNLESS(out != nullptr, ResultNullptrArgument);
+        R_RETURN(this->DoGetFileTimeStampRaw(out, path));
+    }
+
+    Result QueryEntry(char* dst, size_t dst_size, const char* src, size_t src_size, QueryId query,
+                      const Path& path) {
+        R_RETURN(this->DoQueryEntry(dst, dst_size, src, src_size, query, path));
+    }
+
+    // These aren't accessible as commands
+    Result CommitProvisionally(s64 counter) {
+        R_RETURN(this->DoCommitProvisionally(counter));
+    }
+
+    Result Rollback() {
+        R_RETURN(this->DoRollback());
+    }
+
+    Result Flush() {
+        R_RETURN(this->DoFlush());
+    }
+
+private:
+    Result DoCreateFile(const Path& path, s64 size, int flags) {
+        R_RETURN(backend.CreateFile(path.GetString(), size));
+    }
+
+    Result DoDeleteFile(const Path& path) {
+        R_RETURN(backend.DeleteFile(path.GetString()));
+    }
+
+    Result DoCreateDirectory(const Path& path) {
+        R_RETURN(backend.CreateDirectory(path.GetString()));
+    }
+
+    Result DoDeleteDirectory(const Path& path) {
+        R_RETURN(backend.DeleteDirectory(path.GetString()));
+    }
+
+    Result DoDeleteDirectoryRecursively(const Path& path) {
+        R_RETURN(backend.DeleteDirectoryRecursively(path.GetString()));
+    }
+
+    Result DoRenameFile(const Path& old_path, const Path& new_path) {
+        R_RETURN(backend.RenameFile(old_path.GetString(), new_path.GetString()));
+    }
+
+    Result DoRenameDirectory(const Path& old_path, const Path& new_path) {
+        R_RETURN(backend.RenameDirectory(old_path.GetString(), new_path.GetString()));
+    }
+
+    Result DoGetEntryType(DirectoryEntryType* out, const Path& path) {
+        R_RETURN(backend.GetEntryType(out, path.GetString()));
+    }
+
+    Result DoOpenFile(VirtualFile* out_file, const Path& path, OpenMode mode) {
+        R_RETURN(backend.OpenFile(out_file, path.GetString(), mode));
+    }
+
+    Result DoOpenDirectory(VirtualDir* out_directory, const Path& path, OpenDirectoryMode mode) {
+        R_RETURN(backend.OpenDirectory(out_directory, path.GetString()));
+    }
+
+    Result DoCommit() {
+        R_THROW(ResultNotImplemented);
+    }
+
+    Result DoGetFreeSpaceSize(s64* out, const Path& path) {
+        R_THROW(ResultNotImplemented);
+    }
+
+    Result DoGetTotalSpaceSize(s64* out, const Path& path) {
+        R_THROW(ResultNotImplemented);
+    }
+
+    Result DoCleanDirectoryRecursively(const Path& path) {
+        R_RETURN(backend.CleanDirectoryRecursively(path.GetString()));
+    }
+
+    Result DoGetFileTimeStampRaw(FileTimeStampRaw* out, const Path& path) {
+        R_RETURN(backend.GetFileTimeStampRaw(out, path.GetString()));
+    }
+
+    Result DoQueryEntry(char* dst, size_t dst_size, const char* src, size_t src_size, QueryId query,
+                        const Path& path) {
+        R_THROW(ResultNotImplemented);
+    }
+
+    // These aren't accessible as commands
+    Result DoCommitProvisionally(s64 counter) {
+        R_THROW(ResultNotImplemented);
+    }
+
+    Result DoRollback() {
+        R_THROW(ResultNotImplemented);
+    }
+
+    Result DoFlush() {
+        R_THROW(ResultNotImplemented);
+    }
+
+    Service::FileSystem::VfsDirectoryServiceWrapper backend;
+};
+
+} // namespace FileSys::Fsa

src/core/file_sys/fssrv/fssrv_sf_path.h

@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "core/file_sys/fs_directory.h"
+
+namespace FileSys::Sf {
+
+struct Path {
+    char str[EntryNameLengthMax + 1];
+
+    static constexpr Path Encode(const char* p) {
+        Path path = {};
+        for (size_t i = 0; i < sizeof(path) - 1; i++) {
+            path.str[i] = p[i];
+            if (p[i] == '\x00') {
+                break;
+            }
+        }
+        return path;
+    }
+
+    static constexpr size_t GetPathLength(const Path& path) {
+        size_t len = 0;
+        for (size_t i = 0; i < sizeof(path) - 1 && path.str[i] != '\x00'; i++) {
+            len++;
+        }
+        return len;
+    }
+};
+static_assert(std::is_trivially_copyable_v<Path>, "Path must be trivially copyable.");
+
+using FspPath = Path;
+
+} // namespace FileSys::Sf