VideoCore: Update MemoryManager

src/video_core/memory_manager.cpp

@@ -16,36 +16,63 @@
 
 namespace Tegra {
 
-MemoryManager::MemoryManager(Core::System& system_)
-    : system{system_}, page_table(page_table_size) {}
+MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 page_bits_)
+    : system{system_}, address_space_bits{address_space_bits_}, page_bits{page_bits_}, entries{},
+      page_table{address_space_bits, address_space_bits + page_bits - 38, page_bits} {
+    address_space_size = 1ULL << address_space_bits;
+    allocate_start = address_space_bits > 32 ? 1ULL << 32 : 0;
+    page_size = 1ULL << page_bits;
+    page_mask = page_size - 1ULL;
+    const u64 page_table_bits = address_space_bits - cpu_page_bits;
+    const u64 page_table_size = 1ULL << page_table_bits;
+    page_table_mask = page_table_size - 1;
+
+    entries.resize(page_table_size / 32, 0);
+}
 
 MemoryManager::~MemoryManager() = default;
 
-void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
-    rasterizer = rasterizer_;
+MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const {
+    position = position >> page_bits;
+    const u64 entry_mask = entries[position / 32];
+    const size_t sub_index = position % 32;
+    return static_cast<EntryType>((entry_mask >> (2 * sub_index)) & 0x03ULL);
 }
 
-GPUVAddr MemoryManager::UpdateRange(GPUVAddr gpu_addr, PageEntry page_entry, std::size_t size) {
+void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) {
+    position = position >> page_bits;
+    const u64 entry_mask = entries[position / 32];
+    const size_t sub_index = position % 32;
+    entries[position / 32] =
+        (~(3ULL << sub_index * 2) & entry_mask) | (static_cast<u64>(entry) << sub_index * 2);
+}
+
+template <MemoryManager::EntryType entry_type>
+GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr,
+                                    size_t size) {
     u64 remaining_size{size};
+    if constexpr (entry_type == EntryType::Mapped) {
+        page_table.ReserveRange(gpu_addr, size);
+    }
     for (u64 offset{}; offset < size; offset += page_size) {
-        if (remaining_size < page_size) {
-            SetPageEntry(gpu_addr + offset, page_entry + offset, remaining_size);
-        } else {
-            SetPageEntry(gpu_addr + offset, page_entry + offset);
+        const GPUVAddr current_gpu_addr = gpu_addr + offset;
+        SetEntry(current_gpu_addr, entry_type);
+        if constexpr (entry_type == EntryType::Mapped) {
+            const VAddr current_cpu_addr = cpu_addr + offset;
+            const auto index = PageEntryIndex(current_gpu_addr);
+            page_table[index] = static_cast<u32>(current_cpu_addr >> 12ULL);
         }
         remaining_size -= page_size;
     }
     return gpu_addr;
 }
 
+void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
+    rasterizer = rasterizer_;
+}
+
 GPUVAddr MemoryManager::Map(VAddr cpu_addr, GPUVAddr gpu_addr, std::size_t size) {
-    const auto it = std::ranges::lower_bound(map_ranges, gpu_addr, {}, &MapRange::first);
-    if (it != map_ranges.end() && it->first == gpu_addr) {
-        it->second = size;
-    } else {
-        map_ranges.insert(it, MapRange{gpu_addr, size});
-    }
-    return UpdateRange(gpu_addr, cpu_addr, size);
+    return PageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size);
 }
 
 GPUVAddr MemoryManager::MapAllocate(VAddr cpu_addr, std::size_t size, std::size_t align) {
@@ -62,13 +89,6 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
     if (size == 0) {
         return;
     }
-    const auto it = std::ranges::lower_bound(map_ranges, gpu_addr, {}, &MapRange::first);
-    if (it != map_ranges.end()) {
-        ASSERT(it->first == gpu_addr);
-        map_ranges.erase(it);
-    } else {
-        ASSERT_MSG(false, "Unmapping non-existent GPU address=0x{:x}", gpu_addr);
-    }
     const auto submapped_ranges = GetSubmappedRange(gpu_addr, size);
 
     for (const auto& [map_addr, map_size] : submapped_ranges) {
@@ -79,63 +99,23 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
         rasterizer->UnmapMemory(*cpu_addr, map_size);
     }
 
-    UpdateRange(gpu_addr, PageEntry::State::Unmapped, size);
+    PageTableOp<EntryType::Free>(gpu_addr, 0, size);
 }
 
 std::optional<GPUVAddr> MemoryManager::AllocateFixed(GPUVAddr gpu_addr, std::size_t size) {
     for (u64 offset{}; offset < size; offset += page_size) {
-        if (!GetPageEntry(gpu_addr + offset).IsUnmapped()) {
+        if (GetEntry(gpu_addr + offset) != EntryType::Free) {
             return std::nullopt;
         }
     }
 
-    return UpdateRange(gpu_addr, PageEntry::State::Allocated, size);
+    return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size);
 }
 
 GPUVAddr MemoryManager::Allocate(std::size_t size, std::size_t align) {
     return *AllocateFixed(*FindFreeRange(size, align), size);
 }
 
-void MemoryManager::TryLockPage(PageEntry page_entry, std::size_t size) {
-    if (!page_entry.IsValid()) {
-        return;
-    }
-
-    ASSERT(system.CurrentProcess()
-               ->PageTable()
-               .LockForDeviceAddressSpace(page_entry.ToAddress(), size)
-               .IsSuccess());
-}
-
-void MemoryManager::TryUnlockPage(PageEntry page_entry, std::size_t size) {
-    if (!page_entry.IsValid()) {
-        return;
-    }
-
-    ASSERT(system.CurrentProcess()
-               ->PageTable()
-               .UnlockForDeviceAddressSpace(page_entry.ToAddress(), size)
-               .IsSuccess());
-}
-
-PageEntry MemoryManager::GetPageEntry(GPUVAddr gpu_addr) const {
-    return page_table[PageEntryIndex(gpu_addr)];
-}
-
-void MemoryManager::SetPageEntry(GPUVAddr gpu_addr, PageEntry page_entry, std::size_t size) {
-    // TODO(bunnei): We should lock/unlock device regions. This currently causes issues due to
-    // improper tracking, but should be fixed in the future.
-
-    //// Unlock the old page
-    // TryUnlockPage(page_table[PageEntryIndex(gpu_addr)], size);
-
-    //// Lock the new page
-    // TryLockPage(page_entry, size);
-    auto& current_page = page_table[PageEntryIndex(gpu_addr)];
-
-    current_page = page_entry;
-}
-
 std::optional<GPUVAddr> MemoryManager::FindFreeRange(std::size_t size, std::size_t align,
                                                      bool start_32bit_address) const {
     if (!align) {
@@ -145,9 +125,9 @@ std::optional<GPUVAddr> MemoryManager::FindFreeRange(std::size_t size, std::size
     }
 
     u64 available_size{};
-    GPUVAddr gpu_addr{start_32bit_address ? address_space_start_low : address_space_start};
+    GPUVAddr gpu_addr{allocate_start};
     while (gpu_addr + available_size < address_space_size) {
-        if (GetPageEntry(gpu_addr + available_size).IsUnmapped()) {
+        if (GetEntry(gpu_addr + available_size) == EntryType::Free) {
             available_size += page_size;
 
             if (available_size >= size) {
@@ -168,15 +148,12 @@ std::optional<GPUVAddr> MemoryManager::FindFreeRange(std::size_t size, std::size
 }
 
 std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const {
-    if (gpu_addr == 0) {
-        return std::nullopt;
-    }
-    const auto page_entry{GetPageEntry(gpu_addr)};
-    if (!page_entry.IsValid()) {
+    if (GetEntry(gpu_addr) != EntryType::Mapped) {
         return std::nullopt;
     }
 
-    return page_entry.ToAddress() + (gpu_addr & page_mask);
+    const VAddr cpu_addr_base = static_cast<VAddr>(page_table[PageEntryIndex(gpu_addr)]) << 12ULL;
+    return cpu_addr_base + (gpu_addr & page_mask);
 }
 
 std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t size) const {
@@ -227,10 +204,6 @@ template void MemoryManager::Write<u32>(GPUVAddr addr, u32 data);
 template void MemoryManager::Write<u64>(GPUVAddr addr, u64 data);
 
 u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) {
-    if (!GetPageEntry(gpu_addr).IsValid()) {
-        return {};
-    }
-
     const auto address{GpuToCpuAddress(gpu_addr)};
     if (!address) {
         return {};
@@ -240,10 +213,6 @@ u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) {
 }
 
 const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const {
-    if (!GetPageEntry(gpu_addr).IsValid()) {
-        return {};
-    }
-
     const auto address{GpuToCpuAddress(gpu_addr)};
     if (!address) {
         return {};
@@ -252,12 +221,6 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const {
     return system.Memory().GetPointer(*address);
 }
 
-size_t MemoryManager::BytesToMapEnd(GPUVAddr gpu_addr) const noexcept {
-    auto it = std::ranges::upper_bound(map_ranges, gpu_addr, {}, &MapRange::first);
-    --it;
-    return it->second - (gpu_addr - it->first);
-}
-
 void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
                                   bool is_safe) const {
     std::size_t remaining_size{size};
@@ -268,7 +231,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
         const std::size_t copy_amount{
             std::min(static_cast<std::size_t>(page_size) - page_offset, remaining_size)};
         const auto page_addr{GpuToCpuAddress(page_index << page_bits)};
-        if (page_addr && *page_addr != 0) {
+        if (page_addr) {
             const auto src_addr{*page_addr + page_offset};
             if (is_safe) {
                 // Flush must happen on the rasterizer interface, such that memory is always
@@ -307,7 +270,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
         const std::size_t copy_amount{
             std::min(static_cast<std::size_t>(page_size) - page_offset, remaining_size)};
         const auto page_addr{GpuToCpuAddress(page_index << page_bits)};
-        if (page_addr && *page_addr != 0) {
+        if (page_addr) {
             const auto dest_addr{*page_addr + page_offset};
 
             if (is_safe) {
@@ -392,7 +355,7 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons
     size_t page_index{gpu_addr >> page_bits};
     const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits};
     while (page_index < page_last) {
-        if (!page_table[page_index].IsValid() || page_table[page_index].ToAddress() == 0) {
+        if (GetEntry(page_index << page_bits) == EntryType::Free) {
             return false;
         }
         ++page_index;
@@ -408,7 +371,7 @@ std::vector<std::pair<GPUVAddr, std::size_t>> MemoryManager::GetSubmappedRange(
     size_t page_offset{gpu_addr & page_mask};
     std::optional<std::pair<GPUVAddr, std::size_t>> last_segment{};
     std::optional<VAddr> old_page_addr{};
-    const auto extend_size = [&last_segment, &page_index, &page_offset](std::size_t bytes) {
+    const auto extend_size = [this, &last_segment, &page_index, &page_offset](std::size_t bytes) {
         if (!last_segment) {
             const GPUVAddr new_base_addr = (page_index << page_bits) + page_offset;
             last_segment = {new_base_addr, bytes};