INSANEMODE/yuzu
1
0
Fork 0
mirror of https://github.com/yuzu-emu/yuzu.git synced 2025-06-17 21:18:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

SMMU: Initial adaptation to video_core.

Browse Source
This commit is contained in:
Fernando Sahmkow
2023-12-25 07:32:16 +01:00
committed by Liam
parent c85d7ccd79
commit 0a2536a0df
79 changed files with 1262 additions and 1263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/core/CMakeLists.txt
View File

@ -37,6 +37,8 @@ add_library(core STATIC
debugger/gdbstub_arch.h
debugger/gdbstub.cpp
debugger/gdbstub.h
device_memory_manager.h
device_memory_manager.inc
device_memory.cpp
device_memory.h
file_sys/fssystem/fs_i_storage.h

2
src/core/core.cpp
View File

@ -651,7 +651,7 @@ size_t System::GetCurrentHostThreadID() const {
return impl->kernel.GetCurrentHostThreadID();
}
void System::GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback) {
void System::GatherGPUDirtyMemory(std::function<void(PAddr, size_t)>& callback) {
return this->ApplicationProcess()->GatherGPUDirtyMemory(callback);
}

2
src/core/core.h
View File

@ -224,7 +224,7 @@ public:
/// Prepare the core emulation for a reschedule
void PrepareReschedule(u32 core_index);
void GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback);
void GatherGPUDirtyMemory(std::function<void(PAddr, size_t)>& callback);
[[nodiscard]] size_t GetCurrentHostThreadID() const;

43
src/core/device_memory_manager.h
View File

@ -3,10 +3,11 @@
#pragma once
#include <deque>
#include <memory>
#include <array>
#include <atomic>
#include <deque>
#include <memory>
#include <mutex>
#include "common/common_types.h"
#include "common/virtual_buffer.h"
@ -48,26 +49,54 @@ public:
template <typename T>
const T* GetPointer(DAddr address) const;
DAddr GetAddressFromPAddr(PAddr address) const {
DAddr subbits = static_cast<DAddr>(address & page_mask);
return (static_cast<DAddr>(compressed_device_addr[(address >> page_bits)]) << page_bits) + subbits;
}
PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const {
PAddr subbits = static_cast<PAddr>(address & page_mask);
auto paddr = compressed_physical_ptr[(address >> page_bits)];
if (paddr == 0) {
return 0;
}
return (static_cast<PAddr>(paddr - 1) << page_bits) + subbits;
}
template <typename T>
void Write(DAddr address, T value);
template <typename T>
T Read(DAddr address) const;
const u8* GetSpan(const DAddr src_addr, const std::size_t size) const {
return nullptr;
}
u8* GetSpan(const DAddr src_addr, const std::size_t size) {
return nullptr;
}
void ReadBlock(DAddr address, void* dest_pointer, size_t size);
void WriteBlock(DAddr address, void* src_pointer, size_t size);
void ReadBlockUnsafe(DAddr address, void* dest_pointer, size_t size);
void WriteBlock(DAddr address, const void* src_pointer, size_t size);
void WriteBlockUnsafe(DAddr address, const void* src_pointer, size_t size);
size_t RegisterProcess(Memory::Memory* memory);
void UnregisterProcess(size_t id);
void UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta);
static constexpr size_t AS_BITS = Traits::device_virtual_bits;
private:
static constexpr bool supports_pinning = Traits::supports_pinning;
static constexpr size_t device_virtual_bits = Traits::device_virtual_bits;
static constexpr size_t device_as_size = 1ULL << device_virtual_bits;
static constexpr size_t physical_max_bits = 33;
static constexpr size_t page_bits = 12;
static constexpr size_t page_size = 1ULL << page_bits;
static constexpr size_t page_mask = page_size - 1ULL;
static constexpr u32 physical_address_base = 1U << page_bits;
template <typename T>
@ -136,11 +165,15 @@ private:
private:
std::array<std::atomic_uint16_t, subentries> values{};
};
static_assert(sizeof(CounterEntry) == subentries * sizeof(u16), "CounterEntry should be 8 bytes!");
static_assert(sizeof(CounterEntry) == subentries * sizeof(u16),
"CounterEntry should be 8 bytes!");
static constexpr size_t num_counter_entries = (1ULL << (device_virtual_bits - page_bits)) / subentries;
static constexpr size_t num_counter_entries =
(1ULL << (device_virtual_bits - page_bits)) / subentries;
using CachedPages = std::array<CounterEntry, num_counter_entries>;
std::unique_ptr<CachedPages> cached_pages;
std::mutex counter_guard;
std::mutex mapping_guard;
};
} // namespace Core

72
src/core/device_memory_manager.inc
View File

@ -105,7 +105,8 @@ template <typename Traits>
DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_)
: physical_base{reinterpret_cast<const uintptr_t>(device_memory_.buffer.BackingBasePointer())},
interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS),
compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)) {
compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)),
cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) {
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
cached_pages = std::make_unique<CachedPages>();
}
@ -144,10 +145,10 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
Core::Memory::Memory* process_memory = registered_processes[process_id];
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
std::atomic_thread_fence(std::memory_order_acquire);
std::scoped_lock lk(mapping_guard);
for (size_t i = 0; i < num_pages; i++) {
const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE;
auto* ptr = process_memory->GetPointer(Common::ProcessAddress(new_vaddress));
auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress));
if (ptr == nullptr) [[unlikely]] {
compressed_physical_ptr[start_page_d + i] = 0;
continue;
@ -157,14 +158,14 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
compressed_device_addr[phys_addr - 1U] = static_cast<u32>(start_page_d + i);
InsertCPUBacking(start_page_d + i, new_vaddress, process_id);
}
std::atomic_thread_fence(std::memory_order_release);
}
template <typename Traits>
void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
std::atomic_thread_fence(std::memory_order_acquire);
interface->InvalidateRegion(address, size);
std::scoped_lock lk(mapping_guard);
for (size_t i = 0; i < num_pages; i++) {
auto phys_addr = compressed_physical_ptr[start_page_d + i];
compressed_physical_ptr[start_page_d + i] = 0;
@ -173,7 +174,6 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
compressed_device_addr[phys_addr - 1] = 0;
}
}
std::atomic_thread_fence(std::memory_order_release);
}
template <typename Traits>
@ -256,6 +256,7 @@ void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto o
template <typename Traits>
void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, size_t size) {
interface->FlushRegion(address, size);
WalkBlock(
address, size,
[&](size_t copy_amount, DAddr current_vaddr) {
@ -274,7 +275,7 @@ void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, s
}
template <typename Traits>
void DeviceMemoryManager<Traits>::WriteBlock(DAddr address, void* src_pointer, size_t size) {
void DeviceMemoryManager<Traits>::WriteBlock(DAddr address, const void* src_pointer, size_t size) {
WalkBlock(
address, size,
[&](size_t copy_amount, DAddr current_vaddr) {
@ -287,7 +288,46 @@ void DeviceMemoryManager<Traits>::WriteBlock(DAddr address, void* src_pointer, s
std::memcpy(dst_ptr, src_pointer, copy_amount);
},
[&](const std::size_t copy_amount) {
src_pointer = static_cast<u8*>(src_pointer) + copy_amount;
src_pointer = static_cast<const u8*>(src_pointer) + copy_amount;
});
interface->InvalidateRegion(address, size);
}
template <typename Traits>
void DeviceMemoryManager<Traits>::ReadBlockUnsafe(DAddr address, void* dest_pointer, size_t size) {
WalkBlock(
address, size,
[&](size_t copy_amount, DAddr current_vaddr) {
LOG_ERROR(
HW_Memory,
"Unmapped Device ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, address, size);
std::memset(dest_pointer, 0, copy_amount);
},
[&](size_t copy_amount, const u8* const src_ptr) {
std::memcpy(dest_pointer, src_ptr, copy_amount);
},
[&](const std::size_t copy_amount) {
dest_pointer = static_cast<u8*>(dest_pointer) + copy_amount;
});
}
template <typename Traits>
void DeviceMemoryManager<Traits>::WriteBlockUnsafe(DAddr address, const void* src_pointer,
size_t size) {
WalkBlock(
address, size,
[&](size_t copy_amount, DAddr current_vaddr) {
LOG_ERROR(
HW_Memory,
"Unmapped Device WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
current_vaddr, address, size);
},
[&](size_t copy_amount, u8* const dst_ptr) {
std::memcpy(dst_ptr, src_pointer, copy_amount);
},
[&](const std::size_t copy_amount) {
src_pointer = static_cast<const u8*>(src_pointer) + copy_amount;
});
}
@ -313,6 +353,18 @@ void DeviceMemoryManager<Traits>::UnregisterProcess(size_t id) {
template <typename Traits>
void DeviceMemoryManager<Traits>::UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta) {
bool locked = false;
auto lock = [&] {
if (!locked) {
counter_guard.lock();
locked = true;
}
};
SCOPE_EXIT({
if (locked) {
counter_guard.unlock();
}
});
u64 uncache_begin = 0;
u64 cache_begin = 0;
u64 uncache_bytes = 0;
@ -347,6 +399,7 @@ void DeviceMemoryManager<Traits>::UpdatePagesCachedCount(DAddr addr, size_t size
}
uncache_bytes += Memory::YUZU_PAGESIZE;
} else if (uncache_bytes > 0) {
lock();
MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS,
uncache_bytes, false);
uncache_bytes = 0;
@ -357,6 +410,7 @@ void DeviceMemoryManager<Traits>::UpdatePagesCachedCount(DAddr addr, size_t size
}
cache_bytes += Memory::YUZU_PAGESIZE;
} else if (cache_bytes > 0) {
lock();
MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes,
true);
cache_bytes = 0;
@ -364,10 +418,12 @@ void DeviceMemoryManager<Traits>::UpdatePagesCachedCount(DAddr addr, size_t size
vpage++;
}
if (uncache_bytes > 0) {
lock();
MarkRegionCaching(memory_interface, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes,
false);
}
if (cache_bytes > 0) {
lock();
MarkRegionCaching(memory_interface, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes,
true);
}

10
src/core/gpu_dirty_memory_manager.h
View File

@ -23,7 +23,7 @@ public:
~GPUDirtyMemoryManager() = default;
void Collect(VAddr address, size_t size) {
void Collect(PAddr address, size_t size) {
TransformAddress t = BuildTransform(address, size);
TransformAddress tmp, original;
do {
@ -47,7 +47,7 @@ public:
std::memory_order_relaxed));
}
void Gather(std::function<void(VAddr, size_t)>& callback) {
void Gather(std::function<void(PAddr, size_t)>& callback) {
{
std::scoped_lock lk(guard);
TransformAddress t = current.exchange(default_transform, std::memory_order_relaxed);
@ -65,7 +65,7 @@ public:
mask = mask >> empty_bits;
const size_t continuous_bits = std::countr_one(mask);
callback((static_cast<VAddr>(transform.address) << page_bits) + offset,
callback((static_cast<PAddr>(transform.address) << page_bits) + offset,
continuous_bits << align_bits);
mask = continuous_bits < align_size ? (mask >> continuous_bits) : 0;
offset += continuous_bits << align_bits;
@ -89,7 +89,7 @@ private:
constexpr static size_t align_mask = align_size - 1;
constexpr static TransformAddress default_transform = {.address = ~0U, .mask = 0U};
bool IsValid(VAddr address) {
bool IsValid(PAddr address) {
return address < (1ULL << 39);
}
@ -103,7 +103,7 @@ private:
return mask;
}
TransformAddress BuildTransform(VAddr address, size_t size) {
TransformAddress BuildTransform(PAddr address, size_t size) {
const size_t minor_address = address & page_mask;
const size_t minor_bit = minor_address >> align_bits;
const size_t top_bit = (minor_address + size + align_mask) >> align_bits;

218
src/core/guest_memory.h Normal file
View File

@ -0,0 +1,218 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <iterator>
#include <memory>
#include <optional>
#include <span>
#include <vector>
#include "common/scratch_buffer.h"
#include "core/memory.h"
namespace Core::Memory {
enum GuestMemoryFlags : u32 {
Read = 1 << 0,
Write = 1 << 1,
Safe = 1 << 2,
Cached = 1 << 3,
SafeRead = Read | Safe,
SafeWrite = Write | Safe,
SafeReadWrite = SafeRead | SafeWrite,
SafeReadCachedWrite = SafeReadWrite | Cached,
UnsafeRead = Read,
UnsafeWrite = Write,
UnsafeReadWrite = UnsafeRead | UnsafeWrite,
UnsafeReadCachedWrite = UnsafeReadWrite | Cached,
};
namespace {
template <typename M, typename T, GuestMemoryFlags FLAGS>
class GuestMemory {
using iterator = T*;
using const_iterator = const T*;
using value_type = T;
using element_type = T;
using iterator_category = std::contiguous_iterator_tag;
public:
GuestMemory() = delete;
explicit GuestMemory(M& memory, u64 addr, std::size_t size,
Common::ScratchBuffer<T>* backup = nullptr)
: m_memory{memory}, m_addr{addr}, m_size{size} {
static_assert(FLAGS & GuestMemoryFlags::Read || FLAGS & GuestMemoryFlags::Write);
if constexpr (FLAGS & GuestMemoryFlags::Read) {
Read(addr, size, backup);
}
}
~GuestMemory() = default;
T* data() noexcept {
return m_data_span.data();
}
const T* data() const noexcept {
return m_data_span.data();
}
size_t size() const noexcept {
return m_size;
}
size_t size_bytes() const noexcept {
return this->size() * sizeof(T);
}
[[nodiscard]] T* begin() noexcept {
return this->data();
}
[[nodiscard]] const T* begin() const noexcept {
return this->data();
}
[[nodiscard]] T* end() noexcept {
return this->data() + this->size();
}
[[nodiscard]] const T* end() const noexcept {
return this->data() + this->size();
}
T& operator[](size_t index) noexcept {
return m_data_span[index];
}
const T& operator[](size_t index) const noexcept {
return m_data_span[index];
}
void SetAddressAndSize(u64 addr, std::size_t size) noexcept {
m_addr = addr;
m_size = size;
m_addr_changed = true;
}
std::span<T> Read(u64 addr, std::size_t size,
Common::ScratchBuffer<T>* backup = nullptr) noexcept {
m_addr = addr;
m_size = size;
if (m_size == 0) {
m_is_data_copy = true;
return {};
}
if (this->TrySetSpan()) {
if constexpr (FLAGS & GuestMemoryFlags::Safe) {
m_memory.FlushRegion(m_addr, this->size_bytes());
}
} else {
if (backup) {
backup->resize_destructive(this->size());
m_data_span = *backup;
} else {
m_data_copy.resize(this->size());
m_data_span = std::span(m_data_copy);
}
m_is_data_copy = true;
m_span_valid = true;
if constexpr (FLAGS & GuestMemoryFlags::Safe) {
m_memory.ReadBlock(m_addr, this->data(), this->size_bytes());
} else {
m_memory.ReadBlockUnsafe(m_addr, this->data(), this->size_bytes());
}
}
return m_data_span;
}
void Write(std::span<T> write_data) noexcept {
if constexpr (FLAGS & GuestMemoryFlags::Cached) {
m_memory.WriteBlockCached(m_addr, write_data.data(), this->size_bytes());
} else if constexpr (FLAGS & GuestMemoryFlags::Safe) {
m_memory.WriteBlock(m_addr, write_data.data(), this->size_bytes());
} else {
m_memory.WriteBlockUnsafe(m_addr, write_data.data(), this->size_bytes());
}
}
bool TrySetSpan() noexcept {
if (u8* ptr = m_memory.GetSpan(m_addr, this->size_bytes()); ptr) {
m_data_span = {reinterpret_cast<T*>(ptr), this->size()};
m_span_valid = true;
return true;
}
return false;
}
protected:
bool IsDataCopy() const noexcept {
return m_is_data_copy;
}
bool AddressChanged() const noexcept {
return m_addr_changed;
}
M& m_memory;
u64 m_addr{};
size_t m_size{};
std::span<T> m_data_span{};
std::vector<T> m_data_copy{};
bool m_span_valid{false};
bool m_is_data_copy{false};
bool m_addr_changed{false};
};
template <typename M, typename T, GuestMemoryFlags FLAGS>
class GuestMemoryScoped : public GuestMemory<M, T, FLAGS> {
public:
GuestMemoryScoped() = delete;
explicit GuestMemoryScoped(M& memory, u64 addr, std::size_t size,
Common::ScratchBuffer<T>* backup = nullptr)
: GuestMemory<M, T, FLAGS>(memory, addr, size, backup) {
if constexpr (!(FLAGS & GuestMemoryFlags::Read)) {
if (!this->TrySetSpan()) {
if (backup) {
this->m_data_span = *backup;
this->m_span_valid = true;
this->m_is_data_copy = true;
}
}
}
}
~GuestMemoryScoped() {
if constexpr (FLAGS & GuestMemoryFlags::Write) {
if (this->size() == 0) [[unlikely]] {
return;
}
if (this->AddressChanged() || this->IsDataCopy()) {
ASSERT(this->m_span_valid);
if constexpr (FLAGS & GuestMemoryFlags::Cached) {
this->m_memory.WriteBlockCached(this->m_addr, this->data(), this->size_bytes());
} else if constexpr (FLAGS & GuestMemoryFlags::Safe) {
this->m_memory.WriteBlock(this->m_addr, this->data(), this->size_bytes());
} else {
this->m_memory.WriteBlockUnsafe(this->m_addr, this->data(), this->size_bytes());
}
} else if constexpr ((FLAGS & GuestMemoryFlags::Safe) || (FLAGS & GuestMemoryFlags::Cached)) {
this->m_memory.InvalidateRegion(this->m_addr, this->size_bytes());
}
}
}
};
} // namespace
template <typename T, GuestMemoryFlags FLAGS>
using CpuGuestMemory = GuestMemory<Core::Memory::Memory, T, FLAGS>;
template <typename T, GuestMemoryFlags FLAGS>
using CpuGuestMemoryScoped = GuestMemoryScoped<Core::Memory::Memory, T, FLAGS>;
} // namespace Tegra::Memory

61
src/core/hle/service/hle_ipc.cpp
View File

@ -22,19 +22,7 @@
#include "core/hle/service/hle_ipc.h"
#include "core/hle/service/ipc_helpers.h"
#include "core/memory.h"
namespace {
static thread_local std::array read_buffer_data_a{
Common::ScratchBuffer<u8>(),
Common::ScratchBuffer<u8>(),
Common::ScratchBuffer<u8>(),
};
static thread_local std::array read_buffer_data_x{
Common::ScratchBuffer<u8>(),
Common::ScratchBuffer<u8>(),
Common::ScratchBuffer<u8>(),
};
} // Anonymous namespace
#include "core/guest_memory.h"
namespace Service {
@ -343,48 +331,27 @@ std::vector<u8> HLERequestContext::ReadBufferCopy(std::size_t buffer_index) cons
}
std::span<const u8> HLERequestContext::ReadBufferA(std::size_t buffer_index) const {
static thread_local std::array read_buffer_a{
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
};
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::UnsafeRead> gm(memory, 0, 0);
ASSERT_OR_EXECUTE_MSG(
BufferDescriptorA().size() > buffer_index, { return {}; },
"BufferDescriptorA invalid buffer_index {}", buffer_index);
auto& read_buffer = read_buffer_a[buffer_index];
return read_buffer.Read(BufferDescriptorA()[buffer_index].Address(),
BufferDescriptorA()[buffer_index].Size(),
&read_buffer_data_a[buffer_index]);
return gm.Read(BufferDescriptorA()[buffer_index].Address(),
BufferDescriptorA()[buffer_index].Size(), &read_buffer_data_a[buffer_index]);
}
std::span<const u8> HLERequestContext::ReadBufferX(std::size_t buffer_index) const {
static thread_local std::array read_buffer_x{
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
};
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::UnsafeRead> gm(memory, 0, 0);
ASSERT_OR_EXECUTE_MSG(
BufferDescriptorX().size() > buffer_index, { return {}; },
"BufferDescriptorX invalid buffer_index {}", buffer_index);
auto& read_buffer = read_buffer_x[buffer_index];
return read_buffer.Read(BufferDescriptorX()[buffer_index].Address(),
BufferDescriptorX()[buffer_index].Size(),
&read_buffer_data_x[buffer_index]);
return gm.Read(BufferDescriptorX()[buffer_index].Address(),
BufferDescriptorX()[buffer_index].Size(), &read_buffer_data_x[buffer_index]);
}
std::span<const u8> HLERequestContext::ReadBuffer(std::size_t buffer_index) const {
static thread_local std::array read_buffer_a{
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
};
static thread_local std::array read_buffer_x{
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::SafeRead>(memory, 0, 0),
};
Core::Memory::CpuGuestMemory<u8, Core::Memory::GuestMemoryFlags::UnsafeRead> gm(memory, 0, 0);
const bool is_buffer_a{BufferDescriptorA().size() > buffer_index &&
BufferDescriptorA()[buffer_index].Size()};
@ -401,18 +368,14 @@ std::span<const u8> HLERequestContext::ReadBuffer(std::size_t buffer_index) cons
ASSERT_OR_EXECUTE_MSG(
BufferDescriptorA().size() > buffer_index, { return {}; },
"BufferDescriptorA invalid buffer_index {}", buffer_index);
auto& read_buffer = read_buffer_a[buffer_index];
return read_buffer.Read(BufferDescriptorA()[buffer_index].Address(),
BufferDescriptorA()[buffer_index].Size(),
&read_buffer_data_a[buffer_index]);
return gm.Read(BufferDescriptorA()[buffer_index].Address(),
BufferDescriptorA()[buffer_index].Size(), &read_buffer_data_a[buffer_index]);
} else {
ASSERT_OR_EXECUTE_MSG(
BufferDescriptorX().size() > buffer_index, { return {}; },
"BufferDescriptorX invalid buffer_index {}", buffer_index);
auto& read_buffer = read_buffer_x[buffer_index];
return read_buffer.Read(BufferDescriptorX()[buffer_index].Address(),
BufferDescriptorX()[buffer_index].Size(),
&read_buffer_data_x[buffer_index]);
return gm.Read(BufferDescriptorX()[buffer_index].Address(),
BufferDescriptorX()[buffer_index].Size(), &read_buffer_data_x[buffer_index]);
}
}

9
src/core/hle/service/hle_ipc.h
View File

@ -19,8 +19,6 @@
#include "core/hle/ipc.h"
#include "core/hle/kernel/k_handle_table.h"
#include "core/hle/kernel/svc_common.h"
#include "core/hle/kernel/k_auto_object.h"
#include "core/hle/kernel/k_handle_table.h"
union Result;
@ -377,10 +375,6 @@ public:
return nullptr;
}
Kernel::KScopedAutoObject<Kernel::KAutoObject> GetObjectFromHandle(u32 handle) {
return GetClientHandleTable().GetObjectForIpc(handle, thread);
}
[[nodiscard]] std::shared_ptr<SessionRequestManager> GetManager() const {
return manager.lock();
}
@ -432,6 +426,9 @@ private:
Kernel::KernelCore& kernel;
Core::Memory::Memory& memory;
mutable std::array<Common::ScratchBuffer<u8>, 3> read_buffer_data_a{};
mutable std::array<Common::ScratchBuffer<u8>, 3> read_buffer_data_x{};
};
} // namespace Service

64
src/core/hle/service/nvdrv/core/nvmap.cpp
View File

@ -2,6 +2,8 @@
// SPDX-FileCopyrightText: 2022 Skyline Team and Contributors
// SPDX-License-Identifier: GPL-3.0-or-later
#include <functional>
#include "common/alignment.h"
#include "common/assert.h"
#include "common/logging/log.h"
@ -18,6 +20,7 @@ NvMap::Handle::Handle(u64 size_, Id id_)
}
NvResult NvMap::Handle::Alloc(Flags pFlags, u32 pAlign, u8 pKind, u64 pAddress) {
std::scoped_lock lock(mutex);
// Handles cannot be allocated twice
if (allocated) {
return NvResult::AccessDenied;
@ -78,11 +81,9 @@ void NvMap::UnmapHandle(Handle& handle_description) {
// Free and unmap the handle from the SMMU
auto& smmu = host1x.MemoryManager();
smmu.Unmap(static_cast<DAddr>(handle_description.pin_virt_address),
handle_description.aligned_size);
smmu.Free(handle_description.pin_virt_address,
static_cast<size_t>(handle_description.aligned_size));
handle_description.pin_virt_address = 0;
smmu.Unmap(handle_description.d_address, handle_description.aligned_size);
smmu.Free(handle_description.d_address, static_cast<size_t>(handle_description.aligned_size));
handle_description.d_address = 0;
}
bool NvMap::TryRemoveHandle(const Handle& handle_description) {
@ -123,41 +124,16 @@ std::shared_ptr<NvMap::Handle> NvMap::GetHandle(Handle::Id handle) {
}
}
VAddr NvMap::GetHandleAddress(Handle::Id handle) {
DAddr NvMap::GetHandleAddress(Handle::Id handle) {
std::scoped_lock lock(handles_lock);
try {
return handles.at(handle)->address;
return handles.at(handle)->d_address;
} catch (std::out_of_range&) {
return 0;
}
}
NvResult NvMap::AllocateHandle(Handle::Id handle, Handle::Flags pFlags, u32 pAlign, u8 pKind, u64 pAddress, size_t session_id) {
auto handle_description{GetHandle(handle)};
if (!handle_description) [[unlikely]] {
return NvResult::BadParameter;
}
if (handle_description->allocated) [[unlikely]] {
return NvResult::InsufficientMemory;
}
std::scoped_lock lock(handle_description->mutex);
NvResult result = handle_description->Alloc(pFlags, pAlign, pKind, pAddress);
if (result != NvResult::Success) {
return result;
}
auto& smmu = host1x.MemoryManager();
size_t total_size = static_cast<size_t>(handle_description->aligned_size);
handle_description->d_address = smmu.Allocate(total_size);
if (handle_description->d_address == 0) {
return NvResult::InsufficientMemory;
}
smmu.Map(handle_description->d_address, handle_description->address, total_size, session_id);
return NvResult::Success;
}
u32 NvMap::PinHandle(NvMap::Handle::Id handle, size_t session_id) {
DAddr NvMap::PinHandle(NvMap::Handle::Id handle, size_t session_id, bool low_area_pin) {
auto handle_description{GetHandle(handle)};
if (!handle_description) [[unlikely]] {
return 0;
@ -176,35 +152,38 @@ u32 NvMap::PinHandle(NvMap::Handle::Id handle, size_t session_id) {
handle_description->unmap_queue_entry.reset();
handle_description->pins++;
return handle_description->pin_virt_address;
return handle_description->d_address;
}
}
using namespace std::placeholders;
// If not then allocate some space and map it
DAddr address{};
auto& smmu = host1x.MemoryManager();
while ((address = smmu.AllocatePinned(
static_cast<size_t>(handle_description->aligned_size))) == 0) {
auto allocate = std::bind(&Tegra::MaxwellDeviceMemoryManager::Allocate, &smmu, _1);
//: std::bind(&Tegra::MaxwellDeviceMemoryManager::Allocate, &smmu, _1);
while ((address = allocate(static_cast<size_t>(handle_description->aligned_size))) == 0) {
// Free handles until the allocation succeeds
std::scoped_lock queueLock(unmap_queue_lock);
if (auto freeHandleDesc{unmap_queue.front()}) {
// Handles in the unmap queue are guaranteed not to be pinned so don't bother
// checking if they are before unmapping
std::scoped_lock freeLock(freeHandleDesc->mutex);
if (handle_description->pin_virt_address)
if (handle_description->d_address)
UnmapHandle(*freeHandleDesc);
} else {
LOG_CRITICAL(Service_NVDRV, "Ran out of SMMU address space!");
}
}
handle_description->d_address = address;
smmu.Map(address, handle_description->address, handle_description->aligned_size,
session_id);
handle_description->pin_virt_address = static_cast<u32>(address);
}
handle_description->pins++;
return handle_description->pin_virt_address;
return handle_description->d_address;
}
void NvMap::UnpinHandle(Handle::Id handle) {
@ -255,15 +234,10 @@ std::optional<NvMap::FreeInfo> NvMap::FreeHandle(Handle::Id handle, bool interna
LOG_WARNING(Service_NVDRV, "User duplicate count imbalance detected!");
} else if (handle_description->dupes == 0) {
// Force unmap the handle
if (handle_description->pin_virt_address) {
if (handle_description->d_address) {
std::scoped_lock queueLock(unmap_queue_lock);
UnmapHandle(*handle_description);
}
if (handle_description->allocated) {
auto& smmu = host1x.MemoryManager();
smmu.Free(handle_description->d_address, handle_description->aligned_size);
smmu.Unmap(handle_description->d_address, handle_description->aligned_size);
}
handle_description->pins = 0;
}

19
src/core/hle/service/nvdrv/core/nvmap.h
View File

@ -48,7 +48,7 @@ public:
using Id = u32;
Id id; //!< A globally unique identifier for this handle
s32 pins{};
s64 pins{};
u32 pin_virt_address{};
std::optional<typename std::list<std::shared_ptr<Handle>>::iterator> unmap_queue_entry{};
@ -63,15 +63,14 @@ public:
VAddr address{}; //!< The memory location in the guest's AS that this handle corresponds to,
//!< this can also be in the nvdrv tmem
DAddr d_address{}; //!< The memory location in the device's AS that this handle corresponds to,
//!< this can also be in the nvdrv tmem
bool is_shared_mem_mapped{}; //!< If this nvmap has been mapped with the MapSharedMem IPC
//!< call
u8 kind{}; //!< Used for memory compression
bool allocated{}; //!< If the handle has been allocated with `Alloc`
u64 dma_map_addr{}; //! remove me after implementing pinning.
DAddr d_address{}; //!< The memory location in the device's AS that this handle corresponds to,
//!< this can also be in the nvdrv tmem
Handle(u64 size, Id id);
@ -119,7 +118,7 @@ public:
std::shared_ptr<Handle> GetHandle(Handle::Id handle);
VAddr GetHandleAddress(Handle::Id handle);
DAddr GetHandleAddress(Handle::Id handle);
/**
* @brief Maps a handle into the SMMU address space
@ -127,15 +126,7 @@ public:
* number of calls to `UnpinHandle`
* @return The SMMU virtual address that the handle has been mapped to
*/
u32 PinHandle(Handle::Id handle, size_t session_id);
/**
* @brief Maps a handle into the SMMU address space
* @note This operation is refcounted, the number of calls to this must eventually match the
* number of calls to `UnpinHandle`
* @return The SMMU virtual address that the handle has been mapped to
*/
NvResult AllocateHandle(Handle::Id handle, Handle::Flags pFlags, u32 pAlign, u8 pKind, u64 pAddress, size_t session_id);
DAddr PinHandle(Handle::Id handle, size_t session_id, bool low_area_pin);
/**
* @brief When this has been called an equal number of times to `PinHandle` for the supplied

2
src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp
View File

@ -42,7 +42,7 @@ void nvdisp_disp0::flip(u32 buffer_handle, u32 offset, android::PixelFormat form
u32 height, u32 stride, android::BufferTransformFlags transform,
const Common::Rectangle<int>& crop_rect,
std::array<Service::Nvidia::NvFence, 4>& fences, u32 num_fences) {
const VAddr addr = nvmap.GetHandleAddress(buffer_handle);
const DAddr addr = nvmap.GetHandleAddress(buffer_handle);
LOG_TRACE(Service,
"Drawing from address {:X} offset {:08X} Width {} Height {} Stride {} Format {}",
addr, offset, width, height, stride, format);

57
src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
View File

@ -40,15 +40,15 @@ NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> i
case 0x3:
return WrapFixed(this, &nvhost_as_gpu::FreeSpace, input, output);
case 0x5:
return WrapFixed(this, &nvhost_as_gpu::UnmapBuffer, input, output);
return WrapFixed(this, &nvhost_as_gpu::UnmapBuffer, input, output, fd);
case 0x6:
return WrapFixed(this, &nvhost_as_gpu::MapBufferEx, input, output);
return WrapFixed(this, &nvhost_as_gpu::MapBufferEx, input, output, fd);
case 0x8:
return WrapFixed(this, &nvhost_as_gpu::GetVARegions1, input, output);
case 0x9:
return WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output);
case 0x14:
return WrapVariable(this, &nvhost_as_gpu::Remap, input, output);
return WrapVariable(this, &nvhost_as_gpu::Remap, input, output, fd);
default:
break;
}
@ -86,8 +86,15 @@ NvResult nvhost_as_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> i
return NvResult::NotImplemented;
}
void nvhost_as_gpu::OnOpen(size_t session_id, DeviceFD fd) {}
void nvhost_as_gpu::OnClose(DeviceFD fd) {}
void nvhost_as_gpu::OnOpen(size_t session_id, DeviceFD fd) {
sessions[fd] = session_id;
}
void nvhost_as_gpu::OnClose(DeviceFD fd) {
auto it = sessions.find(fd);
if (it != sessions.end()) {
sessions.erase(it);
}
}
NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
LOG_DEBUG(Service_NVDRV, "called, big_page_size=0x{:X}", params.big_page_size);
@ -206,6 +213,8 @@ void nvhost_as_gpu::FreeMappingLocked(u64 offset) {
static_cast<u32>(aligned_size >> page_size_bits));
}
nvmap.UnpinHandle(mapping->handle);
// Sparse mappings shouldn't be fully unmapped, just returned to their sparse state
// Only FreeSpace can unmap them fully
if (mapping->sparse_alloc) {
@ -259,7 +268,7 @@ NvResult nvhost_as_gpu::FreeSpace(IoctlFreeSpace& params) {
return NvResult::Success;
}
NvResult nvhost_as_gpu::Remap(std::span<IoctlRemapEntry> entries) {
NvResult nvhost_as_gpu::Remap(std::span<IoctlRemapEntry> entries, DeviceFD fd) {
LOG_DEBUG(Service_NVDRV, "called, num_entries=0x{:X}", entries.size());
if (!vm.initialised) {
@ -293,19 +302,19 @@ NvResult nvhost_as_gpu::Remap(std::span<IoctlRemapEntry> entries) {
return NvResult::BadValue;
}
VAddr cpu_address{static_cast<VAddr>(
handle->address +
(static_cast<u64>(entry.handle_offset_big_pages) << vm.big_page_size_bits))};
DAddr base = nvmap.PinHandle(entry.handle, sessions[fd], false);
DAddr device_address{static_cast<DAddr>(
base + (static_cast<u64>(entry.handle_offset_big_pages) << vm.big_page_size_bits))};
gmmu->Map(virtual_address, cpu_address, size, static_cast<Tegra::PTEKind>(entry.kind),
use_big_pages);
gmmu->Map(virtual_address, device_address, size,
static_cast<Tegra::PTEKind>(entry.kind), use_big_pages);
}
}
return NvResult::Success;
}
NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params, DeviceFD fd) {
LOG_DEBUG(Service_NVDRV,
"called, flags={:X}, nvmap_handle={:X}, buffer_offset={}, mapping_size={}"
", offset={}",
@ -331,9 +340,9 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
}
u64 gpu_address{static_cast<u64>(params.offset + params.buffer_offset)};
VAddr cpu_address{mapping->ptr + params.buffer_offset};
VAddr device_address{mapping->ptr + params.buffer_offset};
gmmu->Map(gpu_address, cpu_address, params.mapping_size,
gmmu->Map(gpu_address, device_address, params.mapping_size,
static_cast<Tegra::PTEKind>(params.kind), mapping->big_page);
return NvResult::Success;
@ -349,7 +358,8 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
return NvResult::BadValue;
}
VAddr cpu_address{static_cast<VAddr>(handle->address + params.buffer_offset)};
DAddr device_address{static_cast<DAddr>(nvmap.PinHandle(params.handle, sessions[fd], false) +
params.buffer_offset)};
u64 size{params.mapping_size ? params.mapping_size : handle->orig_size};
bool big_page{[&]() {
@ -373,15 +383,14 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
}
const bool use_big_pages = alloc->second.big_pages && big_page;
gmmu->Map(params.offset, cpu_address, size, static_cast<Tegra::PTEKind>(params.kind),
gmmu->Map(params.offset, device_address, size, static_cast<Tegra::PTEKind>(params.kind),
use_big_pages);
auto mapping{std::make_shared<Mapping>(cpu_address, params.offset, size, true,
use_big_pages, alloc->second.sparse)};
auto mapping{std::make_shared<Mapping>(params.handle, device_address, params.offset, size,
true, use_big_pages, alloc->second.sparse)};
alloc->second.mappings.push_back(mapping);
mapping_map[params.offset] = mapping;
} else {
auto& allocator{big_page ? *vm.big_page_allocator : *vm.small_page_allocator};
u32 page_size{big_page ? vm.big_page_size : VM::YUZU_PAGESIZE};
u32 page_size_bits{big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS};
@ -394,18 +403,18 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
return NvResult::InsufficientMemory;
}
gmmu->Map(params.offset, cpu_address, Common::AlignUp(size, page_size),
gmmu->Map(params.offset, device_address, Common::AlignUp(size, page_size),
static_cast<Tegra::PTEKind>(params.kind), big_page);
auto mapping{
std::make_shared<Mapping>(cpu_address, params.offset, size, false, big_page, false)};
auto mapping{std::make_shared<Mapping>(params.handle, device_address, params.offset, size,
false, big_page, false)};
mapping_map[params.offset] = mapping;
}
return NvResult::Success;
}
NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) {
NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params, DeviceFD fd) {
LOG_DEBUG(Service_NVDRV, "called, offset=0x{:X}", params.offset);
std::scoped_lock lock(mutex);
@ -433,6 +442,8 @@ NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) {
gmmu->Unmap(params.offset, mapping->size);
}
nvmap.UnpinHandle(mapping->handle);
mapping_map.erase(params.offset);
} catch (const std::out_of_range&) {
LOG_WARNING(Service_NVDRV, "Couldn't find region to unmap at 0x{:X}", params.offset);

20
src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
View File

@ -141,9 +141,9 @@ private:
NvResult AllocAsEx(IoctlAllocAsEx& params);
NvResult AllocateSpace(IoctlAllocSpace& params);
NvResult Remap(std::span<IoctlRemapEntry> params);
NvResult MapBufferEx(IoctlMapBufferEx& params);
NvResult UnmapBuffer(IoctlUnmapBuffer& params);
NvResult Remap(std::span<IoctlRemapEntry> params, DeviceFD fd);
NvResult MapBufferEx(IoctlMapBufferEx& params, DeviceFD fd);
NvResult UnmapBuffer(IoctlUnmapBuffer& params, DeviceFD fd);
NvResult FreeSpace(IoctlFreeSpace& params);
NvResult BindChannel(IoctlBindChannel& params);
@ -159,16 +159,18 @@ private:
NvCore::NvMap& nvmap;
struct Mapping {
VAddr ptr;
NvCore::NvMap::Handle::Id handle;
DAddr ptr;
u64 offset;
u64 size;
bool fixed;
bool big_page; // Only valid if fixed == false
bool sparse_alloc;
Mapping(VAddr ptr_, u64 offset_, u64 size_, bool fixed_, bool big_page_, bool sparse_alloc_)
: ptr(ptr_), offset(offset_), size(size_), fixed(fixed_), big_page(big_page_),
sparse_alloc(sparse_alloc_) {}
Mapping(NvCore::NvMap::Handle::Id handle_, DAddr ptr_, u64 offset_, u64 size_, bool fixed_,
bool big_page_, bool sparse_alloc_)
: handle(handle_), ptr(ptr_), offset(offset_), size(size_), fixed(fixed_),
big_page(big_page_), sparse_alloc(sparse_alloc_) {}
};
struct Allocation {
@ -212,9 +214,7 @@ private:
bool initialised{};
} vm;
std::shared_ptr<Tegra::MemoryManager> gmmu;
// s32 channel{};
// u32 big_page_size{VM::DEFAULT_BIG_PAGE_SIZE};
std::unordered_map<DeviceFD, size_t> sessions;
};
} // namespace Service::Nvidia::Devices

8
src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp
View File

@ -95,6 +95,9 @@ NvResult nvhost_nvdec_common::Submit(IoctlSubmit& params, std::span<u8> data, De
offset += SliceVectors(data, fence_thresholds, params.fence_count, offset);
auto& gpu = system.GPU();
//auto& device_memory = system.Host1x().MemoryManager();
auto* session = core.GetSession(sessions[fd]);
if (gpu.UseNvdec()) {
for (std::size_t i = 0; i < syncpt_increments.size(); i++) {
const SyncptIncr& syncpt_incr = syncpt_increments[i];
@ -106,7 +109,7 @@ NvResult nvhost_nvdec_common::Submit(IoctlSubmit& params, std::span<u8> data, De
const auto object = nvmap.GetHandle(cmd_buffer.memory_id);
ASSERT_OR_EXECUTE(object, return NvResult::InvalidState;);
Tegra::ChCommandHeaderList cmdlist(cmd_buffer.word_count);
system.ApplicationMemory().ReadBlock(object->address + cmd_buffer.offset, cmdlist.data(),
session->process->GetMemory().ReadBlock(object->address + cmd_buffer.offset, cmdlist.data(),
cmdlist.size() * sizeof(u32));
gpu.PushCommandBuffer(core.Host1xDeviceFile().fd_to_id[fd], cmdlist);
}
@ -136,7 +139,8 @@ NvResult nvhost_nvdec_common::GetWaitbase(IoctlGetWaitbase& params) {
NvResult nvhost_nvdec_common::MapBuffer(IoctlMapBuffer& params, std::span<MapBufferEntry> entries, DeviceFD fd) {
const size_t num_entries = std::min(params.num_entries, static_cast<u32>(entries.size()));
for (size_t i = 0; i < num_entries; i++) {
entries[i].map_address = nvmap.PinHandle(entries[i].map_handle, sessions[fd]);
DAddr pin_address = nvmap.PinHandle(entries[i].map_handle, sessions[fd], true);
entries[i].map_address = static_cast<u32>(pin_address);
}
return NvResult::Success;

4
src/core/hle/service/nvdrv/devices/nvmap.cpp
View File

@ -123,8 +123,8 @@ NvResult nvmap::IocAlloc(IocAllocParams& params, DeviceFD fd) {
return NvResult::InsufficientMemory;
}
const auto result = file.AllocateHandle(params.handle, params.flags, params.align, params.kind,
params.address, sessions[fd]);
const auto result =
handle_description->Alloc(params.flags, params.align, params.kind, params.address);
if (result != NvResult::Success) {
LOG_CRITICAL(Service_NVDRV, "Object failed to allocate, handle={:08X}", params.handle);
return result;

6
src/core/hle/service/nvdrv/nvdrv_interface.cpp
View File

@ -13,8 +13,6 @@
#include "core/hle/service/nvdrv/nvdrv.h"
#include "core/hle/service/nvdrv/nvdrv_interface.h"
#pragma optimize("", off)
namespace Service::Nvidia {
void NVDRV::Open(HLERequestContext& ctx) {
@ -173,8 +171,8 @@ void NVDRV::Initialize(HLERequestContext& ctx) {
[[maybe_unused]] const auto transfer_memory_size = rp.Pop<u32>();
auto& container = nvdrv->GetContainer();
auto process = ctx.GetObjectFromHandle(process_handle);
session_id = container.OpenSession(process->DynamicCast<Kernel::KProcess*>());
auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(process_handle);
session_id = container.OpenSession(process.GetPointerUnsafe());
is_initialized = true;
}

25
src/core/memory.cpp
View File

@ -24,6 +24,8 @@
#include "core/hle/kernel/k_process.h"
#include "core/memory.h"
#include "video_core/gpu.h"
#include "video_core/host1x/gpu_device_memory_manager.h"
#include "video_core/host1x/host1x.h"
#include "video_core/rasterizer_download_area.h"
namespace Core::Memory {
@ -638,15 +640,16 @@ struct Memory::Impl {
base * YUZU_PAGESIZE, (base + size) * YUZU_PAGESIZE);
// During boot, current_page_table might not be set yet, in which case we need not flush
if (system.IsPoweredOn()) {
/*if (system.IsPoweredOn()) {
auto& gpu = system.GPU();
for (u64 i = 0; i < size; i++) {
const auto page = base + i;
if (page_table.pointers[page].Type() == Common::PageType::RasterizerCachedMemory) {
gpu.FlushAndInvalidateRegion(page << YUZU_PAGEBITS, YUZU_PAGESIZE);
}
}
}
}*/
const auto end = base + size;
ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}",
@ -811,10 +814,15 @@ struct Memory::Impl {
return true;
}
void HandleRasterizerDownload(VAddr address, size_t size) {
void HandleRasterizerDownload(VAddr v_address, size_t size) {
const auto* p = GetPointerImpl(
v_address, []() {}, []() {});
auto& gpu_device_memory = system.Host1x().MemoryManager();
DAddr address =
gpu_device_memory.GetAddressFromPAddr(system.DeviceMemory().GetRawPhysicalAddr(p));
const size_t core = system.GetCurrentHostThreadID();
auto& current_area = rasterizer_read_areas[core];
const VAddr end_address = address + size;
const DAddr end_address = address + size;
if (current_area.start_address <= address && end_address <= current_area.end_address)
[[likely]] {
return;
@ -822,7 +830,10 @@ struct Memory::Impl {
current_area = system.GPU().OnCPURead(address, size);
}
void HandleRasterizerWrite(VAddr address, size_t size) {
void HandleRasterizerWrite(VAddr v_address, size_t size) {
const auto* p = GetPointerImpl(
v_address, []() {}, []() {});
PAddr address = system.DeviceMemory().GetRawPhysicalAddr(p);
constexpr size_t sys_core = Core::Hardware::NUM_CPU_CORES - 1;
const size_t core = std::min(system.GetCurrentHostThreadID(),
sys_core); // any other calls threads go to syscore.
@ -836,7 +847,7 @@ struct Memory::Impl {
}
});
auto& current_area = rasterizer_write_areas[core];
VAddr subaddress = address >> YUZU_PAGEBITS;
PAddr subaddress = address >> YUZU_PAGEBITS;
bool do_collection = current_area.last_address == subaddress;
if (!do_collection) [[unlikely]] {
do_collection = system.GPU().OnCPUWrite(address, size);
@ -849,7 +860,7 @@ struct Memory::Impl {
}
struct GPUDirtyState {
VAddr last_address;
PAddr last_address;
};
void InvalidateRegion(Common::ProcessAddress dest_addr, size_t size) {

205
src/core/memory.h
View File

@ -498,209 +498,4 @@ private:
std::unique_ptr<Impl> impl;
};
enum GuestMemoryFlags : u32 {
Read = 1 << 0,
Write = 1 << 1,
Safe = 1 << 2,
Cached = 1 << 3,
SafeRead = Read | Safe,
SafeWrite = Write | Safe,
SafeReadWrite = SafeRead | SafeWrite,
SafeReadCachedWrite = SafeReadWrite | Cached,
UnsafeRead = Read,
UnsafeWrite = Write,
UnsafeReadWrite = UnsafeRead | UnsafeWrite,
UnsafeReadCachedWrite = UnsafeReadWrite | Cached,
};
namespace {
template <typename M, typename T, GuestMemoryFlags FLAGS>
class GuestMemory {
using iterator = T*;
using const_iterator = const T*;
using value_type = T;
using element_type = T;
using iterator_category = std::contiguous_iterator_tag;
public:
GuestMemory() = delete;
explicit GuestMemory(M& memory, u64 addr, std::size_t size,
Common::ScratchBuffer<T>* backup = nullptr)
: m_memory{memory}, m_addr{addr}, m_size{size} {
static_assert(FLAGS & GuestMemoryFlags::Read || FLAGS & GuestMemoryFlags::Write);
if constexpr (FLAGS & GuestMemoryFlags::Read) {
Read(addr, size, backup);
}
}
~GuestMemory() = default;
T* data() noexcept {
return m_data_span.data();
}
const T* data() const noexcept {
return m_data_span.data();
}
size_t size() const noexcept {
return m_size;
}
size_t size_bytes() const noexcept {
return this->size() * sizeof(T);
}
[[nodiscard]] T* begin() noexcept {
return this->data();
}
[[nodiscard]] const T* begin() const noexcept {
return this->data();
}
[[nodiscard]] T* end() noexcept {
return this->data() + this->size();
}
[[nodiscard]] const T* end() const noexcept {
return this->data() + this->size();
}
T& operator[](size_t index) noexcept {
return m_data_span[index];
}
const T& operator[](size_t index) const noexcept {
return m_data_span[index];
}
void SetAddressAndSize(u64 addr, std::size_t size) noexcept {
m_addr = addr;
m_size = size;
m_addr_changed = true;
}
std::span<T> Read(u64 addr, std::size_t size,
Common::ScratchBuffer<T>* backup = nullptr) noexcept {
m_addr = addr;
m_size = size;
if (m_size == 0) {
m_is_data_copy = true;
return {};
}
if (this->TrySetSpan()) {
if constexpr (FLAGS & GuestMemoryFlags::Safe) {
m_memory.FlushRegion(m_addr, this->size_bytes());
}
} else {
if (backup) {
backup->resize_destructive(this->size());
m_data_span = *backup;
} else {
m_data_copy.resize(this->size());
m_data_span = std::span(m_data_copy);
}
m_is_data_copy = true;
m_span_valid = true;
if constexpr (FLAGS & GuestMemoryFlags::Safe) {
m_memory.ReadBlock(m_addr, this->data(), this->size_bytes());
} else {
m_memory.ReadBlockUnsafe(m_addr, this->data(), this->size_bytes());
}
}
return m_data_span;
}
void Write(std::span<T> write_data) noexcept {
if constexpr (FLAGS & GuestMemoryFlags::Cached) {
m_memory.WriteBlockCached(m_addr, write_data.data(), this->size_bytes());
} else if constexpr (FLAGS & GuestMemoryFlags::Safe) {
m_memory.WriteBlock(m_addr, write_data.data(), this->size_bytes());
} else {
m_memory.WriteBlockUnsafe(m_addr, write_data.data(), this->size_bytes());
}
}
bool TrySetSpan() noexcept {
if (u8* ptr = m_memory.GetSpan(m_addr, this->size_bytes()); ptr) {
m_data_span = {reinterpret_cast<T*>(ptr), this->size()};
m_span_valid = true;
return true;
}
return false;
}
protected:
bool IsDataCopy() const noexcept {
return m_is_data_copy;
}
bool AddressChanged() const noexcept {
return m_addr_changed;
}
M& m_memory;
u64 m_addr{};
size_t m_size{};
std::span<T> m_data_span{};
std::vector<T> m_data_copy{};
bool m_span_valid{false};
bool m_is_data_copy{false};
bool m_addr_changed{false};
};
template <typename M, typename T, GuestMemoryFlags FLAGS>
class GuestMemoryScoped : public GuestMemory<M, T, FLAGS> {
public:
GuestMemoryScoped() = delete;
explicit GuestMemoryScoped(M& memory, u64 addr, std::size_t size,
Common::ScratchBuffer<T>* backup = nullptr)
: GuestMemory<M, T, FLAGS>(memory, addr, size, backup) {
if constexpr (!(FLAGS & GuestMemoryFlags::Read)) {
if (!this->TrySetSpan()) {
if (backup) {
this->m_data_span = *backup;
this->m_span_valid = true;
this->m_is_data_copy = true;
}
}
}
}
~GuestMemoryScoped() {
if constexpr (FLAGS & GuestMemoryFlags::Write) {
if (this->size() == 0) [[unlikely]] {
return;
}
if (this->AddressChanged() || this->IsDataCopy()) {
ASSERT(this->m_span_valid);
if constexpr (FLAGS & GuestMemoryFlags::Cached) {
this->m_memory.WriteBlockCached(this->m_addr, this->data(), this->size_bytes());
} else if constexpr (FLAGS & GuestMemoryFlags::Safe) {
this->m_memory.WriteBlock(this->m_addr, this->data(), this->size_bytes());
} else {
this->m_memory.WriteBlockUnsafe(this->m_addr, this->data(), this->size_bytes());
}
} else if constexpr ((FLAGS & GuestMemoryFlags::Safe) ||
(FLAGS & GuestMemoryFlags::Cached)) {
this->m_memory.InvalidateRegion(this->m_addr, this->size_bytes());
}
}
}
};
} // namespace
template <typename T, GuestMemoryFlags FLAGS>
using CpuGuestMemory = GuestMemory<Memory, T, FLAGS>;
template <typename T, GuestMemoryFlags FLAGS>
using CpuGuestMemoryScoped = GuestMemoryScoped<Memory, T, FLAGS>;
template <typename T, GuestMemoryFlags FLAGS>
using GpuGuestMemory = GuestMemory<Tegra::MemoryManager, T, FLAGS>;
template <typename T, GuestMemoryFlags FLAGS>
using GpuGuestMemoryScoped = GuestMemoryScoped<Tegra::MemoryManager, T, FLAGS>;
} // namespace Core::Memory