Merge pull request #3975 from ReinUsesLisp/fast-bufcache
buffer_cache: Replace boost::icl::interval_map with boost::intrusive::set
This commit is contained in:
		| @@ -1,6 +1,7 @@ | |||||||
| add_library(video_core STATIC | add_library(video_core STATIC | ||||||
|     buffer_cache/buffer_block.h |     buffer_cache/buffer_block.h | ||||||
|     buffer_cache/buffer_cache.h |     buffer_cache/buffer_cache.h | ||||||
|  |     buffer_cache/map_interval.cpp | ||||||
|     buffer_cache/map_interval.h |     buffer_cache/map_interval.h | ||||||
|     dirty_flags.cpp |     dirty_flags.cpp | ||||||
|     dirty_flags.h |     dirty_flags.h | ||||||
|   | |||||||
| @@ -12,11 +12,12 @@ | |||||||
| #include <utility> | #include <utility> | ||||||
| #include <vector> | #include <vector> | ||||||
|  |  | ||||||
| #include <boost/icl/interval_map.hpp> | #include <boost/container/small_vector.hpp> | ||||||
| #include <boost/icl/interval_set.hpp> | #include <boost/icl/interval_set.hpp> | ||||||
| #include <boost/range/iterator_range.hpp> | #include <boost/intrusive/set.hpp> | ||||||
|  |  | ||||||
| #include "common/alignment.h" | #include "common/alignment.h" | ||||||
|  | #include "common/assert.h" | ||||||
| #include "common/common_types.h" | #include "common/common_types.h" | ||||||
| #include "common/logging/log.h" | #include "common/logging/log.h" | ||||||
| #include "core/core.h" | #include "core/core.h" | ||||||
| @@ -29,10 +30,12 @@ | |||||||
|  |  | ||||||
| namespace VideoCommon { | namespace VideoCommon { | ||||||
|  |  | ||||||
| using MapInterval = std::shared_ptr<MapIntervalBase>; |  | ||||||
|  |  | ||||||
| template <typename OwnerBuffer, typename BufferType, typename StreamBuffer> | template <typename OwnerBuffer, typename BufferType, typename StreamBuffer> | ||||||
| class BufferCache { | class BufferCache { | ||||||
|  |     using IntervalSet = boost::icl::interval_set<VAddr>; | ||||||
|  |     using IntervalType = typename IntervalSet::interval_type; | ||||||
|  |     using VectorMapInterval = boost::container::small_vector<MapInterval*, 1>; | ||||||
|  |  | ||||||
| public: | public: | ||||||
|     using BufferInfo = std::pair<BufferType, u64>; |     using BufferInfo = std::pair<BufferType, u64>; | ||||||
|  |  | ||||||
| @@ -40,14 +43,12 @@ public: | |||||||
|                             bool is_written = false, bool use_fast_cbuf = false) { |                             bool is_written = false, bool use_fast_cbuf = false) { | ||||||
|         std::lock_guard lock{mutex}; |         std::lock_guard lock{mutex}; | ||||||
|  |  | ||||||
|         const std::optional<VAddr> cpu_addr_opt = |         const auto& memory_manager = system.GPU().MemoryManager(); | ||||||
|             system.GPU().MemoryManager().GpuToCpuAddress(gpu_addr); |         const std::optional<VAddr> cpu_addr_opt = memory_manager.GpuToCpuAddress(gpu_addr); | ||||||
|  |  | ||||||
|         if (!cpu_addr_opt) { |         if (!cpu_addr_opt) { | ||||||
|             return {GetEmptyBuffer(size), 0}; |             return {GetEmptyBuffer(size), 0}; | ||||||
|         } |         } | ||||||
|  |         const VAddr cpu_addr = *cpu_addr_opt; | ||||||
|         VAddr cpu_addr = *cpu_addr_opt; |  | ||||||
|  |  | ||||||
|         // Cache management is a big overhead, so only cache entries with a given size. |         // Cache management is a big overhead, so only cache entries with a given size. | ||||||
|         // TODO: Figure out which size is the best for given games. |         // TODO: Figure out which size is the best for given games. | ||||||
| @@ -77,16 +78,19 @@ public: | |||||||
|             } |             } | ||||||
|         } |         } | ||||||
|  |  | ||||||
|         auto block = GetBlock(cpu_addr, size); |         OwnerBuffer block = GetBlock(cpu_addr, size); | ||||||
|         auto map = MapAddress(block, gpu_addr, cpu_addr, size); |         MapInterval* const map = MapAddress(block, gpu_addr, cpu_addr, size); | ||||||
|  |         if (!map) { | ||||||
|  |             return {GetEmptyBuffer(size), 0}; | ||||||
|  |         } | ||||||
|         if (is_written) { |         if (is_written) { | ||||||
|             map->MarkAsModified(true, GetModifiedTicks()); |             map->MarkAsModified(true, GetModifiedTicks()); | ||||||
|             if (Settings::IsGPULevelHigh() && Settings::values.use_asynchronous_gpu_emulation) { |             if (Settings::IsGPULevelHigh() && Settings::values.use_asynchronous_gpu_emulation) { | ||||||
|                 MarkForAsyncFlush(map); |                 MarkForAsyncFlush(map); | ||||||
|             } |             } | ||||||
|             if (!map->IsWritten()) { |             if (!map->is_written) { | ||||||
|                 map->MarkAsWritten(true); |                 map->is_written = true; | ||||||
|                 MarkRegionAsWritten(map->GetStart(), map->GetEnd() - 1); |                 MarkRegionAsWritten(map->start, map->end - 1); | ||||||
|             } |             } | ||||||
|         } |         } | ||||||
|  |  | ||||||
| @@ -132,12 +136,11 @@ public: | |||||||
|     void FlushRegion(VAddr addr, std::size_t size) { |     void FlushRegion(VAddr addr, std::size_t size) { | ||||||
|         std::lock_guard lock{mutex}; |         std::lock_guard lock{mutex}; | ||||||
|  |  | ||||||
|         std::vector<MapInterval> objects = GetMapsInRange(addr, size); |         VectorMapInterval objects = GetMapsInRange(addr, size); | ||||||
|         std::sort(objects.begin(), objects.end(), [](const MapInterval& a, const MapInterval& b) { |         std::sort(objects.begin(), objects.end(), | ||||||
|             return a->GetModificationTick() < b->GetModificationTick(); |                   [](MapInterval* lhs, MapInterval* rhs) { return lhs->ticks < rhs->ticks; }); | ||||||
|         }); |         for (MapInterval* object : objects) { | ||||||
|         for (auto& object : objects) { |             if (object->is_modified && object->is_registered) { | ||||||
|             if (object->IsModified() && object->IsRegistered()) { |  | ||||||
|                 mutex.unlock(); |                 mutex.unlock(); | ||||||
|                 FlushMap(object); |                 FlushMap(object); | ||||||
|                 mutex.lock(); |                 mutex.lock(); | ||||||
| @@ -148,9 +151,9 @@ public: | |||||||
|     bool MustFlushRegion(VAddr addr, std::size_t size) { |     bool MustFlushRegion(VAddr addr, std::size_t size) { | ||||||
|         std::lock_guard lock{mutex}; |         std::lock_guard lock{mutex}; | ||||||
|  |  | ||||||
|         const std::vector<MapInterval> objects = GetMapsInRange(addr, size); |         const VectorMapInterval objects = GetMapsInRange(addr, size); | ||||||
|         return std::any_of(objects.cbegin(), objects.cend(), [](const MapInterval& map) { |         return std::any_of(objects.cbegin(), objects.cend(), [](const MapInterval* map) { | ||||||
|             return map->IsModified() && map->IsRegistered(); |             return map->is_modified && map->is_registered; | ||||||
|         }); |         }); | ||||||
|     } |     } | ||||||
|  |  | ||||||
| @@ -158,9 +161,8 @@ public: | |||||||
|     void InvalidateRegion(VAddr addr, u64 size) { |     void InvalidateRegion(VAddr addr, u64 size) { | ||||||
|         std::lock_guard lock{mutex}; |         std::lock_guard lock{mutex}; | ||||||
|  |  | ||||||
|         std::vector<MapInterval> objects = GetMapsInRange(addr, size); |         for (auto& object : GetMapsInRange(addr, size)) { | ||||||
|         for (auto& object : objects) { |             if (object->is_registered) { | ||||||
|             if (object->IsRegistered()) { |  | ||||||
|                 Unregister(object); |                 Unregister(object); | ||||||
|             } |             } | ||||||
|         } |         } | ||||||
| @@ -169,10 +171,10 @@ public: | |||||||
|     void OnCPUWrite(VAddr addr, std::size_t size) { |     void OnCPUWrite(VAddr addr, std::size_t size) { | ||||||
|         std::lock_guard lock{mutex}; |         std::lock_guard lock{mutex}; | ||||||
|  |  | ||||||
|         for (const auto& object : GetMapsInRange(addr, size)) { |         for (MapInterval* object : GetMapsInRange(addr, size)) { | ||||||
|             if (object->IsMemoryMarked() && object->IsRegistered()) { |             if (object->is_memory_marked && object->is_registered) { | ||||||
|                 UnmarkMemory(object); |                 UnmarkMemory(object); | ||||||
|                 object->SetSyncPending(true); |                 object->is_sync_pending = true; | ||||||
|                 marked_for_unregister.emplace_back(object); |                 marked_for_unregister.emplace_back(object); | ||||||
|             } |             } | ||||||
|         } |         } | ||||||
| @@ -181,9 +183,9 @@ public: | |||||||
|     void SyncGuestHost() { |     void SyncGuestHost() { | ||||||
|         std::lock_guard lock{mutex}; |         std::lock_guard lock{mutex}; | ||||||
|  |  | ||||||
|         for (const auto& object : marked_for_unregister) { |         for (auto& object : marked_for_unregister) { | ||||||
|             if (object->IsRegistered()) { |             if (object->is_registered) { | ||||||
|                 object->SetSyncPending(false); |                 object->is_sync_pending = false; | ||||||
|                 Unregister(object); |                 Unregister(object); | ||||||
|             } |             } | ||||||
|         } |         } | ||||||
| @@ -192,9 +194,9 @@ public: | |||||||
|  |  | ||||||
|     void CommitAsyncFlushes() { |     void CommitAsyncFlushes() { | ||||||
|         if (uncommitted_flushes) { |         if (uncommitted_flushes) { | ||||||
|             auto commit_list = std::make_shared<std::list<MapInterval>>(); |             auto commit_list = std::make_shared<std::list<MapInterval*>>(); | ||||||
|             for (auto& map : *uncommitted_flushes) { |             for (MapInterval* map : *uncommitted_flushes) { | ||||||
|                 if (map->IsRegistered() && map->IsModified()) { |                 if (map->is_registered && map->is_modified) { | ||||||
|                     // TODO(Blinkhawk): Implement backend asynchronous flushing |                     // TODO(Blinkhawk): Implement backend asynchronous flushing | ||||||
|                     // AsyncFlushMap(map) |                     // AsyncFlushMap(map) | ||||||
|                     commit_list->push_back(map); |                     commit_list->push_back(map); | ||||||
| @@ -228,8 +230,8 @@ public: | |||||||
|             committed_flushes.pop_front(); |             committed_flushes.pop_front(); | ||||||
|             return; |             return; | ||||||
|         } |         } | ||||||
|         for (MapInterval& map : *flush_list) { |         for (MapInterval* map : *flush_list) { | ||||||
|             if (map->IsRegistered()) { |             if (map->is_registered) { | ||||||
|                 // TODO(Blinkhawk): Replace this for reading the asynchronous flush |                 // TODO(Blinkhawk): Replace this for reading the asynchronous flush | ||||||
|                 FlushMap(map); |                 FlushMap(map); | ||||||
|             } |             } | ||||||
| @@ -265,61 +267,60 @@ protected: | |||||||
|     } |     } | ||||||
|  |  | ||||||
|     /// Register an object into the cache |     /// Register an object into the cache | ||||||
|     void Register(const MapInterval& new_map, bool inherit_written = false) { |     MapInterval* Register(MapInterval new_map, bool inherit_written = false) { | ||||||
|         const VAddr cpu_addr = new_map->GetStart(); |         const VAddr cpu_addr = new_map.start; | ||||||
|         if (!cpu_addr) { |         if (!cpu_addr) { | ||||||
|             LOG_CRITICAL(HW_GPU, "Failed to register buffer with unmapped gpu_address 0x{:016x}", |             LOG_CRITICAL(HW_GPU, "Failed to register buffer with unmapped gpu_address 0x{:016x}", | ||||||
|                          new_map->GetGpuAddress()); |                          new_map.gpu_addr); | ||||||
|             return; |             return nullptr; | ||||||
|         } |         } | ||||||
|         const std::size_t size = new_map->GetEnd() - new_map->GetStart(); |         const std::size_t size = new_map.end - new_map.start; | ||||||
|         new_map->MarkAsRegistered(true); |         new_map.is_registered = true; | ||||||
|         const IntervalType interval{new_map->GetStart(), new_map->GetEnd()}; |  | ||||||
|         mapped_addresses.insert({interval, new_map}); |  | ||||||
|         rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1); |         rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1); | ||||||
|         new_map->SetMemoryMarked(true); |         new_map.is_memory_marked = true; | ||||||
|         if (inherit_written) { |         if (inherit_written) { | ||||||
|             MarkRegionAsWritten(new_map->GetStart(), new_map->GetEnd() - 1); |             MarkRegionAsWritten(new_map.start, new_map.end - 1); | ||||||
|             new_map->MarkAsWritten(true); |             new_map.is_written = true; | ||||||
|         } |         } | ||||||
|  |         MapInterval* const storage = mapped_addresses_allocator.Allocate(); | ||||||
|  |         *storage = new_map; | ||||||
|  |         mapped_addresses.insert(*storage); | ||||||
|  |         return storage; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     void UnmarkMemory(const MapInterval& map) { |     void UnmarkMemory(MapInterval* map) { | ||||||
|         if (!map->IsMemoryMarked()) { |         if (!map->is_memory_marked) { | ||||||
|             return; |             return; | ||||||
|         } |         } | ||||||
|         const std::size_t size = map->GetEnd() - map->GetStart(); |         const std::size_t size = map->end - map->start; | ||||||
|         rasterizer.UpdatePagesCachedCount(map->GetStart(), size, -1); |         rasterizer.UpdatePagesCachedCount(map->start, size, -1); | ||||||
|         map->SetMemoryMarked(false); |         map->is_memory_marked = false; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     /// Unregisters an object from the cache |     /// Unregisters an object from the cache | ||||||
|     void Unregister(const MapInterval& map) { |     void Unregister(MapInterval* map) { | ||||||
|         UnmarkMemory(map); |         UnmarkMemory(map); | ||||||
|         map->MarkAsRegistered(false); |         map->is_registered = false; | ||||||
|         if (map->IsSyncPending()) { |         if (map->is_sync_pending) { | ||||||
|  |             map->is_sync_pending = false; | ||||||
|             marked_for_unregister.remove(map); |             marked_for_unregister.remove(map); | ||||||
|             map->SetSyncPending(false); |  | ||||||
|         } |         } | ||||||
|         if (map->IsWritten()) { |         if (map->is_written) { | ||||||
|             UnmarkRegionAsWritten(map->GetStart(), map->GetEnd() - 1); |             UnmarkRegionAsWritten(map->start, map->end - 1); | ||||||
|         } |         } | ||||||
|         const IntervalType delete_interval{map->GetStart(), map->GetEnd()}; |         const auto it = mapped_addresses.find(*map); | ||||||
|         mapped_addresses.erase(delete_interval); |         ASSERT(it != mapped_addresses.end()); | ||||||
|  |         mapped_addresses.erase(it); | ||||||
|  |         mapped_addresses_allocator.Release(map); | ||||||
|     } |     } | ||||||
|  |  | ||||||
| private: | private: | ||||||
|     MapInterval CreateMap(const VAddr start, const VAddr end, const GPUVAddr gpu_addr) { |     MapInterval* MapAddress(const OwnerBuffer& block, GPUVAddr gpu_addr, VAddr cpu_addr, | ||||||
|         return std::make_shared<MapIntervalBase>(start, end, gpu_addr); |                             std::size_t size) { | ||||||
|     } |         const VectorMapInterval overlaps = GetMapsInRange(cpu_addr, size); | ||||||
|  |  | ||||||
|     MapInterval MapAddress(const OwnerBuffer& block, const GPUVAddr gpu_addr, const VAddr cpu_addr, |  | ||||||
|                            const std::size_t size) { |  | ||||||
|         std::vector<MapInterval> overlaps = GetMapsInRange(cpu_addr, size); |  | ||||||
|         if (overlaps.empty()) { |         if (overlaps.empty()) { | ||||||
|             auto& memory_manager = system.GPU().MemoryManager(); |             auto& memory_manager = system.GPU().MemoryManager(); | ||||||
|             const VAddr cpu_addr_end = cpu_addr + size; |             const VAddr cpu_addr_end = cpu_addr + size; | ||||||
|             MapInterval new_map = CreateMap(cpu_addr, cpu_addr_end, gpu_addr); |  | ||||||
|             if (memory_manager.IsGranularRange(gpu_addr, size)) { |             if (memory_manager.IsGranularRange(gpu_addr, size)) { | ||||||
|                 u8* host_ptr = memory_manager.GetPointer(gpu_addr); |                 u8* host_ptr = memory_manager.GetPointer(gpu_addr); | ||||||
|                 UploadBlockData(block, block->GetOffset(cpu_addr), size, host_ptr); |                 UploadBlockData(block, block->GetOffset(cpu_addr), size, host_ptr); | ||||||
| @@ -328,13 +329,12 @@ private: | |||||||
|                 memory_manager.ReadBlockUnsafe(gpu_addr, staging_buffer.data(), size); |                 memory_manager.ReadBlockUnsafe(gpu_addr, staging_buffer.data(), size); | ||||||
|                 UploadBlockData(block, block->GetOffset(cpu_addr), size, staging_buffer.data()); |                 UploadBlockData(block, block->GetOffset(cpu_addr), size, staging_buffer.data()); | ||||||
|             } |             } | ||||||
|             Register(new_map); |             return Register(MapInterval(cpu_addr, cpu_addr_end, gpu_addr)); | ||||||
|             return new_map; |  | ||||||
|         } |         } | ||||||
|  |  | ||||||
|         const VAddr cpu_addr_end = cpu_addr + size; |         const VAddr cpu_addr_end = cpu_addr + size; | ||||||
|         if (overlaps.size() == 1) { |         if (overlaps.size() == 1) { | ||||||
|             MapInterval& current_map = overlaps[0]; |             MapInterval* const current_map = overlaps[0]; | ||||||
|             if (current_map->IsInside(cpu_addr, cpu_addr_end)) { |             if (current_map->IsInside(cpu_addr, cpu_addr_end)) { | ||||||
|                 return current_map; |                 return current_map; | ||||||
|             } |             } | ||||||
| @@ -344,35 +344,39 @@ private: | |||||||
|         bool write_inheritance = false; |         bool write_inheritance = false; | ||||||
|         bool modified_inheritance = false; |         bool modified_inheritance = false; | ||||||
|         // Calculate new buffer parameters |         // Calculate new buffer parameters | ||||||
|         for (auto& overlap : overlaps) { |         for (MapInterval* overlap : overlaps) { | ||||||
|             new_start = std::min(overlap->GetStart(), new_start); |             new_start = std::min(overlap->start, new_start); | ||||||
|             new_end = std::max(overlap->GetEnd(), new_end); |             new_end = std::max(overlap->end, new_end); | ||||||
|             write_inheritance |= overlap->IsWritten(); |             write_inheritance |= overlap->is_written; | ||||||
|             modified_inheritance |= overlap->IsModified(); |             modified_inheritance |= overlap->is_modified; | ||||||
|         } |         } | ||||||
|         GPUVAddr new_gpu_addr = gpu_addr + new_start - cpu_addr; |         GPUVAddr new_gpu_addr = gpu_addr + new_start - cpu_addr; | ||||||
|         for (auto& overlap : overlaps) { |         for (auto& overlap : overlaps) { | ||||||
|             Unregister(overlap); |             Unregister(overlap); | ||||||
|         } |         } | ||||||
|         UpdateBlock(block, new_start, new_end, overlaps); |         UpdateBlock(block, new_start, new_end, overlaps); | ||||||
|         MapInterval new_map = CreateMap(new_start, new_end, new_gpu_addr); |  | ||||||
|  |         const MapInterval new_map{new_start, new_end, new_gpu_addr}; | ||||||
|  |         MapInterval* const map = Register(new_map, write_inheritance); | ||||||
|  |         if (!map) { | ||||||
|  |             return nullptr; | ||||||
|  |         } | ||||||
|         if (modified_inheritance) { |         if (modified_inheritance) { | ||||||
|             new_map->MarkAsModified(true, GetModifiedTicks()); |             map->MarkAsModified(true, GetModifiedTicks()); | ||||||
|             if (Settings::IsGPULevelHigh() && Settings::values.use_asynchronous_gpu_emulation) { |             if (Settings::IsGPULevelHigh() && Settings::values.use_asynchronous_gpu_emulation) { | ||||||
|                 MarkForAsyncFlush(new_map); |                 MarkForAsyncFlush(map); | ||||||
|             } |             } | ||||||
|         } |         } | ||||||
|         Register(new_map, write_inheritance); |         return map; | ||||||
|         return new_map; |  | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     void UpdateBlock(const OwnerBuffer& block, VAddr start, VAddr end, |     void UpdateBlock(const OwnerBuffer& block, VAddr start, VAddr end, | ||||||
|                      std::vector<MapInterval>& overlaps) { |                      const VectorMapInterval& overlaps) { | ||||||
|         const IntervalType base_interval{start, end}; |         const IntervalType base_interval{start, end}; | ||||||
|         IntervalSet interval_set{}; |         IntervalSet interval_set{}; | ||||||
|         interval_set.add(base_interval); |         interval_set.add(base_interval); | ||||||
|         for (auto& overlap : overlaps) { |         for (auto& overlap : overlaps) { | ||||||
|             const IntervalType subtract{overlap->GetStart(), overlap->GetEnd()}; |             const IntervalType subtract{overlap->start, overlap->end}; | ||||||
|             interval_set.subtract(subtract); |             interval_set.subtract(subtract); | ||||||
|         } |         } | ||||||
|         for (auto& interval : interval_set) { |         for (auto& interval : interval_set) { | ||||||
| @@ -386,18 +390,24 @@ private: | |||||||
|         } |         } | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     std::vector<MapInterval> GetMapsInRange(VAddr addr, std::size_t size) { |     VectorMapInterval GetMapsInRange(VAddr addr, std::size_t size) { | ||||||
|  |         VectorMapInterval result; | ||||||
|         if (size == 0) { |         if (size == 0) { | ||||||
|             return {}; |             return result; | ||||||
|         } |         } | ||||||
|  |  | ||||||
|         std::vector<MapInterval> objects{}; |         const VAddr addr_end = addr + size; | ||||||
|         const IntervalType interval{addr, addr + size}; |         auto it = mapped_addresses.lower_bound(addr); | ||||||
|         for (auto& pair : boost::make_iterator_range(mapped_addresses.equal_range(interval))) { |         if (it != mapped_addresses.begin()) { | ||||||
|             objects.push_back(pair.second); |             --it; | ||||||
|         } |         } | ||||||
|  |         while (it != mapped_addresses.end() && it->start < addr_end) { | ||||||
|         return objects; |             if (it->Overlaps(addr, addr_end)) { | ||||||
|  |                 result.push_back(&*it); | ||||||
|  |             } | ||||||
|  |             ++it; | ||||||
|  |         } | ||||||
|  |         return result; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     /// Returns a ticks counter used for tracking when cached objects were last modified |     /// Returns a ticks counter used for tracking when cached objects were last modified | ||||||
| @@ -405,12 +415,12 @@ private: | |||||||
|         return ++modified_ticks; |         return ++modified_ticks; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     void FlushMap(MapInterval map) { |     void FlushMap(MapInterval* map) { | ||||||
|         std::size_t size = map->GetEnd() - map->GetStart(); |         const std::size_t size = map->end - map->start; | ||||||
|         OwnerBuffer block = blocks[map->GetStart() >> block_page_bits]; |         OwnerBuffer block = blocks[map->start >> block_page_bits]; | ||||||
|         staging_buffer.resize(size); |         staging_buffer.resize(size); | ||||||
|         DownloadBlockData(block, block->GetOffset(map->GetStart()), size, staging_buffer.data()); |         DownloadBlockData(block, block->GetOffset(map->start), size, staging_buffer.data()); | ||||||
|         system.Memory().WriteBlockUnsafe(map->GetStart(), staging_buffer.data(), size); |         system.Memory().WriteBlockUnsafe(map->start, staging_buffer.data(), size); | ||||||
|         map->MarkAsModified(false, 0); |         map->MarkAsModified(false, 0); | ||||||
|     } |     } | ||||||
|  |  | ||||||
| @@ -515,7 +525,7 @@ private: | |||||||
|             } else { |             } else { | ||||||
|                 written_pages[page_start] = 1; |                 written_pages[page_start] = 1; | ||||||
|             } |             } | ||||||
|             page_start++; |             ++page_start; | ||||||
|         } |         } | ||||||
|     } |     } | ||||||
|  |  | ||||||
| @@ -531,7 +541,7 @@ private: | |||||||
|                     written_pages.erase(it); |                     written_pages.erase(it); | ||||||
|                 } |                 } | ||||||
|             } |             } | ||||||
|             page_start++; |             ++page_start; | ||||||
|         } |         } | ||||||
|     } |     } | ||||||
|  |  | ||||||
| @@ -542,14 +552,14 @@ private: | |||||||
|             if (written_pages.count(page_start) > 0) { |             if (written_pages.count(page_start) > 0) { | ||||||
|                 return true; |                 return true; | ||||||
|             } |             } | ||||||
|             page_start++; |             ++page_start; | ||||||
|         } |         } | ||||||
|         return false; |         return false; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     void MarkForAsyncFlush(MapInterval& map) { |     void MarkForAsyncFlush(MapInterval* map) { | ||||||
|         if (!uncommitted_flushes) { |         if (!uncommitted_flushes) { | ||||||
|             uncommitted_flushes = std::make_shared<std::unordered_set<MapInterval>>(); |             uncommitted_flushes = std::make_shared<std::unordered_set<MapInterval*>>(); | ||||||
|         } |         } | ||||||
|         uncommitted_flushes->insert(map); |         uncommitted_flushes->insert(map); | ||||||
|     } |     } | ||||||
| @@ -566,10 +576,9 @@ private: | |||||||
|     u64 buffer_offset = 0; |     u64 buffer_offset = 0; | ||||||
|     u64 buffer_offset_base = 0; |     u64 buffer_offset_base = 0; | ||||||
|  |  | ||||||
|     using IntervalSet = boost::icl::interval_set<VAddr>; |     MapIntervalAllocator mapped_addresses_allocator; | ||||||
|     using IntervalCache = boost::icl::interval_map<VAddr, MapInterval>; |     boost::intrusive::set<MapInterval, boost::intrusive::compare<MapIntervalCompare>> | ||||||
|     using IntervalType = typename IntervalCache::interval_type; |         mapped_addresses; | ||||||
|     IntervalCache mapped_addresses; |  | ||||||
|  |  | ||||||
|     static constexpr u64 write_page_bit = 11; |     static constexpr u64 write_page_bit = 11; | ||||||
|     std::unordered_map<u64, u32> written_pages; |     std::unordered_map<u64, u32> written_pages; | ||||||
| @@ -583,10 +592,10 @@ private: | |||||||
|     u64 modified_ticks = 0; |     u64 modified_ticks = 0; | ||||||
|  |  | ||||||
|     std::vector<u8> staging_buffer; |     std::vector<u8> staging_buffer; | ||||||
|     std::list<MapInterval> marked_for_unregister; |     std::list<MapInterval*> marked_for_unregister; | ||||||
|  |  | ||||||
|     std::shared_ptr<std::unordered_set<MapInterval>> uncommitted_flushes{}; |     std::shared_ptr<std::unordered_set<MapInterval*>> uncommitted_flushes; | ||||||
|     std::list<std::shared_ptr<std::list<MapInterval>>> committed_flushes; |     std::list<std::shared_ptr<std::list<MapInterval*>>> committed_flushes; | ||||||
|  |  | ||||||
|     std::recursive_mutex mutex; |     std::recursive_mutex mutex; | ||||||
| }; | }; | ||||||
|   | |||||||
							
								
								
									
										33
									
								
								src/video_core/buffer_cache/map_interval.cpp
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										33
									
								
								src/video_core/buffer_cache/map_interval.cpp
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,33 @@ | |||||||
|  | // Copyright 2020 yuzu Emulator Project | ||||||
|  | // Licensed under GPLv2 or any later version | ||||||
|  | // Refer to the license.txt file included. | ||||||
|  |  | ||||||
|  | #include <algorithm> | ||||||
|  | #include <array> | ||||||
|  | #include <cstddef> | ||||||
|  | #include <memory> | ||||||
|  |  | ||||||
|  | #include "video_core/buffer_cache/map_interval.h" | ||||||
|  |  | ||||||
|  | namespace VideoCommon { | ||||||
|  |  | ||||||
|  | MapIntervalAllocator::MapIntervalAllocator() { | ||||||
|  |     FillFreeList(first_chunk); | ||||||
|  | } | ||||||
|  |  | ||||||
|  | MapIntervalAllocator::~MapIntervalAllocator() = default; | ||||||
|  |  | ||||||
|  | void MapIntervalAllocator::AllocateNewChunk() { | ||||||
|  |     *new_chunk = std::make_unique<Chunk>(); | ||||||
|  |     FillFreeList(**new_chunk); | ||||||
|  |     new_chunk = &(*new_chunk)->next; | ||||||
|  | } | ||||||
|  |  | ||||||
|  | void MapIntervalAllocator::FillFreeList(Chunk& chunk) { | ||||||
|  |     const std::size_t old_size = free_list.size(); | ||||||
|  |     free_list.resize(old_size + chunk.data.size()); | ||||||
|  |     std::transform(chunk.data.rbegin(), chunk.data.rend(), free_list.begin() + old_size, | ||||||
|  |                    [](MapInterval& interval) { return &interval; }); | ||||||
|  | } | ||||||
|  |  | ||||||
|  | } // namespace VideoCommon | ||||||
| @@ -4,104 +4,89 @@ | |||||||
|  |  | ||||||
| #pragma once | #pragma once | ||||||
|  |  | ||||||
|  | #include <array> | ||||||
|  | #include <cstddef> | ||||||
|  | #include <memory> | ||||||
|  | #include <vector> | ||||||
|  |  | ||||||
|  | #include <boost/intrusive/set_hook.hpp> | ||||||
|  |  | ||||||
| #include "common/common_types.h" | #include "common/common_types.h" | ||||||
| #include "video_core/gpu.h" | #include "video_core/gpu.h" | ||||||
|  |  | ||||||
| namespace VideoCommon { | namespace VideoCommon { | ||||||
|  |  | ||||||
| class MapIntervalBase { | struct MapInterval : public boost::intrusive::set_base_hook<boost::intrusive::optimize_size<true>> { | ||||||
| public: |     MapInterval() = default; | ||||||
|     MapIntervalBase(const VAddr start, const VAddr end, const GPUVAddr gpu_addr) |  | ||||||
|         : start{start}, end{end}, gpu_addr{gpu_addr} {} |  | ||||||
|  |  | ||||||
|     void SetCpuAddress(VAddr new_cpu_addr) { |     /*implicit*/ MapInterval(VAddr start_) noexcept : start{start_} {} | ||||||
|         cpu_addr = new_cpu_addr; |  | ||||||
|  |     explicit MapInterval(VAddr start_, VAddr end_, GPUVAddr gpu_addr_) noexcept | ||||||
|  |         : start{start_}, end{end_}, gpu_addr{gpu_addr_} {} | ||||||
|  |  | ||||||
|  |     bool IsInside(VAddr other_start, VAddr other_end) const noexcept { | ||||||
|  |         return start <= other_start && other_end <= end; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     VAddr GetCpuAddress() const { |     bool Overlaps(VAddr other_start, VAddr other_end) const noexcept { | ||||||
|         return cpu_addr; |         return start < other_end && other_start < end; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     GPUVAddr GetGpuAddress() const { |     void MarkAsModified(bool is_modified_, u64 ticks_) noexcept { | ||||||
|         return gpu_addr; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool IsInside(const VAddr other_start, const VAddr other_end) const { |  | ||||||
|         return (start <= other_start && other_end <= end); |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool operator==(const MapIntervalBase& rhs) const { |  | ||||||
|         return std::tie(start, end) == std::tie(rhs.start, rhs.end); |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool operator!=(const MapIntervalBase& rhs) const { |  | ||||||
|         return !operator==(rhs); |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     void MarkAsRegistered(const bool registered) { |  | ||||||
|         is_registered = registered; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool IsRegistered() const { |  | ||||||
|         return is_registered; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     void SetMemoryMarked(bool is_memory_marked_) { |  | ||||||
|         is_memory_marked = is_memory_marked_; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool IsMemoryMarked() const { |  | ||||||
|         return is_memory_marked; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     void SetSyncPending(bool is_sync_pending_) { |  | ||||||
|         is_sync_pending = is_sync_pending_; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool IsSyncPending() const { |  | ||||||
|         return is_sync_pending; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     VAddr GetStart() const { |  | ||||||
|         return start; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     VAddr GetEnd() const { |  | ||||||
|         return end; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     void MarkAsModified(const bool is_modified_, const u64 tick) { |  | ||||||
|         is_modified = is_modified_; |         is_modified = is_modified_; | ||||||
|         ticks = tick; |         ticks = ticks_; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     bool IsModified() const { |     boost::intrusive::set_member_hook<> member_hook_; | ||||||
|         return is_modified; |     VAddr start = 0; | ||||||
|  |     VAddr end = 0; | ||||||
|  |     GPUVAddr gpu_addr = 0; | ||||||
|  |     u64 ticks = 0; | ||||||
|  |     bool is_written = false; | ||||||
|  |     bool is_modified = false; | ||||||
|  |     bool is_registered = false; | ||||||
|  |     bool is_memory_marked = false; | ||||||
|  |     bool is_sync_pending = false; | ||||||
|  | }; | ||||||
|  |  | ||||||
|  | struct MapIntervalCompare { | ||||||
|  |     constexpr bool operator()(const MapInterval& lhs, const MapInterval& rhs) const noexcept { | ||||||
|  |         return lhs.start < rhs.start; | ||||||
|  |     } | ||||||
|  | }; | ||||||
|  |  | ||||||
|  | class MapIntervalAllocator { | ||||||
|  | public: | ||||||
|  |     MapIntervalAllocator(); | ||||||
|  |     ~MapIntervalAllocator(); | ||||||
|  |  | ||||||
|  |     MapInterval* Allocate() { | ||||||
|  |         if (free_list.empty()) { | ||||||
|  |             AllocateNewChunk(); | ||||||
|  |         } | ||||||
|  |         MapInterval* const interval = free_list.back(); | ||||||
|  |         free_list.pop_back(); | ||||||
|  |         return interval; | ||||||
|     } |     } | ||||||
|  |  | ||||||
|     u64 GetModificationTick() const { |     void Release(MapInterval* interval) { | ||||||
|         return ticks; |         free_list.push_back(interval); | ||||||
|     } |  | ||||||
|  |  | ||||||
|     void MarkAsWritten(const bool is_written_) { |  | ||||||
|         is_written = is_written_; |  | ||||||
|     } |  | ||||||
|  |  | ||||||
|     bool IsWritten() const { |  | ||||||
|         return is_written; |  | ||||||
|     } |     } | ||||||
|  |  | ||||||
| private: | private: | ||||||
|     VAddr start; |     struct Chunk { | ||||||
|     VAddr end; |         std::unique_ptr<Chunk> next; | ||||||
|     GPUVAddr gpu_addr; |         std::array<MapInterval, 0x8000> data; | ||||||
|     VAddr cpu_addr{}; |     }; | ||||||
|     bool is_written{}; |  | ||||||
|     bool is_modified{}; |     void AllocateNewChunk(); | ||||||
|     bool is_registered{}; |  | ||||||
|     bool is_memory_marked{}; |     void FillFreeList(Chunk& chunk); | ||||||
|     bool is_sync_pending{}; |  | ||||||
|     u64 ticks{}; |     std::vector<MapInterval*> free_list; | ||||||
|  |     std::unique_ptr<Chunk>* new_chunk = &first_chunk.next; | ||||||
|  |  | ||||||
|  |     Chunk first_chunk; | ||||||
| }; | }; | ||||||
|  |  | ||||||
| } // namespace VideoCommon | } // namespace VideoCommon | ||||||
|   | |||||||
| @@ -8,6 +8,7 @@ | |||||||
|  |  | ||||||
| #include "common/assert.h" | #include "common/assert.h" | ||||||
| #include "common/microprofile.h" | #include "common/microprofile.h" | ||||||
|  | #include "video_core/buffer_cache/buffer_cache.h" | ||||||
| #include "video_core/engines/maxwell_3d.h" | #include "video_core/engines/maxwell_3d.h" | ||||||
| #include "video_core/rasterizer_interface.h" | #include "video_core/rasterizer_interface.h" | ||||||
| #include "video_core/renderer_opengl/gl_buffer_cache.h" | #include "video_core/renderer_opengl/gl_buffer_cache.h" | ||||||
|   | |||||||
| @@ -4,6 +4,7 @@ | |||||||
|  |  | ||||||
| #include "common/assert.h" | #include "common/assert.h" | ||||||
|  |  | ||||||
|  | #include "video_core/renderer_opengl/gl_buffer_cache.h" | ||||||
| #include "video_core/renderer_opengl/gl_fence_manager.h" | #include "video_core/renderer_opengl/gl_fence_manager.h" | ||||||
|  |  | ||||||
| namespace OpenGL { | namespace OpenGL { | ||||||
|   | |||||||
| @@ -7,6 +7,7 @@ | |||||||
| #include <memory> | #include <memory> | ||||||
|  |  | ||||||
| #include "core/core.h" | #include "core/core.h" | ||||||
|  | #include "video_core/buffer_cache/buffer_cache.h" | ||||||
| #include "video_core/renderer_vulkan/vk_buffer_cache.h" | #include "video_core/renderer_vulkan/vk_buffer_cache.h" | ||||||
| #include "video_core/renderer_vulkan/vk_device.h" | #include "video_core/renderer_vulkan/vk_device.h" | ||||||
| #include "video_core/renderer_vulkan/vk_scheduler.h" | #include "video_core/renderer_vulkan/vk_scheduler.h" | ||||||
|   | |||||||
| @@ -7,6 +7,7 @@ | |||||||
| #include <memory> | #include <memory> | ||||||
|  |  | ||||||
| #include "video_core/fence_manager.h" | #include "video_core/fence_manager.h" | ||||||
|  | #include "video_core/renderer_vulkan/vk_buffer_cache.h" | ||||||
| #include "video_core/renderer_vulkan/wrapper.h" | #include "video_core/renderer_vulkan/wrapper.h" | ||||||
|  |  | ||||||
| namespace Core { | namespace Core { | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user
	 bunnei
					bunnei