Custom textures rewrite (#6452)

* common: Add thread pool from yuzu

* Is really useful for asynchronous operations like shader compilation and custom textures, will be used in following PRs

* core: Improve ImageInterface

* Provide a default implementation so frontends don't have to duplicate code registering the lodepng version

* Add a dds version too which we will use in the next commit

* rasterizer_cache: Rewrite custom textures

* There's just too much to talk about here, look at the PR description for more details

* rasterizer_cache: Implement basic pack configuration file

* custom_tex_manager: Flip dumped textures

* custom_tex_manager: Optimize custom texture hashing

* If no convertions are needed then we can hash the decoded data directly removing the needed for duplicate decode

* custom_tex_manager: Implement asynchronous texture loading

* The file loading and decoding is offloaded into worker threads, while the upload itself still occurs in the main thread to avoid having to manage shared contexts

* Address review comments

* custom_tex_manager: Introduce custom material support

* video_core: Move custom textures to separate directory

* Also split the files to make the code cleaner

* gl_texture_runtime: Generate mipmaps for material

* custom_tex_manager: Prevent memory overflow when preloading

* externals: Add dds-ktx as submodule

* string_util: Return vector from SplitString

* No code benefits from passing it as an argument

* custom_textures: Use json config file

* gl_rasterizer: Only bind material for unit 0

* Address review comments

src/video_core/rasterizer_cache/pixel_format.h

@@ -44,7 +44,7 @@ enum class SurfaceType : u32 {
     Invalid = 5,
 };
 
-enum class TextureType : u32 {
+enum class TextureType : u16 {
     Texture2D = 0,
     CubeMap = 1,
 };
@@ -99,10 +99,10 @@ constexpr SurfaceType GetFormatType(PixelFormat format) {
     return FORMAT_MAP[index].type;
 }
 
-std::string_view PixelFormatAsString(PixelFormat format);
-
 bool CheckFormatsBlittable(PixelFormat source_format, PixelFormat dest_format);
 
+std::string_view PixelFormatAsString(PixelFormat format);
+
 PixelFormat PixelFormatFromTextureFormat(Pica::TexturingRegs::TextureFormat format);
 
 PixelFormat PixelFormatFromColorFormat(Pica::FramebufferRegs::ColorFormat format);

src/video_core/rasterizer_cache/rasterizer_cache.cpp

@@ -8,6 +8,7 @@
 #include "common/logging/log.h"
 #include "common/microprofile.h"
 #include "core/memory.h"
+#include "video_core/custom_textures/custom_tex_manager.h"
 #include "video_core/rasterizer_cache/rasterizer_cache.h"
 #include "video_core/regs.h"
 #include "video_core/renderer_base.h"
@@ -19,6 +20,15 @@ namespace {
 
 MICROPROFILE_DEFINE(RasterizerCache_CopySurface, "RasterizerCache", "CopySurface",
                     MP_RGB(128, 192, 64));
+MICROPROFILE_DEFINE(RasterizerCache_UploadSurface, "RasterizerCache", "UploadSurface",
+                    MP_RGB(128, 192, 64));
+MICROPROFILE_DEFINE(RasterizerCache_ComputeHash, "RasterizerCache", "ComputeHash",
+                    MP_RGB(32, 64, 192));
+MICROPROFILE_DEFINE(RasterizerCache_DownloadSurface, "RasterizerCache", "DownloadSurface",
+                    MP_RGB(128, 192, 64));
+MICROPROFILE_DEFINE(RasterizerCache_Invalidation, "RasterizerCache", "Invalidation",
+                    MP_RGB(128, 64, 192));
+MICROPROFILE_DEFINE(RasterizerCache_Flush, "RasterizerCache", "Flush", MP_RGB(128, 64, 192));
 
 constexpr auto RangeFromInterval(const auto& map, const auto& interval) {
     return boost::make_iterator_range(map.equal_range(interval));
@@ -119,11 +129,15 @@ auto FindMatch(const auto& surface_cache, const SurfaceParams& params, ScaleMatc
 
 } // Anonymous namespace
 
-RasterizerCache::RasterizerCache(Memory::MemorySystem& memory_, OpenGL::TextureRuntime& runtime_,
-                                 Pica::Regs& regs_, RendererBase& renderer_)
-    : memory{memory_}, runtime{runtime_}, regs{regs_}, renderer{renderer_},
-      resolution_scale_factor{renderer.GetResolutionScaleFactor()},
-      use_filter{Settings::values.texture_filter.GetValue() != Settings::TextureFilter::None} {}
+RasterizerCache::RasterizerCache(Memory::MemorySystem& memory_,
+                                 CustomTexManager& custom_tex_manager_,
+                                 OpenGL::TextureRuntime& runtime_, Pica::Regs& regs_,
+                                 RendererBase& renderer_)
+    : memory{memory_}, custom_tex_manager{custom_tex_manager_}, runtime{runtime_}, regs{regs_},
+      renderer{renderer_}, resolution_scale_factor{renderer.GetResolutionScaleFactor()},
+      use_filter{Settings::values.texture_filter.GetValue() != Settings::TextureFilter::None},
+      dump_textures{Settings::values.dump_textures.GetValue()},
+      use_custom_textures{Settings::values.custom_textures.GetValue()} {}
 
 RasterizerCache::~RasterizerCache() {
 #ifndef ANDROID
@@ -132,6 +146,32 @@ RasterizerCache::~RasterizerCache() {
 #endif
 }
 
+void RasterizerCache::TickFrame() {
+    custom_tex_manager.TickFrame();
+
+    const u32 scale_factor = renderer.GetResolutionScaleFactor();
+    const bool resolution_scale_changed = resolution_scale_factor != scale_factor;
+    const bool use_custom_texture_changed =
+        Settings::values.custom_textures.GetValue() != use_custom_textures;
+    const bool texture_filter_changed =
+        renderer.Settings().texture_filter_update_requested.exchange(false);
+
+    if (resolution_scale_changed || texture_filter_changed || use_custom_texture_changed) {
+        resolution_scale_factor = scale_factor;
+        use_filter = Settings::values.texture_filter.GetValue() != Settings::TextureFilter::None;
+        use_custom_textures = Settings::values.custom_textures.GetValue();
+        if (use_custom_textures) {
+            custom_tex_manager.FindCustomTextures();
+        }
+        FlushAll();
+        while (!surface_cache.empty()) {
+            UnregisterSurface(*surface_cache.begin()->second.begin());
+        }
+        texture_cube_cache.clear();
+        runtime.Reset();
+    }
+}
+
 bool RasterizerCache::AccelerateTextureCopy(const GPU::Regs::DisplayTransferConfig& config) {
     // Texture copy size is aligned to 16 byte units
     const u32 copy_size = Common::AlignDown(config.texture_copy.size, 16);
@@ -572,21 +612,6 @@ OpenGL::Framebuffer RasterizerCache::GetFramebufferSurfaces(bool using_color_fb,
                                                             bool using_depth_fb) {
     const auto& config = regs.framebuffer.framebuffer;
 
-    // Update resolution_scale_factor and reset cache if changed
-    const u32 scale_factor = renderer.GetResolutionScaleFactor();
-    const bool resolution_scale_changed = resolution_scale_factor != scale_factor;
-    const bool texture_filter_changed =
-        renderer.Settings().texture_filter_update_requested.exchange(false);
-
-    if (resolution_scale_changed || texture_filter_changed) {
-        resolution_scale_factor = scale_factor;
-        use_filter = Settings::values.texture_filter.GetValue() != Settings::TextureFilter::None;
-        FlushAll();
-        while (!surface_cache.empty())
-            UnregisterSurface(*surface_cache.begin()->second.begin());
-        texture_cube_cache.clear();
-    }
-
     const s32 framebuffer_width = config.GetWidth();
     const s32 framebuffer_height = config.GetHeight();
     const auto viewport_rect = regs.rasterizer.GetViewportRect();
@@ -818,11 +843,13 @@ void RasterizerCache::ValidateSurface(const SurfaceRef& surface, PAddr addr, u32
     // Filtered mipmaps often look really bad. We can achieve better quality by
     // generating them from the base level.
     if (surface->res_scale != 1 && level != 0) {
-        runtime.GenerateMipmaps(*surface, surface->levels - 1);
+        runtime.GenerateMipmaps(*surface);
     }
 }
 
 void RasterizerCache::UploadSurface(const SurfaceRef& surface, SurfaceInterval interval) {
+    MICROPROFILE_SCOPE(RasterizerCache_UploadSurface);
+
     const SurfaceParams load_info = surface->FromInterval(interval);
     ASSERT(load_info.addr >= surface->addr && load_info.end <= surface->end);
 
@@ -838,6 +865,18 @@ void RasterizerCache::UploadSurface(const SurfaceRef& surface, SurfaceInterval i
     DecodeTexture(load_info, load_info.addr, load_info.end, upload_data, staging.mapped,
                   runtime.NeedsConversion(surface->pixel_format));
 
+    if (use_custom_textures) {
+        const u64 hash = ComputeCustomHash(load_info, staging.mapped, upload_data);
+        if (UploadCustomSurface(surface, load_info, hash)) {
+            return;
+        }
+    }
+    if (dump_textures && !surface->is_custom) {
+        const u64 hash = Common::ComputeHash64(upload_data.data(), upload_data.size());
+        const u32 level = surface->LevelOf(load_info.addr);
+        custom_tex_manager.DumpTexture(load_info, level, upload_data, hash);
+    }
+
     const BufferTextureCopy upload = {
         .buffer_offset = 0,
         .buffer_size = staging.size,
@@ -847,7 +886,49 @@ void RasterizerCache::UploadSurface(const SurfaceRef& surface, SurfaceInterval i
     surface->Upload(upload, staging);
 }
 
+bool RasterizerCache::UploadCustomSurface(const SurfaceRef& surface, const SurfaceParams& load_info,
+                                          u64 hash) {
+    const u32 level = surface->LevelOf(load_info.addr);
+    const bool is_base_level = level == 0;
+    Material* material = custom_tex_manager.GetMaterial(hash);
+
+    if (!material) {
+        return surface->IsCustom();
+    }
+    if (!is_base_level && custom_tex_manager.SkipMipmaps()) {
+        return true;
+    }
+
+    surface->is_custom = true;
+
+    const auto upload = [this, level, surface, material]() -> bool {
+        if (!surface->IsCustom() && !surface->Swap(material)) {
+            LOG_ERROR(HW_GPU, "Custom compressed format {} unsupported by host GPU",
+                      material->format);
+            return false;
+        }
+        surface->UploadCustom(material, level);
+        if (custom_tex_manager.SkipMipmaps()) {
+            runtime.GenerateMipmaps(*surface);
+        }
+        return true;
+    };
+    return custom_tex_manager.Decode(material, std::move(upload));
+}
+
+u64 RasterizerCache::ComputeCustomHash(const SurfaceParams& load_info, std::span<u8> decoded,
+                                       std::span<u8> upload_data) {
+    MICROPROFILE_SCOPE(RasterizerCache_ComputeHash);
+
+    if (custom_tex_manager.UseNewHash()) {
+        return Common::ComputeHash64(upload_data.data(), upload_data.size());
+    }
+    return Common::ComputeHash64(decoded.data(), decoded.size());
+}
+
 void RasterizerCache::DownloadSurface(const SurfaceRef& surface, SurfaceInterval interval) {
+    MICROPROFILE_SCOPE(RasterizerCache_DownloadSurface);
+
     const SurfaceParams flush_info = surface->FromInterval(interval);
     const u32 flush_start = boost::icl::first(interval);
     const u32 flush_end = boost::icl::last_next(interval);

src/video_core/rasterizer_cache/rasterizer_cache.h

@@ -4,7 +4,9 @@
 
 #pragma once
 
+#include <functional>
 #include <unordered_map>
+#include <vector>
 #include <boost/icl/interval_map.hpp>
 #include <boost/icl/interval_set.hpp>
 #include "video_core/rasterizer_cache/surface_base.h"
@@ -28,6 +30,8 @@ enum class ScaleMatch {
     Ignore   // accept every scaled res
 };
 
+class CustomTexManager;
+struct CustomTexture;
 class RendererBase;
 
 class RasterizerCache : NonCopyable {
@@ -62,10 +66,13 @@ public:
     };
 
 public:
-    RasterizerCache(Memory::MemorySystem& memory, OpenGL::TextureRuntime& runtime, Pica::Regs& regs,
-                    RendererBase& renderer);
+    RasterizerCache(Memory::MemorySystem& memory, CustomTexManager& custom_tex_manager,
+                    OpenGL::TextureRuntime& runtime, Pica::Regs& regs, RendererBase& renderer);
     ~RasterizerCache();
 
+    /// Notify the cache that a new frame has been queued
+    void TickFrame();
+
     /// Perform hardware accelerated texture copy according to the provided configuration
     bool AccelerateTextureCopy(const GPU::Regs::DisplayTransferConfig& config);
 
@@ -126,6 +133,13 @@ private:
     /// Copies pixel data in interval from the guest VRAM to the host GPU surface
     void UploadSurface(const SurfaceRef& surface, SurfaceInterval interval);
 
+    /// Uploads a custom texture identified with hash to the target surface
+    bool UploadCustomSurface(const SurfaceRef& surface, const SurfaceParams& load_info, u64 hash);
+
+    /// Returns the hash used to lookup the custom surface.
+    u64 ComputeCustomHash(const SurfaceParams& load_info, std::span<u8> decoded,
+                          std::span<u8> upload_data);
+
     /// Copies pixel data in interval from the host GPU surface to the guest VRAM
     void DownloadSurface(const SurfaceRef& surface, SurfaceInterval interval);
 
@@ -158,6 +172,7 @@ private:
 
 private:
     Memory::MemorySystem& memory;
+    CustomTexManager& custom_tex_manager;
     OpenGL::TextureRuntime& runtime;
     Pica::Regs& regs;
     RendererBase& renderer;
@@ -168,7 +183,9 @@ private:
     u32 resolution_scale_factor;
     RenderTargets render_targets;
     std::unordered_map<TextureCubeConfig, TextureCube> texture_cube_cache;
-    bool use_filter{};
+    bool use_filter;
+    bool dump_textures;
+    bool use_custom_textures;
 };
 
 } // namespace VideoCore

src/video_core/rasterizer_cache/surface_base.cpp

@@ -3,6 +3,7 @@
 // Refer to the license.txt file included.
 
 #include "common/alignment.h"
+#include "video_core/custom_textures/material.h"
 #include "video_core/rasterizer_cache/surface_base.h"
 #include "video_core/texture/texture_decode.h"
 
@@ -101,6 +102,10 @@ SurfaceInterval SurfaceBase::GetCopyableInterval(const SurfaceParams& params) co
     return result;
 }
 
+bool SurfaceBase::HasNormalMap() const noexcept {
+    return material && material->Map(MapType::Normal) != nullptr;
+}
+
 ClearValue SurfaceBase::MakeClearValue(PAddr copy_addr, PixelFormat dst_format) {
     const SurfaceType dst_type = GetFormatType(dst_format);
     const std::array fill_buffer = MakeFillBuffer(copy_addr);

src/video_core/rasterizer_cache/surface_base.h

@@ -11,6 +11,8 @@ namespace VideoCore {
 
 using SurfaceRegions = boost::icl::interval_set<PAddr, std::less, SurfaceInterval>;
 
+struct Material;
+
 class SurfaceBase : public SurfaceParams {
 public:
     SurfaceBase(const SurfaceParams& params);
@@ -28,10 +30,17 @@ public:
     /// Returns the clear value used to validate another surface from this fill surface
     ClearValue MakeClearValue(PAddr copy_addr, PixelFormat dst_format);
 
+    /// Returns true if the surface contains a custom material with a normal map.
+    bool HasNormalMap() const noexcept;
+
     u64 ModificationTick() const noexcept {
         return modification_tick;
     }
 
+    bool IsCustom() const noexcept {
+        return is_custom && custom_format != CustomPixelFormat::Invalid;
+    }
+
     bool IsRegionValid(SurfaceInterval interval) const {
         return (invalid_regions.find(interval) == invalid_regions.end());
     }
@@ -57,6 +66,8 @@ private:
 
 public:
     bool registered = false;
+    bool is_custom = false;
+    const Material* material = nullptr;
     SurfaceRegions invalid_regions;
     u32 fill_size = 0;
     std::array<u8, 4> fill_data;

src/video_core/rasterizer_cache/surface_params.cpp

@@ -215,12 +215,12 @@ u32 SurfaceParams::LevelOf(PAddr level_addr) const {
     return level;
 }
 
-std::string SurfaceParams::DebugName(bool scaled) const noexcept {
+std::string SurfaceParams::DebugName(bool scaled, bool custom) const noexcept {
     const u32 scaled_width = scaled ? GetScaledWidth() : width;
     const u32 scaled_height = scaled ? GetScaledHeight() : height;
-    return fmt::format("Surface: {}x{} {} {} levels from {:#x} to {:#x} ({})", scaled_width,
+    return fmt::format("Surface: {}x{} {} {} levels from {:#x} to {:#x} ({}{})", scaled_width,
                        scaled_height, PixelFormatAsString(pixel_format), levels, addr, end,
-                       scaled ? "scaled" : "unscaled");
+                       custom ? "custom," : "", scaled ? "scaled" : "unscaled");
 }
 
 } // namespace VideoCore

src/video_core/rasterizer_cache/surface_params.h

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include "video_core/custom_textures/custom_format.h"
 #include "video_core/rasterizer_cache/utils.h"
 
 namespace VideoCore {
@@ -46,7 +47,7 @@ public:
     u32 LevelOf(PAddr addr) const;
 
     /// Returns a string identifier of the params object
-    std::string DebugName(bool scaled) const noexcept;
+    std::string DebugName(bool scaled, bool custom = false) const noexcept;
 
     [[nodiscard]] SurfaceInterval GetInterval() const noexcept {
         return SurfaceInterval{addr, end};
@@ -101,6 +102,7 @@ public:
     bool is_tiled = false;
     TextureType texture_type = TextureType::Texture2D;
     PixelFormat pixel_format = PixelFormat::Invalid;
+    CustomPixelFormat custom_format = CustomPixelFormat::Invalid;
     SurfaceType type = SurfaceType::Invalid;
 
     std::array<u32, MAX_PICA_LEVELS> mipmap_offsets{};