diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp
index 015a2d33d3..afb94082ba 100644
--- a/src/video_core/texture_cache/image_info.cpp
+++ b/src/video_core/texture_cache/image_info.cpp
@@ -16,6 +16,7 @@ namespace VideoCommon {
 using Tegra::Texture::TextureType;
 using Tegra::Texture::TICEntry;
 using VideoCore::Surface::PixelFormat;
+using VideoCore::Surface::SurfaceType;
 
 ImageInfo::ImageInfo(const TICEntry& config) noexcept {
     format = PixelFormatFromTextureInfo(config.format, config.r_type, config.g_type, config.b_type,
@@ -102,6 +103,7 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
         layer_stride = CalculateLayerStride(*this);
         maybe_unaligned_layer_stride = CalculateLayerSize(*this);
         rescaleable &= (block.depth == 0) && resources.levels == 1;
+        rescaleable &= size.height > 256 || GetFormatType(format) != SurfaceType::ColorTexture;
         downscaleable = size.height > 512;
     }
 }
@@ -136,6 +138,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs, size_t index)
         size.depth = rt.depth;
     } else {
         rescaleable = block.depth == 0;
+        rescaleable &= size.height > 256;
         downscaleable = size.height > 512;
         type = ImageType::e2D;
         resources.layers = rt.depth;
@@ -200,6 +203,7 @@ ImageInfo::ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept {
             .depth = 1,
         };
         rescaleable = block.depth == 0;
+        rescaleable &= size.height > 256;
         downscaleable = size.height > 512;
     }
 }