GPU: Ensure coordinates are masked/clamped

src/core/gpu.cpp

@@ -196,7 +196,7 @@ void GPU::SoftReset()
   m_GPUSTAT.vertical_interlace = false;
   m_GPUSTAT.display_disable = true;
   m_GPUSTAT.dma_direction = DMADirection::Off;
-  m_drawing_area.Set(0, 0, 0, 0);
+  m_drawing_area = {};
   m_drawing_area_changed = true;
   m_drawing_offset = {};
   std::memset(&m_crtc_state.regs, 0, sizeof(m_crtc_state.regs));

src/core/gpu.h

@@ -294,7 +294,10 @@ class GPU
   }
 
   /// Returns true if the drawing area is valid (i.e. left <= right, top <= bottom).
-  ALWAYS_INLINE bool IsDrawingAreaIsValid() const { return m_drawing_area.Valid(); }
+  ALWAYS_INLINE bool IsDrawingAreaIsValid() const
+  {
+    return (m_drawing_area.left <= m_drawing_area.right && m_drawing_area.top <= m_drawing_area.bottom);
+  }
 
   /// Clamps the specified coordinates to the drawing area.
   ALWAYS_INLINE void ClampCoordinatesToDrawingArea(s32* x, s32* y)
@@ -457,13 +460,8 @@ class GPU
     ALWAYS_INLINE void ClearTextureWindowChangedFlag() { texture_window_changed = false; }
   } m_draw_mode = {};
 
-  Common::Rectangle<u32> m_drawing_area{0, 0, VRAM_WIDTH, VRAM_HEIGHT};
-
-  struct DrawingOffset
-  {
-    s32 x;
-    s32 y;
-  } m_drawing_offset = {};
+  GPUDrawingArea m_drawing_area = {};
+  GPUDrawingOffset m_drawing_offset = {};
 
   bool m_console_is_pal = false;
   bool m_set_texture_disable_mask = false;

src/core/gpu_backend.h

@@ -63,7 +63,7 @@ class GPUBackend
 
   void HandleCommand(const GPUBackendCommand* cmd);
 
-  Common::Rectangle<u32> m_drawing_area{};
+  GPUDrawingArea m_drawing_area = {};
 
   Threading::KernelSemaphore m_sync_semaphore;
   std::atomic_bool m_gpu_thread_sleeping{false};

src/core/gpu_hw.cpp

@@ -77,7 +77,7 @@ ALWAYS_INLINE static bool IsBlendedTextureFiltering(GPUTextureFilter filter)
 }
 
 /// Computes the area affected by a VRAM transfer, including wrap-around of X.
-static Common::Rectangle<u32> GetVRAMTransferBounds(u32 x, u32 y, u32 width, u32 height)
+ALWAYS_INLINE_RELEASE static Common::Rectangle<u32> GetVRAMTransferBounds(u32 x, u32 y, u32 width, u32 height)
 {
   Common::Rectangle<u32> out_rc = Common::Rectangle<u32>::FromExtents(x % VRAM_WIDTH, y % VRAM_HEIGHT, width, height);
   if (out_rc.right > VRAM_WIDTH)
@@ -244,6 +244,7 @@ void GPU_HW::Reset(bool clear_vram)
   m_batch_ubo_data = {};
   m_batch_ubo_dirty = true;
   m_current_depth = 1;
+  SetClampedDrawingArea();
 
   if (clear_vram)
     ClearFramebuffer();
@@ -500,6 +501,20 @@ void GPU_HW::CheckSettings()
   }
 }
 
+void GPU_HW::SetClampedDrawingArea()
+{
+  if (!IsDrawingAreaIsValid()) [[unlikely]]
+  {
+    m_clamped_drawing_area = {};
+    return;
+  }
+
+  m_clamped_drawing_area.right = std::min(m_drawing_area.right + 1, static_cast<u32>(VRAM_WIDTH));
+  m_clamped_drawing_area.left = std::min(m_drawing_area.left, std::min(m_clamped_drawing_area.right, VRAM_WIDTH - 1));
+  m_clamped_drawing_area.bottom = std::min(m_drawing_area.bottom + 1, static_cast<u32>(VRAM_HEIGHT));
+  m_clamped_drawing_area.top = std::min(m_drawing_area.top, std::min(m_drawing_area.bottom, VRAM_HEIGHT - 1));
+}
+
 u32 GPU_HW::CalculateResolutionScale() const
 {
   const u32 max_resolution_scale = GetMaxResolutionScale();
@@ -592,6 +607,26 @@ void GPU_HW::ClearVRAMDirtyRectangle()
   m_vram_dirty_write_rect.SetInvalid();
 }
 
+void GPU_HW::IncludeDrawnDirtyRectangle(s32 min_x, s32 min_y, s32 max_x, s32 max_y)
+{
+  const u32 clamped_min_x = std::clamp(min_x, static_cast<s32>(m_clamped_drawing_area.left),
+                                       static_cast<s32>(m_clamped_drawing_area.right - 1));
+  const u32 clamped_max_x =
+    std::clamp(max_x, static_cast<s32>(m_clamped_drawing_area.left), static_cast<s32>(m_clamped_drawing_area.right));
+  m_vram_dirty_draw_rect.left = std::min(m_vram_dirty_draw_rect.left, clamped_min_x);
+  m_vram_dirty_draw_rect.right = std::max(m_vram_dirty_draw_rect.right, clamped_max_x);
+
+  const u32 clamped_min_y = std::clamp(min_y, static_cast<s32>(m_clamped_drawing_area.top),
+                                       static_cast<s32>(m_clamped_drawing_area.bottom - 1));
+  const u32 clamped_max_y =
+    std::clamp(max_y, static_cast<s32>(m_clamped_drawing_area.top), static_cast<s32>(m_clamped_drawing_area.bottom));
+  m_vram_dirty_draw_rect.top = std::min(m_vram_dirty_draw_rect.top, clamped_min_y);
+  m_vram_dirty_draw_rect.bottom = std::max(m_vram_dirty_draw_rect.bottom, clamped_max_y);
+
+  DebugAssert(m_vram_dirty_draw_rect.left < VRAM_WIDTH && m_vram_dirty_draw_rect.right <= VRAM_WIDTH);
+  DebugAssert(m_vram_dirty_draw_rect.top < VRAM_HEIGHT && m_vram_dirty_draw_rect.bottom <= VRAM_HEIGHT);
+}
+
 std::tuple<u32, u32> GPU_HW::GetEffectiveDisplayResolution(bool scaled /* = true */)
 {
   const u32 scale = scaled ? m_resolution_scale : 1u;
@@ -2004,13 +2039,9 @@ void GPU_HW::LoadVertices()
       }
       else
       {
-        const u32 clip_left = static_cast<u32>(std::clamp<s32>(min_x, m_drawing_area.left, m_drawing_area.right));
-        const u32 clip_right = static_cast<u32>(std::clamp<s32>(max_x, m_drawing_area.left, m_drawing_area.right)) + 1u;
-        const u32 clip_top = static_cast<u32>(std::clamp<s32>(min_y, m_drawing_area.top, m_drawing_area.bottom));
-        const u32 clip_bottom =
-          static_cast<u32>(std::clamp<s32>(max_y, m_drawing_area.top, m_drawing_area.bottom)) + 1u;
+        // TODO: Cull triangles that fall entirely off-screen.
+        IncludeDrawnDirtyRectangle(min_x, min_y, max_x, max_y);
 
-        m_vram_dirty_draw_rect.Include(clip_left, clip_right, clip_top, clip_bottom);
         AddDrawTriangleTicks(native_vertex_positions[0][0], native_vertex_positions[0][1],
                              native_vertex_positions[1][0], native_vertex_positions[1][1],
                              native_vertex_positions[2][0], native_vertex_positions[2][1], rc.shading_enable,
@@ -2041,14 +2072,8 @@ void GPU_HW::LoadVertices()
         }
         else
         {
-          const u32 clip_left = static_cast<u32>(std::clamp<s32>(min_x_123, m_drawing_area.left, m_drawing_area.right));
-          const u32 clip_right =
-            static_cast<u32>(std::clamp<s32>(max_x_123, m_drawing_area.left, m_drawing_area.right)) + 1u;
-          const u32 clip_top = static_cast<u32>(std::clamp<s32>(min_y_123, m_drawing_area.top, m_drawing_area.bottom));
-          const u32 clip_bottom =
-            static_cast<u32>(std::clamp<s32>(max_y_123, m_drawing_area.top, m_drawing_area.bottom)) + 1u;
-
-          m_vram_dirty_draw_rect.Include(clip_left, clip_right, clip_top, clip_bottom);
+          IncludeDrawnDirtyRectangle(min_x_123, min_y_123, max_x_123, max_y_123);
+
           AddDrawTriangleTicks(native_vertex_positions[2][0], native_vertex_positions[2][1],
                                native_vertex_positions[1][0], native_vertex_positions[1][1],
                                native_vertex_positions[3][0], native_vertex_positions[3][1], rc.shading_enable,
@@ -2187,14 +2212,14 @@ void GPU_HW::LoadVertices()
         tex_top = 0;
       }
 
+      IncludeDrawnDirtyRectangle(pos_x, pos_y, pos_x + rectangle_width, pos_y + rectangle_height);
+
       const u32 clip_left = static_cast<u32>(std::clamp<s32>(pos_x, m_drawing_area.left, m_drawing_area.right));
       const u32 clip_right =
         static_cast<u32>(std::clamp<s32>(pos_x + rectangle_width, m_drawing_area.left, m_drawing_area.right)) + 1u;
       const u32 clip_top = static_cast<u32>(std::clamp<s32>(pos_y, m_drawing_area.top, m_drawing_area.bottom));
       const u32 clip_bottom =
         static_cast<u32>(std::clamp<s32>(pos_y + rectangle_height, m_drawing_area.top, m_drawing_area.bottom)) + 1u;
-
-      m_vram_dirty_draw_rect.Include(clip_left, clip_right, clip_top, clip_bottom);
       AddDrawRectangleTicks(clip_right - clip_left, clip_bottom - clip_top, rc.texture_enable, rc.transparency_enable);
 
       if (m_sw_renderer)
@@ -2251,13 +2276,14 @@ void GPU_HW::LoadVertices()
           return;
         }
 
+        IncludeDrawnDirtyRectangle(min_x, min_y, max_x, max_y);
+
         const u32 clip_left = static_cast<u32>(std::clamp<s32>(min_x, m_drawing_area.left, m_drawing_area.right));
         const u32 clip_right = static_cast<u32>(std::clamp<s32>(max_x, m_drawing_area.left, m_drawing_area.right)) + 1u;
         const u32 clip_top = static_cast<u32>(std::clamp<s32>(min_y, m_drawing_area.top, m_drawing_area.bottom));
         const u32 clip_bottom =
           static_cast<u32>(std::clamp<s32>(max_y, m_drawing_area.top, m_drawing_area.bottom)) + 1u;
 
-        m_vram_dirty_draw_rect.Include(clip_left, clip_right, clip_top, clip_bottom);
         AddDrawLineTicks(clip_right - clip_left, clip_bottom - clip_top, rc.shading_enable);
 
         // TODO: Should we do a PGXP lookup here? Most lines are 2D.
@@ -2317,14 +2343,15 @@ void GPU_HW::LoadVertices()
           }
           else
           {
+            IncludeDrawnDirtyRectangle(min_x, min_y, max_x, max_y);
+
             const u32 clip_left = static_cast<u32>(std::clamp<s32>(min_x, m_drawing_area.left, m_drawing_area.right));
             const u32 clip_right =
               static_cast<u32>(std::clamp<s32>(max_x, m_drawing_area.left, m_drawing_area.right)) + 1u;
             const u32 clip_top = static_cast<u32>(std::clamp<s32>(min_y, m_drawing_area.top, m_drawing_area.bottom));
             const u32 clip_bottom =
               static_cast<u32>(std::clamp<s32>(max_y, m_drawing_area.top, m_drawing_area.bottom)) + 1u;
 
-            m_vram_dirty_draw_rect.Include(clip_left, clip_right, clip_top, clip_bottom);
             AddDrawLineTicks(clip_right - clip_left, clip_bottom - clip_top, rc.shading_enable);
 
             // TODO: Should we do a PGXP lookup here? Most lines are 2D.
@@ -3087,6 +3114,7 @@ void GPU_HW::DispatchRenderCommand()
     if (m_drawing_area_changed)
     {
       m_drawing_area_changed = false;
+      SetClampedDrawingArea();
       SetScissor();
 
       if (m_pgxp_depth_buffer && m_last_depth_z < 1.0f)

src/core/gpu_hw.h

@@ -131,6 +131,7 @@ class GPU_HW final : public GPU
   void PrintSettingsToLog();
   void CheckSettings();
 
+  void SetClampedDrawingArea();
   void UpdateVRAMReadTexture(bool drawn, bool written);
   void UpdateDepthBufferFromMaskBit();
   void ClearDepthBuffer();
@@ -149,6 +150,7 @@ class GPU_HW final : public GPU
   void SetFullVRAMDirtyRectangle();
   void ClearVRAMDirtyRectangle();
   void IncludeVRAMDirtyRectangle(Common::Rectangle<u32>& rect, const Common::Rectangle<u32>& new_rect);
+  void IncludeDrawnDirtyRectangle(s32 min_x, s32 min_y, s32 max_x, s32 max_y);
   void CheckForTexPageOverlap(u32 texpage, u32 min_u, u32 min_v, u32 max_u, u32 max_v);
 
   bool IsFlushed() const;
@@ -252,6 +254,7 @@ class GPU_HW final : public GPU
   BatchUBOData m_batch_ubo_data = {};
 
   // Bounding box of VRAM area that the GPU has drawn into.
+  GPUDrawingArea m_clamped_drawing_area = {};
   Common::Rectangle<u32> m_vram_dirty_draw_rect;
   Common::Rectangle<u32> m_vram_dirty_write_rect;
   Common::Rectangle<u32> m_current_uv_range;

src/core/gpu_sw_backend.cpp

@@ -214,6 +214,7 @@ void ALWAYS_INLINE_RELEASE GPU_SW_Backend::ShadePixel(const GPUBackendDrawComman
   if ((bg_color.bits & mask_and) != 0)
     return;
 
+  DebugAssert(static_cast<u32>(x) < VRAM_WIDTH && static_cast<u32>(y) < VRAM_HEIGHT);
   SetPixel(static_cast<u32>(x), static_cast<u32>(y), color.bits | cmd->params.GetMaskOR());
 }
 
@@ -234,6 +235,7 @@ void GPU_SW_Backend::DrawRectangle(const GPUBackendDrawRectangleCommand* cmd)
       continue;
     }
 
+    const u32 draw_y = static_cast<u32>(y) & VRAM_HEIGHT_MASK;
     const u8 texcoord_y = Truncate8(ZeroExtend32(origin_texcoord_y) + offset_y);
 
     for (u32 offset_x = 0; offset_x < cmd->width; offset_x++)
@@ -244,8 +246,8 @@ void GPU_SW_Backend::DrawRectangle(const GPUBackendDrawRectangleCommand* cmd)
 
       const u8 texcoord_x = Truncate8(ZeroExtend32(origin_texcoord_x) + offset_x);
 
-      ShadePixel<texture_enable, raw_texture_enable, transparency_enable, false>(
-        cmd, static_cast<u32>(x), static_cast<u32>(y), r, g, b, texcoord_x, texcoord_y);
+      ShadePixel<texture_enable, raw_texture_enable, transparency_enable, false>(cmd, static_cast<u32>(x), draw_y, r, g,
+                                                                                 b, texcoord_x, texcoord_y);
     }
   }
 }
@@ -569,7 +571,7 @@ void GPU_SW_Backend::DrawTriangle(const GPUBackendDrawPolygonCommand* cmd,
           continue;
 
         DrawSpan<shading_enable, texture_enable, raw_texture_enable, transparency_enable, dithering_enable>(
-          cmd, yi, GetPolyXFP_Int(lc), GetPolyXFP_Int(rc), ig, idl);
+          cmd, y & VRAM_HEIGHT_MASK, GetPolyXFP_Int(lc), GetPolyXFP_Int(rc), ig, idl);
       }
     }
     else
@@ -583,9 +585,8 @@ void GPU_SW_Backend::DrawTriangle(const GPUBackendDrawPolygonCommand* cmd,
 
         if (y >= static_cast<s32>(m_drawing_area.top))
         {
-
           DrawSpan<shading_enable, texture_enable, raw_texture_enable, transparency_enable, dithering_enable>(
-            cmd, yi, GetPolyXFP_Int(lc), GetPolyXFP_Int(rc), ig, idl);
+            cmd, y & VRAM_HEIGHT_MASK, GetPolyXFP_Int(lc), GetPolyXFP_Int(rc), ig, idl);
         }
 
         yi++;
@@ -698,8 +699,8 @@ void GPU_SW_Backend::DrawLine(const GPUBackendDrawLineCommand* cmd, const GPUBac
       const u8 g = shading_enable ? static_cast<u8>(cur_point.g >> Line_RGB_FractBits) : p0->g;
       const u8 b = shading_enable ? static_cast<u8>(cur_point.b >> Line_RGB_FractBits) : p0->b;
 
-      ShadePixel<false, false, transparency_enable, dithering_enable>(cmd, static_cast<u32>(x), static_cast<u32>(y), r,
-                                                                      g, b, 0, 0);
+      ShadePixel<false, false, transparency_enable, dithering_enable>(
+        cmd, static_cast<u32>(x), static_cast<u32>(y) & VRAM_HEIGHT_MASK, r, g, b, 0, 0);
     }
 
     cur_point.x += step.dx_dk;

src/core/gpu_types.h

@@ -83,6 +83,21 @@ enum class GPUInterlacedDisplayMode : u8
   SeparateFields
 };
 
+// NOTE: Inclusive, not exclusive on the upper bounds.
+struct GPUDrawingArea
+{
+  u32 left;
+  u32 top;
+  u32 right;
+  u32 bottom;
+};
+
+struct GPUDrawingOffset
+{
+  s32 x;
+  s32 y;
+};
+
 union GPURenderCommand
 {
   u32 bits;
@@ -318,7 +333,7 @@ struct GPUBackendCopyVRAMCommand : public GPUBackendCommand
 
 struct GPUBackendSetDrawingAreaCommand : public GPUBackendCommand
 {
-  Common::Rectangle<u32> new_area;
+  GPUDrawingArea new_area;
 };
 
 struct GPUBackendDrawCommand : public GPUBackendCommand