Fix VT mouse capture issues in Terminal and conhost (#7166)

This pull request fixes capture and event generation in VT mouse mode
for both conhost and terminal.

Fixes #6401.

[1/3] Terminal: clamp mouse events to the viewport, don't throw them away

 gnome-terminal (at least) sends mouse events whose x/y are at the
 extreme ends of the buffer when a drag starts inside the terminal and
 then exits it.

 We would previously discard any mouse events that exited the borders of
 the viewport. Now we will keep emitting events where X/Y=0/w/h.

[2/3] conhost: clamp VT mouse to viewport, capture pointer

 This is the same as (1), but for conhost. conhost wasn't already
 capturing the pointer when VT mouse mode was in use. By capturing, we
 ensure that events that happen outside the screen still result in events
 sent to an application (like a release after a drag)

[3/3] wpf: capture the pointer when VT mouse is enabled

 This is the same as (2), but for the WPF control. Clamping is handled
 in TerminalCore in (1), so we didn't need to do it in WPF.

src/cascadia/PublicTerminalCore/HwndTerminal.cpp

@@ -55,10 +55,29 @@ try
 
     if (terminal)
     {
-        if (_IsMouseMessage(uMsg) && terminal->_CanSendVTMouseInput())
+        if (_IsMouseMessage(uMsg))
         {
-            if (terminal->_SendMouseEvent(uMsg, wParam, lParam))
+            if (terminal->_CanSendVTMouseInput() && terminal->_SendMouseEvent(uMsg, wParam, lParam))
             {
+                // GH#6401: Capturing the mouse ensures that we get drag/release events
+                // even if the user moves outside the window.
+                // _SendMouseEvent returns false if the terminal's not in VT mode, so we'll
+                // fall through to release the capture.
+                switch (uMsg)
+                {
+                case WM_LBUTTONDOWN:
+                case WM_MBUTTONDOWN:
+                case WM_RBUTTONDOWN:
+                    SetCapture(hwnd);
+                    break;
+                case WM_LBUTTONUP:
+                case WM_MBUTTONUP:
+                case WM_RBUTTONUP:
+                    ReleaseCapture();
+                    break;
+                }
+
+                // Suppress all mouse events that made it into the terminal.
                 return 0;
             }
         }
@@ -76,6 +95,10 @@ try
             return 0;
         case WM_LBUTTONUP:
             terminal->_singleClickTouchdownPos = std::nullopt;
+            [[fallthrough]];
+        case WM_MBUTTONUP:
+        case WM_RBUTTONUP:
+            ReleaseCapture();
             break;
         case WM_MOUSEMOVE:
             if (WI_IsFlagSet(wParam, MK_LBUTTON))

src/cascadia/TerminalCore/Terminal.cpp

@@ -486,14 +486,14 @@ bool Terminal::SendKeyEvent(const WORD vkey,
 // - false if we did not translate the key, and it should be processed into a character.
 bool Terminal::SendMouseEvent(const COORD viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta)
 {
-    // viewportPos must be within the dimensions of the viewport
-    const auto viewportDimensions = _mutableViewport.Dimensions();
-    if (viewportPos.X < 0 || viewportPos.X >= viewportDimensions.X || viewportPos.Y < 0 || viewportPos.Y >= viewportDimensions.Y)
-    {
-        return false;
-    }
-
-    return _terminalInput->HandleMouse(viewportPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta);
+    // GH#6401: VT applications should be able to receive mouse events from outside the
+    // terminal buffer. This is likely to happen when the user drags the cursor offscreen.
+    // We shouldn't throw away perfectly good events when they're offscreen, so we just
+    // clamp them to be within the range [(0, 0), (W, H)].
+#pragma warning(suppress : 26496) // analysis can't tell we're assigning through a reference below
+    auto clampedPos{ viewportPos };
+    _mutableViewport.ToOrigin().Clamp(clampedPos);
+    return _terminalInput->HandleMouse(clampedPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta);
 }
 
 // Method Description:

src/interactivity/win32/windowio.cpp

@@ -123,7 +123,14 @@ bool HandleTerminalMouseEvent(const COORD cMousePosition,
     // Virtual terminal input mode
     if (IsInVirtualTerminalInputMode())
     {
-        fWasHandled = gci.GetActiveInputBuffer()->GetTerminalInput().HandleMouse(cMousePosition, uiButton, sModifierKeystate, sWheelDelta);
+        // GH#6401: VT applications should be able to receive mouse events from outside the
+        // terminal buffer. This is likely to happen when the user drags the cursor offscreen.
+        // We shouldn't throw away perfectly good events when they're offscreen, so we just
+        // clamp them to be within the range [(0, 0), (W, H)].
+        auto clampedPosition{ cMousePosition };
+        const auto clampViewport{ gci.GetActiveOutputBuffer().GetViewport().ToOrigin() };
+        clampViewport.Clamp(clampedPosition);
+        fWasHandled = gci.GetActiveInputBuffer()->GetTerminalInput().HandleMouse(clampedPosition, uiButton, sModifierKeystate, sWheelDelta);
     }
 
     return fWasHandled;
@@ -635,6 +642,25 @@ BOOL HandleMouseEvent(const SCREEN_INFORMATION& ScreenInfo,
 
         if (HandleTerminalMouseEvent(MousePosition, Message, GET_KEYSTATE_WPARAM(wParam), sDelta))
         {
+            // GH#6401: Capturing the mouse ensures that we get drag/release events
+            // even if the user moves outside the window.
+            // HandleTerminalMouseEvent returns false if the terminal's not in VT mode,
+            // so capturing/releasing here should not impact other console mouse event
+            // consumers.
+            switch (Message)
+            {
+            case WM_LBUTTONDOWN:
+            case WM_MBUTTONDOWN:
+            case WM_RBUTTONDOWN:
+                SetCapture(ServiceLocator::LocateConsoleWindow()->GetWindowHandle());
+                break;
+            case WM_LBUTTONUP:
+            case WM_MBUTTONUP:
+            case WM_RBUTTONUP:
+                ReleaseCapture();
+                break;
+            }
+
             return FALSE;
         }
     }