Synthesize VT mouse events and add mouse support to Terminal (#4859)

## Summary of the Pull Request
Make TerminalControl synthesize mouse events and Terminal send them to
the TerminalInput's MouseInput module.

The implementation here takes significant inspiration from how we handle
KeyEvents.

## References
Closes #545 - VT Mouse Mode (Terminal)
References #376 - VT Mouse Mode (ConPty)

### TerminalControl
- `_TrySendMouseEvent` attempts to send a mouse event via TermInput.
  Similar to `_TrySendKeyEvent`
- Use the above function to try and send the mouse event _before_
  deciding to modify the selection

### TerminalApi
- Hookup (re)setting the various modes to handle VT Input
- Terminal is _always_ in VT Input mode (important for #4856)

### TerminalDispatch
- Hookup (re)setting the various modes to handle VT Input

### TerminalInput
- Convert the mouse input position from viewport position to buffer
  position
- Then send it over to the MouseInput in TerminalInput to actually do it
  (#4848)

## Validation Steps Performed
Tests should still pass.
This commit is contained in:
Carlos Zamora 2020-03-12 17:44:28 -07:00 committed by GitHub
parent 93b31f6e3f
commit ae71dce2ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 421 additions and 0 deletions

View file

@ -22,6 +22,7 @@ using namespace winrt::Windows::UI::Xaml::Input;
using namespace winrt::Windows::UI::Xaml::Automation::Peers;
using namespace winrt::Windows::UI::Core;
using namespace winrt::Windows::UI::ViewManagement;
using namespace winrt::Windows::UI::Input;
using namespace winrt::Windows::System;
using namespace winrt::Microsoft::Terminal::Settings;
using namespace winrt::Windows::ApplicationModel::DataTransfer;
@ -812,6 +813,72 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
e.Handled(true);
}
// Method Description:
// - Send this particular mouse event to the terminal.
// See Terminal::SendMouseEvent for more information.
// Arguments:
// - point: the PointerPoint object representing a mouse event from our XAML input handler
bool TermControl::_TrySendMouseEvent(Windows::UI::Input::PointerPoint const& point)
{
const auto props = point.Properties();
// Get the terminal position relative to the viewport
const auto terminalPosition = _GetTerminalPosition(point.Position());
// Which mouse button changed state (and how)
unsigned int uiButton{};
switch (props.PointerUpdateKind())
{
case PointerUpdateKind::LeftButtonPressed:
uiButton = WM_LBUTTONDOWN;
break;
case PointerUpdateKind::LeftButtonReleased:
uiButton = WM_LBUTTONUP;
break;
case PointerUpdateKind::MiddleButtonPressed:
uiButton = WM_MBUTTONDOWN;
break;
case PointerUpdateKind::MiddleButtonReleased:
uiButton = WM_MBUTTONUP;
break;
case PointerUpdateKind::RightButtonPressed:
uiButton = WM_RBUTTONDOWN;
break;
case PointerUpdateKind::RightButtonReleased:
uiButton = WM_RBUTTONUP;
break;
default:
uiButton = WM_MOUSEMOVE;
}
// Mouse wheel data
const short sWheelDelta = ::base::saturated_cast<short>(props.MouseWheelDelta());
if (sWheelDelta != 0 && !props.IsHorizontalMouseWheel())
{
// if we have a mouse wheel delta and it wasn't a horizontal wheel motion
uiButton = WM_MOUSEWHEEL;
}
const auto modifiers = _GetPressedModifierKeys();
return _terminal->SendMouseEvent(terminalPosition, uiButton, modifiers, sWheelDelta);
}
// Method Description:
// - Checks if we can send vt mouse input.
// Arguments:
// - point: the PointerPoint object representing a mouse event from our XAML input handler
bool TermControl::_CanSendVTMouseInput()
{
// If the user is holding down Shift, suppress mouse events
// TODO GH#4875: disable/customize this functionality
const auto modifiers = _GetPressedModifierKeys();
if (modifiers.IsShiftPressed())
{
return false;
}
return _terminal->IsTrackingMouseInput();
}
// Method Description:
// - handle a mouse click event. Begin selection process.
// Arguments:
@ -847,6 +914,13 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
const auto altEnabled = WI_IsFlagSet(modifiers, static_cast<uint32_t>(VirtualKeyModifiers::Menu));
const auto shiftEnabled = WI_IsFlagSet(modifiers, static_cast<uint32_t>(VirtualKeyModifiers::Shift));
if (_CanSendVTMouseInput())
{
_TrySendMouseEvent(point);
args.Handled(true);
return;
}
if (point.Properties().IsLeftButtonPressed())
{
// A single left click from out of focus should only focus.
@ -927,8 +1001,21 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
const auto ptr = args.Pointer();
const auto point = args.GetCurrentPoint(*this);
if (!_focused)
{
args.Handled(true);
return;
}
if (ptr.PointerDeviceType() == Windows::Devices::Input::PointerDeviceType::Mouse || ptr.PointerDeviceType() == Windows::Devices::Input::PointerDeviceType::Pen)
{
if (_CanSendVTMouseInput())
{
_TrySendMouseEvent(point);
args.Handled(true);
return;
}
if (point.Properties().IsLeftButtonPressed())
{
_clickDrag = true;
@ -1015,6 +1102,13 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
if (ptr.PointerDeviceType() == Windows::Devices::Input::PointerDeviceType::Mouse || ptr.PointerDeviceType() == Windows::Devices::Input::PointerDeviceType::Pen)
{
if (_CanSendVTMouseInput())
{
_TrySendMouseEvent(point);
args.Handled(true);
return;
}
// Only a left click release when copy on select is active should perform a copy.
// Right clicks and middle clicks should not need to do anything when released.
if (_terminal->IsCopyOnSelectActive() && point.Properties().PointerUpdateKind() == Windows::UI::Input::PointerUpdateKind::LeftButtonReleased)
@ -1056,6 +1150,14 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
Input::PointerRoutedEventArgs const& args)
{
const auto point = args.GetCurrentPoint(*this);
if (_CanSendVTMouseInput())
{
_TrySendMouseEvent(point);
args.Handled(true);
return;
}
const auto delta = point.Properties().MouseWheelDelta();
// Get the state of the Ctrl & Shift keys

View file

@ -211,6 +211,8 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
::Microsoft::Terminal::Core::ControlKeyStates _GetPressedModifierKeys() const;
bool _TrySendKeyEvent(const WORD vkey, const WORD scanCode, ::Microsoft::Terminal::Core::ControlKeyStates modifiers);
bool _TrySendMouseEvent(Windows::UI::Input::PointerPoint const& point);
bool _CanSendVTMouseInput();
const COORD _GetTerminalPosition(winrt::Windows::Foundation::Point cursorPosition);
const unsigned int _NumberOfClicks(winrt::Windows::Foundation::Point clickPos, Timestamp clickTime);

View file

@ -45,6 +45,17 @@ namespace Microsoft::Terminal::Core
virtual bool SetDefaultForeground(const DWORD color) noexcept = 0;
virtual bool SetDefaultBackground(const DWORD color) noexcept = 0;
virtual bool SetCursorKeysMode(const bool applicationMode) noexcept = 0;
virtual bool SetKeypadMode(const bool applicationMode) noexcept = 0;
virtual bool EnableVT200MouseMode(const bool enabled) noexcept = 0;
virtual bool EnableUTF8ExtendedMouseMode(const bool enabled) noexcept = 0;
virtual bool EnableSGRExtendedMouseMode(const bool enabled) noexcept = 0;
virtual bool EnableButtonEventMouseMode(const bool enabled) noexcept = 0;
virtual bool EnableAnyEventMouseMode(const bool enabled) noexcept = 0;
virtual bool EnableAlternateScrollMode(const bool enabled) noexcept = 0;
virtual bool IsVtInputEnabled() const = 0;
protected:
ITerminalApi() = default;
};

View file

@ -17,6 +17,7 @@ namespace Microsoft::Terminal::Core
ITerminalInput& operator=(ITerminalInput&&) = default;
virtual bool SendKeyEvent(const WORD vkey, const WORD scanCode, const ControlKeyStates states) = 0;
virtual bool SendMouseEvent(const COORD viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta) = 0;
virtual bool SendCharEvent(const wchar_t ch) = 0;
// void SendMouseEvent(uint row, uint col, KeyModifiers modifiers);

View file

@ -341,6 +341,17 @@ void Terminal::TrySnapOnInput()
}
}
// Routine Description:
// - Relays if we are tracking mouse input
// Parameters:
// - <none>
// Return value:
// - true, if we are tracking mouse input. False, otherwise
bool Terminal::IsTrackingMouseInput() const noexcept
{
return _terminalInput->IsTrackingMouseInput();
}
// Method Description:
// - Send this particular key event to the terminal. The terminal will translate
// the key and the modifiers pressed into the appropriate VT sequence for that
@ -403,6 +414,32 @@ bool Terminal::SendKeyEvent(const WORD vkey, const WORD scanCode, const ControlK
return translated && manuallyHandled;
}
// Method Description:
// - Send this particular mouse event to the terminal. The terminal will translate
// the button and the modifiers pressed into the appropriate VT sequence for that
// mouse event. If we do translate the key, we'll return true. In that case, the
// event should NOT be processed any further. If we return false, the event
// was NOT translated, and we should instead use the event normally
// Arguments:
// - viewportPos: the position of the mouse event relative to the viewport origin.
// - uiButton: the WM mouse button event code
// - states: The Microsoft::Terminal::Core::ControlKeyStates representing the modifier key states.
// - wheelDelta: the amount that the scroll wheel changed (should be 0 unless button is a WM_MOUSE*WHEEL)
// Return Value:
// - true if we translated the key event, and it should not be processed any further.
// - false if we did not translate the key, and it should be processed into a character.
bool Terminal::SendMouseEvent(const COORD viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta)
{
// viewportPos must be within the dimensions of the viewport
const auto viewportDimensions = _mutableViewport.Dimensions();
if (viewportPos.X < 0 || viewportPos.X >= viewportDimensions.X || viewportPos.Y < 0 || viewportPos.Y >= viewportDimensions.Y)
{
return false;
}
return _terminalInput->HandleMouse(viewportPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta);
}
bool Terminal::SendCharEvent(const wchar_t ch)
{
return _terminalInput->HandleChar(ch);

View file

@ -95,11 +95,23 @@ public:
bool SetCursorStyle(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::CursorStyle cursorStyle) noexcept override;
bool SetDefaultForeground(const COLORREF color) noexcept override;
bool SetDefaultBackground(const COLORREF color) noexcept override;
bool SetCursorKeysMode(const bool applicationMode) noexcept override;
bool SetKeypadMode(const bool applicationMode) noexcept override;
bool EnableVT200MouseMode(const bool enabled) noexcept override;
bool EnableUTF8ExtendedMouseMode(const bool enabled) noexcept override;
bool EnableSGRExtendedMouseMode(const bool enabled) noexcept override;
bool EnableButtonEventMouseMode(const bool enabled) noexcept override;
bool EnableAnyEventMouseMode(const bool enabled) noexcept override;
bool EnableAlternateScrollMode(const bool enabled) noexcept override;
bool IsVtInputEnabled() const noexcept override;
#pragma endregion
#pragma region ITerminalInput
// These methods are defined in Terminal.cpp
bool SendKeyEvent(const WORD vkey, const WORD scanCode, const Microsoft::Terminal::Core::ControlKeyStates states) override;
bool SendMouseEvent(const COORD viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta) override;
bool SendCharEvent(const wchar_t ch) override;
[[nodiscard]] HRESULT UserResize(const COORD viewportSize) noexcept override;
@ -107,6 +119,7 @@ public:
int GetScrollOffset() noexcept override;
void TrySnapOnInput() override;
bool IsTrackingMouseInput() const noexcept;
#pragma endregion
#pragma region IBaseData(base to IRenderData and IUiaData)

View file

@ -519,3 +519,57 @@ try
return true;
}
CATCH_LOG_RETURN_FALSE()
bool Terminal::SetCursorKeysMode(const bool applicationMode) noexcept
{
_terminalInput->ChangeCursorKeysMode(applicationMode);
return true;
}
bool Terminal::SetKeypadMode(const bool applicationMode) noexcept
{
_terminalInput->ChangeKeypadMode(applicationMode);
return true;
}
bool Terminal::EnableVT200MouseMode(const bool enabled) noexcept
{
_terminalInput->EnableDefaultTracking(enabled);
return true;
}
bool Terminal::EnableUTF8ExtendedMouseMode(const bool enabled) noexcept
{
_terminalInput->SetUtf8ExtendedMode(enabled);
return true;
}
bool Terminal::EnableSGRExtendedMouseMode(const bool enabled) noexcept
{
_terminalInput->SetSGRExtendedMode(enabled);
return true;
}
bool Terminal::EnableButtonEventMouseMode(const bool enabled) noexcept
{
_terminalInput->EnableButtonEventTracking(enabled);
return true;
}
bool Terminal::EnableAnyEventMouseMode(const bool enabled) noexcept
{
_terminalInput->EnableAnyEventTracking(enabled);
return true;
}
bool Terminal::EnableAlternateScrollMode(const bool enabled) noexcept
{
_terminalInput->EnableAlternateScroll(enabled);
return true;
}
bool Terminal::IsVtInputEnabled() const noexcept
{
// We should never be getting this call in Terminal.
FAIL_FAST();
}

View file

@ -213,3 +213,175 @@ try
return _terminalApi.EraseInDisplay(eraseType);
}
CATCH_LOG_RETURN_FALSE()
// - DECKPAM, DECKPNM - Sets the keypad input mode to either Application mode or Numeric mode (true, false respectively)
// Arguments:
// - applicationMode - set to true to enable Application Mode Input, false for Numeric Mode Input.
// Return Value:
// - True if handled successfully. False otherwise.
bool TerminalDispatch::SetKeypadMode(const bool fApplicationMode) noexcept
{
_terminalApi.SetKeypadMode(fApplicationMode);
return true;
}
// - DECCKM - Sets the cursor keys input mode to either Application mode or Normal mode (true, false respectively)
// Arguments:
// - applicationMode - set to true to enable Application Mode Input, false for Normal Mode Input.
// Return Value:
// - True if handled successfully. False otherwise.
bool TerminalDispatch::SetCursorKeysMode(const bool applicationMode) noexcept
{
_terminalApi.SetCursorKeysMode(applicationMode);
return true;
}
//Routine Description:
// Enable VT200 Mouse Mode - Enables/disables the mouse input handler in default tracking mode.
//Arguments:
// - enabled - true to enable, false to disable.
// Return value:
// True if handled successfully. False otherwise.
bool TerminalDispatch::EnableVT200MouseMode(const bool enabled) noexcept
{
_terminalApi.EnableVT200MouseMode(enabled);
return true;
}
//Routine Description:
// Enable UTF-8 Extended Encoding - this changes the encoding scheme for sequences
// emitted by the mouse input handler. Does not enable/disable mouse mode on its own.
//Arguments:
// - enabled - true to enable, false to disable.
// Return value:
// True if handled successfully. False otherwise.
bool TerminalDispatch::EnableUTF8ExtendedMouseMode(const bool enabled) noexcept
{
_terminalApi.EnableUTF8ExtendedMouseMode(enabled);
return true;
}
//Routine Description:
// Enable SGR Extended Encoding - this changes the encoding scheme for sequences
// emitted by the mouse input handler. Does not enable/disable mouse mode on its own.
//Arguments:
// - enabled - true to enable, false to disable.
// Return value:
// True if handled successfully. False otherwise.
bool TerminalDispatch::EnableSGRExtendedMouseMode(const bool enabled) noexcept
{
_terminalApi.EnableSGRExtendedMouseMode(enabled);
return true;
}
//Routine Description:
// Enable Button Event mode - send mouse move events WITH A BUTTON PRESSED to the input.
//Arguments:
// - enabled - true to enable, false to disable.
// Return value:
// True if handled successfully. False otherwise.
bool TerminalDispatch::EnableButtonEventMouseMode(const bool enabled) noexcept
{
_terminalApi.EnableButtonEventMouseMode(enabled);
return true;
}
//Routine Description:
// Enable Any Event mode - send all mouse events to the input.
//Arguments:
// - enabled - true to enable, false to disable.
// Return value:
// True if handled successfully. False otherwise.
bool TerminalDispatch::EnableAnyEventMouseMode(const bool enabled) noexcept
{
_terminalApi.EnableAnyEventMouseMode(enabled);
return true;
}
//Routine Description:
// Enable Alternate Scroll Mode - When in the Alt Buffer, send CUP and CUD on
// scroll up/down events instead of the usual sequences
//Arguments:
// - enabled - true to enable, false to disable.
// Return value:
// True if handled successfully. False otherwise.
bool TerminalDispatch::EnableAlternateScroll(const bool enabled) noexcept
{
_terminalApi.EnableAlternateScrollMode(enabled);
return true;
}
bool TerminalDispatch::SetPrivateModes(const std::basic_string_view<DispatchTypes::PrivateModeParams> params) noexcept
{
return _SetResetPrivateModes(params, true);
}
bool TerminalDispatch::ResetPrivateModes(const std::basic_string_view<DispatchTypes::PrivateModeParams> params) noexcept
{
return _SetResetPrivateModes(params, false);
}
// Routine Description:
// - Generalized handler for the setting/resetting of DECSET/DECRST parameters.
// All params in the rgParams will attempt to be executed, even if one
// fails, to allow us to successfully re/set params that are chained with
// params we don't yet support.
// Arguments:
// - params - array of params to set/reset
// - enable - True for set, false for unset.
// Return Value:
// - True if ALL params were handled successfully. False otherwise.
bool TerminalDispatch::_SetResetPrivateModes(const std::basic_string_view<DispatchTypes::PrivateModeParams> params, const bool enable) noexcept
{
// because the user might chain together params we don't support with params we DO support, execute all
// params in the sequence, and only return failure if we failed at least one of them
size_t failures = 0;
for (const auto& p : params)
{
failures += _PrivateModeParamsHelper(p, enable) ? 0 : 1; // increment the number of failures if we fail.
}
return failures == 0;
}
// Routine Description:
// - Support routine for routing private mode parameters to be set/reset as flags
// Arguments:
// - params - array of params to set/reset
// - enable - True for set, false for unset.
// Return Value:
// - True if handled successfully. False otherwise.
bool TerminalDispatch::_PrivateModeParamsHelper(const DispatchTypes::PrivateModeParams param, const bool enable) noexcept
{
bool success = false;
switch (param)
{
case DispatchTypes::PrivateModeParams::DECCKM_CursorKeysMode:
// set - Enable Application Mode, reset - Normal mode
success = SetCursorKeysMode(enable);
break;
case DispatchTypes::PrivateModeParams::VT200_MOUSE_MODE:
success = EnableVT200MouseMode(enable);
break;
case DispatchTypes::PrivateModeParams::BUTTON_EVENT_MOUSE_MODE:
success = EnableButtonEventMouseMode(enable);
break;
case DispatchTypes::PrivateModeParams::ANY_EVENT_MOUSE_MODE:
success = EnableAnyEventMouseMode(enable);
break;
case DispatchTypes::PrivateModeParams::UTF8_EXTENDED_MODE:
success = EnableUTF8ExtendedMouseMode(enable);
break;
case DispatchTypes::PrivateModeParams::SGR_EXTENDED_MODE:
success = EnableSGRExtendedMouseMode(enable);
break;
case DispatchTypes::PrivateModeParams::ALTERNATE_SCROLL:
success = EnableAlternateScroll(enable);
break;
default:
// If no functions to call, overall dispatch was a failure.
success = false;
break;
}
return success;
}

View file

@ -38,6 +38,19 @@ public:
bool InsertCharacter(const size_t count) noexcept override;
bool EraseInDisplay(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::EraseType eraseType) noexcept override;
bool SetCursorKeysMode(const bool applicationMode) noexcept override; // DECCKM
bool SetKeypadMode(const bool applicationMode) noexcept override; // DECKPAM, DECKPNM
bool EnableVT200MouseMode(const bool enabled) noexcept override; // ?1000
bool EnableUTF8ExtendedMouseMode(const bool enabled) noexcept override; // ?1005
bool EnableSGRExtendedMouseMode(const bool enabled) noexcept override; // ?1006
bool EnableButtonEventMouseMode(const bool enabled) noexcept override; // ?1002
bool EnableAnyEventMouseMode(const bool enabled) noexcept override; // ?1003
bool EnableAlternateScroll(const bool enabled) noexcept override; // ?1007
bool SetPrivateModes(const std::basic_string_view<::Microsoft::Console::VirtualTerminal::DispatchTypes::PrivateModeParams> /*params*/) noexcept override; // DECSET
bool ResetPrivateModes(const std::basic_string_view<::Microsoft::Console::VirtualTerminal::DispatchTypes::PrivateModeParams> /*params*/) noexcept override; // DECRST
private:
::Microsoft::Terminal::Core::ITerminalApi& _terminalApi;
@ -46,4 +59,7 @@ private:
bool _SetBoldColorHelper(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::GraphicsOptions opt) noexcept;
bool _SetDefaultColorHelper(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::GraphicsOptions opt) noexcept;
void _SetGraphicsOptionHelper(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::GraphicsOptions opt) noexcept;
bool _SetResetPrivateModes(const std::basic_string_view<::Microsoft::Console::VirtualTerminal::DispatchTypes::PrivateModeParams> params, const bool enable) noexcept;
bool _PrivateModeParamsHelper(const ::Microsoft::Console::VirtualTerminal::DispatchTypes::PrivateModeParams param, const bool enable) noexcept;
};

View file

@ -264,6 +264,17 @@ static constexpr short _encodeDefaultCoordinate(const short sCoordinateValue) no
return sCoordinateValue + 32;
}
// Routine Description:
// - Relays if we are tracking mouse input
// Parameters:
// - <none>
// Return value:
// - true, if we are tracking mouse input. False, otherwise
bool TerminalInput::IsTrackingMouseInput() const noexcept
{
return (_mouseInputState.trackingMode != TrackingMode::None);
}
// Routine Description:
// - Attempt to handle the given mouse coordinates and windows button as a VT-style mouse event.
// If the event should be transmitted in the selected mouse mode, then we'll try and

View file

@ -44,6 +44,8 @@ namespace Microsoft::Console::VirtualTerminal
const unsigned int button,
const short modifierKeyState,
const short delta);
bool IsTrackingMouseInput() const noexcept;
#pragma endregion
#pragma region MouseInputState Management