From 2b8b034b89a1d29414d4a014a10a809f96477ece Mon Sep 17 00:00:00 2001 From: Carlos Zamora Date: Wed, 11 Dec 2019 13:52:49 -0800 Subject: [PATCH] Attach UiaRenderer and Fire Selection Changed Events (#2989) This PR makes use of the UiaRenderer by attaching it to the TerminalControl and setting up selectionChanged events for accessibility. Part 1: attaching the UiaRenderer The uiaRenderer is treated very similarly to the dxRenderer. We have a unique_ptr ref to it in the TermControl. This gets populated when the TermControlAutomationPeer is created (thus enabling accessibility). To prevent every TermControl from sending signals simultaneously, we specifically only enable whichever one is in an active pane. The UiaRenderer needs to send encoded events to the automation provider (in this case, TermControlAutomationPeer). We needed our own automation events so that we can reuse this model for ConHost. This is the purpose of IUiaEventDispatcher. We need a dispatcher for the UiaRenderer. Otherwise, we would do a lot of work to find out when to fire an event, but we wouldn't have a way of doing that. Part 2: hooking up selection events This provides a little bit of polish to hooking it up before. Primarily to actually make it work. This includes returning S_FALSE instead of E_NOTIMPL. The main thing here really is just how to detect if a selection has changed. This also shows how clean adding more events will be in the future! --- src/cascadia/TerminalControl/TermControl.cpp | 54 +++++++--- src/cascadia/TerminalControl/TermControl.h | 2 + .../TermControlAutomationPeer.cpp | 42 ++++++++ .../TermControlAutomationPeer.h | 12 ++- .../TermControlAutomationPeer.idl | 7 +- .../TerminalControl/TerminalControl.vcxproj | 1 + src/renderer/uia/UiaRenderer.cpp | 101 ++++++++++++------ src/renderer/uia/UiaRenderer.hpp | 8 +- src/types/IUiaEventDispatcher.h | 27 +++++ src/types/lib/types.vcxproj | 1 + src/types/lib/types.vcxproj.filters | 5 +- 11 files changed, 204 insertions(+), 56 deletions(-) create mode 100644 src/types/IUiaEventDispatcher.h diff --git a/src/cascadia/TerminalControl/TermControl.cpp b/src/cascadia/TerminalControl/TermControl.cpp index 63f55945f..703dca615 100644 --- a/src/cascadia/TerminalControl/TermControl.cpp +++ b/src/cascadia/TerminalControl/TermControl.cpp @@ -17,6 +17,7 @@ using namespace ::Microsoft::Console::Types; using namespace ::Microsoft::Terminal::Core; using namespace winrt::Windows::UI::Xaml; +using namespace winrt::Windows::UI::Xaml::Automation::Peers; using namespace winrt::Windows::UI::Core; using namespace winrt::Windows::System; using namespace winrt::Microsoft::Terminal::Settings; @@ -393,20 +394,31 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation Close(); } + // Method Description: + // - Creates an automation peer for the Terminal Control, enabling accessibility on our control. + // Arguments: + // - None + // Return Value: + // - The automation peer for our control Windows::UI::Xaml::Automation::Peers::AutomationPeer TermControl::OnCreateAutomationPeer() + try { - Windows::UI::Xaml::Automation::Peers::AutomationPeer autoPeer{ nullptr }; if (GetUiaData()) { - try - { - // create a custom automation peer with this code pattern: - // (https://docs.microsoft.com/en-us/windows/uwp/design/accessibility/custom-automation-peers) - autoPeer = winrt::make(this); - } - CATCH_LOG(); + // create a custom automation peer with this code pattern: + // (https://docs.microsoft.com/en-us/windows/uwp/design/accessibility/custom-automation-peers) + auto autoPeer = winrt::make_self(this); + + _uiaEngine = std::make_unique<::Microsoft::Console::Render::UiaEngine>(autoPeer.get()); + _renderer->AddRenderEngine(_uiaEngine.get()); + return *autoPeer; } - return autoPeer; + return nullptr; + } + catch (...) + { + LOG_CAUGHT_EXCEPTION(); + return nullptr; } ::Microsoft::Console::Types::IUiaData* TermControl::GetUiaData() const @@ -780,22 +792,20 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation if (multiClickMapper == 3) { _terminal->TripleClickSelection(terminalPosition); - _renderer->TriggerSelection(); } else if (multiClickMapper == 2) { _terminal->DoubleClickSelection(terminalPosition); - _renderer->TriggerSelection(); } else { // save location before rendering _terminal->SetSelectionAnchor(terminalPosition); - _renderer->TriggerSelection(); _lastMouseClick = point.Timestamp(); _lastMouseClickPos = cursorPosition; } + _renderer->TriggerSelection(); } else if (point.Properties().IsRightButtonPressed()) { @@ -1195,8 +1205,9 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation } // Method Description: - // - Event handler for the GotFocus event. This is used to start - // blinking the cursor when the window is focused. + // - Event handler for the GotFocus event. This is used to... + // - enable accessibility notifications for this TermControl + // - start blinking the cursor when the window is focused void TermControl::_GotFocusHandler(Windows::Foundation::IInspectable const& /* sender */, RoutedEventArgs const& /* args */) { @@ -1206,6 +1217,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation } _focused = true; + if (_uiaEngine.get()) + { + THROW_IF_FAILED(_uiaEngine->Enable()); + } + if (_tsfInputControl != nullptr) { _tsfInputControl.NotifyFocusEnter(); @@ -1218,8 +1234,9 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation } // Method Description: - // - Event handler for the LostFocus event. This is used to hide - // and stop blinking the cursor when the window loses focus. + // - Event handler for the LostFocus event. This is used to... + // - disable accessibility notifications for this TermControl + // - hide and stop blinking the cursor when the window loses focus. void TermControl::_LostFocusHandler(Windows::Foundation::IInspectable const& /* sender */, RoutedEventArgs const& /* args */) { @@ -1229,6 +1246,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation } _focused = false; + if (_uiaEngine.get()) + { + THROW_IF_FAILED(_uiaEngine->Disable()); + } + if (_tsfInputControl != nullptr) { _tsfInputControl.NotifyFocusLeave(); diff --git a/src/cascadia/TerminalControl/TermControl.h b/src/cascadia/TerminalControl/TermControl.h index fcf7d22db..5aea2d861 100644 --- a/src/cascadia/TerminalControl/TermControl.h +++ b/src/cascadia/TerminalControl/TermControl.h @@ -10,6 +10,7 @@ #include #include "../../renderer/base/Renderer.hpp" #include "../../renderer/dx/DxRenderer.hpp" +#include "../../renderer/uia/UiaRenderer.hpp" #include "../../cascadia/TerminalCore/Terminal.hpp" #include "cppwinrt_utils.h" @@ -113,6 +114,7 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation std::unique_ptr<::Microsoft::Console::Render::Renderer> _renderer; std::unique_ptr<::Microsoft::Console::Render::DxEngine> _renderEngine; + std::unique_ptr<::Microsoft::Console::Render::UiaEngine> _uiaEngine; Settings::IControlSettings _settings; bool _focused; diff --git a/src/cascadia/TerminalControl/TermControlAutomationPeer.cpp b/src/cascadia/TerminalControl/TermControlAutomationPeer.cpp index bf7b2e770..40ae55f4b 100644 --- a/src/cascadia/TerminalControl/TermControlAutomationPeer.cpp +++ b/src/cascadia/TerminalControl/TermControlAutomationPeer.cpp @@ -33,6 +33,48 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation THROW_IF_FAILED(::Microsoft::WRL::MakeAndInitialize<::Microsoft::Terminal::TermControlUiaProvider>(&_uiaProvider, owner, std::bind(&TermControlAutomationPeer::GetBoundingRectWrapped, this))); }; + // Method Description: + // - Signals the ui automation client that the terminal's selection has changed and should be updated + // Arguments: + // - + // Return Value: + // - + void TermControlAutomationPeer::SignalSelectionChanged() + { + Dispatcher().RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [&]() { + // The event that is raised when the text selection is modified. + RaiseAutomationEvent(AutomationEvents::TextPatternOnTextSelectionChanged); + }); + } + + // Method Description: + // - Signals the ui automation client that the terminal's output has changed and should be updated + // Arguments: + // - + // Return Value: + // - + void TermControlAutomationPeer::SignalTextChanged() + { + Dispatcher().RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [&]() { + // The event that is raised when textual content is modified. + RaiseAutomationEvent(AutomationEvents::TextPatternOnTextChanged); + }); + } + + // Method Description: + // - Signals the ui automation client that the cursor's state has changed and should be updated + // Arguments: + // - + // Return Value: + // - + void TermControlAutomationPeer::SignalCursorChanged() + { + Dispatcher().RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [&]() { + // The event that is raised when the text was changed in an edit control. + RaiseAutomationEvent(AutomationEvents::TextEditTextChanged); + }); + } + winrt::hstring TermControlAutomationPeer::GetClassNameCore() const { return L"TermControl"; diff --git a/src/cascadia/TerminalControl/TermControlAutomationPeer.h b/src/cascadia/TerminalControl/TermControlAutomationPeer.h index 65c0e3231..d5bfc4b09 100644 --- a/src/cascadia/TerminalControl/TermControlAutomationPeer.h +++ b/src/cascadia/TerminalControl/TermControlAutomationPeer.h @@ -27,14 +27,14 @@ Author(s): #include "TermControl.h" #include "TermControlAutomationPeer.g.h" #include -#include "../../renderer/inc/IRenderData.hpp" -#include "../types/WindowUiaProviderBase.hpp" #include "TermControlUiaProvider.hpp" +#include "../types/IUiaEventDispatcher.h" namespace winrt::Microsoft::Terminal::TerminalControl::implementation { struct TermControlAutomationPeer : - public TermControlAutomationPeerT + public TermControlAutomationPeerT, + ::Microsoft::Console::Types::IUiaEventDispatcher { public: TermControlAutomationPeer(winrt::Microsoft::Terminal::TerminalControl::implementation::TermControl* owner); @@ -44,6 +44,12 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation winrt::hstring GetLocalizedControlTypeCore() const; winrt::Windows::Foundation::IInspectable GetPatternCore(winrt::Windows::UI::Xaml::Automation::Peers::PatternInterface patternInterface) const; +#pragma region IUiaEventDispatcher + void SignalSelectionChanged() override; + void SignalTextChanged() override; + void SignalCursorChanged() override; +#pragma endregion + #pragma region ITextProvider Pattern Windows::UI::Xaml::Automation::Provider::ITextRangeProvider RangeFromPoint(Windows::Foundation::Point screenLocation); Windows::UI::Xaml::Automation::Provider::ITextRangeProvider RangeFromChild(Windows::UI::Xaml::Automation::Provider::IRawElementProviderSimple childElement); diff --git a/src/cascadia/TerminalControl/TermControlAutomationPeer.idl b/src/cascadia/TerminalControl/TermControlAutomationPeer.idl index d979c8247..18ada03e7 100644 --- a/src/cascadia/TerminalControl/TermControlAutomationPeer.idl +++ b/src/cascadia/TerminalControl/TermControlAutomationPeer.idl @@ -5,10 +5,9 @@ import "TermControl.idl"; namespace Microsoft.Terminal.TerminalControl { - [default_interface] - runtimeclass TermControlAutomationPeer : + [default_interface] runtimeclass TermControlAutomationPeer : Windows.UI.Xaml.Automation.Peers.FrameworkElementAutomationPeer, Windows.UI.Xaml.Automation.Provider.ITextProvider - { - } + { + } } diff --git a/src/cascadia/TerminalControl/TerminalControl.vcxproj b/src/cascadia/TerminalControl/TerminalControl.vcxproj index cc52c8931..5ec8cd426 100644 --- a/src/cascadia/TerminalControl/TerminalControl.vcxproj +++ b/src/cascadia/TerminalControl/TerminalControl.vcxproj @@ -71,6 +71,7 @@ + diff --git a/src/renderer/uia/UiaRenderer.cpp b/src/renderer/uia/UiaRenderer.cpp index cba61ff18..7b1ca85ca 100644 --- a/src/renderer/uia/UiaRenderer.cpp +++ b/src/renderer/uia/UiaRenderer.cpp @@ -13,9 +13,12 @@ using namespace Microsoft::Console::Types; // Routine Description: // - Constructs a UIA engine for console text // which primarily notifies automation clients of any activity -UiaEngine::UiaEngine() noexcept : +UiaEngine::UiaEngine(IUiaEventDispatcher* dispatcher) : + _dispatcher{ THROW_HR_IF_NULL(E_INVALIDARG, dispatcher) }, _isPainting{ false }, - _isEnabled{ false }, + _selectionChanged{ false }, + _isEnabled{ true }, + _prevSelection{}, RenderEngineBase() { } @@ -53,7 +56,7 @@ UiaEngine::UiaEngine() noexcept : // - S_OK, else an appropriate HRESULT for failing to allocate or write. [[nodiscard]] HRESULT UiaEngine::Invalidate(const SMALL_RECT* const /*psrRegion*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -87,9 +90,41 @@ UiaEngine::UiaEngine() noexcept : // - rectangles - One or more rectangles describing character positions on the grid // Return Value: // - S_OK -[[nodiscard]] HRESULT UiaEngine::InvalidateSelection(const std::vector& /*rectangles*/) noexcept +[[nodiscard]] HRESULT UiaEngine::InvalidateSelection(const std::vector& rectangles) noexcept { - return E_NOTIMPL; + // early exit: different number of rows + if (_prevSelection.size() != rectangles.size()) + { + try + { + _selectionChanged = true; + _prevSelection = rectangles; + } + CATCH_LOG_RETURN_HR(E_FAIL); + return S_OK; + } + + for (size_t i = 0; i < rectangles.size(); i++) + { + try + { + const auto prevRect = _prevSelection.at(i); + const auto newRect = rectangles.at(i); + + // if any value is different, selection has changed + if (prevRect.Top != newRect.Top || prevRect.Right != newRect.Right || prevRect.Left != newRect.Left || prevRect.Bottom != newRect.Bottom) + { + _selectionChanged = true; + _prevSelection = rectangles; + return S_OK; + } + } + CATCH_LOG_RETURN_HR(E_FAIL); + } + + // assume selection has not changed + _selectionChanged = false; + return S_OK; } // Routine Description: @@ -102,7 +137,7 @@ UiaEngine::UiaEngine() noexcept : // - S_OK [[nodiscard]] HRESULT UiaEngine::InvalidateScroll(const COORD* const /*pcoordDelta*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -115,7 +150,7 @@ UiaEngine::UiaEngine() noexcept : // - S_OK, else an appropriate HRESULT for failing to allocate or write. [[nodiscard]] HRESULT UiaEngine::InvalidateAll() noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -156,19 +191,13 @@ UiaEngine::UiaEngine() noexcept : { RETURN_HR_IF(S_FALSE, !_isEnabled); + // add more events here + // bool somethingToDo = _selectionChanged; + // If there's nothing to do, quick return - bool somethingToDo = false; - - if (_isEnabled) - { - somethingToDo = false; - - if (somethingToDo) - { - _isPainting = true; - } - } + RETURN_HR_IF(S_FALSE, !_selectionChanged); + _isPainting = true; return S_OK; } @@ -180,15 +209,23 @@ UiaEngine::UiaEngine() noexcept : // - S_OK, else an appropriate HRESULT for failing to allocate or write. [[nodiscard]] HRESULT UiaEngine::EndPaint() noexcept { + RETURN_HR_IF(S_FALSE, !_isEnabled); RETURN_HR_IF(E_INVALIDARG, !_isPainting); // invalid to end paint when we're not painting - if (_isEnabled) + // Fire UIA Events here + if (_selectionChanged) { - _isPainting = false; - - // fire UIA events here + try + { + _dispatcher->SignalSelectionChanged(); + } + CATCH_LOG(); } + _selectionChanged = false; + _prevSelection.clear(); + _isPainting = false; + return S_OK; } @@ -230,17 +267,18 @@ UiaEngine::UiaEngine() noexcept : // Routine Description: // - Places one line of text onto the screen at the given position +// For UIA, this doesn't mean anything. So do nothing. // Arguments: // - clusters - Iterable collection of cluster information (text and columns it should consume) // - coord - Character coordinate position in the cell grid // - fTrimLeft - Whether or not to trim off the left half of a double wide character // Return Value: -// - E_NOTIMPL +// - S_FALSE [[nodiscard]] HRESULT UiaEngine::PaintBufferLine(std::basic_string_view const /*clusters*/, COORD const /*coord*/, const bool /*trimLeft*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -277,6 +315,7 @@ UiaEngine::UiaEngine() noexcept : // Routine Description: // - Draws the cursor on the screen +// For UIA, this doesn't mean anything. So do nothing. // Arguments: // - options - Packed options relevant to how to draw the cursor // Return Value: @@ -288,7 +327,7 @@ UiaEngine::UiaEngine() noexcept : // Routine Description: // - Updates the default brush colors used for drawing -// - Not currently used by UiaEngine. +// For UIA, this doesn't mean anything. So do nothing. // Arguments: // - colorForeground - // - colorBackground - @@ -315,7 +354,7 @@ UiaEngine::UiaEngine() noexcept : // - S_FALSE since we do nothing [[nodiscard]] HRESULT UiaEngine::UpdateFont(const FontInfoDesired& /*pfiFontInfoDesired*/, FontInfo& /*fiFontInfo*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -327,7 +366,7 @@ UiaEngine::UiaEngine() noexcept : // - S_OK [[nodiscard]] HRESULT UiaEngine::UpdateDpi(int const /*iDpi*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Method Description: @@ -338,7 +377,7 @@ UiaEngine::UiaEngine() noexcept : // - HRESULT S_OK [[nodiscard]] HRESULT UiaEngine::UpdateViewport(const SMALL_RECT /*srNewViewport*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -376,7 +415,7 @@ UiaEngine::UiaEngine() noexcept : // - S_OK [[nodiscard]] HRESULT UiaEngine::GetFontSize(_Out_ COORD* const /*pFontSize*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Routine Description: @@ -385,10 +424,10 @@ UiaEngine::UiaEngine() noexcept : // - glyph - The glyph run to process for column width. // - pResult - True if it should take two columns. False if it should take one. // Return Value: -// - E_NOTIMPL +// - S_OK or relevant DirectWrite error. [[nodiscard]] HRESULT UiaEngine::IsGlyphWideByFont(const std::wstring_view /*glyph*/, _Out_ bool* const /*pResult*/) noexcept { - return E_NOTIMPL; + return S_FALSE; } // Method Description: diff --git a/src/renderer/uia/UiaRenderer.hpp b/src/renderer/uia/UiaRenderer.hpp index 1e93e8ba8..056a63b00 100644 --- a/src/renderer/uia/UiaRenderer.hpp +++ b/src/renderer/uia/UiaRenderer.hpp @@ -17,6 +17,7 @@ Author(s): #include "../../renderer/inc/RenderEngineBase.hpp" +#include "../../types/IUiaEventDispatcher.h" #include "../../types/inc/Viewport.hpp" namespace Microsoft::Console::Render @@ -24,7 +25,7 @@ namespace Microsoft::Console::Render class UiaEngine final : public RenderEngineBase { public: - UiaEngine() noexcept; + UiaEngine(Microsoft::Console::Types::IUiaEventDispatcher* dispatcher); // Only one UiaEngine may present information at a time. // This ensures that an automation client isn't overwhelmed @@ -79,5 +80,10 @@ namespace Microsoft::Console::Render private: bool _isEnabled; bool _isPainting; + bool _selectionChanged; + + Microsoft::Console::Types::IUiaEventDispatcher* _dispatcher; + + std::vector _prevSelection; }; } diff --git a/src/types/IUiaEventDispatcher.h b/src/types/IUiaEventDispatcher.h new file mode 100644 index 000000000..1d9fc11e8 --- /dev/null +++ b/src/types/IUiaEventDispatcher.h @@ -0,0 +1,27 @@ +/*++ +Copyright (c) Microsoft Corporation +Licensed under the MIT license. + +Module Name: +- IUiaEventDispatcher.h + +Abstract: +- This serves as the interface allowing accessibility providers to detect and respond to accessibility events +- This interface should allow accessibility events to be supported across different implementation patterns + +Author(s): +- Carlos Zamora (CaZamor) Sept-2019 +--*/ + +#pragma once + +namespace Microsoft::Console::Types +{ + class IUiaEventDispatcher + { + public: + virtual void SignalSelectionChanged() = 0; + virtual void SignalTextChanged() = 0; + virtual void SignalCursorChanged() = 0; + }; +} diff --git a/src/types/lib/types.vcxproj b/src/types/lib/types.vcxproj index 8206e301e..45a0054c1 100644 --- a/src/types/lib/types.vcxproj +++ b/src/types/lib/types.vcxproj @@ -47,6 +47,7 @@ + diff --git a/src/types/lib/types.vcxproj.filters b/src/types/lib/types.vcxproj.filters index c999e57a2..d39aaabf0 100644 --- a/src/types/lib/types.vcxproj.filters +++ b/src/types/lib/types.vcxproj.filters @@ -152,6 +152,9 @@ Header Files + + Header Files + Header Files @@ -162,4 +165,4 @@ - \ No newline at end of file +