Attach UiaRenderer and Fire Selection Changed Events (#2989)

This PR makes use of the UiaRenderer by attaching it to the TerminalControl and setting up selectionChanged events for accessibility.

Part 1: attaching the UiaRenderer
The uiaRenderer is treated very similarly to the dxRenderer. We have a unique_ptr ref to it in the TermControl. This gets populated when the TermControlAutomationPeer is created (thus enabling accessibility).

To prevent every TermControl from sending signals simultaneously, we specifically only enable whichever one is in an active pane.

The UiaRenderer needs to send encoded events to the automation provider (in this case, TermControlAutomationPeer). We needed our own automation events so that we can reuse this model for ConHost. This is the purpose of IUiaEventDispatcher.

We need a dispatcher for the UiaRenderer. Otherwise, we would do a lot of work to find out when to fire an event, but we wouldn't have a way of doing that.

Part 2: hooking up selection events
This provides a little bit of polish to hooking it up before. Primarily to actually make it work. This includes returning S_FALSE instead of E_NOTIMPL.

The main thing here really is just how to detect if a selection has changed. This also shows how clean adding more events will be in the future!

src/cascadia/TerminalControl/TermControl.cpp

@@ -17,6 +17,7 @@
 using namespace ::Microsoft::Console::Types;
 using namespace ::Microsoft::Terminal::Core;
 using namespace winrt::Windows::UI::Xaml;
+using namespace winrt::Windows::UI::Xaml::Automation::Peers;
 using namespace winrt::Windows::UI::Core;
 using namespace winrt::Windows::System;
 using namespace winrt::Microsoft::Terminal::Settings;
@@ -393,20 +394,31 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         Close();
     }
 
+    // Method Description:
+    // - Creates an automation peer for the Terminal Control, enabling accessibility on our control.
+    // Arguments:
+    // - None
+    // Return Value:
+    // - The automation peer for our control
     Windows::UI::Xaml::Automation::Peers::AutomationPeer TermControl::OnCreateAutomationPeer()
+    try
     {
-        Windows::UI::Xaml::Automation::Peers::AutomationPeer autoPeer{ nullptr };
         if (GetUiaData())
         {
-            try
-            {
-                // create a custom automation peer with this code pattern:
-                // (https://docs.microsoft.com/en-us/windows/uwp/design/accessibility/custom-automation-peers)
-                autoPeer = winrt::make<winrt::Microsoft::Terminal::TerminalControl::implementation::TermControlAutomationPeer>(this);
-            }
-            CATCH_LOG();
+            // create a custom automation peer with this code pattern:
+            // (https://docs.microsoft.com/en-us/windows/uwp/design/accessibility/custom-automation-peers)
+            auto autoPeer = winrt::make_self<winrt::Microsoft::Terminal::TerminalControl::implementation::TermControlAutomationPeer>(this);
+
+            _uiaEngine = std::make_unique<::Microsoft::Console::Render::UiaEngine>(autoPeer.get());
+            _renderer->AddRenderEngine(_uiaEngine.get());
+            return *autoPeer;
         }
-        return autoPeer;
+        return nullptr;
+    }
+    catch (...)
+    {
+        LOG_CAUGHT_EXCEPTION();
+        return nullptr;
     }
 
     ::Microsoft::Console::Types::IUiaData* TermControl::GetUiaData() const
@@ -780,22 +792,20 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
                 if (multiClickMapper == 3)
                 {
                     _terminal->TripleClickSelection(terminalPosition);
-                    _renderer->TriggerSelection();
                 }
                 else if (multiClickMapper == 2)
                 {
                     _terminal->DoubleClickSelection(terminalPosition);
-                    _renderer->TriggerSelection();
                 }
                 else
                 {
                     // save location before rendering
                     _terminal->SetSelectionAnchor(terminalPosition);
 
-                    _renderer->TriggerSelection();
                     _lastMouseClick = point.Timestamp();
                     _lastMouseClickPos = cursorPosition;
                 }
+                _renderer->TriggerSelection();
             }
             else if (point.Properties().IsRightButtonPressed())
             {
@@ -1195,8 +1205,9 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
     }
 
     // Method Description:
-    // - Event handler for the GotFocus event. This is used to start
-    //   blinking the cursor when the window is focused.
+    // - Event handler for the GotFocus event. This is used to...
+    //   - enable accessibility notifications for this TermControl
+    //   - start blinking the cursor when the window is focused
     void TermControl::_GotFocusHandler(Windows::Foundation::IInspectable const& /* sender */,
                                        RoutedEventArgs const& /* args */)
     {
@@ -1206,6 +1217,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         }
         _focused = true;
 
+        if (_uiaEngine.get())
+        {
+            THROW_IF_FAILED(_uiaEngine->Enable());
+        }
+
         if (_tsfInputControl != nullptr)
         {
             _tsfInputControl.NotifyFocusEnter();
@@ -1218,8 +1234,9 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
     }
 
     // Method Description:
-    // - Event handler for the LostFocus event. This is used to hide
-    //   and stop blinking the cursor when the window loses focus.
+    // - Event handler for the LostFocus event. This is used to...
+    //   - disable accessibility notifications for this TermControl
+    //   - hide and stop blinking the cursor when the window loses focus.
     void TermControl::_LostFocusHandler(Windows::Foundation::IInspectable const& /* sender */,
                                         RoutedEventArgs const& /* args */)
     {
@@ -1229,6 +1246,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         }
         _focused = false;
 
+        if (_uiaEngine.get())
+        {
+            THROW_IF_FAILED(_uiaEngine->Disable());
+        }
+
         if (_tsfInputControl != nullptr)
         {
             _tsfInputControl.NotifyFocusLeave();

src/cascadia/TerminalControl/TermControl.h

@@ -10,6 +10,7 @@
 #include <winrt/Microsoft.Terminal.Settings.h>
 #include "../../renderer/base/Renderer.hpp"
 #include "../../renderer/dx/DxRenderer.hpp"
+#include "../../renderer/uia/UiaRenderer.hpp"
 #include "../../cascadia/TerminalCore/Terminal.hpp"
 #include "cppwinrt_utils.h"
 
@@ -113,6 +114,7 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 
         std::unique_ptr<::Microsoft::Console::Render::Renderer> _renderer;
         std::unique_ptr<::Microsoft::Console::Render::DxEngine> _renderEngine;
+        std::unique_ptr<::Microsoft::Console::Render::UiaEngine> _uiaEngine;
 
         Settings::IControlSettings _settings;
         bool _focused;

src/cascadia/TerminalControl/TermControlAutomationPeer.cpp

@@ -33,6 +33,48 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         THROW_IF_FAILED(::Microsoft::WRL::MakeAndInitialize<::Microsoft::Terminal::TermControlUiaProvider>(&_uiaProvider, owner, std::bind(&TermControlAutomationPeer::GetBoundingRectWrapped, this)));
     };
 
+    // Method Description:
+    // - Signals the ui automation client that the terminal's selection has changed and should be updated
+    // Arguments:
+    // - <none>
+    // Return Value:
+    // - <none>
+    void TermControlAutomationPeer::SignalSelectionChanged()
+    {
+        Dispatcher().RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [&]() {
+            // The event that is raised when the text selection is modified.
+            RaiseAutomationEvent(AutomationEvents::TextPatternOnTextSelectionChanged);
+        });
+    }
+
+    // Method Description:
+    // - Signals the ui automation client that the terminal's output has changed and should be updated
+    // Arguments:
+    // - <none>
+    // Return Value:
+    // - <none>
+    void TermControlAutomationPeer::SignalTextChanged()
+    {
+        Dispatcher().RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [&]() {
+            // The event that is raised when textual content is modified.
+            RaiseAutomationEvent(AutomationEvents::TextPatternOnTextChanged);
+        });
+    }
+
+    // Method Description:
+    // - Signals the ui automation client that the cursor's state has changed and should be updated
+    // Arguments:
+    // - <none>
+    // Return Value:
+    // - <none>
+    void TermControlAutomationPeer::SignalCursorChanged()
+    {
+        Dispatcher().RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [&]() {
+            // The event that is raised when the text was changed in an edit control.
+            RaiseAutomationEvent(AutomationEvents::TextEditTextChanged);
+        });
+    }
+
     winrt::hstring TermControlAutomationPeer::GetClassNameCore() const
     {
         return L"TermControl";

src/cascadia/TerminalControl/TermControlAutomationPeer.h

@@ -27,14 +27,14 @@ Author(s):
 #include "TermControl.h"
 #include "TermControlAutomationPeer.g.h"
 #include <winrt/Microsoft.Terminal.TerminalControl.h>
-#include "../../renderer/inc/IRenderData.hpp"
-#include "../types/WindowUiaProviderBase.hpp"
 #include "TermControlUiaProvider.hpp"
+#include "../types/IUiaEventDispatcher.h"
 
 namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 {
     struct TermControlAutomationPeer :
-        public TermControlAutomationPeerT<TermControlAutomationPeer>
+        public TermControlAutomationPeerT<TermControlAutomationPeer>,
+        ::Microsoft::Console::Types::IUiaEventDispatcher
     {
     public:
         TermControlAutomationPeer(winrt::Microsoft::Terminal::TerminalControl::implementation::TermControl* owner);
@@ -44,6 +44,12 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         winrt::hstring GetLocalizedControlTypeCore() const;
         winrt::Windows::Foundation::IInspectable GetPatternCore(winrt::Windows::UI::Xaml::Automation::Peers::PatternInterface patternInterface) const;
 
+#pragma region IUiaEventDispatcher
+        void SignalSelectionChanged() override;
+        void SignalTextChanged() override;
+        void SignalCursorChanged() override;
+#pragma endregion
+
 #pragma region ITextProvider Pattern
         Windows::UI::Xaml::Automation::Provider::ITextRangeProvider RangeFromPoint(Windows::Foundation::Point screenLocation);
         Windows::UI::Xaml::Automation::Provider::ITextRangeProvider RangeFromChild(Windows::UI::Xaml::Automation::Provider::IRawElementProviderSimple childElement);

src/cascadia/TerminalControl/TermControlAutomationPeer.idl

@@ -5,10 +5,9 @@ import "TermControl.idl";
 
 namespace Microsoft.Terminal.TerminalControl
 {
-    [default_interface]
-    runtimeclass TermControlAutomationPeer :
+    [default_interface] runtimeclass TermControlAutomationPeer :
         Windows.UI.Xaml.Automation.Peers.FrameworkElementAutomationPeer,
         Windows.UI.Xaml.Automation.Provider.ITextProvider
-        {
-        }
+    {
+    }
 }

src/cascadia/TerminalControl/TerminalControl.vcxproj

@@ -71,6 +71,7 @@
     <ProjectReference Include="..\..\buffer\out\lib\bufferout.vcxproj" />
     <ProjectReference Include="$(OpenConsoleDir)src\renderer\base\lib\base.vcxproj" />
     <ProjectReference Include="..\..\renderer\dx\lib\dx.vcxproj" />
+    <ProjectReference Include="..\..\renderer\uia\lib\uia.vcxproj" />
     <ProjectReference Include="..\..\terminal\parser\lib\parser.vcxproj" />
     <ProjectReference Include="$(OpenConsoleDir)src\terminal\input\lib\terminalinput.vcxproj" />
     <ProjectReference Include="$(OpenConsoleDir)src\cascadia\TerminalSettings\TerminalSettings.vcxproj">

src/renderer/uia/UiaRenderer.cpp

@@ -13,9 +13,12 @@ using namespace Microsoft::Console::Types;
 // Routine Description:
 // - Constructs a UIA engine for console text
 //   which primarily notifies automation clients of any activity
-UiaEngine::UiaEngine() noexcept :
+UiaEngine::UiaEngine(IUiaEventDispatcher* dispatcher) :
+    _dispatcher{ THROW_HR_IF_NULL(E_INVALIDARG, dispatcher) },
     _isPainting{ false },
-    _isEnabled{ false },
+    _selectionChanged{ false },
+    _isEnabled{ true },
+    _prevSelection{},
     RenderEngineBase()
 {
 }
@@ -53,7 +56,7 @@ UiaEngine::UiaEngine() noexcept :
 // - S_OK, else an appropriate HRESULT for failing to allocate or write.
 [[nodiscard]] HRESULT UiaEngine::Invalidate(const SMALL_RECT* const /*psrRegion*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -87,9 +90,41 @@ UiaEngine::UiaEngine() noexcept :
 // - rectangles - One or more rectangles describing character positions on the grid
 // Return Value:
 // - S_OK
-[[nodiscard]] HRESULT UiaEngine::InvalidateSelection(const std::vector<SMALL_RECT>& /*rectangles*/) noexcept
+[[nodiscard]] HRESULT UiaEngine::InvalidateSelection(const std::vector<SMALL_RECT>& rectangles) noexcept
 {
-    return E_NOTIMPL;
+    // early exit: different number of rows
+    if (_prevSelection.size() != rectangles.size())
+    {
+        try
+        {
+            _selectionChanged = true;
+            _prevSelection = rectangles;
+        }
+        CATCH_LOG_RETURN_HR(E_FAIL);
+        return S_OK;
+    }
+
+    for (size_t i = 0; i < rectangles.size(); i++)
+    {
+        try
+        {
+            const auto prevRect = _prevSelection.at(i);
+            const auto newRect = rectangles.at(i);
+
+            // if any value is different, selection has changed
+            if (prevRect.Top != newRect.Top || prevRect.Right != newRect.Right || prevRect.Left != newRect.Left || prevRect.Bottom != newRect.Bottom)
+            {
+                _selectionChanged = true;
+                _prevSelection = rectangles;
+                return S_OK;
+            }
+        }
+        CATCH_LOG_RETURN_HR(E_FAIL);
+    }
+
+    // assume selection has not changed
+    _selectionChanged = false;
+    return S_OK;
 }
 
 // Routine Description:
@@ -102,7 +137,7 @@ UiaEngine::UiaEngine() noexcept :
 // - S_OK
 [[nodiscard]] HRESULT UiaEngine::InvalidateScroll(const COORD* const /*pcoordDelta*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -115,7 +150,7 @@ UiaEngine::UiaEngine() noexcept :
 // - S_OK, else an appropriate HRESULT for failing to allocate or write.
 [[nodiscard]] HRESULT UiaEngine::InvalidateAll() noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -156,19 +191,13 @@ UiaEngine::UiaEngine() noexcept :
 {
     RETURN_HR_IF(S_FALSE, !_isEnabled);
 
+    // add more events here
+    // bool somethingToDo = _selectionChanged;
+
     // If there's nothing to do, quick return
-    bool somethingToDo = false;
-
-    if (_isEnabled)
-    {
-        somethingToDo = false;
-
-        if (somethingToDo)
-        {
-            _isPainting = true;
-        }
-    }
+    RETURN_HR_IF(S_FALSE, !_selectionChanged);
 
+    _isPainting = true;
     return S_OK;
 }
 
@@ -180,15 +209,23 @@ UiaEngine::UiaEngine() noexcept :
 // - S_OK, else an appropriate HRESULT for failing to allocate or write.
 [[nodiscard]] HRESULT UiaEngine::EndPaint() noexcept
 {
+    RETURN_HR_IF(S_FALSE, !_isEnabled);
     RETURN_HR_IF(E_INVALIDARG, !_isPainting); // invalid to end paint when we're not painting
 
-    if (_isEnabled)
+    // Fire UIA Events here
+    if (_selectionChanged)
     {
-        _isPainting = false;
-
-        // fire UIA events here
+        try
+        {
+            _dispatcher->SignalSelectionChanged();
+        }
+        CATCH_LOG();
     }
 
+    _selectionChanged = false;
+    _prevSelection.clear();
+    _isPainting = false;
+
     return S_OK;
 }
 
@@ -230,17 +267,18 @@ UiaEngine::UiaEngine() noexcept :
 
 // Routine Description:
 // - Places one line of text onto the screen at the given position
+//  For UIA, this doesn't mean anything. So do nothing.
 // Arguments:
 // - clusters - Iterable collection of cluster information (text and columns it should consume)
 // - coord - Character coordinate position in the cell grid
 // - fTrimLeft - Whether or not to trim off the left half of a double wide character
 // Return Value:
-// - E_NOTIMPL
+// - S_FALSE
 [[nodiscard]] HRESULT UiaEngine::PaintBufferLine(std::basic_string_view<Cluster> const /*clusters*/,
                                                  COORD const /*coord*/,
                                                  const bool /*trimLeft*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -277,6 +315,7 @@ UiaEngine::UiaEngine() noexcept :
 
 // Routine Description:
 // - Draws the cursor on the screen
+//  For UIA, this doesn't mean anything. So do nothing.
 // Arguments:
 // - options - Packed options relevant to how to draw the cursor
 // Return Value:
@@ -288,7 +327,7 @@ UiaEngine::UiaEngine() noexcept :
 
 // Routine Description:
 // - Updates the default brush colors used for drawing
-// - Not currently used by UiaEngine.
+//  For UIA, this doesn't mean anything. So do nothing.
 // Arguments:
 // - colorForeground - <unused>
 // - colorBackground - <unused>
@@ -315,7 +354,7 @@ UiaEngine::UiaEngine() noexcept :
 // - S_FALSE since we do nothing
 [[nodiscard]] HRESULT UiaEngine::UpdateFont(const FontInfoDesired& /*pfiFontInfoDesired*/, FontInfo& /*fiFontInfo*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -327,7 +366,7 @@ UiaEngine::UiaEngine() noexcept :
 // - S_OK
 [[nodiscard]] HRESULT UiaEngine::UpdateDpi(int const /*iDpi*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Method Description:
@@ -338,7 +377,7 @@ UiaEngine::UiaEngine() noexcept :
 // - HRESULT S_OK
 [[nodiscard]] HRESULT UiaEngine::UpdateViewport(const SMALL_RECT /*srNewViewport*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -376,7 +415,7 @@ UiaEngine::UiaEngine() noexcept :
 // - S_OK
 [[nodiscard]] HRESULT UiaEngine::GetFontSize(_Out_ COORD* const /*pFontSize*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Routine Description:
@@ -385,10 +424,10 @@ UiaEngine::UiaEngine() noexcept :
 // - glyph - The glyph run to process for column width.
 // - pResult - True if it should take two columns. False if it should take one.
 // Return Value:
-// - E_NOTIMPL
+// - S_OK or relevant DirectWrite error.
 [[nodiscard]] HRESULT UiaEngine::IsGlyphWideByFont(const std::wstring_view /*glyph*/, _Out_ bool* const /*pResult*/) noexcept
 {
-    return E_NOTIMPL;
+    return S_FALSE;
 }
 
 // Method Description:

src/renderer/uia/UiaRenderer.hpp

@@ -17,6 +17,7 @@ Author(s):
 
 #include "../../renderer/inc/RenderEngineBase.hpp"
 
+#include "../../types/IUiaEventDispatcher.h"
 #include "../../types/inc/Viewport.hpp"
 
 namespace Microsoft::Console::Render
@@ -24,7 +25,7 @@ namespace Microsoft::Console::Render
     class UiaEngine final : public RenderEngineBase
     {
     public:
-        UiaEngine() noexcept;
+        UiaEngine(Microsoft::Console::Types::IUiaEventDispatcher* dispatcher);
 
         // Only one UiaEngine may present information at a time.
         // This ensures that an automation client isn't overwhelmed
@@ -79,5 +80,10 @@ namespace Microsoft::Console::Render
     private:
         bool _isEnabled;
         bool _isPainting;
+        bool _selectionChanged;
+
+        Microsoft::Console::Types::IUiaEventDispatcher* _dispatcher;
+
+        std::vector<SMALL_RECT> _prevSelection;
     };
 }

src/types/IUiaEventDispatcher.h

@@ -0,0 +1,27 @@
+/*++
+Copyright (c) Microsoft Corporation
+Licensed under the MIT license.
+
+Module Name:
+- IUiaEventDispatcher.h
+
+Abstract:
+- This serves as the interface allowing accessibility providers to detect and respond to accessibility events
+- This interface should allow accessibility events to be supported across different implementation patterns
+
+Author(s):
+- Carlos Zamora (CaZamor) Sept-2019
+--*/
+
+#pragma once
+
+namespace Microsoft::Console::Types
+{
+    class IUiaEventDispatcher
+    {
+    public:
+        virtual void SignalSelectionChanged() = 0;
+        virtual void SignalTextChanged() = 0;
+        virtual void SignalCursorChanged() = 0;
+    };
+}

src/types/lib/types.vcxproj

@@ -47,6 +47,7 @@
     <ClInclude Include="..\inc\Viewport.hpp" />
     <ClInclude Include="..\inc\Utf16Parser.hpp" />
     <ClInclude Include="..\IUiaData.h" />
+    <ClInclude Include="..\IUiaEventDispatcher.h" />
     <ClInclude Include="..\IUiaWindow.h" />
     <ClInclude Include="..\precomp.h" />
     <ClInclude Include="..\ScreenInfoUiaProviderBase.h" />

src/types/lib/types.vcxproj.filters

@@ -152,6 +152,9 @@
     <ClInclude Include="..\IBaseData.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\IUiaEventDispatcher.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\inc\utils.hpp">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -162,4 +165,4 @@
   <ItemGroup>
     <Natvis Include="$(SolutionDir)tools\ConsoleTypes.natvis" />
   </ItemGroup>
-</Project>
+</Project>