terminal/src/types/TermControlUiaProvider.cpp

178 lines
6.7 KiB
C++
Raw Permalink Normal View History

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
#include "precomp.h"
#include "TermControlUiaProvider.hpp"
using namespace Microsoft::Terminal;
using namespace Microsoft::Console::Types;
using namespace Microsoft::WRL;
#pragma warning(suppress : 26434) // WRL RuntimeClassInitialize base is a no-op and we need this for MakeAndInitialize
HRESULT TermControlUiaProvider::RuntimeClassInitialize(_In_ ::Microsoft::Console::Types::IUiaData* const uiaData,
_In_ ::Microsoft::Console::Types::IControlAccessibilityInfo* controlInfo) noexcept
{
RETURN_HR_IF_NULL(E_INVALIDARG, uiaData);
RETURN_IF_FAILED(ScreenInfoUiaProviderBase::RuntimeClassInitialize(uiaData));
_controlInfo = controlInfo;
return S_OK;
}
IFACEMETHODIMP TermControlUiaProvider::Navigate(_In_ NavigateDirection direction,
_COM_Outptr_result_maybenull_ IRawElementProviderFragment** ppProvider) noexcept
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppProvider);
*ppProvider = nullptr;
if (direction == NavigateDirection_Parent)
{
try
{
// TODO GitHub #2102: UIA Tree Navigation
//_pUiaParent->QueryInterface(IID_PPV_ARGS(ppProvider));
}
catch (...)
{
*ppProvider = nullptr;
return wil::ResultFromCaughtException();
}
RETURN_IF_NULL_ALLOC(*ppProvider);
}
// For the other directions the default of nullptr is correct
return S_OK;
}
IFACEMETHODIMP TermControlUiaProvider::get_BoundingRectangle(_Out_ UiaRect* pRect) noexcept
{
// TODO GitHub #1914: Re-attach Tracing to UIA Tree
//Tracing::s_TraceUia(this, ApiCall::GetBoundingRectangle, nullptr);
const RECT rc = _controlInfo->GetBounds();
pRect->left = rc.left;
pRect->top = rc.top;
pRect->width = static_cast<double>(rc.right) - static_cast<double>(rc.left);
pRect->height = static_cast<double>(rc.bottom) - static_cast<double>(rc.top);
return S_OK;
}
IFACEMETHODIMP TermControlUiaProvider::get_HostRawElementProvider(_COM_Outptr_result_maybenull_ IRawElementProviderSimple** ppProvider) noexcept
{
try
{
return _controlInfo->GetHostUiaProvider(ppProvider);
}
CATCH_RETURN();
}
IFACEMETHODIMP TermControlUiaProvider::get_FragmentRoot(_COM_Outptr_result_maybenull_ IRawElementProviderFragmentRoot** ppProvider) noexcept
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppProvider);
// TODO GitHub #1914: Re-attach Tracing to UIA Tree
//Tracing::s_TraceUia(this, ApiCall::GetFragmentRoot, nullptr);
try
{
// TODO GitHub #2102: UIA Tree Navigation - the special fragments that knows about all of its descendants is called a fragment root
//_pUiaParent->QueryInterface(IID_PPV_ARGS(ppProvider));
*ppProvider = nullptr;
}
catch (...)
{
*ppProvider = nullptr;
return wil::ResultFromCaughtException();
}
RETURN_IF_NULL_ALLOC(*ppProvider);
return S_OK;
}
const COORD TermControlUiaProvider::GetFontSize() const noexcept
{
return _controlInfo->GetFontSize();
}
const RECT TermControlUiaProvider::GetPadding() const noexcept
{
return _controlInfo->GetPadding();
}
const double TermControlUiaProvider::GetScaleFactor() const noexcept
{
return _controlInfo->GetScaleFactor();
}
void TermControlUiaProvider::ChangeViewport(const SMALL_RECT NewWindow)
{
_controlInfo->ChangeViewport(NewWindow);
}
HRESULT TermControlUiaProvider::GetSelectionRange(_In_ IRawElementProviderSimple* pProvider, const std::wstring_view wordDelimiters, _COM_Outptr_result_maybenull_ UiaTextRangeBase** ppUtr)
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppUtr);
*ppUtr = nullptr;
const auto start = _pData->GetSelectionAnchor();
// we need to make end exclusive
Move rect expansion to textbuffer; refactor selection code (#4560) - When performing chunk selection, the expansion now occurs at the time of the selection, not the rendering of the selection - `GetSelectionRects()` was moved to the `TextBuffer` and is now shared between ConHost and Windows Terminal - Some of the selection variables were renamed for clarity - Selection COORDs are now in the Text Buffer coordinate space - Fixes an issue with Shift+Click after performing a Multi-Click Selection ## References This also contributes to... - #4509: UIA Box Selection - #2447: UIA Signaling for Selection - #1354: UIA support for Wide Glyphs Now that the expansion occurs at before render-time, the selection anchors are an accurate representation of what is selected. We just need to move `GetText` to the `TextBuffer`. Then we can have those three issues just rely on code from the text buffer. This also means ConHost gets some of this stuff for free 😀 ### TextBuffer - `GetTextRects` is the abstracted form of `GetSelectionRects` - `_ExpandTextRow` is still needed to handle wide glyphs properly ### Terminal - Rename... - `_boxSelection` --> `_blockSelection` for consistency with ConHost - `_selectionAnchor` --> `_selectionStart` for consistency with UIA - `_endSelectionPosition` --> `_selectionEnd` for consistency with UIA - Selection anchors are in Text Buffer coordinates now - Really rely on `SetSelectionEnd` to accomplish appropriate chunk selection and shift+click actions ## Validation Steps Performed - Shift+Click - Multi-Click --> Shift+Click - Chunk Selection at... - top of buffer - bottom of buffer - random region in scrollback Closes #4465 Closes #4547
2020-02-28 01:42:26 +01:00
auto end = _pData->GetSelectionEnd();
_pData->GetTextBuffer().GetSize().IncrementInBounds(end, true);
TermControlUiaTextRange* result = nullptr;
RETURN_IF_FAILED(MakeAndInitialize<TermControlUiaTextRange>(&result, _pData, pProvider, start, end, _pData->IsBlockSelection(), wordDelimiters));
*ppUtr = result;
return S_OK;
}
Enable Word Navigation in UiaTextRange (#3659) Enables support for word navigation when using an automation client (i.e.: Narrator, etc...). Specifically, adds this functionality to the UiaTextRange class. The only delimiter used is whitespace because that's how words are separated in English. # General "Word Movement" Expectations The resulting text range should include any word break characters that are present at the end of the word, but before the start of the next word. (Source) If you already are on a word, getting the "next word" means you skip the word you are on, and highlight the upcoming word appropriately. (similar idea when moving backwards) # Word Expansion Since word selection is supposed to detect word delimiters already, I figured I'd reuse that code. I moved it from TerminalCore to the TextBuffer. Then I built on top of it by adding an optional additional parameter that decides if you want to include... - the delimiter run when moving forward - the character run when moving backwards It defaults to false so that we don't have to care when using it in selection. But we change it to true when using it in our UiaTextRange # UiaTextRange The code is based on character movement. This allows us to actually work with boundary conditions. The main thing to remember here is that each text range is recorded as a MoveState. The text range is most easily defined when you think about the start Endpoint and the end Endpoint. An Endpoint is just a linear 1-dimensional indexing of the text buffer. Examples: - Endpoint 0 --> (0,0) - Endpoint 79 --> (79,0) (when the buffer width is 80) - Endpoint 80 -->(0,1) (when the buffer width is 80) - When moving forward, the strategy is to focus on moving the end Endpoint. That way, we properly get the indexing for the "next" word (this also fixes a wrapping issue). Then, we update the start Endpoint. (This is reversed for moving backwards). - When moving a specific Endpoint, we just have a few extra if statements to properly adjust for moving start vs end. # Hooking it up All we really had to do is add an enum. This part was super easy :) I originally wanted the delimiters to be able to be defined. I'm not so sure about that anymore. Either way, I hardcoded our delimiter into a variable so if we ever want to expand on it or make that customizable, we just modify that variable. # Defining your own word delimiters - Import a word delimiter into the constructor of the ScreenInfoUiaProvider (SIUP) - This defines a word delimiter for all the UiaTextRanges (UTR) created by in this context - import a word delimiter into the UTR directly - this provides more control over what a "word" is - this can be useful if you have an idea of what text a particular UTR will encounter and you want to customize the word navigation for it (i.e consider adding / or \\ for file paths) The default param of " " is scattered throughout because this is the word delimiter used in the English language.
2019-12-13 00:22:12 +01:00
HRESULT TermControlUiaProvider::CreateTextRange(_In_ IRawElementProviderSimple* const pProvider, const std::wstring_view wordDelimiters, _COM_Outptr_result_maybenull_ UiaTextRangeBase** ppUtr)
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppUtr);
*ppUtr = nullptr;
TermControlUiaTextRange* result = nullptr;
RETURN_IF_FAILED(MakeAndInitialize<TermControlUiaTextRange>(&result, _pData, pProvider, wordDelimiters));
*ppUtr = result;
return S_OK;
}
HRESULT TermControlUiaProvider::CreateTextRange(_In_ IRawElementProviderSimple* const pProvider,
const Cursor& cursor,
Enable Word Navigation in UiaTextRange (#3659) Enables support for word navigation when using an automation client (i.e.: Narrator, etc...). Specifically, adds this functionality to the UiaTextRange class. The only delimiter used is whitespace because that's how words are separated in English. # General "Word Movement" Expectations The resulting text range should include any word break characters that are present at the end of the word, but before the start of the next word. (Source) If you already are on a word, getting the "next word" means you skip the word you are on, and highlight the upcoming word appropriately. (similar idea when moving backwards) # Word Expansion Since word selection is supposed to detect word delimiters already, I figured I'd reuse that code. I moved it from TerminalCore to the TextBuffer. Then I built on top of it by adding an optional additional parameter that decides if you want to include... - the delimiter run when moving forward - the character run when moving backwards It defaults to false so that we don't have to care when using it in selection. But we change it to true when using it in our UiaTextRange # UiaTextRange The code is based on character movement. This allows us to actually work with boundary conditions. The main thing to remember here is that each text range is recorded as a MoveState. The text range is most easily defined when you think about the start Endpoint and the end Endpoint. An Endpoint is just a linear 1-dimensional indexing of the text buffer. Examples: - Endpoint 0 --> (0,0) - Endpoint 79 --> (79,0) (when the buffer width is 80) - Endpoint 80 -->(0,1) (when the buffer width is 80) - When moving forward, the strategy is to focus on moving the end Endpoint. That way, we properly get the indexing for the "next" word (this also fixes a wrapping issue). Then, we update the start Endpoint. (This is reversed for moving backwards). - When moving a specific Endpoint, we just have a few extra if statements to properly adjust for moving start vs end. # Hooking it up All we really had to do is add an enum. This part was super easy :) I originally wanted the delimiters to be able to be defined. I'm not so sure about that anymore. Either way, I hardcoded our delimiter into a variable so if we ever want to expand on it or make that customizable, we just modify that variable. # Defining your own word delimiters - Import a word delimiter into the constructor of the ScreenInfoUiaProvider (SIUP) - This defines a word delimiter for all the UiaTextRanges (UTR) created by in this context - import a word delimiter into the UTR directly - this provides more control over what a "word" is - this can be useful if you have an idea of what text a particular UTR will encounter and you want to customize the word navigation for it (i.e consider adding / or \\ for file paths) The default param of " " is scattered throughout because this is the word delimiter used in the English language.
2019-12-13 00:22:12 +01:00
const std::wstring_view wordDelimiters,
_COM_Outptr_result_maybenull_ UiaTextRangeBase** ppUtr)
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppUtr);
*ppUtr = nullptr;
TermControlUiaTextRange* result = nullptr;
RETURN_IF_FAILED(MakeAndInitialize<TermControlUiaTextRange>(&result, _pData, pProvider, cursor, wordDelimiters));
*ppUtr = result;
return S_OK;
}
HRESULT TermControlUiaProvider::CreateTextRange(_In_ IRawElementProviderSimple* const pProvider,
Refactor UiaTextRange For Improved Navigation and Reliability (#4018) ## Summary of the Pull Request This pull request is intended to achieve the following goals... 1) reduce duplicate code 2) remove static functions 3) improve readability 4) improve reliability 5) improve code-coverage for testing 6) establish functioning text buffer navigation in Narrator and NVDA This also required a change to the wrapper class `XamlUiaTextRange` that has been causing issues with Narrator and NVDA. See below for additional context. ## References #3976 - I believe this might have been a result of improperly handling degenerate ranges. Fixed here. #3895 - reduced the duplicate code. No need to separate into different files #2160 - same as #3976 above #1993 - I think just about everything is no longer static ## PR Checklist * [x] Closes #3895, Closes #1993, Closes #3976, Closes #2160 * [x] CLA signed * [x] Tests added/passed ## Detailed Description of the Pull Request / Additional comments ### UiaTextRange - converted endpoints into the COORD system in the TextBuffer coordinate space - `start` is inclusive, `end` is exclusive. A degenerate range is when start == end. - all functions are no longer static - `MoveByUnit()` functions now rely on `MoveEndpointByUnit()` functions - removed unnecessary typedefs like `Endpoint`, `ScreenInfoRow`, etc.. - relied more heavily on existing functionality from `TextBuffer` and `Viewport` ### XamlUiaTextRange - `GetAttributeValue()` must return a special HRESULT that signifies that the requested attribute is not supported. This was the cause of a number of inconsistencies between Narrator and NVDA. - `FindText()` should return `nullptr` if nothing was found. #4373 properly fixes this functionality now that Search is a shared module ### TextBuffer - Word navigation functionality is entirely in `TextBuffer` for proper abstraction - a total of 6 functions are now dedicated to word navigation to get a good understanding of the differences between a "word" in Accessibility and a "word" in selection As an example, consider a buffer with this text in it: " word other " In selection, a "word" is defined as the range between two delimiters, so the words in the example include [" ", "word", " ", "other", " "]. In accessibility , a "word" includes the delimiters after a range of readable characters, so the words in the example include ["word ", "other "]. Additionally, accessibility word navigation must be able to detect if it is on the first or last word. This resulted in a slight variant of word navigation functions that return a boolean instead of a COORD. Ideally, these functions can be consolidated, but that is too risky for a PR of this size as it can have an effect on selection. ### Viewport - the concept of `EndExclusive` is added. This is used by UiaTextRange's `end` anchor as it is exclusive. To signify that the last character in the buffer is included in this buffer, `end` must be one past the end of the buffer. This is `EndExclusive` - Since many functions check if the given `COORD` is in bounds, a flag must be set to allow `EndExclusive` as a valid `COORD` that is in bounds. ### Testing - word navigation testing relies more heavily on TextBuffer tests - additional testing was created for non-movement focused functions of UiaTextRange - The results have been compared to Microsoft Word and some have been verified by UiAutomation/Narrator contacts as expected results. ## Validation Steps Performed Tests pass Narrator works NVDA works
2020-01-31 21:59:39 +01:00
const COORD start,
const COORD end,
Enable Word Navigation in UiaTextRange (#3659) Enables support for word navigation when using an automation client (i.e.: Narrator, etc...). Specifically, adds this functionality to the UiaTextRange class. The only delimiter used is whitespace because that's how words are separated in English. # General "Word Movement" Expectations The resulting text range should include any word break characters that are present at the end of the word, but before the start of the next word. (Source) If you already are on a word, getting the "next word" means you skip the word you are on, and highlight the upcoming word appropriately. (similar idea when moving backwards) # Word Expansion Since word selection is supposed to detect word delimiters already, I figured I'd reuse that code. I moved it from TerminalCore to the TextBuffer. Then I built on top of it by adding an optional additional parameter that decides if you want to include... - the delimiter run when moving forward - the character run when moving backwards It defaults to false so that we don't have to care when using it in selection. But we change it to true when using it in our UiaTextRange # UiaTextRange The code is based on character movement. This allows us to actually work with boundary conditions. The main thing to remember here is that each text range is recorded as a MoveState. The text range is most easily defined when you think about the start Endpoint and the end Endpoint. An Endpoint is just a linear 1-dimensional indexing of the text buffer. Examples: - Endpoint 0 --> (0,0) - Endpoint 79 --> (79,0) (when the buffer width is 80) - Endpoint 80 -->(0,1) (when the buffer width is 80) - When moving forward, the strategy is to focus on moving the end Endpoint. That way, we properly get the indexing for the "next" word (this also fixes a wrapping issue). Then, we update the start Endpoint. (This is reversed for moving backwards). - When moving a specific Endpoint, we just have a few extra if statements to properly adjust for moving start vs end. # Hooking it up All we really had to do is add an enum. This part was super easy :) I originally wanted the delimiters to be able to be defined. I'm not so sure about that anymore. Either way, I hardcoded our delimiter into a variable so if we ever want to expand on it or make that customizable, we just modify that variable. # Defining your own word delimiters - Import a word delimiter into the constructor of the ScreenInfoUiaProvider (SIUP) - This defines a word delimiter for all the UiaTextRanges (UTR) created by in this context - import a word delimiter into the UTR directly - this provides more control over what a "word" is - this can be useful if you have an idea of what text a particular UTR will encounter and you want to customize the word navigation for it (i.e consider adding / or \\ for file paths) The default param of " " is scattered throughout because this is the word delimiter used in the English language.
2019-12-13 00:22:12 +01:00
const std::wstring_view wordDelimiters,
_COM_Outptr_result_maybenull_ UiaTextRangeBase** ppUtr)
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppUtr);
*ppUtr = nullptr;
TermControlUiaTextRange* result = nullptr;
RETURN_IF_FAILED(MakeAndInitialize<TermControlUiaTextRange>(&result, _pData, pProvider, start, end, false, wordDelimiters));
*ppUtr = result;
return S_OK;
}
HRESULT TermControlUiaProvider::CreateTextRange(_In_ IRawElementProviderSimple* const pProvider,
const UiaPoint point,
Enable Word Navigation in UiaTextRange (#3659) Enables support for word navigation when using an automation client (i.e.: Narrator, etc...). Specifically, adds this functionality to the UiaTextRange class. The only delimiter used is whitespace because that's how words are separated in English. # General "Word Movement" Expectations The resulting text range should include any word break characters that are present at the end of the word, but before the start of the next word. (Source) If you already are on a word, getting the "next word" means you skip the word you are on, and highlight the upcoming word appropriately. (similar idea when moving backwards) # Word Expansion Since word selection is supposed to detect word delimiters already, I figured I'd reuse that code. I moved it from TerminalCore to the TextBuffer. Then I built on top of it by adding an optional additional parameter that decides if you want to include... - the delimiter run when moving forward - the character run when moving backwards It defaults to false so that we don't have to care when using it in selection. But we change it to true when using it in our UiaTextRange # UiaTextRange The code is based on character movement. This allows us to actually work with boundary conditions. The main thing to remember here is that each text range is recorded as a MoveState. The text range is most easily defined when you think about the start Endpoint and the end Endpoint. An Endpoint is just a linear 1-dimensional indexing of the text buffer. Examples: - Endpoint 0 --> (0,0) - Endpoint 79 --> (79,0) (when the buffer width is 80) - Endpoint 80 -->(0,1) (when the buffer width is 80) - When moving forward, the strategy is to focus on moving the end Endpoint. That way, we properly get the indexing for the "next" word (this also fixes a wrapping issue). Then, we update the start Endpoint. (This is reversed for moving backwards). - When moving a specific Endpoint, we just have a few extra if statements to properly adjust for moving start vs end. # Hooking it up All we really had to do is add an enum. This part was super easy :) I originally wanted the delimiters to be able to be defined. I'm not so sure about that anymore. Either way, I hardcoded our delimiter into a variable so if we ever want to expand on it or make that customizable, we just modify that variable. # Defining your own word delimiters - Import a word delimiter into the constructor of the ScreenInfoUiaProvider (SIUP) - This defines a word delimiter for all the UiaTextRanges (UTR) created by in this context - import a word delimiter into the UTR directly - this provides more control over what a "word" is - this can be useful if you have an idea of what text a particular UTR will encounter and you want to customize the word navigation for it (i.e consider adding / or \\ for file paths) The default param of " " is scattered throughout because this is the word delimiter used in the English language.
2019-12-13 00:22:12 +01:00
const std::wstring_view wordDelimiters,
_COM_Outptr_result_maybenull_ UiaTextRangeBase** ppUtr)
{
RETURN_HR_IF_NULL(E_INVALIDARG, ppUtr);
*ppUtr = nullptr;
TermControlUiaTextRange* result = nullptr;
RETURN_IF_FAILED(MakeAndInitialize<TermControlUiaTextRange>(&result, _pData, pProvider, point, wordDelimiters));
*ppUtr = result;
return S_OK;
}