terminal/src/cascadia/WindowsTerminal_UIATests/Elements/TerminalApp.cs
Michael Niksa 7dadde5dd6
Implement PGO in pipelines for AMD64 architecture; supply training test scenarios (#10071)
Implement PGO in pipelines for AMD64 architecture; supply training test scenarios

## References
- #3075 - Relevant to speed interests there and other linked issues.

## PR Checklist
* [x] Closes #6963
* [x] I work here.
* [x] New UIA Tests added and passed. Manual build runs also tested.

## Detailed Description of the Pull Request / Additional comments
- Creates a new pipeline run for creating instrumented binaries for Profile Guided Optimization (PGO).
- Creates a new suite of UIA tests on the full Windows Terminal app to run PGO training scenarios on instrumented binaries (and incidentally can be used to write other UIA tests later for the full Terminal app.)
- Creates a new NuGet artifact to store trained PGO databases (PGD files) at `Microsoft.Internal.Windows.Terminal.PGODatabase`
- Creates a new NuGet artifact to supply large-scale test content for automated tests at `Microsoft.Internal.Windows.Terminal.TestContent`
- Adjusts the release pipeline to run binaries in PGO optimized mode where content from PGO databases is leveraged at link time to optimize the final release build

The following binaries are trained:
- OpenConsole.exe
- WindowsTerminal.exe
- TerminalApp.dll
- TerminalConnection.dll
- Microsoft.Terminal.Control.dll
- Microsoft.Terminal.Remoting.dll
- Microsoft.Terminal.Settings.Editor.dll
- Microsoft.Terminal.Settings.Model.dll

In the future, adding `<PgoTarget>true</PgoTarget>` to a new `vcxproj` file will automatically enroll the DLL/EXE for PGO instrumentation and optimization going forward.

Two training test scenarios are implemented:
- Smoke test the Terminal by just opening it and typing a bit of text then exiting. (Should help focus on the standard launch path.)
- Optimize bulk text output by launching terminal, outputting `big.txt`, then exiting.

Additional scenarios can be contributed to the `WindowsTerminal_UIATests` project with the `[TestProperty("IsPGO", "true")]` annotation to add them to the suite of scenarios for PGO.

**NOTE:** There are currently no weights applied to the various test scenarios. We will revisit that in the future when/if necessary.

## Validation Steps Performed
- [x] - Training run completed at https://dev.azure.com/ms/terminal/_build?definitionId=492&_a=summary
- [x] - Optimization run completed locally (by forcing `PGOBuildMode` to `Optimize` on my local machine, manually retrieving the databases with NuGet, and building).
- [x] - Validated locally that x86 and ARM64 do not get trained and automatically skip optimization as databases are not present for them.
- [x] - Smoke tested optimized binary versus latest releases. `big.txt` output through CMD is ~11-12seconds prior to PGO and just over 8 seconds with PGO.
2021-05-13 21:12:30 +00:00

177 lines
6.5 KiB
C#

//----------------------------------------------------------------------------------------------------------------------
// <copyright file="TerminalApp.cs" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
// <summary>Helper and wrapper for generating the base test application and its UI Root node.</summary>
//----------------------------------------------------------------------------------------------------------------------
namespace WindowsTerminal.UIA.Tests.Elements
{
using System;
using System.IO;
using WindowsTerminal.UIA.Tests.Common;
using WindowsTerminal.UIA.Tests.Common.NativeMethods;
using OpenQA.Selenium.Remote;
using OpenQA.Selenium.Appium;
using OpenQA.Selenium.Appium.iOS;
using OpenQA.Selenium.Interactions;
using WEX.Logging.Interop;
using WEX.TestExecution;
using WEX.TestExecution.Markup;
using System.Runtime.InteropServices;
using System.Security.Principal;
using OpenQA.Selenium;
public class TerminalApp : IDisposable
{
protected const string AppDriverUrl = "http://127.0.0.1:4723";
private IntPtr job;
public IOSDriver<IOSElement> Session { get; private set; }
public Actions Actions { get; private set; }
public AppiumWebElement UIRoot { get; private set; }
private bool isDisposed = false;
private TestContext context;
public string ContentPath { get; private set; }
public string GetFullTestContentPath(string filename)
{
return Path.GetFullPath(Path.Combine(ContentPath, filename));
}
public TerminalApp(TestContext context, string shellToLaunch = "powershell.exe")
{
this.context = context;
// If running locally, set WTPath to where we can find a loose deployment of Windows Terminal
// On the build machines, the scripts lay it out at the appx\ subfolder of the test deployment directory
string path = Path.GetFullPath(Path.Combine(context.TestDeploymentDir, @"appx\WindowsTerminal.exe"));
if (context.Properties.Contains("WTPath"))
{
path = (string)context.Properties["WTPath"];
}
Log.Comment($"Windows Terminal will be launched from '{path}'");
// Same goes for the content directory. Set WTTestContent for where the content files are
// for running tests.
// On the build machines, the scripts lay it out at the content\ subfolder.
ContentPath = @"content";
if (context.Properties.Contains("WTTestContent"))
{
ContentPath = (string)context.Properties["WTTestContent"];
}
Log.Comment($"Test Content will be loaded from '{Path.GetFullPath(ContentPath)}'");
this.CreateProcess(path, shellToLaunch);
}
~TerminalApp()
{
this.Dispose(false);
}
public void Dispose()
{
this.Dispose(true);
GC.SuppressFinalize(this);
}
public AppiumWebElement GetRoot()
{
return this.UIRoot;
}
protected virtual void Dispose(bool disposing)
{
if (!this.isDisposed)
{
// ensure we're exited when this is destroyed or disposed of explicitly
this.ExitProcess();
this.isDisposed = true;
}
}
private void CreateProcess(string path, string shellToLaunch)
{
string WindowTitleToFind = "WindowsTerminal.UIA.Tests";
job = WinBase.CreateJobObject(IntPtr.Zero, IntPtr.Zero);
NativeMethods.Win32NullHelper(job, "Creating job object to hold binaries under test.");
Log.Comment("Attempting to launch command-line application at '{0}'", path);
string binaryToRunPath = path;
string args = $"new-tab --title \"{WindowTitleToFind}\" --suppressApplicationTitle \"{shellToLaunch}\"";
string launchArgs = $"{binaryToRunPath} {args}";
WinBase.STARTUPINFO si = new WinBase.STARTUPINFO();
si.cb = Marshal.SizeOf(si);
WinBase.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION();
NativeMethods.Win32BoolHelper(WinBase.CreateProcess(null,
launchArgs,
IntPtr.Zero,
IntPtr.Zero,
false,
WinBase.CP_CreationFlags.CREATE_SUSPENDED,
IntPtr.Zero,
null,
ref si,
out pi),
"Attempting to create child host window process.");
Log.Comment($"Host window PID: {pi.dwProcessId}");
NativeMethods.Win32BoolHelper(WinBase.AssignProcessToJobObject(job, pi.hProcess), "Assigning new host window (suspended) to job object.");
NativeMethods.Win32BoolHelper(-1 != WinBase.ResumeThread(pi.hThread), "Resume host window process now that it is attached and its launch of the child application will be caught in the job object.");
Globals.WaitForTimeout();
DesiredCapabilities appCapabilities = new DesiredCapabilities();
appCapabilities.SetCapability("app", @"Root");
Session = new IOSDriver<IOSElement>(new Uri(AppDriverUrl), appCapabilities);
Verify.IsNotNull(Session);
Actions = new Actions(Session);
Verify.IsNotNull(Session);
Globals.WaitForLongTimeout();
UIRoot = Session.FindElementByName(WindowTitleToFind);
// Set the timeout to 15 seconds after we found the initial window.
Session.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(15);
}
private bool IsRunningAsAdmin()
{
return new WindowsPrincipal(WindowsIdentity.GetCurrent()).IsInRole(WindowsBuiltInRole.Administrator);
}
private void ExitProcess()
{
Globals.SweepAllModules(this.context);
// Release attachment to the child process console.
WinCon.FreeConsole();
this.UIRoot = null;
if (this.job != IntPtr.Zero)
{
WinBase.TerminateJobObject(this.job, 0);
}
this.job = IntPtr.Zero;
}
}
}