maxmustermann/PowerShell
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Experimental feature: Implicit remoting batching perf improvement (#8038)

Browse source
This commit is contained in:
Paul Higinbotham 2018-10-30 09:55:39 -07:00 committed by Aditya Patwardhan
parent 1aa5bb3576
commit 20f3a6a337
9 changed files with 580 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs
View file

@ -1985,6 +1985,7 @@ namespace Microsoft.PowerShell.Commands
PrivateData = @{{
ImplicitRemoting = $true
ImplicitSessionId = '{4}'
}}
}}
";
@ -2003,7 +2004,8 @@ namespace Microsoft.PowerShell.Commands
CodeGeneration.EscapeSingleQuotedStringContent(_moduleGuid.ToString()),
CodeGeneration.EscapeSingleQuotedStringContent(StringUtil.Format(ImplicitRemotingStrings.ProxyModuleDescription, this.GetConnectionString())),
CodeGeneration.EscapeSingleQuotedStringContent(Path.GetFileName(psm1fileName)),
CodeGeneration.EscapeSingleQuotedStringContent(Path.GetFileName(formatPs1xmlFileName)));
CodeGeneration.EscapeSingleQuotedStringContent(Path.GetFileName(formatPs1xmlFileName)),
this._remoteRunspaceInfo.InstanceId);
}
#endregion

18
src/Microsoft.PowerShell.ConsoleHost/host/msh/Executor.cs
View file

@ -3,10 +3,12 @@
using System;
using System.Collections.ObjectModel;
using System.Collections.Generic;
using System.Management.Automation;
using System.Management.Automation.Runspaces;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Management.Automation.Language;
using Dbg = System.Management.Automation.Diagnostics;
@ -320,6 +322,22 @@ namespace Microsoft.PowerShell
{
Dbg.Assert(!String.IsNullOrEmpty(command), "command should have a value");
// Experimental:
// Check for implicit remoting commands that can be batched, and execute as batched if able.
if (ExperimentalFeature.IsEnabled("PSImplicitRemotingBatching"))
{
var addOutputter = ((options & ExecutionOptions.AddOutputter) > 0);
if (addOutputter &&
!_parent.RunspaceRef.IsRunspaceOverridden &&
_parent.RunspaceRef.Runspace.ExecutionContext.Modules != null &&
_parent.RunspaceRef.Runspace.ExecutionContext.Modules.IsImplicitRemotingModuleLoaded &&
Utils.TryRunAsImplicitBatch(command, _parent.RunspaceRef.Runspace))
{
exceptionThrown = null;
return null;
}
}
Pipeline tempPipeline = CreatePipeline(command, (options & ExecutionOptions.AddToHistory) > 0);
return ExecuteCommandHelper(tempPipeline, out exceptionThrown, options);

4
src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs
View file

@ -88,6 +88,10 @@ namespace System.Management.Automation
source: EngineSource,
isEnabled: false),
*/
new ExperimentalFeature(name: "PSImplicitRemotingBatching",
description: "Batch implicit remoting proxy commands to improve performance",
source: EngineSource,
isEnabled: false)
};
EngineExperimentalFeatures = new ReadOnlyCollection<ExperimentalFeature>(engineFeatures);

22
src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs
View file

@ -5034,6 +5034,21 @@ namespace Microsoft.PowerShell.Commands
// And the appdomain level module path cache.
PSModuleInfo.RemoveFromAppDomainLevelCache(module.Name);
// Update implicit module loaded property
if (Context.Modules.IsImplicitRemotingModuleLoaded)
{
Context.Modules.IsImplicitRemotingModuleLoaded = false;
foreach (var modInfo in Context.Modules.ModuleTable.Values)
{
var privateData = modInfo.PrivateData as Hashtable;
if ((privateData != null) && privateData.ContainsKey("ImplicitRemoting"))
{
Context.Modules.IsImplicitRemotingModuleLoaded = true;
break;
}
}
}
}
}
}
@ -6883,6 +6898,13 @@ namespace Microsoft.PowerShell.Commands
{
targetSessionState.Module.AddNestedModule(module);
}
var privateDataHashTable = module.PrivateData as Hashtable;
if (context.Modules.IsImplicitRemotingModuleLoaded == false &&
privateDataHashTable != null && privateDataHashTable.ContainsKey("ImplicitRemoting"))
{
context.Modules.IsImplicitRemotingModuleLoaded = true;
}
}
/// <summary>

9
src/System.Management.Automation/engine/Modules/ModuleIntrinsics.cs
View file

@ -50,6 +50,15 @@ namespace System.Management.Automation
private const int MaxModuleNestingDepth = 10;
/// <summary>
/// Gets and sets boolean that indicates when an implicit remoting module is loaded.
/// </summary>
internal bool IsImplicitRemotingModuleLoaded
{
get;
set;
}
internal void IncrementModuleNestingDepth(PSCmdlet cmdlet, string path)
{
if (++ModuleNestingDepth > MaxModuleNestingDepth)

385
src/System.Management.Automation/engine/Utils.cs
View file

@ -6,6 +6,8 @@ using System.Runtime.InteropServices;
using System.Diagnostics.CodeAnalysis;
using System.Management.Automation.Configuration;
using System.Management.Automation.Internal;
using System.Management.Automation.Language;
using System.Management.Automation.Runspaces;
using System.Management.Automation.Security;
using System.Reflection;
using Microsoft.PowerShell.Commands;
@ -1359,7 +1361,377 @@ namespace System.Management.Automation
return obj != null && Marshal.IsComObject(obj);
#endif
}
#region Implicit Remoting Batching
// Commands allowed to run on target remote session along with implicit remote commands
private static readonly HashSet<string> AllowedCommands = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
"ForEach-Object",
"Measure-Command",
"Measure-Object",
"Sort-Object",
"Where-Object"
};
// Determines if the typed command invokes implicit remoting module proxy functions in such
// a way as to allow simple batching, to reduce round trips between client and server sessions.
// Requirements:
// a. All commands must be implicit remoting module proxy commands targeted to the same remote session
// b. Except for *allowed* commands that can be safely run on remote session rather than client session
// c. Commands must be in a simple pipeline
internal static bool TryRunAsImplicitBatch(string command, Runspace runspace)
{
try
{
var scriptBlock = ScriptBlock.Create(command);
var scriptBlockAst = scriptBlock.Ast as ScriptBlockAst;
if (scriptBlockAst == null)
{
return false;
}
// Make sure that this is a simple pipeline
string errorId;
string errorMsg;
scriptBlockAst.GetSimplePipeline(true, out errorId, out errorMsg);
if (errorId != null)
{
return false;
}
// Run checker
var checker = new PipelineForBatchingChecker { ScriptBeingConverted = scriptBlockAst };
scriptBlockAst.InternalVisit(checker);
// If this is just a single command, there is no point in batching it
if (checker.Commands.Count < 2)
{
return false;
}
// We have a valid batching candidate
using (var ps = System.Management.Automation.PowerShell.Create())
{
ps.Runspace = runspace;
// Check commands
if (!TryGetCommandInfoList(ps, checker.Commands, out Collection<CommandInfo> cmdInfoList))
{
return false;
}
// All command modules must be implicit remoting modules from the same PSSession
var success = true;
var psSessionId = Guid.Empty;
foreach (var cmdInfo in cmdInfoList)
{
// Check for allowed command
string cmdName = (cmdInfo is AliasInfo aliasInfo) ? aliasInfo.ReferencedCommand.Name : cmdInfo.Name;
if (AllowedCommands.Contains(cmdName))
{
continue;
}
// Commands must be from implicit remoting module
if (cmdInfo.Module == null || string.IsNullOrEmpty(cmdInfo.ModuleName))
{
success = false;
break;
}
// Commands must be from modules imported into the same remote session
if (cmdInfo.Module.PrivateData is System.Collections.Hashtable privateData)
{
var sessionIdString = privateData["ImplicitSessionId"] as string;
if (string.IsNullOrEmpty(sessionIdString))
{
success = false;
break;
}
var sessionId = new Guid(sessionIdString);
if (psSessionId == Guid.Empty)
{
psSessionId = sessionId;
}
else if (psSessionId != sessionId)
{
success = false;
break;
}
}
else
{
success = false;
break;
}
}
if (success)
{
//
// Invoke command pipeline as entire pipeline on remote session
//
// Update script to declare variables via Using keyword
if (checker.ValidVariables.Count > 0)
{
foreach (var variableName in checker.ValidVariables)
{
command = command.Replace(variableName, ("Using:" + variableName), StringComparison.OrdinalIgnoreCase);
}
scriptBlock = ScriptBlock.Create(command);
}
// Retrieve the PSSession runspace in which to run the batch script on
ps.Commands.Clear();
ps.Commands.AddCommand("Get-PSSession").AddParameter("InstanceId", psSessionId);
var psSession = ps.Invoke<System.Management.Automation.Runspaces.PSSession>().FirstOrDefault();
if (psSession == null || (ps.Streams.Error.Count > 0) || (psSession.Availability != RunspaceAvailability.Available))
{
return false;
}
// Create and invoke implicit remoting command pipeline
ps.Commands.Clear();
ps.AddCommand("Invoke-Command").AddParameter("Session", psSession).AddParameter("ScriptBlock", scriptBlock).AddParameter("HideComputerName", true)
.AddCommand("Out-Default");
try
{
ps.Invoke();
}
catch (Exception ex)
{
var errorRecord = new ErrorRecord(ex, "ImplicitRemotingBatchExecutionTerminatingError", ErrorCategory.InvalidOperation, null);
ps.Commands.Clear();
ps.AddCommand("Write-Error").AddParameter("InputObject", errorRecord).Invoke();
}
return true;
}
}
}
catch (Exception) { }
return false;
}
private const string WhereObjectCommandAlias = "?";
private static bool TryGetCommandInfoList(PowerShell ps, HashSet<string> commandNames, out Collection<CommandInfo> cmdInfoList)
{
if (commandNames.Count == 0)
{
cmdInfoList = null;
return false;
}
bool specialCaseWhereCommandAlias = commandNames.Contains(WhereObjectCommandAlias);
if (specialCaseWhereCommandAlias)
{
commandNames.Remove(WhereObjectCommandAlias);
}
// Use Get-Command to collect CommandInfo from candidate commands, with correct precedence so
// that implicit remoting proxy commands will appear when available.
ps.Commands.Clear();
ps.Commands.AddCommand("Get-Command").AddParameter("Name", commandNames.ToArray());
cmdInfoList = ps.Invoke<CommandInfo>();
if (ps.Streams.Error.Count > 0)
{
return false;
}
// For special case '?' alias don't use Get-Command to retrieve command info, and instead
// use the GetCommand API.
if (specialCaseWhereCommandAlias)
{
var cmdInfo = ps.Runspace.ExecutionContext.SessionState.InvokeCommand.GetCommand(WhereObjectCommandAlias, CommandTypes.Alias);
if (cmdInfo == null)
{
return false;
}
cmdInfoList.Add(cmdInfo);
}
return true;
}
#endregion
}
#region ImplicitRemotingBatching
// A visitor to walk an AST and validate that it is a candidate for implicit remoting batching.
// Based on ScriptBlockToPowerShellChecker.
internal class PipelineForBatchingChecker : AstVisitor
{
internal readonly HashSet<string> ValidVariables = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
internal readonly HashSet<string> Commands = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
internal ScriptBlockAst ScriptBeingConverted { get; set; }
public override AstVisitAction VisitVariableExpression(VariableExpressionAst variableExpressionAst)
{
if (!variableExpressionAst.VariablePath.IsAnyLocal())
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"VariableTypeNotSupported"),
variableExpressionAst);
}
if (variableExpressionAst.VariablePath.UnqualifiedPath != "_")
{
ValidVariables.Add(variableExpressionAst.VariablePath.UnqualifiedPath);
}
return AstVisitAction.Continue;
}
public override AstVisitAction VisitPipeline(PipelineAst pipelineAst)
{
if (pipelineAst.PipelineElements[0] is CommandExpressionAst)
{
// If the first element is a CommandExpression, this pipeline should be the value
// of a parameter. We want to avoid a scriptblock that contains only a pure expression.
// The check "pipelineAst.Parent.Parent == ScriptBeingConverted" guarantees we throw
// error on that kind of scriptblock.
// Disallow pure expressions at the "top" level, but allow them otherwise.
// We want to catch:
// 1 | echo
// But we don't want to error out on:
// echo $(1)
// See the comment in VisitCommand on why it's safe to check Parent.Parent, we
// know that we have at least:
// * a NamedBlockAst (the end block)
// * a ScriptBlockAst (the ast we're comparing to)
if (pipelineAst.GetPureExpression() == null || pipelineAst.Parent.Parent == ScriptBeingConverted)
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"PipelineStartingWithExpressionNotSupported"),
pipelineAst);
}
}
return AstVisitAction.Continue;
}
public override AstVisitAction VisitCommand(CommandAst commandAst)
{
if (commandAst.InvocationOperator == TokenKind.Dot)
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"DotSourcingNotSupported"),
commandAst);
}
/*
// Up front checking ensures that we have a simple script block,
// so we can safely assume that the parents are:
// * a PipelineAst
// * a NamedBlockAst (the end block)
// * a ScriptBlockAst (the ast we're comparing to)
// If that isn't the case, the conversion isn't allowed. It
// is also safe to assume that we have at least 3 parents, a script block can't be simpler.
if (commandAst.Parent.Parent.Parent != ScriptBeingConverted)
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"CantConvertWithCommandInvocations not supported"),
commandAst);
}
*/
if (commandAst.CommandElements[0] is ScriptBlockExpressionAst)
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"ScriptBlockInvocationNotSupported"),
commandAst);
}
var commandName = commandAst.GetCommandName();
if (commandName != null)
{
Commands.Add(commandName);
}
return AstVisitAction.Continue;
}
public override AstVisitAction VisitMergingRedirection(MergingRedirectionAst redirectionAst)
{
if (redirectionAst.ToStream != RedirectionStream.Output)
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"MergeRedirectionNotSupported"),
redirectionAst);
}
return AstVisitAction.Continue;
}
public override AstVisitAction VisitFileRedirection(FileRedirectionAst redirectionAst)
{
ThrowError(
new ImplicitRemotingBatchingNotSupportedException(
"FileRedirectionNotSupported"),
redirectionAst);
return AstVisitAction.Continue;
}
/*
public override AstVisitAction VisitScriptBlockExpression(ScriptBlockExpressionAst scriptBlockExpressionAst)
{
ThrowError(new ImplicitRemotingBatchingNotSupportedException(
"ScriptBlocks not supported"),
scriptBlockExpressionAst);
return AstVisitAction.SkipChildren;
}
*/
public override AstVisitAction VisitUsingExpression(UsingExpressionAst usingExpressionAst)
{
// Using expressions are not expected in Implicit remoting commands.
ThrowError(new ImplicitRemotingBatchingNotSupportedException(
"UsingExpressionNotSupported"),
usingExpressionAst);
return AstVisitAction.SkipChildren;
}
internal static void ThrowError(ImplicitRemotingBatchingNotSupportedException ex, Ast ast)
{
InterpreterError.UpdateExceptionErrorRecordPosition(ex, ast.Extent);
throw ex;
}
}
internal class ImplicitRemotingBatchingNotSupportedException : Exception
{
internal string ErrorId
{
get;
private set;
}
internal ImplicitRemotingBatchingNotSupportedException(string errorId) : base(
ParserStrings.ImplicitRemotingPipelineBatchingNotSupported)
{
ErrorId = errorId;
}
}
#endregion
}
namespace System.Management.Automation.Internal
@ -1404,6 +1776,19 @@ namespace System.Management.Automation.Internal
fieldInfo.SetValue(null, value);
}
}
/// <summary>
/// Test hook used to test implicit remoting batching. A local runspace must be provided that has imported a
/// remote session, i.e., has run the Import-PSSession cmdlet. This hook will return true if the provided commandPipeline
/// is successfully batched and run in the remote session, and false if it is rejected for batching.
/// </summary>
/// <param name="commandPipeline">Command pipeline to test</param>
/// <param name="runspace">Runspace with imported remote session</param>
/// <returns>True if commandPipeline is batched successfully</returns>
public static bool TestImplicitRemotingBatching(string commandPipeline, System.Management.Automation.Runspaces.Runspace runspace)
{
return Utils.TryRunAsImplicitBatch(commandPipeline, runspace);
}
}
/// <summary>

3
src/System.Management.Automation/resources/ParserStrings.resx
View file

@ -1467,4 +1467,7 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent
<data name="ParserError" xml:space="preserve">
<value>{0}</value>
</data>
<data name="ImplicitRemotingPipelineBatchingNotSupported" xml:space="preserve">
<value>Command pipeline not supported for implicit remoting batching.</value>
</data>
</root>

135
test/powershell/engine/Remoting/ImplicitRemotingBatching.Tests.ps1 Normal file
View file

@ -0,0 +1,135 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
Describe "TestImplicitRemotingBatching hook should correctly batch simple remote command pipelines" -Tags 'Feature','RequireAdminOnWindows' {
BeforeAll {
if (! $isWindows) { return }
[powershell] $powerShell = [powershell]::Create([System.Management.Automation.RunspaceMode]::NewRunspace)
# Create remote session in new PowerShell session
$powerShell.AddScript('Import-Module -Name HelpersRemoting; $remoteSession = New-RemoteSession').Invoke()
if ($powerShell.Streams.Error.Count -gt 0) { throw "Unable to create remote session for test" }
# Import implicit commands from remote session
$powerShell.Commands.Clear()
$powerShell.AddScript('Import-PSSession -Session $remoteSession -CommandName Get-Process,Write-Output -AllowClobber').Invoke()
if ($powerShell.Streams.Error.Count -gt 0) { throw "Unable to import pssession for test" }
# Define $filter variable in local session
$powerShell.Commands.Clear()
$powerShell.AddScript('$filter = "pwsh","powershell"').Invoke()
$localRunspace = $powerShell.Runspace
[powershell] $psInvoke = [powershell]::Create([System.Management.Automation.RunspaceMode]::NewRunspace)
$testCases = @(
@{
Name = 'Two implicit commands should be successfully batched'
CommandLine = 'Get-Process -Name "pwsh" | Write-Output'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Where-Object should be successfully batched'
CommandLine = 'Get-Process | Write-Output | Where-Object { $_.Name -like "*pwsh*" }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Where-Object alias (?) should be successfully batched'
CommandLine = 'Get-Process | Write-Output | ? { $_.Name -like "*pwsh*" }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Where-Object alias (where) should be successfully batched'
CommandLine = 'Get-Process | Write-Output | where { $_.Name -like "*pwsh*" }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Sort-Object should be successfully batched'
CommandLine = 'Get-Process -Name "pwsh" | Sort-Object -Property Name | Write-Output'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Sort-Object alias (sort) should be successfully batched'
CommandLine = 'Get-Process -Name "pwsh" | sort -Property Name | Write-Output'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with ForEach-Object should be successfully batched'
CommandLine = 'Get-Process -Name "pwsh" | Write-Output | ForEach-Object { $_ }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with ForEach-Object alias (%) should be successfully batched'
CommandLine = 'Get-Process -Name "pwsh" | Write-Output | % { $_ }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with ForEach-Object alias (foreach) should be successfully batched'
CommandLine = 'Get-Process -Name "pwsh" | Write-Output | foreach { $_ }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Measure-Command should be successfully batched'
CommandLine = 'Measure-Command { Get-Process | Write-Output }'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Measure-Object should be successfully batched'
CommandLine = 'Get-Process | Write-Output | Measure-Object'
ExpectedOutput = $true
},
@{
Name = 'Two implicit commands with Measure-Object alias (measure) should be successfully batched'
CommandLine = 'Get-Process | Write-Output | measure'
ExpectedOutput = $true
},
@{
Name = 'Implicit commands with variable arguments should be successfully batched'
CommandLine = 'Get-Process -Name $filter | Write-Output'
ExpectedOutput = $true
},
@{
Name = 'Pipeline with non-implicit command should not be batched'
CommandLine = 'Get-Process | Write-Output | Select-Object -Property Name'
ExpectedOutput = $false
},
@{
Name = 'Non-simple pipeline should not be batched'
CommandLine = '1..2 | % { Get-Process pwsh | Write-Output }'
ExpectedOutput = $false
}
@{
Name = 'Pipeline with single command should not be batched'
CommandLine = 'Get-Process pwsh'
ExpectedOutput = $false
},
@{
Name = 'Pipeline without any implicit commands should not be batched'
CommandLine = 'Get-PSSession | Out-Default'
ExpectedOutput = $false
}
)
}
AfterAll {
if (! $isWindows) { return }
if ($remoteSession -ne $null) { Remove-PSSession $remoteSession -ErrorAction Ignore }
if ($powershell -ne $null) { $powershell.Dispose() }
if ($psInvoke -ne $null) { $psInvoke.Dispose() }
}
It "<Name>" -TestCases $testCases -Skip:(! $IsWindows) {
param ($CommandLine, $ExpectedOutput)
$psInvoke.Commands.Clear()
$psInvoke.Commands.AddScript('param ($cmdLine, $runspace) [System.Management.Automation.Internal.InternalTestHooks]::TestImplicitRemotingBatching($cmdLine, $runspace)').AddArgument($CommandLine).AddArgument($localRunspace)
$result = $psInvoke.Invoke()
$result | Should Be $ExpectedOutput
}
}

1
test/tools/Modules/HelpersRemoting/HelpersRemoting.psm1
View file

@ -88,6 +88,7 @@ function CreateParameters
return $parameters
}
function New-RemoteSession
{
param (