Add benchmark to test compiler performance (#16083)

test/perf/benchmarks/Engine.Compiler.cs

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+#if NET6_0
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Management.Automation;
+using System.Management.Automation.Language;
+
+using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
+
+namespace Engine
+{
+    [BenchmarkCategory(Categories.Engine, Categories.Internal)]
+    public class Compiler
+    {
+        private static readonly Dictionary<string, ScriptBlockAst> s_scriptBlocksDict;
+        private static readonly List<string> s_functionNames;
+        private ScriptBlockAst _currentAst;
+
+        static Compiler()
+        {
+            string pattern = string.Format("{0}test{0}perf{0}benchmarks", Path.DirectorySeparatorChar);
+            string location = typeof(Compiler).Assembly.Location;
+            string testFilePath = null;
+
+            int start = location.IndexOf(pattern, StringComparison.Ordinal);
+            if (start > 0)
+            {
+                testFilePath = Path.Join(location.AsSpan(0, start + pattern.Length), "assets", "compiler.test.ps1");
+            }
+
+            var topScriptBlockAst = Parser.ParseFile(testFilePath, tokens: out _, errors: out _);
+            var allFunctions = topScriptBlockAst.FindAll(ast => ast is FunctionDefinitionAst, searchNestedScriptBlocks: false);
+
+            s_scriptBlocksDict = new Dictionary<string, ScriptBlockAst>(capacity: 16);
+            s_functionNames = new List<string>(capacity: 16);
+
+            foreach (FunctionDefinitionAst function in allFunctions)
+            {
+                s_functionNames.Add(function.Name);
+                s_scriptBlocksDict.Add(function.Name, function.Body);
+            }
+        }
+
+        [ParamsSource(nameof(FunctionName))]
+        public string FunctionsToCompile { get; set; }
+
+        public IEnumerable<string> FunctionName() => s_functionNames;
+
+        [GlobalSetup(Target = nameof(CompileFunction))]
+        public void GlobalSetup()
+        {
+            _currentAst = s_scriptBlocksDict[FunctionsToCompile];
+
+            // Run it once to get the C# code jitted.
+            // The first call to this takes relatively too long, which makes the BDN's heuristic incorrectly
+            // believe that there is no need to run many ops in each interation. However, the subsequent runs
+            // of this method is much faster than the first run, and this causes 'MinIterationTime' warnings
+            // to our benchmarks and make the benchmark results not reliable.
+            // Calling this method once in 'GlobalSetup' is a workaround. 
+            // See https://github.com/dotnet/BenchmarkDotNet/issues/837#issuecomment-828600157
+            CompileFunction();
+        }
+
+        [Benchmark]
+        public bool CompileFunction()
+        {
+            var compiledData = new CompiledScriptBlockData(_currentAst, isFilter: false);
+            return compiledData.Compile(true);
+        }
+    }
+}
+
+#endif

test/perf/benchmarks/Program.cs

@@ -10,7 +10,7 @@ using BenchmarkDotNet.Extensions;
 
 namespace MicroBenchmarks
 {
-    public class Program
+    public sealed class Program
     {
         public static int Main(string[] args)
         {

test/perf/benchmarks/powershell-perf.csproj

@@ -46,6 +46,7 @@
   <ItemGroup Condition="'$(PerfTargetVersion)' == ''">
     <ProjectReference Include="../../../src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj" />
     <ProjectReference Include="../../../src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj" />
+    <ProjectReference Include="../../../src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(PerfTargetVersion)' != ''">
@@ -59,6 +60,8 @@
     <PackageReference Include="Microsoft.PowerShell.ConsoleHost" Version="$(PerfTargetVersion)" />
     <PackageReference Include="Microsoft.PowerShell.Security" Version="$(PerfTargetVersion)" />
     <PackageReference Include="Microsoft.PowerShell.CoreCLR.Eventing" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.WSMan.Management" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.WSMan.Runtime" Version="$(PerfTargetVersion)" />
     <PackageReference Include="System.Management.Automation" Version="$(PerfTargetVersion)" />
   </ItemGroup>
 

test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj

@@ -6,8 +6,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.12.1.1521" />
-    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.1.1521" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.1" />
   </ItemGroup>
 
   <ItemGroup>

test/perf/dotnet-tools/ResultsComparer/Program.cs

@@ -20,7 +20,7 @@ using Newtonsoft.Json;
 
 namespace ResultsComparer
 {
-    public class Program
+    public sealed class Program
     {
         private const string FullBdnJsonFileExtension = "full.json";
 

tools/UpdateDotnetRuntime.ps1

@@ -79,6 +79,7 @@ function Update-PackageVersion {
         "$PSScriptRoot/packaging/projects/reference/Microsoft.PowerShell.ConsoleHost/Microsoft.PowerShell.ConsoleHost.csproj"
         "$PSScriptRoot/../src/"
         "$PSScriptRoot/../test/tools/"
+        "$PSScriptRoot/../test/perf/dotnet-tools/"
     )
 
     Get-ChildItem -Path $paths -Recurse -Filter "*.csproj" -Exclude 'PSGalleryModules.csproj', 'PSGalleryTestModules.csproj' | ForEach-Object {