From 2280bea11e0e21313ad668da8363f70ca1e4baae Mon Sep 17 00:00:00 2001 From: Sergei Vorobev Date: Thu, 28 Jul 2016 15:47:38 -0700 Subject: [PATCH 1/4] Consolidate map.json files under Microsoft.PowerShell.Commands.Utility Fix #1489 --- .../map.json | 3 -- .../commands/utility/WebCmdlet/map.json | 39 ------------------- .../map.json | 38 ++++++++++++++++++ 3 files changed, 38 insertions(+), 42 deletions(-) delete mode 100644 src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/map.json diff --git a/src/Microsoft.Management.Infrastructure.CimCmdlets/map.json b/src/Microsoft.Management.Infrastructure.CimCmdlets/map.json index 439b349e8..b6a1acc7b 100644 --- a/src/Microsoft.Management.Infrastructure.CimCmdlets/map.json +++ b/src/Microsoft.Management.Infrastructure.CimCmdlets/map.json @@ -34,8 +34,5 @@ "wmi/WMIv2/client/CIMCmdlets/RemoveCimSessionCommand.cs" : "RemoveCimSessionCommand.cs", "wmi/WMIv2/client/CIMCmdlets/SetCimInstanceCommand.cs" : "SetCimInstanceCommand.cs", "wmi/WMIv2/client/CIMCmdlets/Utils.cs" : "Utils.cs", - "wmi/WMIv2/client/CIMCmdlets/BinaryMiLogBase.cs" : "BinaryMiLogBase.cs", - "wmi/WMIv2/client/CIMCmdlets/ExportBinaryMiLogCommand.cs" : "ExportBinaryMiLogCommand.cs", - "wmi/WMIv2/client/CIMCmdlets/ImportBinaryMiLogCommand.cs" : "ImportBinaryMiLogCommand.cs", "wmi/WMIv2/client/CIMCmdlets/Strings.resx" : "resources/Microsoft.Management.Infrastructure.CimCmdlets.Strings.resx" } \ No newline at end of file diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/map.json b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/map.json deleted file mode 100644 index 917a64958..000000000 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/map.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "monad/src/commands/utility/WebCmdlet/ConvertFromJsonCommand.cs": "ConvertFromJsonCommand.cs", - "monad/src/commands/utility/WebCmdlet/ConvertToJsonCommand.cs": "ConvertToJsonCommand.cs", - "monad/src/commands/utility/WebCmdlet/FormObject.cs": "FormObject.cs", - "monad/src/commands/utility/WebCmdlet/FormObjectCollection.cs": "FormObjectCollection.cs", - "monad/src/commands/utility/WebCmdlet/JsonObject.cs": "JsonObject.cs", - "monad/src/commands/utility/WebCmdlet/PSUserAgent.cs": "PSUserAgent.cs", - "monad/src/commands/utility/WebCmdlet/StreamHelper.cs": "StreamHelper.cs", - "monad/src/commands/utility/WebCmdlet/WebCmdletElementCollection.cs": "WebCmdletElementCollection.cs", - "monad/src/commands/utility/WebCmdlet/WebRequestMethod.cs": "WebRequestMethod.cs", - "monad/src/commands/utility/WebCmdlet/WebRequestSession.cs": "WebRequestSession.cs", - "monad/src/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs": "Common/BasicHtmlWebResponseObject.Common.cs", - "monad/src/commands/utility/WebCmdlet/Common/ContentHelper.Common.cs": "Common/ContentHelper.Common.cs", - "monad/src/commands/utility/WebCmdlet/Common/HtmlWebResponseObject.Common.cs": "Common/HtmlWebResponseObject.Common.cs", - "monad/src/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs": "Common/InvokeRestMethodCommand.Common.cs", - "monad/src/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs": "Common/WebRequestPSCmdlet.Common.cs", - "monad/src/commands/utility/WebCmdlet/Common/WebResponseObject.Common.cs": "Common/WebResponseObject.Common.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/BasicHtmlWebResponseObject.CoreClr.cs": "CoreCLR/BasicHtmlWebResponseObject.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/ContentHelper.CoreClr.cs": "CoreCLR/ContentHelper.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/HtmlWebResponseObject.CoreClr.cs": "CoreCLR/HtmlWebResponseObject.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/HttpKnownHeaderNames.cs": "CoreCLR/HttpKnownHeaderNames.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/InvokeRestMethodCommand.CoreClr.cs": "CoreCLR/InvokeRestMethodCommand.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs": "CoreCLR/InvokeWebRequestCommand.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/WebProxy.cs": "CoreCLR/WebProxy.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs": "CoreCLR/WebRequestPSCmdlet.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs": "CoreCLR/WebResponseHelper.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs": "CoreCLR/WebResponseObject.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/CoreCLR/WebResponseObjectFactory.CoreClr.cs": "CoreCLR/WebResponseObjectFactory.CoreClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/BasicHtmlWebResponseObject.FullClr.cs": "FullClr/BasicHtmlWebResponseObject.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/ContentHelper.FullClr.cs": "FullClr/ContentHelper.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/HtmlWebResponseObject.FullClr.cs": "FullClr/HtmlWebResponseObject.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/InvokeRestMethodCommand.FullClr.cs": "FullClr/InvokeRestMethodCommand.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/InvokeWebRequestCommand.FullClr.cs": "FullClr/InvokeWebRequestCommand.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/JsonObjectTypeResolver.cs": "FullClr/JsonObjectTypeResolver.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/WebRequestPSCmdlet.FullClr.cs": "FullClr/WebRequestPSCmdlet.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/WebResponseHelper.FullClr.cs": "FullClr/WebResponseHelper.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/WebResponseObject.FullClr.cs": "FullClr/WebResponseObject.FullClr.cs", - "monad/src/commands/utility/WebCmdlet/FullClr/WebResponseObjectFactory.FullClr.cs": "FullClr/WebResponseObjectFactory.FullClr.cs" -} \ No newline at end of file diff --git a/src/Microsoft.PowerShell.Commands.Utility/map.json b/src/Microsoft.PowerShell.Commands.Utility/map.json index eac3a5593..a351c8d42 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/map.json +++ b/src/Microsoft.PowerShell.Commands.Utility/map.json @@ -134,4 +134,42 @@ "monad/src/commands/utility/WriteProgressCmdlet.cs": "commands/utility/WriteProgressCmdlet.cs", "monad/src/commands/utility/XmlCommands.cs": "commands/utility/XmlCommands.cs", "monad/src/singleshell/installer/MshUtilityMshSnapin.cs": "singleshell/installer/MshUtilityMshSnapin.cs", + + "monad/src/commands/utility/WebCmdlet/ConvertFromJsonCommand.cs": "ConvertFromJsonCommand.cs", + "monad/src/commands/utility/WebCmdlet/ConvertToJsonCommand.cs": "ConvertToJsonCommand.cs", + "monad/src/commands/utility/WebCmdlet/FormObject.cs": "FormObject.cs", + "monad/src/commands/utility/WebCmdlet/FormObjectCollection.cs": "FormObjectCollection.cs", + "monad/src/commands/utility/WebCmdlet/JsonObject.cs": "JsonObject.cs", + "monad/src/commands/utility/WebCmdlet/PSUserAgent.cs": "PSUserAgent.cs", + "monad/src/commands/utility/WebCmdlet/StreamHelper.cs": "StreamHelper.cs", + "monad/src/commands/utility/WebCmdlet/WebCmdletElementCollection.cs": "WebCmdletElementCollection.cs", + "monad/src/commands/utility/WebCmdlet/WebRequestMethod.cs": "WebRequestMethod.cs", + "monad/src/commands/utility/WebCmdlet/WebRequestSession.cs": "WebRequestSession.cs", + "monad/src/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs": "Common/BasicHtmlWebResponseObject.Common.cs", + "monad/src/commands/utility/WebCmdlet/Common/ContentHelper.Common.cs": "Common/ContentHelper.Common.cs", + "monad/src/commands/utility/WebCmdlet/Common/HtmlWebResponseObject.Common.cs": "Common/HtmlWebResponseObject.Common.cs", + "monad/src/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs": "Common/InvokeRestMethodCommand.Common.cs", + "monad/src/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs": "Common/WebRequestPSCmdlet.Common.cs", + "monad/src/commands/utility/WebCmdlet/Common/WebResponseObject.Common.cs": "Common/WebResponseObject.Common.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/BasicHtmlWebResponseObject.CoreClr.cs": "CoreCLR/BasicHtmlWebResponseObject.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/ContentHelper.CoreClr.cs": "CoreCLR/ContentHelper.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/HtmlWebResponseObject.CoreClr.cs": "CoreCLR/HtmlWebResponseObject.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/HttpKnownHeaderNames.cs": "CoreCLR/HttpKnownHeaderNames.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/InvokeRestMethodCommand.CoreClr.cs": "CoreCLR/InvokeRestMethodCommand.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs": "CoreCLR/InvokeWebRequestCommand.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/WebProxy.cs": "CoreCLR/WebProxy.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs": "CoreCLR/WebRequestPSCmdlet.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs": "CoreCLR/WebResponseHelper.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs": "CoreCLR/WebResponseObject.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/CoreCLR/WebResponseObjectFactory.CoreClr.cs": "CoreCLR/WebResponseObjectFactory.CoreClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/BasicHtmlWebResponseObject.FullClr.cs": "FullClr/BasicHtmlWebResponseObject.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/ContentHelper.FullClr.cs": "FullClr/ContentHelper.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/HtmlWebResponseObject.FullClr.cs": "FullClr/HtmlWebResponseObject.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/InvokeRestMethodCommand.FullClr.cs": "FullClr/InvokeRestMethodCommand.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/InvokeWebRequestCommand.FullClr.cs": "FullClr/InvokeWebRequestCommand.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/JsonObjectTypeResolver.cs": "FullClr/JsonObjectTypeResolver.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/WebRequestPSCmdlet.FullClr.cs": "FullClr/WebRequestPSCmdlet.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/WebResponseHelper.FullClr.cs": "FullClr/WebResponseHelper.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/WebResponseObject.FullClr.cs": "FullClr/WebResponseObject.FullClr.cs", + "monad/src/commands/utility/WebCmdlet/FullClr/WebResponseObjectFactory.FullClr.cs": "FullClr/WebResponseObjectFactory.FullClr.cs" } From 0dd425da1994bb3c0580ca1bb0d0856d0aa7161b Mon Sep 17 00:00:00 2001 From: PowerShell Team Date: Thu, 28 Jul 2016 15:58:00 -0700 Subject: [PATCH 2/4] Integrate changes between [SD:717473] and [SD:725290] --- .../Bootstrap/BootstrapProvider.cs | 201 +++++++----------- .../Bootstrap/BootstrapRequest.cs | 65 +++++- ...nagement.Providers.Resources.Messages.resx | 10 + .../Repository/LocalPackageRepository.cs | 12 +- .../PackageList/ExePackageInstaller.cs | 6 +- .../PackageList/PackageListParser.cs | 54 ++++- .../PackageList/PackageListProvider.cs | 134 +++++++++--- .../PackageList/PackageListRequest.cs | 195 ++++++++--------- .../PackageList/PackageQuery.cs | 18 ++ .../PackageList/WebDownloader.cs | 111 +++++++--- .../PackageList/ZipPackageInstaller.cs | 6 +- .../Sdk/Constants.cs | 10 +- ...SourceListProvider.Resources.Messages.resx | 48 ++++- .../Activities/WorkflowJobConverter.cs | 43 +++- .../security/CertificateProvider.cs | 2 +- .../security/SignatureCommands.cs | 1 + .../WorkflowCore/ImportWorkflowCommand.cs | 157 ++++++++++++-- .../WorkflowCore/LocalRunspaceProvider.cs | 41 +++- .../ServiceCore/WorkflowCore/WorkflowJob2.cs | 2 + .../resources/Resources.resx | 2 +- .../Microsoft.PowerShell.Utility.psd1 | 15 +- .../PSWorkflowUtility/PSWorkflowUtility.psm1 | 3 +- .../engine/Modules/ModuleCmdletBase.cs | 26 ++- .../engine/parser/ast.cs | 3 + .../engine/runtime/CompiledScriptBlock.cs | 6 +- .../engine/runtime/Operations/MiscOps.cs | 3 +- .../resources/Modules.resx | 2 +- .../security/SecuritySupport.cs | 14 +- 28 files changed, 848 insertions(+), 342 deletions(-) diff --git a/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapProvider.cs b/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapProvider.cs index aa849feb5..13d5b6448 100644 --- a/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapProvider.cs +++ b/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapProvider.cs @@ -32,7 +32,6 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { using Directory = System.IO.Directory; using ErrorCategory = PackageManagement.Internal.ErrorCategory; using File = System.IO.File; - using System.IO.Compression; public class BootstrapProvider { private static readonly Dictionary _features = new Dictionary { @@ -324,127 +323,12 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { return false; } - case Iso19770_2.MediaType.NuGetPackage: - return InstallNugetPackage(provider, link, fastPath, request); - default: request.Warning("Provider '{0}' with link '{1}' has unknown media type '{2}'.", provider.Name, link.HRef, link.MediaType); return false; } } - private bool InstallNugetPackage(Package provider, Link link, string fastPath, BootstrapRequest request) - { - // download the nuget package - string downloadedNupkg = request.DownloadAndValidateFile(provider._swidtag); - - if (downloadedNupkg != null) - { - // extracted folder - string extractedFolder = String.Concat(FilesystemExtensions.GenerateTemporaryFileOrDirectoryNameInTempDirectory()); - - try - { - //unzip the file - ZipFile.ExtractToDirectory(downloadedNupkg, extractedFolder); - - if (Directory.Exists(extractedFolder)) - { - string versionFolder = Path.Combine(request.DestinationPath(request), provider.Name, provider.Version); - // tool folder is where we find things like nuget.exe - string toolFolder = Path.Combine(extractedFolder, "tools"); - string libFolder = Path.Combine(extractedFolder, "lib"); - - // create the directory version folder if not exist - if (!Directory.Exists(versionFolder)) - { - Directory.CreateDirectory(versionFolder); - } - - // copy the tools directory - if (Directory.Exists(toolFolder)) - { - string destinationToolFolder = Path.Combine(versionFolder, "tools"); - - if (!Directory.Exists(destinationToolFolder)) - { - Directory.CreateDirectory(destinationToolFolder); - } - - foreach (string child in Directory.EnumerateFiles(toolFolder)) - { - try - { - // try copy and overwrite - File.Copy(child, Path.Combine(destinationToolFolder, Path.GetFileName(child)), true); - } - catch (Exception e) - { - request.Debug(e.StackTrace); - if (!(e is UnauthorizedAccessException || e is IOException)) - { - // something wrong, delete the version folder - versionFolder.TryHardToDelete(); - return false; - } - - // otherwise this means the file is just being used. so just moves on to copy other files - } - } - } - - // copy files from lib - if (Directory.Exists(libFolder)) - { - // check that the lib folder has at most 1 dll - if (Directory.EnumerateFiles(libFolder).Count(file => String.Equals(Path.GetExtension(file), ".dll", StringComparison.OrdinalIgnoreCase)) > 1) - { - request.Warning(String.Format(CultureInfo.CurrentCulture, Resources.Messages.MoreThanOneDllExists, provider.Name)); - return false; - } - - foreach (string child in Directory.EnumerateFiles(libFolder)) - { - try - { - File.Copy(child, Path.Combine(versionFolder, Path.GetFileName(child)), true); - } - catch (Exception e) - { - request.Debug(e.StackTrace); - if (!(e is UnauthorizedAccessException || e is IOException)) - { - // something wrong, delete the version folder - versionFolder.TryHardToDelete(); - return false; - } - - // otherwise this means the file is just being used. so just moves on to copy other files - } - } - } - - // target file name is the assembly provider - string targetFile = Path.Combine(versionFolder, Path.GetFileName(link.Attributes[Iso19770_2.Discovery.TargetFilename])); - - if (File.Exists(targetFile)) - { - request.Verbose(Resources.Messages.InstalledPackage, provider.Name, targetFile); - request.YieldFromSwidtag(provider, fastPath); - return true; - } - } - } - finally - { - downloadedNupkg.TryHardToDelete(); - extractedFolder.TryHardToDelete(); - } - } - - return false; - } - private bool InstallPackageFile(Package provider, string fastPath, BootstrapRequest request) { // we can download and verify this package and get the core to install it. var file = request.DownloadAndValidateFile(provider._swidtag); @@ -522,7 +406,6 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { return false; } - string targetFilename = fastPath; string file = fastPath; @@ -532,7 +415,7 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { targetFilename = link.Attributes[Iso19770_2.Discovery.TargetFilename]; // download the file - file = request.DownloadAndValidateFile(provider._swidtag); + file = request.DownloadAndValidateFile(provider._swidtag); } @@ -551,22 +434,23 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { //... providername\version\.dll var versionFolder = Path.Combine(request.DestinationPath(request), provider.Name, provider.Version); - if (!Directory.Exists(versionFolder)) { - //we create it + // if version folder exists, remove it + if (Directory.Exists(versionFolder)) + { + RemoveDirectory(versionFolder); + } + + // create the directory if we successfully deleted it + if (!Directory.Exists(versionFolder)) + { Directory.CreateDirectory(versionFolder); } var targetFile = Path.Combine(versionFolder, targetFilename); - + if (file != null) { try { - // looks good! let's keep it - if (File.Exists(targetFile)) { - request.Debug("Removing old file '{0}'", targetFile); - targetFile.TryHardToDelete(); - } - // is that file still there? if (File.Exists(targetFile)) { request.Error(ErrorCategory.InvalidOperation, fastPath, Constants.Messages.UnableToRemoveFile, targetFile); @@ -575,7 +459,16 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { request.Debug("Copying file '{0}' to '{1}'", file, targetFile); try { - File.Copy(file, targetFile); + if (File.Exists(file)) + { + // if this is a file + File.Copy(file, targetFile); + } + else if (Directory.Exists(file)) + { + // if this is a directory, copy items over + CopyDirectory(file, versionFolder); + } } catch (Exception ex) { request.Debug(ex.StackTrace); @@ -599,11 +492,61 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { file.TryHardToDelete(); } } - } + } return false; } + private void RemoveDirectory(string directoryFolder) + { + // remove all files + foreach (var fileToBeRemoved in Directory.EnumerateFiles(directoryFolder)) + { + fileToBeRemoved.TryHardToDelete(); + } + + // remove all subdirectories + foreach (var folderToBeRemoved in Directory.EnumerateDirectories(directoryFolder)) + { + RemoveDirectory(folderToBeRemoved); + } + + try + { + // now try to remove the directory + Directory.Delete(directoryFolder); + } + catch { } + } + + private void CopyDirectory(string sourceFolder, string destinationFolder) + { + // check that source and destination folders exist + if (!sourceFolder.DirectoryExists() || !destinationFolder.DirectoryExists()) + { + return; + } + + // copy the files over + foreach (var file in Directory.EnumerateFiles(sourceFolder)) + { + File.Copy(file, Path.Combine(destinationFolder, Path.GetFileName(file)), true); + } + + // copy the directories over + foreach (var directory in Directory.EnumerateDirectories(sourceFolder)) + { + var destinationDirName = Path.Combine(destinationFolder, Path.GetFileName(directory)); + + if (!Directory.Exists(destinationDirName)) + { + Directory.CreateDirectory(destinationDirName); + } + + CopyDirectory(directory, destinationDirName); + } + } + private void InstallPackageFromFile(string fastPath, BootstrapRequest request) { var filePath = new Uri(fastPath).LocalPath; diff --git a/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapRequest.cs b/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapRequest.cs index 0c0f636f5..09cf02ca3 100644 --- a/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapRequest.cs +++ b/src/Microsoft.PackageManagement.CoreProviders/Bootstrap/BootstrapRequest.cs @@ -29,6 +29,9 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { using PackageManagement.Internal.Utility.Collections; using PackageManagement.Internal.Utility.Extensions; using ErrorCategory = PackageManagement.Internal.ErrorCategory; + using System.IO.Compression; + using File = System.IO.File; + using Directory = System.IO.Directory; public abstract class BootstrapRequest : Request { internal Uri[] _urls @@ -201,6 +204,51 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { } } + /// + /// Extract zipped package and return the unzipped folder + /// + /// + /// + private string ExtractZipPackage(string zippedPackagePath) + { + if (zippedPackagePath != null && zippedPackagePath.FileExists()) + { + // extracted folder + string extractedFolder = FilesystemExtensions.GenerateTemporaryFileOrDirectoryNameInTempDirectory(); + + try + { + //unzip the file + ZipFile.ExtractToDirectory(zippedPackagePath, extractedFolder); + + // extraction fails + if (!Directory.Exists(extractedFolder)) + { + Verbose(string.Format(CultureInfo.CurrentCulture, Resources.Messages.FailToExtract, zippedPackagePath, extractedFolder)); + return string.Empty; + } + + // the zipped folder + var zippedDirectory = Directory.EnumerateDirectories(extractedFolder).FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(zippedDirectory) && Directory.Exists(zippedDirectory)) + { + return zippedDirectory; + } + } + catch (Exception ex) + { + Verbose(string.Format(CultureInfo.CurrentCulture, Resources.Messages.FailToInstallZipFolder, zippedPackagePath, ex.Message)); + Debug(ex.StackTrace); + + // remove the extracted folder + extractedFolder.TryHardToDelete(); + } + } + + return string.Empty; + } + /// /// Helper function to retry downloading a file. /// downloadFileFunction is the main function that is used to download the file when given a uri @@ -257,7 +305,22 @@ namespace Microsoft.PackageManagement.Providers.Internal.Bootstrap { link.HRef); // got a valid file! - if (file != null && file.FileExists()) { + if (file != null && file.FileExists()) { + // if file is zip, unpack it and return the unpacked folder + if (link.MediaType == Iso19770_2.MediaType.ZipPackage) + { + try + { + // let's extract the zipped file + return ExtractZipPackage(file); + } + finally + { + // delete the zipped file + file.TryHardToDelete(); + } + } + return file; } } diff --git a/src/Microsoft.PackageManagement.CoreProviders/resources/Microsoft.PackageManagement.Providers.Resources.Messages.resx b/src/Microsoft.PackageManagement.CoreProviders/resources/Microsoft.PackageManagement.Providers.Resources.Messages.resx index e39083f72..7cd8c5796 100644 --- a/src/Microsoft.PackageManagement.CoreProviders/resources/Microsoft.PackageManagement.Providers.Resources.Messages.resx +++ b/src/Microsoft.PackageManagement.CoreProviders/resources/Microsoft.PackageManagement.Providers.Resources.Messages.resx @@ -121,6 +121,16 @@ Fail to extract and copy to '{0}'. 0 - The directory we are copying to + + Cannot delete existing provider folder '{0}'. + + + Cannot extract from '{0}' to '{1}' + + + Fail to extract zip package '{0}': '{1}' + 0 - package path, 1 - exception message + Finding the package '{0}'. 0 - package name diff --git a/src/Microsoft.PackageManagement.NuGetProvider/Repository/LocalPackageRepository.cs b/src/Microsoft.PackageManagement.NuGetProvider/Repository/LocalPackageRepository.cs index 64b0a793a..fc1c77995 100644 --- a/src/Microsoft.PackageManagement.NuGetProvider/Repository/LocalPackageRepository.cs +++ b/src/Microsoft.PackageManagement.NuGetProvider/Repository/LocalPackageRepository.cs @@ -240,7 +240,17 @@ var partialManifestNameMatches = GetPackageFiles(partialManifestName).Where( path => FileNameMatchesPattern(packageId, version, path)); - return filesMatchingFullName.Concat(partialNameMatches).Concat(partialManifestNameMatches); + filesMatchingFullName = filesMatchingFullName.Concat(partialNameMatches).Concat(partialManifestNameMatches); + } + + // cannot find matching files, we should try to search for just packageid.nupkg + if (filesMatchingFullName.Count() == 0) + { + // exclude version + var packageWithoutVersionName = FileUtility.MakePackageFileName(true, packageId, null, NuGetConstant.PackageExtension); + var packageWithoutVersionManifest = Path.ChangeExtension(packageWithoutVersionName, NuGetConstant.ManifestExtension); + + return GetPackageFiles(packageWithoutVersionName).Concat(GetPackageFiles(packageWithoutVersionManifest)); } return filesMatchingFullName; diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/ExePackageInstaller.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/ExePackageInstaller.cs index d3539d802..18e6ca1d7 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/ExePackageInstaller.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/ExePackageInstaller.cs @@ -44,10 +44,8 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider request.Verbose("Package: '{0}'", exePackage); // validate the file - if (!WebDownloader.PerformSecurityScan(exePackage)) - { - //TODO Security Scan work - request.WriteError(ErrorCategory.InvalidOperation, Constants.ProviderName, "The package download from '{0}' failed in the security scan.", package.Source); + if (!WebDownloader.VerifyHash(exePackage,package, request)) + { return false; } diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs index f1194779b..ef2cbea48 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs @@ -180,6 +180,11 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider { item.Common = common; item.FilePath = packageSpecPath; + + if (string.IsNullOrWhiteSpace(item.Version)) + { + throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.Messages.VersionNotFound, item.Name)); + } } item.PackageSource = packageSource; @@ -201,6 +206,11 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider else { convertedPackage.Common = new CommonDefinition(); + + if (String.IsNullOrWhiteSpace(convertedPackage.Version)) + { + throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.Messages.VersionNotFound, convertedPackage.Name, packageSpecPath)); + } } convertedPackage.FilePath = packageSpecPath; @@ -249,6 +259,13 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider public List installationOption { get; set; } } + public class PackageHash + { + public string algorithm { get; set; } + public string hashCode { get; set; } + + } + public class CommonDefinition { public string Name { get; set; } @@ -285,16 +302,20 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider private string _destination; private List _dependencies; private List _dependencyObjects; - private OSRequirement _osRequirement; + private OSRequirement _osRequirement; private string _type; private string _isTrustedSource; private bool _isPackageProvider; private string _installArguments; private string _unInstallAdditionalArguments; + private string _source; public CommonDefinition Common { get; set; } public string Version { get; set; } + + public PackageHash Hash { get; set; } + public SemanticVersion SemVer { get @@ -306,7 +327,36 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider return null; } } - public string Source { get; set; } + public string Source + { + get + { + return _source; + } + set + { + if (value != null) + { + Uri uri; + + // check whether source is http instead of https + if (!Uri.TryCreate(value, UriKind.Absolute, out uri)) + { + throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.Messages.UnsuportedUriFormat, Constants.ProviderName, value)); + } + + if (!uri.IsFile) + { + if (uri.Scheme != Uri.UriSchemeHttps) + { + throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.Messages.UriSchemeNotSupported, uri.Scheme, Uri.UriSchemeHttps)); + } + } + + _source = value; + } + } + } public string Name { diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListProvider.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListProvider.cs index bb1333230..8e4fe99d7 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListProvider.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListProvider.cs @@ -22,10 +22,12 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider using System.Globalization; using System.IO; using System.Linq; + using System.Text; using System.Management.Automation; using Microsoft.PackageManagement.Provider.Utility; using PackageManagement.Packaging; - using ErrorCategory = PackageManagement.Internal.ErrorCategory; + using ErrorCategory = PackageManagement.Internal.ErrorCategory; + using System.Security.Cryptography; public class PackageSourceListProvider { @@ -37,6 +39,7 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider private const string RequiredPackageManagementVersion = "1.0.0.1"; private static bool _doesPackageManagementVersionMatch = false; private static Version _currentPackageManagementVersion; + private static string _pslDirLocation = Path.Combine(Environment.GetEnvironmentVariable("appdata"), Constants.ProviderName); /// /// The features that this package supports. @@ -125,6 +128,7 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider case "install": request.YieldDynamicOption("Scope", "String", false, new[] {"CurrentUser", "AllUsers"}); + request.YieldDynamicOption("SkipHashValidation", Constants.OptionType.Switch, false); break; } } @@ -189,23 +193,16 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider return; } - request.Debug(Resources.Messages.DebugInfoCallMethod, PackageProviderName, "GetOptionValue"); - // if this is supposed to be an update, there will be a dynamic parameter set for IsUpdatePackageSource + // Set-PackageSource will update the existing package source. In that case IsUpdate = true. var isUpdate = request.GetOptionValue(Constants.Parameters.IsUpdate).IsTrue(); request.Debug(Resources.Messages.VariableCheck, "IsUpdate", isUpdate); - // if your source supports credentials you get get them too: - // string username =request.Username; - // SecureString password = request.Password; - // feel free to send back an error here if your provider requires credentials for package sources. // check first that we're not clobbering an existing source, unless this is an update - request.Debug(Resources.Messages.DebugInfoCallMethod, PackageProviderName, string.Format(CultureInfo.InvariantCulture, "FindRegisteredSource -name'{0}'", name)); var src = request.FindRegisteredSource(name); - if (src != null && !isUpdate) { // tell the user that there's one here already request.WriteError(ErrorCategory.InvalidArgument, name, Constants.Messages.PackageSourceExists, name); @@ -213,29 +210,72 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider } // conversely, if it didn't find one, and it is an update, that's bad too: - if (src == null && isUpdate) { + if (src == null && isUpdate) + { // you can't find that package source? Tell that to the user - request.WriteError(ErrorCategory.ObjectNotFound, name, Constants.Messages.UnableToResolveSource, Constants.ProviderName, name); + request.WriteError(ErrorCategory.ObjectNotFound, name, Constants.Messages.UnableToResolveSource, name); return; } // ok, we know that we're ok to save this source // next we check if the location is valid (if we support that kind of thing) - var validated = false; - - if (!request.SkipValidate.Value) { - // the user has not opted to skip validating the package source location, so check if the source is valid - validated = request.ValidateSourceLocation(location); - - if (!validated) { - request.WriteError(ErrorCategory.InvalidData, name, Constants.Messages.SourceLocationNotValid, location); - return; - } - + validated = request.ValidateSourceLocation(location); + if (!validated) { + request.WriteError(ErrorCategory.InvalidData, name, Constants.Messages.SourceLocationNotValid, location); + return; + } + else + { request.Verbose(Resources.Messages.SuccessfullyValidated, name); } + bool force = request.GetOptionValue("Force") != null; + //if source is UNC location/ copy it to local path; + Uri uri; + if (Uri.TryCreate(location, UriKind.Absolute, out uri)) + { + if (uri.IsFile && uri.IsUnc) + { + string fileName = Path.GetFileNameWithoutExtension(location); + string directory = Path.GetDirectoryName(location); + string catalogFilePath = Path.Combine(directory, fileName+".cat"); + if (!File.Exists(catalogFilePath)) + { + request.WriteError(ErrorCategory.InvalidData, location, Resources.Messages.CatalogFileMissing, location); + return; + } + if (!TestCatalogFile(location, catalogFilePath, request)) + { + return; + } + if (force || request.ShouldContinue(Resources.Messages.QueryDownloadPackageSourceList.format(location), Resources.Messages.PackageSourceListNotTrusted)) + { + string destination = Path.Combine(_pslDirLocation, Path.GetFileName(uri.LocalPath)); + if (File.Exists(destination)) + { + if (force || request.ShouldContinue(Resources.Messages.OverwriteFile, Resources.Messages.FileExists)) + { + File.Copy(location, destination, true); + location = destination; + } + else + { + return; + } + } + else { + File.Copy(location, destination); + location = destination; + } + } + else + { + return; + } + } + } + // it's good to check just before you actaully write something to see if the user has cancelled the operation if (request.IsCanceled) { return; @@ -314,13 +354,13 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider request.Warning(Resources.Messages.WildCardCharsAreNotSupported, name); return; } - + var packages = request.GetPackages(name); if (request.GetOptionValue("AllVersions").IsTrue()) { // if version is specified then name can not be empty or with wildcard. in this case the cmdlet has been errored out already. // here we just return all packages we can find - if (request.FilterOnVersion(packages, requiredVersion, minimumVersion, maximumVersion, minInclusive: true, maxInclusive: true, latest: false).Any(p => !request.YieldFromSwidtag(p, p.Name))) + if (request.FilterOnVersion(packages, requiredVersion, minimumVersion, maximumVersion, minInclusive: true, maxInclusive: true, latest: false).OrderBy(p => p.Name).Any(p => !request.YieldFromSwidtag(p, p.Name))) { return; } @@ -329,7 +369,7 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider //return the latest version if (packages.GroupBy(p => p.Name) - .Select(each => each.OrderByDescending(pp => pp.Version).FirstOrDefault()).Any( item =>!request.YieldFromSwidtag(item, item.Name))) + .Select(each => each.OrderByDescending(pp => pp.Version).FirstOrDefault()).OrderBy(p=>p.Name).Any( item =>!request.YieldFromSwidtag(item, item.Name))) { return; } @@ -717,20 +757,19 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider //download the msi package to the temp file WebDownloader.DownloadFile(package.Source, destination, request, null); - + if (!File.Exists(destination)) { return; } + + // validate the file - if (!WebDownloader.PerformSecurityScan(destination)) + if (!WebDownloader.VerifyHash(destination,package, request)) { - //TODO Security Scan work - request.WriteError(ErrorCategory.InvalidOperation, Constants.ProviderName, "The package download from '{0}' failed in the security scan.", package.Source); return; } - - + if (!package.IsTrustedSource) { if (!request.ShouldContinueWithUntrustedPackageSource(package.Name, package.Source)) @@ -868,7 +907,38 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider return Enumerable.Empty(); - } + } + + internal static bool TestCatalogFile(string jsonFile, string catalogFile, PackageSourceListRequest request) + { + try + { + PSObject result = null; + using (PowerShell powershell = PowerShell.Create()) + { + if (powershell != null) + { + result = powershell + .AddCommand("Test-FileCatalog") + .AddParameter("CatalogFilePath", catalogFile) + .AddParameter("Path", jsonFile) + .Invoke().FirstOrDefault(); + } + if (result.ToString().EqualsIgnoreCase("Valid")) + return true; + } + } + catch(Exception ex) + { + request.WriteError(ErrorCategory.InvalidData, catalogFile, Resources.Messages.CatalogFileVerificationFailedWithError, catalogFile, ex.Message.ToString()); + return false; + } + + request.WriteError(ErrorCategory.InvalidData, catalogFile, Resources.Messages.CatalogFileVerificationFailed, jsonFile); + return false; + + } + } } diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListRequest.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListRequest.cs index e04536575..4c8282017 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListRequest.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListRequest.cs @@ -37,12 +37,16 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider using SemanticVersion = Microsoft.PackageManagement.Provider.Utility.SemanticVersion; public abstract class PackageSourceListRequest : Request { - + private IEnumerable _packageQuery; private static IDictionary _registeredPackageSources; private string _configurationFileLocation; private XDocument _config; private string _defaultConfig; + private string PowerShellSourceURL = @"http://go.microsoft.com/fwlink/?LinkID=821777&clcid=0x409"; + private string PowerShellNanoSourceURL = @"http://go.microsoft.com/fwlink/?LinkID=821783&clcid=0x409"; + private string PowerShellSourceCatalogURL = @"http://go.microsoft.com/fwlink/?LinkID=823093&clcid=0x409"; + private string PowerShellNanoSourceCatalogURL = @"http://go.microsoft.com/fwlink/?LinkID=823094&clcid=0x409"; private IEnumerable _packageSources; private const string _PackageSourceListRequest = "PackageSourceListRequest"; private HttpClient _httpClient; @@ -57,6 +61,7 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider internal Lazy SkipValidate; internal readonly Lazy AllVersions; internal readonly Lazy Headers; + internal Lazy SkipHashValidation; internal const WildcardOptions WildcardOptions = System.Management.Automation.WildcardOptions.CultureInvariant | System.Management.Automation.WildcardOptions.IgnoreCase; @@ -67,6 +72,12 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider "; + internal const string EmptyConfig = @" + + + +"; + /// /// Ctor required by the PackageManagement Platform /// @@ -75,20 +86,48 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider AllVersions = new Lazy(() => GetOptionValue("AllVersions").IsTrue()); SkipValidate = new Lazy(() => GetOptionValue("SkipValidate").IsTrue()); Headers = new Lazy(() => (GetOptionValues("Headers") ?? new string[0]).ToArray()); + SkipHashValidation = new Lazy(() => GetOptionValue("SkipHashValidation").IsTrue()); } - internal string DefaultConfigLocation + + internal string DefaultJSONFileLocation + { + get + { + return Path.Combine(Environment.GetEnvironmentVariable("appdata"), Constants.ProviderName, Constants.JSONFileName); + } + } + + internal string DefaultCatlogFileLocation + { + get + { + return Path.Combine(Environment.GetEnvironmentVariable("appdata"), Constants.ProviderName, Constants.CatFileName); + } + } + + internal string DefaultJSONSourceLocation { get { #if CORECLR - return Path.Combine(Environment.GetEnvironmentVariable("ProgramFiles"), "PackageManagement\\ProviderAssemblies\\PSL", "PSL.json"); + return PowerShellNanoSourceURL; #else - return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "PackageManagement\\ProviderAssemblies\\PSL", "PSL.json"); + return PowerShellSourceURL; #endif } } - + internal string DefaultJSONCatalogFileLocation + { + get + { +#if CORECLR + return PowerShellNanoSourceCatalogURL; +#else + return PowerShellSourceCatalogURL; +#endif + } + } internal string DefaultConfig { @@ -96,13 +135,13 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider { if (_defaultConfig == null) { - _defaultConfig = DefaultConfigDefinition.Replace("##", DefaultConfigLocation); - + _defaultConfig = DefaultConfigDefinition.Replace("##", DefaultJSONFileLocation); } return _defaultConfig; } } - + + internal IEnumerable PackageSources { get @@ -790,66 +829,23 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider Debug(Resources.Messages.NotFoundRegisteredSource, src, Constants.ProviderName); - // doesn't look like we have this as a source. - if (Uri.IsWellFormedUriString(src, UriKind.Absolute)) - { - // we have been passed in an URI - var srcUri = new Uri(src); - if (SupportedSchemes.Contains(srcUri.Scheme.ToLower())) - { - // it's one of our supported uri types. - var isValidated = false; - - //if (!SkipValidate.Value) - //{ - // isValidated = PathUtility.ValidateSourceUri(SupportedSchemes, srcUri, this); - //} - - if (SkipValidate.Value || isValidated) - { - Debug(Resources.Messages.SuccessfullyValidated, src); - - yield return new PackageSource - { - Request = this, - Location = srcUri.ToString(), - Name = srcUri.ToString(), - Trusted = false, - IsRegistered = false, - IsValidated = isValidated, - }; - continue; - } - if (userSpecifiesArrayOfSources) - { - Verbose(Constants.Messages.UnableToResolveSource, Constants.ProviderName, src); - } - else - { - Warning(Constants.Messages.UnableToResolveSource, Constants.ProviderName, src); - } - - continue; - } - - // Not a valid location? - if (userSpecifiesArrayOfSources) - { - Verbose(Constants.Messages.UnableToResolveSource, Constants.ProviderName, src); - } - else - { - Warning(Constants.Messages.UnableToResolveSource, Constants.ProviderName, src); - } - - continue; - } - // is it a file path? if (System.IO.Directory.Exists(src)) { Debug(Resources.Messages.SourceIsADirectory, src); - + PackageSource newSource = new PackageSource + { + Request = this, + Location = src, + Name = src, + Trusted = true, + IsRegistered = false, + IsValidated = true, + }; + yield return newSource; + } + else if (File.Exists(src) && (Path.GetExtension(src).EqualsIgnoreCase(".json")) ) + { PackageSource newSource = new PackageSource { Request = this, @@ -947,47 +943,52 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider { get { - if (String.IsNullOrWhiteSpace(_configurationFileLocation)) - { - // get the value from the request - var path = GetOptionValue("ConfigFile"); + if (string.IsNullOrWhiteSpace(_configurationFileLocation)) + { + var appdataFolder = Environment.GetEnvironmentVariable("appdata"); + _configurationFileLocation = Path.Combine(appdataFolder, Constants.ProviderName, Constants.SettingsFileName); - if (!String.IsNullOrWhiteSpace(path)) + //create directory if does not exist + string dir = Path.GetDirectoryName(_configurationFileLocation); + if (dir != null && !System.IO.Directory.Exists(dir)) { - _configurationFileLocation = path; - - if (!System.IO.File.Exists(_configurationFileLocation)) - { - WriteError(Internal.ErrorCategory.InvalidArgument, _configurationFileLocation, Resources.Messages.FileNotFound); - } - + Debug(Resources.Messages.CreateDirectory, dir); + System.IO.Directory.CreateDirectory(dir); } - else + //create place holder config file + if (!System.IO.File.Exists(_configurationFileLocation)) { - var appdataFolder = Environment.GetEnvironmentVariable("appdata"); - _configurationFileLocation = Path.Combine(appdataFolder, Constants.ProviderName, Constants.SettingsFileName); + Debug(Resources.Messages.CreateFile, _configurationFileLocation); + bool addDefaultConfig = false; + if (System.IO.File.Exists(DefaultJSONFileLocation)) + { + addDefaultConfig = true; + } + else + { + bool force = this.GetOptionValue("Force") != null; + if (force || this.ShouldContinue(Resources.Messages.QueryDownloadPackageSourceList.format(DefaultJSONSourceLocation), Resources.Messages.PackageSourceListNotFound.format(DefaultJSONFileLocation))) + { + WebDownloader.DownloadFile(DefaultJSONSourceLocation, DefaultJSONFileLocation, this, null); + WebDownloader.DownloadFile(DefaultJSONCatalogFileLocation, DefaultCatlogFileLocation, this, null); + if (System.IO.File.Exists(DefaultJSONFileLocation) && System.IO.File.Exists(DefaultCatlogFileLocation) && + PackageSourceListProvider.TestCatalogFile(DefaultJSONFileLocation, DefaultCatlogFileLocation, this)) + { + addDefaultConfig = true; + } + } + } - //create directory if does not exist - string dir = Path.GetDirectoryName(_configurationFileLocation); - if (dir != null && !System.IO.Directory.Exists(dir)) - { - Debug(Resources.Messages.CreateDirectory, dir); - System.IO.Directory.CreateDirectory(dir); - } - //create place holder config file - if (!System.IO.File.Exists(_configurationFileLocation)) - { - Debug(Resources.Messages.CreateFile, _configurationFileLocation); + if(addDefaultConfig) System.IO.File.WriteAllText(_configurationFileLocation, DefaultConfig); - } + else + System.IO.File.WriteAllText(_configurationFileLocation, EmptyConfig); } } - return _configurationFileLocation; } } - internal bool WriteError(Internal.ErrorCategory category, string targetObjectValue, string messageText, params object[] args) { return Error(messageText, category.ToString(), targetObjectValue, base.FormatMessageString(messageText, args)); @@ -1001,9 +1002,11 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider internal bool ValidateSourceLocation(string location) { Debug(Resources.Messages.DebugInfoCallMethod3, _PackageSourceListRequest, "ValidateSourceLocation", location); - - //TODO this is needed for register-packagesource - return true; + if (File.Exists(location) && (Path.GetExtension(location).EqualsIgnoreCase(".json"))) + { + return true; + } + return false; } internal static IRequest ExtendRequest(Dictionary options, string[] sources, bool isTrusted, PackageSourceListRequest request) { diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageQuery.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageQuery.cs index 2e1fbdb4d..81947104d 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageQuery.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageQuery.cs @@ -43,6 +43,12 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider throw new ArgumentNullException("packageSource"); } + if(string.IsNullOrWhiteSpace(packageSource.Location) || !System.IO.File.Exists(packageSource.Location)) + { + request.Warning(Resources.Messages.PackageSourceManifestNotFound, packageSource.Location, Constants.ProviderName); + return; + } + Uri uri; if (Uri.TryCreate(packageSource.Location, UriKind.Absolute, out uri)) @@ -56,6 +62,12 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider } catch (Exception ex) { + request.Warning(ex.Message); + while (ex.InnerException != null) + { + ex = ex.InnerException; + request.Warning(ex.Message); + } request.Warning(string.Format(CultureInfo.CurrentCulture, Resources.Messages.InvalidPackageListFormat, uri.AbsolutePath)); ex.Dump(request); } @@ -82,6 +94,12 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider } catch (Exception ex) { + request.Warning(ex.Message); + while (ex.InnerException != null) + { + ex = ex.InnerException; + request.Warning(ex.Message); + } request.Warning(string.Format(CultureInfo.CurrentCulture, Resources.Messages.InvalidPackageListFormat, file)); ex.Dump(request); } diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/WebDownloader.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/WebDownloader.cs index 1fc2e7e58..fab3e4bcb 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/WebDownloader.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/WebDownloader.cs @@ -25,9 +25,13 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider using File = System.IO.File; using System.Threading.Tasks; using Microsoft.PackageManagement.Provider.Utility; + using System.Security.Cryptography; + using System.Linq; + using ErrorCategory = PackageManagement.Internal.ErrorCategory; + + public class WebDownloader + { - public class WebDownloader { - /// /// Download data from remote via uri query. /// @@ -44,11 +48,11 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider // try downloading for 3 times int remainingTry = 3; long totalDownloaded = 0; - + CancellationTokenSource cts; - Stream input = null; - FileStream output = null; - + Stream input = null; + FileStream output = null; + while (remainingTry > 0) { // if user cancel the request, no need to do anything @@ -67,24 +71,27 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider // decrease try by 1 remainingTry -= 1; - var httpClient = request.Client; + var httpClient = request.Client; httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "text/html; charset=iso-8859-1"); input = await httpClient.GetStreamAsync(query); - + // buffer size of 64 KB, this seems to be preferable buffer size, not too small and not too big byte[] bytes = new byte[1024 * 64]; output = File.Open(fileName, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read); int current = 0; - + // here we read content that we got from the http response stream into the bytes array current = await input.ReadAsync(bytes, 0, bytes.Length, cts.Token); + int progressPercentage = progressTracker.StartPercent; // report initial progress - request.Progress(progressTracker.ProgressID, progressTracker.StartPercent, Resources.Messages.BytesRead, current); + request.Progress(progressTracker.ProgressID, progressPercentage, Resources.Messages.BytesRead, current); + + int i = progressTracker.StartPercent; while (current > 0) { @@ -92,10 +99,10 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider // here we write out the bytes array into the file await output.WriteAsync(bytes, 0, current, cts.Token); - + // report the progress - request.Progress(progressTracker.ProgressID, progressTracker.StartPercent, Resources.Messages.BytesRead, totalDownloaded); - + request.Progress(progressTracker.ProgressID, progressPercentage public static readonly string SettingsFileName = "PSL.config"; - + + /// + /// Sample JSON file containing open powershell entry + /// + public static readonly string JSONFileName = "PSL.json"; + public static readonly string CatFileName = "PSL.cat"; + internal static class MediaType { public const string MsiPackage = "msi"; @@ -79,7 +85,7 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider public const string ProviderSwidtagUnavailable = "MSG:ProviderSwidtagUnavailable"; public const string RemoveEnvironmentVariableRequiresElevation = "MSG:RemoveEnvironmentVariableRequiresElevation"; public const string SchemeNotSupported = "MSG:SchemeNotSupported"; - public const string SourceLocationNotValid = "MSG:SourceLocationNotValid"; + public const string SourceLocationNotValid = "MSG:SourceLocationNotValid_Location"; public const string UnableToCopyFileTo = "MSG:UnableToCopyFileTo"; public const string UnableToCreateShortcutTargetDoesNotExist = "MSG:UnableToCreateShortcutTargetDoesNotExist"; public const string UnableToDownload = "MSG:UnableToDownload"; diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/resources/Microsoft.PackageManagement.PackageSourceListProvider.Resources.Messages.resx b/src/Microsoft.PackageManagement.PackageSourceListProvider/resources/Microsoft.PackageManagement.PackageSourceListProvider.Resources.Messages.resx index 78d8ce1ca..c6d21c122 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/resources/Microsoft.PackageManagement.PackageSourceListProvider.Resources.Messages.resx +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/resources/Microsoft.PackageManagement.PackageSourceListProvider.Resources.Messages.resx @@ -277,7 +277,7 @@ Unknown category for '{0}'::'{1}': '{2}' - Uri Scheme '{0}' is not supported. + Uri Scheme '{0}' is not supported. Only '{1}' is supported. Use the default configuration. @@ -392,4 +392,50 @@ Package List Source '{0}' has incorrect format. + + Do you want to download package source list from '{0}'? + 0 - source list path + + + A file with this name already exists in target directory + + + Do you want to overwrite it? + + + PackageSourceList is not trusted + + + Package '{0}' in json file '{1}' is missing version information. + + + Invalid hash for package '{0}'. Please correct the values in json file or use -SkipHashValidation switch to skip Hash validation. + + + Hash validation was successful. + + + Hash verification failed for package '{0}' downloaded from '{1}'. + + + Hash algorithm '{0}' is not supported. We only support sha512, md5 and sha256. + + + Skipping Hash Validation. + + + Catalog file not found for '{0}'. Catalog files are required for non local json files and should have the same name and located in the same directory as json file. + + + Catalog File verification failed for '{0}'. + + + Catalog File '{0}' verification failed with '{1}'. + + + Cannot find source list file '{0}'. + + + File '{0}' is registered as a package source location for the provider '{1}'. But it does not exist. Run 'Get-PackageSource' to view the registered package sources, 'Unregister-PackageSource' to unregister the source if you do not wish to use it or 'Set-PackageSource' to fix the source location. + \ No newline at end of file diff --git a/src/Microsoft.PowerShell.Activities/Activities/WorkflowJobConverter.cs b/src/Microsoft.PowerShell.Activities/Activities/WorkflowJobConverter.cs index b3819423d..6aaf318fd 100644 --- a/src/Microsoft.PowerShell.Activities/Activities/WorkflowJobConverter.cs +++ b/src/Microsoft.PowerShell.Activities/Activities/WorkflowJobConverter.cs @@ -198,6 +198,43 @@ namespace Microsoft.PowerShell.Workflow /// based on the definition provided by this script block. /// public List CompileWorkflows(ScriptBlockAst ast, PSModuleInfo definingModule, InitialSessionState initialSessionState, out ParseException parsingErrors, string rootWorkflowName) + { + return CompileWorkflowsImpl(ast, definingModule, initialSessionState, null, out parsingErrors, rootWorkflowName); + } + + /// + /// Converts a PowerShell AST into a script block that represents + /// the workflow to run. + /// + /// The PowerShell AST correpsponding to the job's definition. + /// The module that is defining this command (if any). + /// The initial session state of a runspace. + /// Language mode of source that is creating the workflow. + /// Optional, once assigned, only root Workflow will be compiled. + /// + /// A PowerShell script block that invokes an underlying job, + /// based on the definition provided by this script block. + /// + public List CompileWorkflows(ScriptBlockAst ast, PSModuleInfo definingModule, InitialSessionState initialSessionState, PSLanguageMode? sourceLanguageMode, out ParseException parsingErrors) + { + return CompileWorkflowsImpl(ast, definingModule, initialSessionState, sourceLanguageMode, out parsingErrors, null); + } + + /// + /// Converts a PowerShell AST into a script block that represents + /// the workflow to run. + /// + /// The PowerShell AST correpsponding to the job's definition. + /// The module that is defining this command (if any) + /// The initial session state of a runspace. + /// Language mode of source that is creating the workflow. + /// parsing errors + /// Optional, once assigned, only root Workflow will be compiled + /// + /// A PowerShell script block that invokes an underlying job, + /// based on the definition provided by this script block. + /// + private List CompileWorkflowsImpl(ScriptBlockAst ast, PSModuleInfo definingModule, InitialSessionState initialSessionState, PSLanguageMode? sourceLanguageMode, out ParseException parsingErrors, string rootWorkflowName) { List errorList = new List(); @@ -327,7 +364,7 @@ namespace Microsoft.PowerShell.Workflow try { - entry.workflowInfo = CompileSingleWorkflow(entry.scope, func, scriptBlockTokenCache, definingModule, requiredAssemblies, activityMap, processedActivityLibraries, invoker, rootWorkflowName); + entry.workflowInfo = CompileSingleWorkflow(entry.scope, func, scriptBlockTokenCache, definingModule, requiredAssemblies, activityMap, processedActivityLibraries, invoker, sourceLanguageMode, rootWorkflowName); result.Add(entry.workflowInfo); } catch (ParseException e) @@ -365,6 +402,7 @@ namespace Microsoft.PowerShell.Workflow Dictionary activityMap, HashSet processedActivityLibraries, System.Management.Automation.PowerShell invoker, + PSLanguageMode? sourceLanguageMode = (PSLanguageMode?)null, string rootWorkflowName = null) { Dictionary parameterValidation; @@ -399,10 +437,11 @@ namespace Microsoft.PowerShell.Workflow // Pass either the workflow script file path if available or the full source otherwise. string scriptFile = parentAst.Extent.File; string scriptSource = string.IsNullOrEmpty(scriptFile) ? parentAst.Extent.StartScriptPosition.GetFullScript() : null; + ReadOnlyCollection attributeAstCollection = (func.Body.ParamBlock != null) ? func.Body.ParamBlock.Attributes : null; var functionDefinition = ImportWorkflowCommand.CreateFunctionFromXaml(func.Name, xaml, referencedAssemblies, calledWorkflows.Select(wfi => wfi.NestedXamlDefinition).ToArray(), null, parameterValidation, modulePath, true, workflowAttributes, - scriptFile, scriptSource, rootWorkflowName); + scriptFile, scriptSource, rootWorkflowName, sourceLanguageMode, attributeAstCollection); var helpContent = func.GetHelpContent(scriptBlockTokenCache); if (helpContent != null) diff --git a/src/Microsoft.PowerShell.Security/security/CertificateProvider.cs b/src/Microsoft.PowerShell.Security/security/CertificateProvider.cs index f1c993950..5a24d97de 100644 --- a/src/Microsoft.PowerShell.Security/security/CertificateProvider.cs +++ b/src/Microsoft.PowerShell.Security/security/CertificateProvider.cs @@ -3545,7 +3545,7 @@ namespace Microsoft.PowerShell.Commands { if (DownLevelHelper.NativeFilteringSupported()) { - Collection ekuCollection = System.Management.Automation.SecuritySupport.GetCertEKU(cert); + Collection ekuCollection = System.Management.Automation.Internal.SecuritySupport.GetCertEKU(cert); foreach (string oidString in ekuCollection) { diff --git a/src/Microsoft.PowerShell.Security/security/SignatureCommands.cs b/src/Microsoft.PowerShell.Security/security/SignatureCommands.cs index c9f05a235..971edd0f2 100644 --- a/src/Microsoft.PowerShell.Security/security/SignatureCommands.cs +++ b/src/Microsoft.PowerShell.Security/security/SignatureCommands.cs @@ -5,6 +5,7 @@ Copyright (c) Microsoft Corporation. All rights reserved. using System; using System.Management.Automation; +using System.Management.Automation.Internal; using Dbg=System.Management.Automation.Diagnostics; using System.Collections.Generic; using System.Collections; diff --git a/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/ImportWorkflowCommand.cs b/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/ImportWorkflowCommand.cs index 11a94fa1c..0f4d801a2 100644 --- a/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/ImportWorkflowCommand.cs +++ b/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/ImportWorkflowCommand.cs @@ -110,17 +110,15 @@ namespace Microsoft.PowerShell.Commands protected override void ProcessRecord() { // In ConstrainedLanguage, XAML workflows are not supported (even from a trusted FullLanguage state), - // since we can't prevent tampering. + // unless they are signed in-box OS binaries. + bool checkSignatures = false; if ((SystemPolicy.GetSystemLockdownPolicy() == SystemEnforcementMode.Enforce) || (this.SessionState.LanguageMode == PSLanguageMode.ConstrainedLanguage)) { // However, this static internal property can be changed by tests that can already run a script // in full-language mode. PropertyInfo xamlProperty = typeof(SystemPolicy).GetProperty("XamlWorkflowSupported", BindingFlags.NonPublic | BindingFlags.Static); - if (! ((bool) xamlProperty.GetValue(null, null))) - { - throw new NotSupportedException(Resources.XamlWorkflowsNotSupported); - } + checkSignatures = !((bool)xamlProperty.GetValue(null, null)); } string dependentWorkflowAssemblyPath = string.Empty; @@ -174,8 +172,10 @@ namespace Microsoft.PowerShell.Commands WriteError(er); Tracer.TraceErrorRecord(er); continue; - } + + CheckFileSignatureAsNeeded(checkSignatures, resolvedPath); + try { // Finally load the file. If there is an access violation, write a @@ -251,6 +251,8 @@ namespace Microsoft.PowerShell.Commands continue; } + CheckFileSignatureAsNeeded(checkSignatures, resolvedPath); + FunctionDetails detailsToUseForUpdate = null; try { @@ -302,7 +304,8 @@ namespace Microsoft.PowerShell.Commands requiredAssemblies, dependentWorkflowContent.ToArray(), dependentWorkflowAssemblyPath, - resolvedPath); + resolvedPath, + this.SessionState.LanguageMode); // check if the function cache already has the entry to this file // If detailsToUseForUpdate is not null it is a forced import module @@ -342,6 +345,14 @@ namespace Microsoft.PowerShell.Commands } //ProcessRecord + private static void CheckFileSignatureAsNeeded(bool checkSignatures, string filePath) + { + if (checkSignatures && !System.Management.Automation.Internal.SecuritySupport.IsProductBinary(filePath)) + { + throw new NotSupportedException(Resources.XamlWorkflowsNotSupported); + } + } + private static readonly ConcurrentDictionary FunctionCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); @@ -358,7 +369,15 @@ namespace Microsoft.PowerShell.Commands /// Any workflows required by this workflow. /// Path to the dependent assembly. /// Resolved Path of the xaml - private static FunctionDetails GenerateFunctionFromXaml(string name, string xaml, Dictionary requiredAssemblies, string[] dependentWorkflows, string dependentAssemblyPath, string resolvedPath) + /// Language mode of source in which workflow should run + private static FunctionDetails GenerateFunctionFromXaml( + string name, + string xaml, + Dictionary requiredAssemblies, + string[] dependentWorkflows, + string dependentAssemblyPath, + string resolvedPath, + PSLanguageMode sourceLanguageMode) { if (name == null) { @@ -368,7 +387,21 @@ namespace Microsoft.PowerShell.Commands } string modulePath = System.IO.Path.GetDirectoryName(resolvedPath); - string functionDefinition = CreateFunctionFromXaml(name, xaml, requiredAssemblies, dependentWorkflows, dependentAssemblyPath, null, modulePath, false, "[CmdletBinding()]"); + string functionDefinition = CreateFunctionFromXaml( + name, + xaml, + requiredAssemblies, + dependentWorkflows, + dependentAssemblyPath, + null, + modulePath, + false, + "[CmdletBinding()]", + null, /* scriptContent */ + null, /* fullScript */ + null, /* rootWorkflowName */ + sourceLanguageMode, + null); FunctionDetails details = new FunctionDetails {Name = name, FunctionDefinition = functionDefinition, Xaml = xaml}; @@ -438,6 +471,24 @@ namespace Microsoft.PowerShell.Commands [SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly", MessageId = "workFlowDefinition")] public static ContainerParentJob StartWorkflowApplication(PSCmdlet command, string jobName, string workflowGuid, bool startAsync, bool parameterCollectionProcessed, Hashtable[] parameters, bool debuggerActive) + { + return StartWorkflowApplication(command, jobName, workflowGuid, startAsync, parameterCollectionProcessed, parameters, false, null); + } + + /// + /// Executes an instance of the workflow object graph identified by the passed + /// GUID, binding parameters from the Parameters hastable. + /// + /// The powershell command. + /// The GUID used to identify the workflow to run. + /// The parameters to pass to the workflow instance. + /// The friendly name for the job + /// True if there was a PSParameters collection + /// + /// True if debugger is in active state. + /// Language mode of source creating workflow. + public static ContainerParentJob StartWorkflowApplication(PSCmdlet command, string jobName, string workflowGuid, bool startAsync, + bool parameterCollectionProcessed, Hashtable[] parameters, bool debuggerActive, string SourceLanguageMode) { Guid trackingGuid = Guid.NewGuid(); _structuredTracer.BeginStartWorkflowApplication(trackingGuid); @@ -573,6 +624,15 @@ namespace Microsoft.PowerShell.Commands myJob = command.JobManager.NewJob(specification) as ContainerParentJob; _structuredTracer.EndCreateNewJob(trackingGuid); + // Pass the source language mode to the workflow job so that it can be + // applied during activity execution. + PSLanguageMode sourceLanguageModeValue; + PSLanguageMode? sourceLanguageMode = null; + if (!string.IsNullOrEmpty(SourceLanguageMode) && Enum.TryParse(SourceLanguageMode, out sourceLanguageModeValue)) + { + sourceLanguageMode = sourceLanguageModeValue; + } + // Raise engine event of new WF job start for debugger, if // debugger is active (i.e., has breakpoints set). if (debuggerActive) @@ -582,12 +642,13 @@ namespace Microsoft.PowerShell.Commands if (startAsync) { - if (!PSSessionConfigurationData.IsServerManager) + foreach(PSWorkflowJob childjob in myJob.ChildJobs) { - foreach(PSWorkflowJob childjob in myJob.ChildJobs) + if (!PSSessionConfigurationData.IsServerManager) { childjob.EnableStreamUnloadOnPersistentState(); } + childjob.SourceLanguageMode = sourceLanguageMode; } myJob.StartJobAsync(); } @@ -598,6 +659,7 @@ namespace Microsoft.PowerShell.Commands foreach (PSWorkflowJob childJob in myJob.ChildJobs) { childJob.SynchronousExecution = true; + childJob.SourceLanguageMode = sourceLanguageMode; } myJob.StartJob(); } @@ -605,8 +667,8 @@ namespace Microsoft.PowerShell.Commands // write an event specifying that job creation is done _structuredTracer.EndStartWorkflowApplication(trackingGuid); _structuredTracer.TrackingGuidContainerParentJobCorrelation(trackingGuid, myJob.InstanceId); - return myJob; + return myJob; } private static void RaiseWFJobEvent(PSCmdlet command, ContainerParentJob job, bool startAsync) @@ -1010,6 +1072,46 @@ namespace Microsoft.PowerShell.Commands string fullScript, string rootWorkflowName ) + { + return CreateFunctionFromXaml(name, xaml, requiredAssemblies, dependentWorkflows, dependentAssemblyPath, parameterValidation, modulePath, + scriptWorkflow, workflowAttributes, scriptContent, fullScript, rootWorkflowName, null, null); + } + + /// + /// Creates a workflow activity based on the provided Xaml and returns PowerShell script that will + /// run the workflow. + /// + /// Workflow name + /// Workflow Xaml definition + /// Required assemblies + /// Dependent workflows + /// Path for dependent assemblies + /// Workflow parameters + /// Module path + /// True if this is script workflow + /// the attribute string to use for the workflow, should be '[CmdletBinding()]' + /// File path containing script content. + /// Full source script. + /// Only root Workflow will be compiled + /// Language mode of source that is creating the workflow + /// Optional collection of parameter attribute Asts + /// + public static string CreateFunctionFromXaml( + string name, + string xaml, + Dictionary requiredAssemblies, + string[] dependentWorkflows, + string dependentAssemblyPath, + Dictionary parameterValidation, + string modulePath, + bool scriptWorkflow, + string workflowAttributes, + string scriptContent, + string fullScript, + string rootWorkflowName, + PSLanguageMode? sourceLanguageMode, + ReadOnlyCollection attributeAstCollection + ) { // check to see if the specified name is allowed if (!Regex.IsMatch(name, functionNamePattern)) @@ -1304,9 +1406,15 @@ namespace Microsoft.PowerShell.Commands completeFunctionDefinitionTemplate.AppendLine(" }}"); completeFunctionDefinitionTemplate.AppendLine(FunctionBodyTemplate); + // Mark the function definition with sourceLanguageMode (language mode that function can run under, i.e., + // as trusted or not trusted), unless the workflow script is marked with the "SecurityCritical" attribute in + // which case the function will always be run under the current system lock down setting. + bool isSecurityCritical = ContainsSecurityCriticalAttribute(attributeAstCollection); + string sourceLanguageModeStr = (!isSecurityCritical && (sourceLanguageMode != null)) ? sourceLanguageMode.ToString() : string.Empty; + // Combine the pieces to create the complete function string functionDefinition = String.Format(CultureInfo.InvariantCulture, completeFunctionDefinitionTemplate.ToString(), - defaultUpdates, workflowGuid, modulePath); + defaultUpdates, workflowGuid, modulePath, sourceLanguageModeStr); #if DEBUG // Verify that the generated function is valid powershell. This is only an issue when changing the @@ -1334,6 +1442,23 @@ namespace Microsoft.PowerShell.Commands return functionDefinition; } + private static Type securityCriticalAttributeType = typeof(System.Security.SecurityCriticalAttribute); + private static bool ContainsSecurityCriticalAttribute(ReadOnlyCollection attributeAsts) + { + if (attributeAsts != null) + { + foreach (var attributeAst in attributeAsts) + { + if (attributeAst.TypeName.GetReflectionAttributeType() == securityCriticalAttributeType) + { + return true; + } + } + } + + return false; + } + /// /// @@ -1515,6 +1640,9 @@ namespace Microsoft.PowerShell.Commands # which uses this as a base path to find localized content files. $psBoundParameters['" + Constants.ModulePath + @"'] = '{2}' + # Variable that contains the source language mode. + [string] $SourceLanguageMode = '{3}' + if (Test-Path variable:\PSSenderInfo) {{ $psBoundParameters['" + Constants.PSSenderInfo + @"'] = $PSSenderInfo @@ -1693,7 +1821,8 @@ namespace Microsoft.PowerShell.Commands $AsJob, $parameterCollectionProcessed, $finalParameterCollection, - $debuggerActive) + $debuggerActive, + $SourceLanguageMode) }} catch {{ diff --git a/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/LocalRunspaceProvider.cs b/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/LocalRunspaceProvider.cs index ced8359fe..3e2fc5ded 100644 --- a/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/LocalRunspaceProvider.cs +++ b/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/LocalRunspaceProvider.cs @@ -80,23 +80,29 @@ namespace Microsoft.PowerShell.Workflow return EndGetRunspace(asyncResult); } - private Runspace AssignRunspaceIfPossible() + private Runspace AssignRunspaceIfPossible(PSLanguageMode? sourceLanguageMode = null) { Runspace runspace = null; + PSLanguageMode languageMode = (sourceLanguageMode != null) ? sourceLanguageMode.Value : + (_languageMode != null) ? _languageMode.Value : GetSystemLanguageMode(); lock (_runspaceCache.TimerServicingSyncObject) { + // Retrieve or create a local runspace having the same language mode as the source, if provided. foreach (Item item in _runspaceCache.Cast>().Where(item => !item.Busy)) { - item.Idle = false; - item.Busy = true; - runspace = item.Value; - break; + if (item.Value.SessionStateProxy.LanguageMode == languageMode) + { + item.Idle = false; + item.Busy = true; + runspace = item.Value; + break; + } } if ((runspace == null || runspace.RunspaceStateInfo.State != RunspaceState.Opened) && (_maxRunspaces == MaxRunspacesPossible || _runspaceCache.Cache.Count < _maxRunspaces)) { - runspace = CreateLocalActivityRunspace(_languageMode); + runspace = CreateLocalActivityRunspace(languageMode); runspace.Open(); _tracer.WriteMessage("New local runspace created"); @@ -108,6 +114,11 @@ namespace Microsoft.PowerShell.Workflow return runspace; } + private static PSLanguageMode GetSystemLanguageMode() + { + return (SystemPolicy.GetSystemLockdownPolicy() == SystemEnforcementMode.Enforce) ? + PSLanguageMode.ConstrainedLanguage : PSLanguageMode.FullLanguage; + } private void TraceThreadPoolInfo(string message) { @@ -208,7 +219,23 @@ namespace Microsoft.PowerShell.Workflow LocalRunspaceAsyncResult asyncResult = new LocalRunspaceAsyncResult(state, callback, Guid.Empty); - Runspace runspace = AssignRunspaceIfPossible(); + // Get the source language mode from the activity arguments if available and pass to runspace fetching. + PSLanguageMode? sourceLanguageMode = null; + RunCommandsArguments args = state as RunCommandsArguments; + if (args != null) + { + PSWorkflowRuntime wfRuntime = args.WorkflowHost as PSWorkflowRuntime; + if (wfRuntime != null) + { + PSWorkflowJob wfJob = wfRuntime.JobManager.GetJob(args.PSActivityContext.JobInstanceId); + if (wfJob != null) + { + sourceLanguageMode = wfJob.SourceLanguageMode; + } + } + } + + Runspace runspace = AssignRunspaceIfPossible(sourceLanguageMode); if (runspace != null) { asyncResult.Runspace = runspace; diff --git a/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/WorkflowJob2.cs b/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/WorkflowJob2.cs index e2cb65fe0..da2669f5a 100644 --- a/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/WorkflowJob2.cs +++ b/src/Microsoft.PowerShell.Workflow.ServiceCore/ServiceCore/WorkflowCore/WorkflowJob2.cs @@ -1205,6 +1205,8 @@ namespace Microsoft.PowerShell.Workflow internal bool? IsSuspendable = null; + internal PSLanguageMode? SourceLanguageMode { get; set; } + #endregion Internal Accessors #region Internal Methods diff --git a/src/Microsoft.PowerShell.Workflow.ServiceCore/resources/Resources.resx b/src/Microsoft.PowerShell.Workflow.ServiceCore/resources/Resources.resx index 4dbc8e5c8..9ac565af3 100644 --- a/src/Microsoft.PowerShell.Workflow.ServiceCore/resources/Resources.resx +++ b/src/Microsoft.PowerShell.Workflow.ServiceCore/resources/Resources.resx @@ -413,7 +413,7 @@ For more information about how to add checkpoints, see the help topics for Windo {0} is a number and should not be localized. - Cannot load workflow. XAML-based workflows are not supported in this language mode. Only script-based workflows are supported. + Cannot load the workflow. Only signed in-box XAML-based workflows or script-based workflows are supported in the current language mode. Retrying activity connection: attempt {0} of {1}. diff --git a/src/Modules/Windows+Unix-Core/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1 b/src/Modules/Windows+Unix-Core/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1 index 180cda817..34c87b2d5 100644 --- a/src/Modules/Windows+Unix-Core/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1 +++ b/src/Modules/Windows+Unix-Core/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1 @@ -1,17 +1,17 @@ -@{ +@{ GUID="1DA87E53-152B-403E-98DC-74D7B4D63D59" Author="Microsoft Corporation" CompanyName="Microsoft Corporation" -Copyright="© Microsoft Corporation. All rights reserved." +Copyright="© Microsoft Corporation. All rights reserved." ModuleVersion="3.1.0.0" PowerShellVersion="3.0" CmdletsToExport= "Format-List", "Format-Custom", "Format-Table", "Format-Wide", "Out-File", "Out-String", "Get-FormatData", "Export-FormatData", "ConvertFrom-Json", "ConvertTo-Json", - "Invoke-RestMethod", "Invoke-WebRequest", - "Register-ObjectEvent", "Register-EngineEvent", "Wait-Event", "Get-Event", "Remove-Event", - "Get-EventSubscriber", "Unregister-Event", "New-Event", "Add-Member", "Add-Type", "Compare-Object", - "ConvertFrom-StringData", "Export-Csv", "Import-Csv", "ConvertTo-Csv", "ConvertFrom-Csv", "Export-Alias", - "Invoke-Expression", "Get-Alias", "Get-Culture", "Get-Date", "Get-Host", "Get-Member", "Get-Random", + "Invoke-RestMethod", "Invoke-WebRequest", "Register-ObjectEvent", "Register-EngineEvent", + "Wait-Event", "Get-Event", "Remove-Event", "Get-EventSubscriber", "Unregister-Event", + "New-Event", "Add-Member", "Add-Type", "Compare-Object", "ConvertFrom-StringData", + "Export-Csv", "Import-Csv", "ConvertTo-Csv", "ConvertFrom-Csv", "Export-Alias", "Invoke-Expression", + "Get-Alias", "Get-Culture", "Get-Date", "Get-Host", "Get-Member", "Get-Random", "Get-UICulture", "Get-Unique", "Import-Alias", "Import-LocalizedData", "Select-String", "Measure-Object", "New-Alias", "New-TimeSpan", "Read-Host", "Set-Alias", "Set-Date", "Start-Sleep", "Tee-Object", "Measure-Command", "Update-TypeData", "Update-FormatData", @@ -20,7 +20,6 @@ CmdletsToExport= "Format-List", "Format-Custom", "Format-Table", "Format-Wide", "Clear-Variable", "Export-Clixml", "Import-Clixml", "ConvertTo-Xml", "Select-Xml", "Write-Debug", "Write-Verbose", "Write-Warning", "Write-Error", "Write-Information", "Write-Output", "Set-PSBreakpoint", "Get-PSBreakpoint", "Remove-PSBreakpoint", "Enable-PSBreakpoint", "Disable-PSBreakpoint", "Get-PSCallStack", - "Get-TraceSource", "Set-TraceSource", "Trace-Command", "Unblock-File", "Get-Runspace", "Debug-Runspace", "Enable-RunspaceDebug", "Disable-RunspaceDebug", "Get-RunspaceDebug", "Wait-Debugger" FunctionsToExport= "Get-FileHash", "New-TemporaryFile", "New-Guid", "Format-Hex", "Import-PowerShellDataFile", diff --git a/src/Modules/Windows-Full/PSWorkflowUtility/PSWorkflowUtility.psm1 b/src/Modules/Windows-Full/PSWorkflowUtility/PSWorkflowUtility.psm1 index 779e01d89..3eaa04411 100644 --- a/src/Modules/Windows-Full/PSWorkflowUtility/PSWorkflowUtility.psm1 +++ b/src/Modules/Windows-Full/PSWorkflowUtility/PSWorkflowUtility.psm1 @@ -1,8 +1,9 @@ -workflow Invoke-AsWorkflow +workflow Invoke-AsWorkflow { <# .EXTERNALHELP Microsoft.PowerShell.Workflow.ServiceCore.dll-help.xml #> + [System.Security.SecurityCritical()] [CmdletBinding(DefaultParameterSetName='Command', HelpUri='http://go.microsoft.com/fwlink/?LinkId=238267')] param( [Parameter(Mandatory=$true,ParameterSetName="Command")] diff --git a/src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs b/src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs index 4aa04d1f5..648674beb 100644 --- a/src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs +++ b/src/System.Management.Automation/engine/Modules/ModuleCmdletBase.cs @@ -3777,7 +3777,7 @@ namespace Microsoft.PowerShell.Commands if (workflowsToProcess != null && workflowsToProcess.Count > 0) { // In ConstrainedLanguage, XAML workflows are not supported (even from a trusted FullLanguage state), - // since we can't prevent tampering. + // unless they are signed in-box OS binaries. if ((SystemPolicy.GetSystemLockdownPolicy() == SystemEnforcementMode.Enforce) || (Context.LanguageMode == PSLanguageMode.ConstrainedLanguage)) { @@ -3785,15 +3785,30 @@ namespace Microsoft.PowerShell.Commands // in full-language mode. if (! SystemPolicy.XamlWorkflowSupported) { - throw new NotSupportedException(Modules.XamlWorkflowsNotSupported); + foreach (string workflowPath in ResolveWorkflowFiles(moduleBase, workflowsToProcess)) + { + if (!SecuritySupport.IsProductBinary(workflowPath)) + { + throw new NotSupportedException(Modules.XamlWorkflowsNotSupported); + } + } } } SessionStateInternal oldSessionStateWF = Context.EngineSessionState; + PSLanguageMode? savedLanguageMode = null; try { Context.EngineSessionState = ss.Internal; + // Always run workflow import script as trusted since only signed in-box files can be imported + // on locked down machines. + if (Context.LanguageMode != PSLanguageMode.FullLanguage) + { + savedLanguageMode = Context.LanguageMode; + Context.LanguageMode = PSLanguageMode.FullLanguage; + } + if (dependentWorkflows != null && dependentWorkflows.Count > 0) { ScriptBlock importWorkflow = ScriptBlock.Create(Context, @@ -3811,7 +3826,7 @@ namespace Microsoft.PowerShell.Commands else { ScriptBlock importWorkflow = ScriptBlock.Create(Context, - "param($files, $dependentFiles) Microsoft.PowerShell.Workflow.ServiceCore\\Import-PSWorkflow -Path \"$files\" -Force:$"+BaseForce + "param($files, $dependentFiles) Microsoft.PowerShell.Workflow.ServiceCore\\Import-PSWorkflow -Path \"$files\" -Force:$" + BaseForce ); foreach (string workflowPath in ResolveWorkflowFiles(moduleBase, workflowsToProcess)) @@ -3830,6 +3845,11 @@ namespace Microsoft.PowerShell.Commands finally { Context.EngineSessionState = oldSessionStateWF; + + if (savedLanguageMode != null) + { + Context.LanguageMode = savedLanguageMode.Value; + } } } } diff --git a/src/System.Management.Automation/engine/parser/ast.cs b/src/System.Management.Automation/engine/parser/ast.cs index 3856c3b28..f08415b64 100644 --- a/src/System.Management.Automation/engine/parser/ast.cs +++ b/src/System.Management.Automation/engine/parser/ast.cs @@ -11084,6 +11084,9 @@ namespace System.Management.Automation.Internal /// List CompileWorkflows(ScriptBlockAst ast, PSModuleInfo definingModule, InitialSessionState initialSessionState, out ParseException parsingErrors); + /// + List CompileWorkflows(ScriptBlockAst ast, PSModuleInfo definingModule, InitialSessionState initialSessionState, PSLanguageMode? languageMode, out ParseException parsingErrors); + /// List CompileWorkflows(ScriptBlockAst ast, PSModuleInfo definingModule, string rootWorkflowName); diff --git a/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs b/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs index ebf9b90dd..65d9c32c6 100644 --- a/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs +++ b/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs @@ -1828,7 +1828,7 @@ namespace System.Management.Automation if (_scriptBlock.HasBeginBlock) { - RunClause(_runOptimized ? _scriptBlock.BeginBlock : _scriptBlock.UnoptimizedBeginBlock, AutomationNull.Value, _input.GetEnumerator()); + RunClause(_runOptimized ? _scriptBlock.BeginBlock : _scriptBlock.UnoptimizedBeginBlock, AutomationNull.Value, _input); } } @@ -1851,7 +1851,7 @@ namespace System.Management.Automation } if (_scriptBlock.HasProcessBlock) { - RunClause(_runOptimized ? _scriptBlock.ProcessBlock : _scriptBlock.UnoptimizedProcessBlock, dollarUnder, _input.GetEnumerator()); + RunClause(_runOptimized ? _scriptBlock.ProcessBlock : _scriptBlock.UnoptimizedProcessBlock, dollarUnder, _input); _input.Clear(); } } @@ -1865,7 +1865,7 @@ namespace System.Management.Automation if (_scriptBlock.HasEndBlock) { - RunClause(_runOptimized ? _scriptBlock.EndBlock : _scriptBlock.UnoptimizedEndBlock, AutomationNull.Value, _input.ToArray().GetEnumerator()); + RunClause(_runOptimized ? _scriptBlock.EndBlock : _scriptBlock.UnoptimizedEndBlock, AutomationNull.Value, _input.ToArray()); } } diff --git a/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs b/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs index 95b475da8..bc9b2161b 100644 --- a/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs +++ b/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs @@ -1125,7 +1125,8 @@ namespace System.Management.Automation try { var converterInstance = Utils.GetAstToWorkflowConverterAndEnsureWorkflowModuleLoaded(context); - var workflows = converterInstance.CompileWorkflows(scriptBlockAst, context.EngineSessionState.Module, null, out parseErrors); + PSLanguageMode? languageMode = (context != null) ? context.LanguageMode : (PSLanguageMode?) null; + var workflows = converterInstance.CompileWorkflows(scriptBlockAst, context.EngineSessionState.Module, null, languageMode, out parseErrors); foreach (var workflow in workflows) { context.EngineSessionState.SetWorkflowRaw(workflow, diff --git a/src/System.Management.Automation/resources/Modules.resx b/src/System.Management.Automation/resources/Modules.resx index 2a0d13659..c8febde19 100644 --- a/src/System.Management.Automation/resources/Modules.resx +++ b/src/System.Management.Automation/resources/Modules.resx @@ -597,7 +597,7 @@ Cannot define the workflow. The language mode for this session is incompatible with the system-wide language mode. - Cannot load the workflow. XAML-based workflows are not supported in the current language mode. Only script-based workflows are supported in this language mode. + Cannot load the workflow. Only signed in-box XAML-based workflows or script-based workflows are supported in the current language mode. Cannot verify the Microsoft .NET Framework version {0} because it is not included in the list of permitted versions. diff --git a/src/System.Management.Automation/security/SecuritySupport.cs b/src/System.Management.Automation/security/SecuritySupport.cs index c9f38774c..bb4174863 100644 --- a/src/System.Management.Automation/security/SecuritySupport.cs +++ b/src/System.Management.Automation/security/SecuritySupport.cs @@ -100,7 +100,7 @@ namespace Microsoft.PowerShell } } -namespace System.Management.Automation +namespace System.Management.Automation.Internal { /// /// The SAFER policy associated with this file @@ -118,7 +118,10 @@ namespace System.Management.Automation Disallowed = 2 } - internal static class SecuritySupport + /// + /// Security Support APIs + /// + public static class SecuritySupport { #region execution policy @@ -414,7 +417,12 @@ namespace System.Management.Automation } } - internal static bool IsProductBinary(string file) + /// + /// Returns true if file has product binary signature + /// + /// Name of file to check + /// True when file has product binary signature + public static bool IsProductBinary(string file) { if(String.IsNullOrEmpty(file) || (! IO.File.Exists(file))) { From 0ac0f1962ea106d0af9a8097fedba7bc154e9241 Mon Sep 17 00:00:00 2001 From: Sergei Vorobev Date: Thu, 28 Jul 2016 17:31:17 -0700 Subject: [PATCH 3/4] Replace Uri.UriSchemeHttps by https in PackageListParser Uri.UriSchemeHttps is not available on CoreCLR --- .../PackageList/PackageListParser.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs index ef2cbea48..a20856c2d 100644 --- a/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs +++ b/src/Microsoft.PackageManagement.PackageSourceListProvider/PackageList/PackageListParser.cs @@ -347,9 +347,9 @@ namespace Microsoft.PackageManagement.PackageSourceListProvider if (!uri.IsFile) { - if (uri.Scheme != Uri.UriSchemeHttps) + if (uri.Scheme != "https") { - throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.Messages.UriSchemeNotSupported, uri.Scheme, Uri.UriSchemeHttps)); + throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.Messages.UriSchemeNotSupported, uri.Scheme, "https")); } } From dac906db3d26bd1dbaba81251e41a761b7ea5309 Mon Sep 17 00:00:00 2001 From: Sergei Vorobev Date: Thu, 28 Jul 2016 17:51:46 -0700 Subject: [PATCH 4/4] Skip $input type test per #1563 --- test/powershell/Language/Parser/AutomaticVariables.Tests.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/AutomaticVariables.Tests.ps1 b/test/powershell/Language/Parser/AutomaticVariables.Tests.ps1 index ee6a57f6f..b73d68dfa 100644 --- a/test/powershell/Language/Parser/AutomaticVariables.Tests.ps1 +++ b/test/powershell/Language/Parser/AutomaticVariables.Tests.ps1 @@ -1,6 +1,8 @@ Describe 'Automatic variable $input' -Tags "CI" { - It '$input Type should be enumerator' { + # Skip on hold for discussion on https://github.com/PowerShell/PowerShell/issues/1563 + # $input type in advanced functions + It '$input Type should be enumerator' -Skip { function from_begin { [cmdletbinding()]param() begin { Write-Output -NoEnumerate $input } } function from_process { [cmdletbinding()]param() process { Write-Output -NoEnumerate $input } } function from_end { [cmdletbinding()]param() end { Write-Output -NoEnumerate $input } }