Compare commits

...

12 commits

Author SHA1 Message Date
Fraser Waters c8336bdc38 rm loggign 2021-11-12 18:55:41 +00:00
Fraser Waters 5db461b6be loggign 2021-11-12 18:52:27 +00:00
Fraser Waters c2c08cb6a6 fix test proj 2021-11-12 18:52:08 +00:00
Fraser Waters 1580b522d3 fixes 2021-11-12 18:37:21 +00:00
Fraser Waters 636f2c9a73 fixes 2021-11-12 18:22:42 +00:00
Fraser Waters f1f1b80deb serialise sdk side 2021-11-12 17:45:20 +00:00
Fraser Waters 85886b31ca serialise one side 2021-11-12 15:55:13 +00:00
Fraser Waters fa11f3aac1 working hardcoded 2021-11-12 15:39:53 +00:00
Fraser Waters 98e9e80d81 wi 2021-11-12 12:58:15 +00:00
Fraser Waters 69496beead wip 2021-11-11 17:48:15 +00:00
Fraser Waters 08414a7b99 wip 2021-11-11 17:16:15 +00:00
Fraser Waters e20ffbebba Dynamic provider test 2021-11-10 21:05:03 +00:00
19 changed files with 452 additions and 15 deletions

View file

@ -56,6 +56,9 @@ test_all:: dotnet_test auto_test
dist::
go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version=${DOTNET_VERSION}" ${LANGHOST_PKG}
dotnet publish Pulumi.Dynamic/Pulumi.Dynamic.csproj /p:Version=${DOTNET_VERSION} -r linux-x64
cp ./Pulumi.Dynamic/bin/Debug/netcoreapp3.1/linux-x64/publish/Pulumi.Dynamic "$$(go env GOPATH)"/bin/pulumi-resource-pulumi-dotnet
brew:: BREW_VERSION := $(shell ../../scripts/get-version HEAD)
brew::
go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version=${BREW_VERSION}" ${LANGHOST_PKG}

View file

@ -0,0 +1,175 @@
using Pulumirpc;
using Grpc.Core;
using Google.Protobuf.WellKnownTypes;
using System.Linq;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Collections.Immutable;
public static class Program
{
class DynamicResourceProviderServicer : ResourceProvider.ResourceProviderBase
{
private static (Pulumi.DynamicResourceProvider, ImmutableDictionary<string, object?>) GetProvider(Struct properties)
{
var fields = properties.Fields;
if (!fields.TryGetValue("__provider", out var providerValue))
{
var props = string.Concat(fields.Select(item => string.Format("{0} = {1}", item.Key, item.Value)));
var msg = string.Format("Dynamic resource had no '__provider' property, was: {0}", props);
throw new RpcException(new Status(StatusCode.Unknown, msg));
}
var providerString = providerValue.StringValue;
if(providerString == null)
{
throw new RpcException(new Status(StatusCode.Unknown, "Dynamic resource '__provider' property was not a string"));
}
var pickler = new Ibasa.Pikala.Pickler();
var memoryStream = new System.IO.MemoryStream(System.Convert.FromBase64String(providerString));
var provider = pickler.Deserialize(memoryStream) as Pulumi.DynamicResourceProvider;
if (provider == null)
{
throw new RpcException(new Status(StatusCode.Unknown, "Dynamic resource could not deserialise provider implementation"));
}
return (provider, Pulumi.Serialization.Rpc.DeserialiseProperties(properties));
}
public override Task<CheckResponse> CheckConfig(CheckRequest request, ServerCallContext context)
{
throw new RpcException(new Status(StatusCode.Unimplemented, "CheckConfig is not implemented by the dynamic provider"));
}
public override Task<DiffResponse> DiffConfig(DiffRequest request, ServerCallContext context)
{
throw new RpcException(new Status(StatusCode.Unimplemented, "DiffConfig is not implemented by the dynamic provider"));
}
public override Task<InvokeResponse> Invoke(InvokeRequest request, ServerCallContext context)
{
throw new RpcException(new Status(StatusCode.Unimplemented, "Invoke is not implemented by the dynamic provider"));
}
public override Task<GetSchemaResponse> GetSchema(GetSchemaRequest request, ServerCallContext context)
{
throw new RpcException(new Status(StatusCode.Unimplemented, "GetSchema is not implemented by the dynamic provider"));
}
public override Task<ConfigureResponse> Configure(ConfigureRequest request, ServerCallContext context)
{
var response = new ConfigureResponse();
response.AcceptSecrets = false;
return Task.FromResult(response);
}
public override Task<PluginInfo> GetPluginInfo(Empty request, ServerCallContext context)
{
var response = new PluginInfo();
response.Version = "0.1.0";
return Task.FromResult(response);
}
public override Task<Empty> Cancel(Empty request, ServerCallContext context)
{
return Task.FromResult(new Empty());
}
public override async Task<CreateResponse> Create(CreateRequest request, ServerCallContext context)
{
try {
var (provider, inputs) = GetProvider(request.Properties);
var (id, outputs) = await provider.Create(inputs);
var response = new CreateResponse();
response.Id = id;
response.Properties = Pulumi.Serialization.Rpc.SerialiseProperties(outputs);
// Readd provider
response.Properties.Fields.Add("__provider", request.Properties.Fields["__provider"]);
return response;
} catch (System.Exception ex)
{
throw new RpcException(new Status(StatusCode.Unknown, ex.Message));
}
}
public override Task<ReadResponse> Read(ReadRequest request, ServerCallContext context)
{
var id = request.Id;
var props = request.Properties;
var response = new ReadResponse();
response.Id = id;
response.Properties = props;
return Task.FromResult(response);
}
public override Task<CheckResponse> Check(CheckRequest request, ServerCallContext context)
{
var response = new CheckResponse();
response.Inputs = request.News;
return Task.FromResult(response);
}
public override Task<DiffResponse> Diff(DiffRequest request, ServerCallContext context)
{
var response = new DiffResponse();
return Task.FromResult(response);
//fields = {}
//if result.changes is not None:
// if result.changes:
// fields["changes"] = proto.DiffResponse.DIFF_SOME # pylint: disable=no-member
// else:
// fields["changes"] = proto.DiffResponse.DIFF_NONE # pylint: disable=no-member
//else:
// fields["changes"] = proto.DiffResponse.DIFF_UNKNOWN # pylint: disable=no-member
//if result.replaces is not None:
// fields["replaces"] = result.replaces
//if result.delete_before_replace is not None:
// fields["deleteBeforeReplace"] = result.delete_before_replace
}
public override Task<UpdateResponse> Update(UpdateRequest request, ServerCallContext context)
{
var response = new UpdateResponse();
response.Properties = request.News;
return Task.FromResult(response);
}
public override Task<Empty> Delete(DeleteRequest request, ServerCallContext context)
{
return Task.FromResult(new Empty());
}
}
public static void Main(string[] args)
{
var monitor = new DynamicResourceProviderServicer();
// maxRpcMessageSize raises the gRPC Max Message size from `4194304` (4mb) to `419430400` (400mb)
var maxRpcMessageSize = 400 * 1024 * 1024;
var grpcChannelOptions = new List<ChannelOption> { new ChannelOption(ChannelOptions.MaxReceiveMessageLength, maxRpcMessageSize)};
var server = new Server(grpcChannelOptions)
{
Services = { ResourceProvider.BindService(monitor) },
Ports = { new ServerPort("0.0.0.0", 0, ServerCredentials.Insecure) }
};
server.Start();
var port = server.Ports.First();
System.Console.WriteLine(port.BoundPort.ToString());
Task? shutdownTask = null;
var exitEvent = new System.Threading.ManualResetEventSlim();
System.Console.CancelKeyPress += (System.ConsoleCancelEventHandler)((sender, e) => {
shutdownTask = server.ShutdownAsync();
exitEvent.Set();
});
exitEvent.Wait();
shutdownTask!.Wait();
}
}

View file

@ -0,0 +1,39 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<PublishSingleFile>true</PublishSingleFile>
<TargetFramework>netcoreapp3.1</TargetFramework>
<Nullable>enable</Nullable>
<Authors>Pulumi</Authors>
<Company>Pulumi Corp.</Company>
<Description>The Pulumi .NET dynamic provider runs dynamic resources providers from C#, F#, and VB.NET.</Description>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<PropertyGroup>
<AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
<EmbedUntrackedSources>true</EmbedUntrackedSources>
<PublishRepositoryUrl>true</PublishRepositoryUrl>
</PropertyGroup>
<PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">
<ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Grpc.Tools" Version="2.37.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Pulumi\Pulumi.csproj" />
</ItemGroup>
<ItemGroup>
<Protobuf Include="..\..\proto\*.proto" Access="internal" />
</ItemGroup>
</Project>

View file

@ -112,7 +112,7 @@ namespace Pulumi
public SerializationResult ToSerializationResult()
=> new SerializationResult(
Serializer.CreateStruct(PropertyValues),
Serializer.CreateStruct(PropertyValues!),
PropertyToDependentResources);
}
}

View file

@ -36,10 +36,10 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.CodeAnalysis.PublicApiAnalyzers" Version="2.9.6">
<!--<PackageReference Include="Microsoft.CodeAnalysis.PublicApiAnalyzers" Version="2.9.6">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</PackageReference>-->
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.16" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
<PackageReference Include="OneOf" Version="2.1.151" />
@ -47,6 +47,7 @@
<PackageReference Include="semver" Version="2.0.6" />
<PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
<PackageReference Include="Ibasa.Pikala" Version="0.0.9" />
</ItemGroup>
<ItemGroup>

View file

@ -1,5 +1,6 @@
// Copyright 2016-2019, Pulumi Corporation
using System;
using Pulumi.Serialization;
namespace Pulumi
@ -62,5 +63,10 @@ namespace Pulumi
remote: false, dependency: dependency)
{
}
private protected CustomResource( Func<Resource, string> typeProvider, string name, ResourceArgs? args, CustomResourceOptions? options = null)
: base(typeProvider, name, custom: true, args ?? ResourceArgs.Empty, options ?? new CustomResourceOptions(), remote: false)
{
}
}
}

View file

@ -0,0 +1,55 @@
// Copyright 2016-2019, Pulumi Corporation
using System;
namespace Pulumi
{
public class DynamicResourceArgs : ResourceArgs
{
[Input("__provider", required: true)]
public Input<string> Provider { get; set; } = null!;
}
public class DynamicResource : CustomResource
{
private static string GetTypeName(Resource resource)
{
var type = resource.GetType();
var typeName = string.IsNullOrEmpty(type.Namespace) ? $"dynamic:{type.Name}" : $"dynamic/{type.Namespace}:{type.Name}";;
return $"pulumi-dotnet:{typeName}";
}
private static bool ByValueFilter(System.Reflection.Assembly assembly)
{
// Assemblies known to be used for defining dynamic providers
var knownAssemblies = new string [] {
"Pulumi", "System.Collections.Immutable"
};
var assemblyName = assembly.GetName().Name;
return !Array.Exists(knownAssemblies, name => name == assemblyName);
}
private static ResourceArgs SetProvider(DynamicResourceProvider provider, DynamicResourceArgs? args)
{
if (args == null)
{
args = new DynamicResourceArgs();
}
var pickler = new Ibasa.Pikala.Pickler(ByValueFilter);
var memoryStream = new System.IO.MemoryStream();
pickler.Serialize(memoryStream, provider);
var base64String = System.Convert.ToBase64String(memoryStream.ToArray());
args.Provider = base64String;
return args;
}
#pragma warning disable RS0022 // Constructor make noninheritable base class inheritable
public DynamicResource(DynamicResourceProvider provider, string name, DynamicResourceArgs? args, CustomResourceOptions? options = null)
: base((Func<Resource, string>)GetTypeName, name, SetProvider(provider, args), options)
#pragma warning restore RS0022 // Constructor make noninheritable base class inheritable
{
}
}
}

View file

@ -0,0 +1,17 @@
// Copyright 2016-2019, Pulumi Corporation
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Threading.Tasks;
namespace Pulumi
{
public abstract class DynamicResourceProvider
{
public virtual Task<(string, IDictionary<string, object?>)> Create(ImmutableDictionary<string, object?> properties)
{
throw new NotImplementedException();
}
}
}

View file

@ -20,33 +20,33 @@ namespace Pulumi
/// The child resources of this resource. We use these (only from a ComponentResource) to
/// allow code to dependOn a ComponentResource and have that effectively mean that it is
/// depending on all the CustomResource children of that component.
///
///
/// Important! We only walk through ComponentResources.They're the only resources that
/// serve as an aggregation of other primitive(i.e.custom) resources.While a custom resource
/// can be a parent of other resources, we don't want to ever depend on those child
/// resource. If we do, it's simple to end up in a situation where we end up depending on a
/// child resource that has a data cycle dependency due to the data passed into it. An
/// example of how this would be bad is:
///
///
/// <c>
/// var c1 = new CustomResource("c1");
/// var c2 = new CustomResource("c2", { parentId = c1.id }, { parent = c1 });
/// var c3 = new CustomResource("c3", { parentId = c1.id }, { parent = c1 });
/// </c>
///
///
/// The problem here is that 'c2' has a data dependency on 'c1'. If it tries to wait on
/// 'c1' it will walk to the children and wait on them.This will mean it will wait on 'c3'.
/// But 'c3' will be waiting in the same manner on 'c2', and a cycle forms. This normally
/// does not happen with ComponentResources as they do not have any data flowing into
/// them.The only way you would be able to have a problem is if you had this sort of coding
/// pattern:
///
///
/// <c>
/// var c1 = new ComponentResource("c1");
/// var c2 = new CustomResource("c2", { parentId = c1.urn }, { parent: c1 });
/// var c3 = new CustomResource("c3", { parentId = c1.urn }, { parent: c1 });
/// </c>
///
///
/// However, this would be pretty nonsensical as there is zero need for a custom resource to
/// ever need to reference the urn of a component resource. So it's acceptable if that sort
/// of pattern failed in practice.
@ -120,8 +120,32 @@ namespace Pulumi
private protected Resource(
string type, string name, bool custom,
ResourceArgs args, ResourceOptions options,
bool remote = false, bool dependency = false) :
this(_ => type, name, custom, args, options, remote, dependency)
{
}
/// <summary>
/// Creates and registers a new resource object. <paramref name="typeProvider"/> is the fully
/// qualified type token and <paramref name="name"/> is the "name" part to use in creating a
/// stable and globally unique URN for the object. dependsOn is an optional list of other
/// resources that this resource depends on, controlling the order in which we perform
/// resource operations.
/// </summary>
/// <param name="typeProvider">The type of the resource.</param>
/// <param name="name">The unique name of the resource.</param>
/// <param name="custom">True to indicate that this is a custom resource, managed by a plugin.</param>
/// <param name="args">The arguments to use to populate the new resource.</param>
/// <param name="options">A bag of options that control this resource's behavior.</param>
/// <param name="remote">True if this is a remote component resource.</param>
/// <param name="dependency">True if this is a synthetic resource used internally for dependency tracking.</param>
private protected Resource(
Func<Resource, string> typeProvider, string name, bool custom,
ResourceArgs args, ResourceOptions options,
bool remote = false, bool dependency = false)
{
var type = typeProvider(this);
if (dependency)
{
_type = "";

View file

@ -100,7 +100,7 @@ namespace Pulumi.Serialization
});
public static OutputData<object?> Deserialize(Value value)
=> DeserializeCore(value,
=> DeserializeCore(value,
v => v.KindCase switch
{
Value.KindOneofCase.NumberValue => DeserializerDouble(v),

View file

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using Google.Protobuf.WellKnownTypes;
namespace Pulumi.Serialization
{
public static class Rpc {
public static ImmutableDictionary<string, object?> DeserialiseProperties(Struct properties)
{
var output = Deserializer.Deserialize(Value.ForStruct(properties));
if (!output.IsKnown || output.IsSecret)
{
throw new Exception("Deserialize of a Struct should always be known and not secret!");
}
var result = output.Value as ImmutableDictionary<string, object?>;
if (result == null)
{
throw new Exception("Deserialize of a Struct should always return an ImmutableDictionary!");
}
return result;
}
public static Struct SerialiseProperties(IDictionary<string, object?> properties)
{
var dictionary = ImmutableDictionary.CreateRange(properties);
return Serializer.CreateStruct(dictionary);
}
}
}

View file

@ -381,7 +381,7 @@ $"Tasks are not allowed inside ResourceArgs. Please wrap your Task in an Output:
bool b => Value.ForBool(b),
string s => Value.ForString(s),
ImmutableArray<object> list => Value.ForList(list.Select(CreateValue).ToArray()),
ImmutableDictionary<string, object> dict => Value.ForStruct(CreateStruct(dict)),
ImmutableDictionary<string, object?> dict => Value.ForStruct(CreateStruct(dict)),
_ => throw new InvalidOperationException("Unsupported value when converting to protobuf: " + value.GetType().FullName),
};
@ -410,7 +410,7 @@ $"Tasks are not allowed inside ResourceArgs. Please wrap your Task in an Output:
/// Given a <see cref="ImmutableDictionary{TKey, TValue}"/> produced by <see cref="SerializeAsync"/>,
/// produces the equivalent <see cref="Struct"/> that can be passed to the Pulumi engine.
/// </summary>
public static Struct CreateStruct(ImmutableDictionary<string, object> serializedDictionary)
public static Struct CreateStruct(ImmutableDictionary<string, object?> serializedDictionary)
{
var result = new Struct();
foreach (var key in serializedDictionary.Keys.OrderBy(k => k))

View file

@ -47,7 +47,7 @@ namespace Pulumi.Testing
}
return new InvokeResponse { Return = await SerializeAsync(registeredResource).ConfigureAwait(false) };
}
var result = await _mocks.CallAsync(new MockCallArgs
{
Token = request.Tok,
@ -183,13 +183,13 @@ namespace Pulumi.Testing
return builder.ToImmutable();
}
private async Task<ImmutableDictionary<string, object>> SerializeToDictionary(object o)
private async Task<ImmutableDictionary<string, object?>> SerializeToDictionary(object o)
{
if (o is IDictionary<string, object> d)
if (o is IDictionary<string, object?> d)
{
o = d.ToImmutableDictionary();
}
return await _serializer.SerializeAsync("", o, true).ConfigureAwait(false) as ImmutableDictionary<string, object>
return await _serializer.SerializeAsync("", o, true).ConfigureAwait(false) as ImmutableDictionary<string, object?>
?? throw new InvalidOperationException($"{o.GetType().FullName} is not a supported argument type");
}

View file

@ -0,0 +1,3 @@
/.pulumi/
[Bb]in/
[Oo]bj/

View file

@ -0,0 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
</Project>

View file

@ -0,0 +1,51 @@
// Copyright 2016-2021, Pulumi Corporation. All rights reserved.
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Threading.Tasks;
using Pulumi;
public sealed class RandomResourceProvider : DynamicResourceProvider
{
public override async Task<(string, IDictionary<string, object>)> Create(ImmutableDictionary<string, object> properties)
{
var random = new System.Random();
var buffer = new byte[15];
random.NextBytes(buffer);
var val = System.Convert.ToBase64String(buffer);
return (val, new Dictionary<string, object>{ { "val", val } });
}
}
public sealed class RandomArgs : DynamicResourceArgs
{
[Input("val")]
public Input<string> Val { get; set; }
}
class Random : DynamicResource
{
[Output("val")]
public Output<string> Val { get; private set; }
public Random(string name, CustomResourceOptions options = null)
: base(new RandomResourceProvider(), name, new RandomArgs() { Val = "" }, options)
{
}
}
class Program
{
static Task<int> Main(string[] args)
{
return Deployment.RunAsync(() => {
var r = new Random("foo");
return new Dictionary<string, object> {
{ "random_id", r.Id },
{ "random_val", r.Val }
};
});
}
}

View file

@ -0,0 +1,3 @@
name: dynamic_dotnet
description: A simple dotnet program that uses dynamic providers.
runtime: dotnet

View file

@ -0,0 +1 @@
Intentionally make no changes.

View file

@ -591,3 +591,22 @@ func TestAboutDotnet(t *testing.T) {
// This one doesn't have a current stack. Assert that we caught it.
assert.Contains(t, stderr, "No current stack")
}
// Tests dynamic provider in Dotnet.
func TestDynamicDotnet(t *testing.T) {
var randomVal string
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: filepath.Join("dynamic", "dotnet"),
Dependencies: []string{"Pulumi"},
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
randomVal = stack.Outputs["random_val"].(string)
},
EditDirs: []integration.EditDir{{
Dir: "step1",
Additive: true,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
assert.Equal(t, randomVal, stack.Outputs["random_val"].(string))
},
}},
})
}