Convert the AWS Lambda module to CIDLC

This commit is contained in:
joeduffy 2017-04-28 12:27:19 -07:00
parent 5ae168d23c
commit 1489a73b18
7 changed files with 228 additions and 191 deletions

25
lib/aws/idl/kms/key.go Normal file
View file

@ -0,0 +1,25 @@
// Copyright 2017 Pulumi, Inc. All rights reserved.
package kms
import (
"github.com/pulumi/coconut/pkg/resource/idl"
)
// The Key resource creates a customer master key (CMK) in AWS Key Management Service (AWS KMS). Users (customers) can
// use the master key to encrypt their data stored in AWS services that are integrated with AWS KMS or within their
// applications. For more information, see http://docs.aws.amazon.com/kms/latest/developerguide/.
type Key struct {
idl.NamedResource
// KeyPolicy attaches a KMS policy to this key. Use a policy to specify who has permission to use the key and which
// actions they can perform. For more information, see
// http://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html.
KeyPolicy interface{} `coco:"keyPolicy"` // TODO: map the schema.
// Description is an optional description of the key. Use a description that helps your users decide whether the
// key is appropriate for a particular task.
Description *string `coco:"description,optional"`
// Enabled indicates whether the key is available for use. This value is `true` by default.
Enabled *bool `coco:"enabled,optional"`
// EnableKeyRotation indicates whether AWS KMS rotates the key. This value is `false` by default.
EnableKeyRotation *bool `coco:"enableKeyRotation,optional"`
}

View file

@ -0,0 +1,100 @@
// Copyright 2017 Pulumi, Inc. All rights reserved.
package lambda
import (
"github.com/pulumi/coconut/pkg/resource/idl"
aws "github.com/pulumi/coconut/lib/aws/idl"
"github.com/pulumi/coconut/lib/aws/idl/ec2"
"github.com/pulumi/coconut/lib/aws/idl/iam"
"github.com/pulumi/coconut/lib/aws/idl/kms"
)
// The Function resource creates an AWS Lambda function that can run code in response to events.
type Function struct {
idl.NamedResource
// code is the source code of your Lambda function. This supports all the usual Coconut asset schemes, in addition
// to Amazon Simple Storage Service (S3) bucket locations, indicating with a URI scheme of s3//<bucket>/<object>.
Code *idl.Asset `coco:"code"`
// handler is the name of the function (within your source code) that Lambda calls to start running your code.
Handler string `coco:"handler"`
// role is the AWS Identity and Access Management (IAM) execution role that Lambda assumes when it runs your code
// to access AWS services.
Role *iam.Role `coco:"role"`
// runtime is the runtime environment for the Lambda function that you are uploading.
Runtime Runtime `coco:"runtime"`
// functionName is a name for the function. If you don't specify a name, a unique physical ID is used instead.
FunctionName *string `coco:"functionName,optional"`
// deadLetterConfig configures how Lambda handles events that it can't process. If you don't specify a Dead Letter
// Queue (DLQ) configuration, Lambda discards events after the maximum number of retries.
DeadLetterConfig *DeadLetterConfig `coco:"deadLetterConfig,optional"`
// description is an optional description of the function.
Description *string `coco:"description,optional"`
// environment contains key-value pairs that Lambda caches and makes available for your Lambda functions. Use
// environment variables to apply configuration changes, such as test and production environment configurations,
// without changing your Lambda function source code.
Environment *Environment `coco:"environment,optional"`
// kmsKey is a AWS Key Management Service (AMS KMS) key that Lambda uses to encrypt and decrypt environment
// variables.
KMSKey *kms.Key `coco:"kmsKey,optional"`
// memorySize is the amount of memory, in MB, that is allocated to your Lambda function. Lambda uses this value to
// proportionally allocate the amount of CPU power. Your function use case determines your CPU and memory
// requirements. For example, a database operation might need less memory than an image processing function. You
// must specify a value that is greater than or equal to `128` and it must be a multiple of `64`. You cannot
// specify a size larger than `1536`. The default value is `128` MB.
MemorySize *float64 `coco:"memorySize,optional"`
// timeout is the function execution time (in seconds) after which Lambda terminates the function. Because the
// execution time affects cost, set this value based on the function's expected execution time. By default, timeout
// is set to `3` seconds.
Timeout *float64 `coco:"timeout,optional"`
// vpcConfig specifies a VPC configuration that Lambda uses to set up an elastic network interface (ENI). The ENI
// enables your function to connect to other resources in your VPC, but it doesn't provide public Internet access.
// If your function requires Internet access (for example, to access AWS services that don't have VPC endpoints),
// configure a Network Address Translation (NAT) instance inside your VPC or use an Amazon Virtual Private Cloud
// (Amazon VPC) NAT gateway.
VPCConfig *VPCConfig `coco:"vpcConfig,optional"`
// The ARN of the Lambda function, such as `arn:aws:lambda:us-west-2:123456789012:MyStack-AMILookUp-NT5EUXTNTXXD`.
ARN aws.ARN `coco:"arn,out"`
}
// Runtime represents the legal runtime environments for Lambdas.
type Runtime string
const (
NodeJSRuntime Runtime = "nodejs"
NodeJS4d3Runtime Runtime = "nodejs4.3"
NodeJS4d3EdgeRuntime Runtime = "nodejs4.3-edge"
NodeJS6d10Runtime Runtime = "nodejs6.10"
Java8Runtime Runtime = "java8"
Python2d7Runtime Runtime = "python2.7"
DotnetCore1d0Runtime Runtime = "dotnetcore1.0"
)
// DeadLetterConfig is a property of an AWS Lambda Function resource that specifies a Dead Letter Queue (DLQ) that
// events are sent to when functions cannot be processed. For example, you can send unprocessed events to an Amazon
// Simple Notification Service (Amazon SQS) topic, where you can take further action.
type DeadLetterConfig struct {
// target is the target resource where Lambda delivers unprocessed events. It may be an Amazon SNS topic or Amazon
// Simple Queue Service (SQS) queue. For the Lambda function-execution role, you must explicitly provide the
// relevant permissions so that access to your DLQ resource is part of the execution role for your Lambda function.
Target *idl.Resource `coco:"target"` // TODO: sns.Topic | sqs.Queue;
}
// Environment is a property of an AWS Lambda Function resource that specifies key-value pairs that the function can
// access so that you can apply configuration changes, such as test and production environment configurations, without
// changing the function code.
type Environment map[string]string
// VPCConfig is a property of an AWS Lambda Function resource that enables it to access resources in a VPC. For more
// information, see http://docs.aws.amazon.com/lambda/latest/dg/vpc.html.
type VPCConfig struct {
// securityGroups is a list of one or more security groups in the VPC that include the resources to which your
// Lambda function requires access.
SecurityGroups []*ec2.SecurityGroup `coco:"securityGroups"`
// subnets is a list of one or more subnet IDs in the VPC that includes the resources to which your Lambda function
// requires access.
Subnets []*ec2.Subnet `coco:"subnets"`
}

View file

@ -20,6 +20,8 @@ import (
rpc "github.com/pulumi/coconut/lib/aws/rpc/ec2"
)
const InstanceToken = rpc.InstanceToken
// NewInstanceProvider creates a provider that handles EC2 instance operations.
func NewInstanceProvider(ctx *awsctx.Context) cocorpc.ResourceProviderServer {
ops := &instanceProvider{ctx}

View file

@ -21,6 +21,8 @@ import (
rpc "github.com/pulumi/coconut/lib/aws/rpc/ec2"
)
const SecurityGroupToken = rpc.SecurityGroupToken
// constants for the various security group limits.
const (
maxSecurityGroupName = 255

View file

@ -6,7 +6,6 @@ import (
"archive/zip"
"bytes"
"crypto/sha1"
"errors"
"fmt"
"io"
"reflect"
@ -16,23 +15,40 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/lambda"
pbempty "github.com/golang/protobuf/ptypes/empty"
pbstruct "github.com/golang/protobuf/ptypes/struct"
"github.com/pkg/errors"
"github.com/pulumi/coconut/pkg/resource"
"github.com/pulumi/coconut/pkg/tokens"
"github.com/pulumi/coconut/pkg/util/contract"
"github.com/pulumi/coconut/pkg/util/mapper"
"github.com/pulumi/coconut/sdk/go/pkg/cocorpc"
"golang.org/x/net/context"
"github.com/pulumi/coconut/lib/aws/provider/awsctx"
awsrpc "github.com/pulumi/coconut/lib/aws/rpc"
rpc "github.com/pulumi/coconut/lib/aws/rpc/lambda"
)
const Function = tokens.Type("aws:lambda/function:Function")
const FunctionToken = rpc.FunctionToken
// constants for the various function limits.
const (
maxFunctionName = 64
maxFunctionNameARN = 140
functionNameARNPrefix = "arn:aws:lambda:"
)
var functionRuntimes = map[rpc.Runtime]bool{
rpc.NodeJSRuntime: true,
rpc.NodeJS4d3Runtime: true,
rpc.NodeJS4d3EdgeRuntime: true,
rpc.NodeJS6d10Runtime: true,
rpc.Java8Runtime: true,
rpc.Python2d7Runtime: true,
rpc.DotnetCore1d0Runtime: true,
}
// NewFunctionProvider creates a provider that handles Lambda function operations.
func NewFunctionProvider(ctx *awsctx.Context) cocorpc.ResourceProviderServer {
return &funcProvider{ctx}
ops := &funcProvider{ctx}
return rpc.NewFunctionProvider(ops)
}
type funcProvider struct {
@ -40,57 +56,56 @@ type funcProvider struct {
}
// Check validates that the given property bag is valid for a resource of the given type.
func (p *funcProvider) Check(ctx context.Context, req *cocorpc.CheckRequest) (*cocorpc.CheckResponse, error) {
// Read in the properties, create and validate a new group, and return the failures (if any).
contract.Assert(req.GetType() == string(Function))
_, _, result := unmarshalFunction(req.GetProperties())
return resource.NewCheckResponse(result), nil
}
// Name names a given resource. Sometimes this will be assigned by a developer, and so the provider
// simply fetches it from the property bag; other times, the provider will assign this based on its own algorithm.
// In any case, resources with the same name must be safe to use interchangeably with one another.
func (p *funcProvider) Name(ctx context.Context, req *cocorpc.NameRequest) (*cocorpc.NameResponse, error) {
return nil, nil // use the AWS provider default name
func (p *funcProvider) Check(ctx context.Context, obj *rpc.Function) ([]mapper.FieldError, error) {
var failures []mapper.FieldError
if _, has := functionRuntimes[obj.Runtime]; !has {
failures = append(failures,
mapper.NewFieldErr(reflect.TypeOf(obj), rpc.Function_Runtime,
fmt.Errorf("%v is not a valid runtime", obj.Runtime)))
}
if name := obj.FunctionName; name != nil {
var maxName int
if strings.HasPrefix(*name, functionNameARNPrefix) {
maxName = maxFunctionNameARN
} else {
maxName = maxFunctionName
}
if len(*name) > maxName {
failures = append(failures,
mapper.NewFieldErr(reflect.TypeOf(obj), rpc.Function_FunctionName,
fmt.Errorf("exceeded maximum length of %v", maxName)))
}
}
return failures, nil
}
// Create allocates a new instance of the provided resource and returns its unique ID afterwards. (The input ID
// must be blank.) If this call fails, the resource must not have been created (i.e., it is "transacational").
func (p *funcProvider) Create(ctx context.Context, req *cocorpc.CreateRequest) (*cocorpc.CreateResponse, error) {
contract.Assert(req.GetType() == string(Function))
// Read in the properties given by the request, validating as we go; if any fail, reject the request.
fun, _, decerr := unmarshalFunction(req.GetProperties())
if decerr != nil {
// TODO: this is a good example of a "benign" (StateOK) error; handle it accordingly.
return nil, decerr
}
func (p *funcProvider) Create(ctx context.Context, obj *rpc.Function) (string, *rpc.FunctionOuts, error) {
// If an explicit name is given, use it. Otherwise, auto-generate a name in part based on the resource name.
// TODO: use the URN, not just the name, to enhance global uniqueness.
// TODO: even for explicit names, we should consider mangling it somehow, to reduce multi-instancing conflicts.
var name string
if fun.FunctionName != nil {
name = *fun.FunctionName
if obj.FunctionName != nil {
name = *obj.FunctionName
} else {
name = resource.NewUniqueHex(fun.Name+"-", maxFunctionName, sha1.Size)
name = resource.NewUniqueHex(obj.Name+"-", maxFunctionName, sha1.Size)
}
// Fetch the IAM role's ARN.
// TODO[coconut/pulumi#90]: as soon as we can read output properties, this shouldn't be necessary.
role, err := p.ctx.IAM().GetRole(&iam.GetRoleInput{RoleName: fun.Role.StringPtr()})
if err != nil {
return nil, err
var roleARN *string
if role, err := p.ctx.IAM().GetRole(&iam.GetRoleInput{RoleName: obj.Role.StringPtr()}); err != nil {
return "", nil, err
} else {
roleARN = role.Role.Arn
}
roleARN := role.Role.Arn
// Figure out the kind of asset. In addition to the usual suspects, we permit s3:// references.
var code *lambda.FunctionCode
uri, isuri, err := fun.Code.GetURIURL()
if err != nil {
return nil, err
}
if isuri && uri.Scheme == "s3" {
if uri, isuri, err := obj.Code.GetURIURL(); err != nil {
return "", nil, err
} else if isuri && uri.Scheme == "s3" {
// TODO: it's odd that an S3 reference must *already* be a zipfile, whereas others are zipped on the fly.
code = &lambda.FunctionCode{
S3Bucket: aws.String(uri.Host),
@ -98,11 +113,12 @@ func (p *funcProvider) Create(ctx context.Context, req *cocorpc.CreateRequest) (
// TODO: S3ObjectVersion; encode as the #?
}
} else {
zip, err := zipCodeAsset(fun.Code, "index.js") // TODO: don't hard-code the filename.
if err != nil {
return nil, err
// TODO: assets need filenames; don't hard code it.
if zip, err := zipCodeAsset(*obj.Code, "index.js"); err != nil {
return "", nil, err
} else {
code = &lambda.FunctionCode{ZipFile: zip}
}
code = &lambda.FunctionCode{ZipFile: zip}
}
var dlqcfg *lambda.DeadLetterConfig
@ -111,40 +127,39 @@ func (p *funcProvider) Create(ctx context.Context, req *cocorpc.CreateRequest) (
// Convert float fields to in64 if they are non-nil.
var memsize *int64
if fun.MemorySize != nil {
sz := int64(*fun.MemorySize)
if obj.MemorySize != nil {
sz := int64(*obj.MemorySize)
memsize = &sz
}
var timeout *int64
if fun.Timeout != nil {
to := int64(*fun.Timeout)
if obj.Timeout != nil {
to := int64(*obj.Timeout)
timeout = &to
}
// Now go ahead and create the resource. Note that IAM profiles can take several seconds to propagate; see
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role.
fmt.Printf("Creating Lambda Function '%v' with name '%v'\n", fun.Name, name)
fmt.Printf("Creating Lambda Function '%v' with name '%v'\n", obj.Name, name)
create := &lambda.CreateFunctionInput{
Code: code,
DeadLetterConfig: dlqcfg,
Description: fun.Description,
Description: obj.Description,
Environment: env,
FunctionName: aws.String(name),
Handler: aws.String(fun.Handler),
KMSKeyArn: fun.KMSKeyID.StringPtr(),
Handler: aws.String(obj.Handler),
KMSKeyArn: obj.KMSKey.StringPtr(),
MemorySize: memsize,
Publish: nil, // ???
Role: role.Role.Arn,
Runtime: aws.String(fun.Runtime),
Role: roleARN,
Runtime: aws.String(string(obj.Runtime)),
Timeout: timeout,
VpcConfig: vpccfg,
}
var result *lambda.FunctionConfiguration
succ, err := awsctx.RetryProgUntil(
var out *rpc.FunctionOuts
if succ, err := awsctx.RetryProgUntil(
p.ctx,
func() (bool, error) {
var err error
result, err = p.ctx.Lambda().CreateFunction(create)
result, err := p.ctx.Lambda().CreateFunction(create)
if err != nil {
if erraws, iserraws := err.(awserr.Error); iserraws {
if erraws.Code() == "InvalidParameterValueException" &&
@ -154,168 +169,58 @@ func (p *funcProvider) Create(ctx context.Context, req *cocorpc.CreateRequest) (
}
return false, err
}
out = &rpc.FunctionOuts{ARN: awsrpc.ARN(*result.FunctionArn)}
return true, nil
},
func(n int) bool {
fmt.Printf("Lambda IAM role '%v' not yet ready; waiting for it to become usable...\n", *roleARN)
return true
},
)
if err != nil {
return nil, err
}
if !succ {
return nil, fmt.Errorf("Lambda IAM role '%v' did not become useable", *roleARN)
); err != nil {
return "", nil, err
} else if !succ {
return "", nil, fmt.Errorf("Lambda IAM role '%v' did not become useable", *roleARN)
}
// Wait for the function to be ready and then return the function name as the ID.
fmt.Printf("Lambda Function created: %v; waiting for it to become active\n", name)
if err = p.waitForFunctionState(name, true); err != nil {
return nil, err
if err := p.waitForFunctionState(name, true); err != nil {
return "", nil, err
}
return &cocorpc.CreateResponse{
Id: name,
Outputs: resource.MarshalProperties(
nil,
resource.NewPropertyMap(
functionOutput{
ARN: *result.FunctionArn,
},
),
resource.MarshalOptions{},
),
}, nil
return name, out, nil
}
// Read reads the instance state identified by ID, returning a populated resource object, or an error if not found.
func (p *funcProvider) Get(ctx context.Context, req *cocorpc.GetRequest) (*cocorpc.GetResponse, error) {
contract.Assert(req.GetType() == string(Function))
return nil, errors.New("Not yet implemented")
func (p *funcProvider) Get(ctx context.Context, id string) (*rpc.Function, error) {
return nil, nil
}
// InspectChange checks what impacts a hypothetical update will have on the resource's properties.
func (p *funcProvider) InspectChange(
ctx context.Context, req *cocorpc.ChangeRequest) (*cocorpc.InspectChangeResponse, error) {
contract.Assert(req.GetType() == string(Function))
func (p *funcProvider) InspectChange(ctx context.Context, id string,
old *rpc.Function, new *rpc.Function, diff *resource.ObjectDiff) ([]string, error) {
return nil, errors.New("Not yet implemented")
}
// Update updates an existing resource with new values. Only those values in the provided property bag are updated
// to new values. The resource ID is returned and may be different if the resource had to be recreated.
func (p *funcProvider) Update(ctx context.Context, req *cocorpc.ChangeRequest) (*pbempty.Empty, error) {
contract.Assert(req.GetType() == string(Function))
return nil, errors.New("Not yet implemented")
func (p *funcProvider) Update(ctx context.Context, id string,
old *rpc.Function, new *rpc.Function, diff *resource.ObjectDiff) error {
return errors.New("Not yet implemented")
}
// Delete tears down an existing resource with the given ID. If it fails, the resource is assumed to still exist.
func (p *funcProvider) Delete(ctx context.Context, req *cocorpc.DeleteRequest) (*pbempty.Empty, error) {
contract.Assert(req.GetType() == string(Function))
func (p *funcProvider) Delete(ctx context.Context, id string) error {
// First, perform the deletion.
id := req.GetId()
fmt.Printf("Deleting Lambda Function '%v'\n", id)
if _, err := p.ctx.Lambda().DeleteFunction(&lambda.DeleteFunctionInput{
FunctionName: aws.String(id),
}); err != nil {
return nil, err
return err
}
// Wait for the function to actually become deleted before returning.
fmt.Printf("Lambda Function delete request submitted; waiting for it to delete\n")
if err := p.waitForFunctionState(id, false); err != nil {
return nil, err
}
return &pbempty.Empty{}, nil
}
// function represents the state associated with an Lambda function.
type function struct {
Name string `json:"name"` // the function resource's name.
Code resource.Asset `json:"code"` // the function's code.
Handler string `json:"handler"` // the name of the function's handler.
Role resource.ID `json:"role"` // the AWS IAM execution role.
Runtime string `json:"runtime"` // the language runtime.
FunctionName *string `json:"functionName,omitempty"` // the function's published name.
DeadLetterConfig *deadLetterConfig `json:"deadLetterConfig,omitempty"` // a dead letter queue/topic config.
Description *string `json:"description,omitempty"` // an optional friendly description.
Environment *environment `json:"environment,omitempty"` // environment variables.
KMSKeyID *resource.ID `json:"kmsKey,omitempty"` // a KMS key for encrypting/decrypting.
MemorySize *float64 `json:"memorySize,omitempty"` // maximum amount of memory in MB.
Timeout *float64 `json:"timeout,omitempty"` // maximum execution time in seconds.
VPCConfig *vpcConfig `json:"vpcConfig,omitempty"` // optional VPC config for an ENI.
}
type deadLetterConfig struct {
Target resource.ID `json:"target"` // the target SNS topic or SQS queue.
}
type environment map[string]string
type vpcConfig struct {
SecurityGroups []resource.ID `json:"securityGroups"` // security groups for resources this function uses.
Subnets []resource.ID `json:"subnets"` // subnets for resources this function uses.
}
// constants for function property names.
const (
FunctionName = "name"
FunctionRuntime = "runtime"
FunctionFunctionName = "functionName"
)
// constants for the various function limits.
const (
maxFunctionName = 64
maxFunctionNameARN = 140
functionNameARNPrefix = "arn:aws:lambda:"
)
type functionOutput struct {
ARN string `json:"arn"`
}
var functionRuntimes = map[string]bool{
"nodejs": true,
"nodejs4.3": true,
"nodejs4.3-edge": true,
"nodejs6.10": true,
"java8": true,
"python2.7": true,
"dotnetcore1.0": true,
}
// unmarshalFunction decodes and validates a function property bag.
func unmarshalFunction(v *pbstruct.Struct) (function, resource.PropertyMap, mapper.DecodeError) {
var fun function
props := resource.UnmarshalProperties(v)
result := mapper.MapIU(props.Mappable(), &fun)
if _, has := functionRuntimes[fun.Runtime]; !has {
if result == nil {
result = mapper.NewDecodeErr(nil)
}
result.AddFailure(
mapper.NewFieldErr(reflect.TypeOf(fun), FunctionRuntime,
fmt.Errorf("%v is not a valid runtime", fun.Runtime)),
)
}
if name := fun.FunctionName; name != nil {
var maxName int
if strings.HasPrefix(*name, functionNameARNPrefix) {
maxName = maxFunctionNameARN
} else {
maxName = maxFunctionName
}
if len(*name) > maxName {
if result == nil {
result = mapper.NewDecodeErr(nil)
}
result.AddFailure(
mapper.NewFieldErr(reflect.TypeOf(fun), FunctionFunctionName,
fmt.Errorf("exceeded maximum length of %v", maxName)),
)
}
}
return fun, props, result
return p.waitForFunctionState(id, false)
}
func (p *funcProvider) waitForFunctionState(id string, exist bool) error {

View file

@ -33,7 +33,7 @@ func NewProvider() (*Provider, error) {
impls: map[tokens.Type]cocorpc.ResourceProviderServer{
ec2.InstanceToken: ec2.NewInstanceProvider(ctx),
ec2.SecurityGroupToken: ec2.NewSecurityGroupProvider(ctx),
lambda.Function: lambda.NewFunctionProvider(ctx),
lambda.FunctionToken: lambda.NewFunctionProvider(ctx),
iam.RoleToken: iam.NewRoleProvider(ctx),
s3.Bucket: s3.NewBucketProvider(ctx),
s3.Object: s3.NewObjectProvider(ctx),

View file

@ -472,14 +472,17 @@ func (g *RPCGenerator) GenTypeName(t types.Type) string {
case *types.Named:
obj := u.Obj()
// For resource types, simply emit an ID, since that is what will have been serialized.
if res, _ := IsResource(obj, u); res {
return "resource.ID"
}
// For references to the special predefined types, use the runtime provider representation.
if spec, kind := IsSpecial(obj); spec {
switch kind {
case SpecialAssetType:
return "resource.Asset"
case SpecialResourceType, SpecialNamedResourceType:
return "resource.ID"
default:
contract.Failf("Unrecognized special kind: %v", kind)
contract.Failf("Unexpected special kind: %v", kind)
}
}