maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add content properties for aws.s3.Object

Browse source 
Adds the `Content*` properties on S3 Objects to the
Object resource.  Allow update of Objects with new
sources and/or content properties.

Also adds AWS provider test support for validating
resource output properties.

Contributes to #218.
This commit is contained in:
Luke Hoban 2017-06-25 13:21:41 -07:00
parent 5362536396
commit ee2c165a17
7 changed files with 300 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
lib/aws/idl/s3/object.go
View file

@ -26,5 +26,20 @@ type Object struct {
// The Bucket this object belongs to.
Bucket *Bucket `lumi:"bucket,replaces"`
// The Source of content for this object.
Source *idl.Asset `lumi:"source,replaces,in"`
Source *idl.Asset `lumi:"source,in"`
// A standard MIME type describing the format of the object data.
ContentType *string `lumi:"contentType,optional"`
// Specifies presentational information for the object.
ContentDisposition *string `lumi:"contentDisposition,optional"`
// Specifies caching behavior along the request/reply chain.
CacheControl *string `lumi:"cacheControl,optional"`
// Specifies what content encodings have been applied to the object and thus
// what decoding mechanisms must be applied to obtain the media-type referenced
// by the Content-Type header field.
ContentEncoding *string `lumi:"contentEncoding,optional"`
// The language the content is in.
ContentLanguage *string `lumi:"contentLanguage,optional"`
// Size of the body in bytes. This parameter is useful when the size of the
// body cannot be determined automatically.
ContentLength *float64 `lumi:"contentLength,optional"`
}

22
lib/aws/pack/s3/object.ts
View file

@ -9,7 +9,13 @@ import {Bucket} from "./bucket";
export class Object extends lumi.Resource implements ObjectArgs {
public readonly key: string;
public readonly bucket: Bucket;
public readonly source: lumi.asset.Asset;
public source: lumi.asset.Asset;
public contentType?: string;
public contentDisposition?: string;
public cacheControl?: string;
public contentEncoding?: string;
public contentLanguage?: string;
public contentLength?: number;
public static get(id: lumi.ID): Object {
return <any>undefined; // functionality provided by the runtime
@ -33,12 +39,24 @@ export class Object extends lumi.Resource implements ObjectArgs {
throw new Error("Missing required argument 'source'");
}
this.source = args.source;
this.contentType = args.contentType;
this.contentDisposition = args.contentDisposition;
this.cacheControl = args.cacheControl;
this.contentEncoding = args.contentEncoding;
this.contentLanguage = args.contentLanguage;
this.contentLength = args.contentLength;
}
}
export interface ObjectArgs {
readonly key: string;
readonly bucket: Bucket;
readonly source: lumi.asset.Asset;
source: lumi.asset.Asset;
contentType?: string;
contentDisposition?: string;
cacheControl?: string;
contentEncoding?: string;
contentLanguage?: string;
contentLength?: number;
}

27
lib/aws/provider/lambda/function_test.go
View file

@ -15,6 +15,8 @@
package lambda
import (
"crypto/sha256"
"encoding/base64"
"fmt"
"strings"
"testing"
@ -45,6 +47,13 @@ func Test(t *testing.T) {
}()
sourceARN := rpc.ARN("arn:aws:s3:::elasticbeanstalk-us-east-1-111111111111")
code := resource.Archive{
Assets: &map[string]*resource.Asset{
"index.js": {
Text: aws.String("exports.handler = (ev, ctx, cb) => { console.log(ev); console.log(ctx); }"),
},
},
}
resources := map[string]testutil.Resource{
"role": {Provider: iamprovider.NewRoleProvider(ctx), Token: iam.RoleToken},
@ -81,14 +90,8 @@ func Test(t *testing.T) {
Name: "f",
Creator: func(ctx testutil.Context) interface{} {
return &lambda.Function{
Name: aws.String(prefix),
Code: resource.Archive{
Assets: &map[string]*resource.Asset{
"index.js": {
Text: aws.String("exports.handler = (ev, ctx, cb) => { console.log(ev); console.log(ctx); }"),
},
},
},
Name: aws.String(prefix),
Code: code,
Handler: "index.handler",
Runtime: lambda.NodeJS6d10Runtime,
Role: ctx.GetResourceID("role"),
@ -111,8 +114,14 @@ func Test(t *testing.T) {
},
}
testutil.ProviderTest(t, resources, steps)
props := testutil.ProviderTest(t, resources, steps)
// Returned SHA256 must match what we uploaded
byts, err := code.Bytes(resource.ZIPArchive)
assert.NoError(t, err)
sum := sha256.Sum256(byts)
codeSHA256 := base64.StdEncoding.EncodeToString(sum[:])
assert.Equal(t, codeSHA256, props["f"].Fields["codeSHA256"].GetStringValue())
}
func cleanupFunctions(prefix string, ctx *awsctx.Context) error {

43
lib/aws/provider/s3/object.go
View file

@ -90,11 +90,22 @@ func (p *objProvider) Create(ctx context.Context, obj *s3.Object) (resource.ID,
if err != nil {
return "", err
}
var contentLength *int64
if obj.ContentLength != nil {
temp := int64(*obj.ContentLength)
contentLength = &temp
}
fmt.Printf("Creating S3 Object '%v' in bucket '%v'\n", obj.Key, buck)
if _, err := p.ctx.S3().PutObject(&awss3.PutObjectInput{
Bucket: aws.String(buck),
Key: aws.String(obj.Key),
Body: body,
Bucket: aws.String(buck),
Key: aws.String(obj.Key),
Body: body,
ContentType: obj.ContentType,
ContentDisposition: obj.ContentDisposition,
CacheControl: obj.CacheControl,
ContentEncoding: obj.ContentEncoding,
ContentLanguage: obj.ContentLanguage,
ContentLength: contentLength,
}); err != nil {
return "", err
}
@ -113,18 +124,30 @@ func (p *objProvider) Get(ctx context.Context, id resource.ID) (*s3.Object, erro
if err != nil {
return nil, err
}
if _, err := p.ctx.S3().GetObject(&awss3.GetObjectInput{
resp, err := p.ctx.S3().GetObject(&awss3.GetObjectInput{
Bucket: aws.String(buck),
Key: aws.String(key),
}); err != nil {
})
if err != nil {
if awsctx.IsAWSError(err, "NotFound", "NoSuchKey") {
return nil, nil
}
return nil, err
}
var contentLength *float64
if resp.ContentLength != nil {
temp := float64(*resp.ContentLength)
contentLength = &temp
}
return &s3.Object{
Bucket: resource.ID(arn.NewS3Bucket(buck)),
Key: key,
Bucket: resource.ID(arn.NewS3Bucket(buck)),
Key: key,
ContentType: resp.ContentType,
ContentDisposition: resp.ContentDisposition,
CacheControl: resp.CacheControl,
ContentEncoding: resp.ContentEncoding,
ContentLanguage: resp.ContentLanguage,
ContentLength: contentLength,
}, nil
}
@ -138,7 +161,11 @@ func (p *objProvider) InspectChange(ctx context.Context, id resource.ID,
// to new values. The resource ID is returned and may be different if the resource had to be recreated.
func (p *objProvider) Update(ctx context.Context, id resource.ID,
old *s3.Object, new *s3.Object, diff *resource.ObjectDiff) error {
return errors.New("Not yet implemented")
// The id is uniquely determined by `replace` properties, so update is the same as create, and we can expect
// the resulting id to be unchanged.
newid, err := p.Create(ctx, new)
contract.Assert(id == newid)
return err
}
// Delete tears down an existing resource with the given ID. If it fails, the resource is assumed to still exist.

130
lib/aws/provider/s3/object_test.go Normal file
View file

@ -0,0 +1,130 @@
// Copyright 2016-2017, Pulumi Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package s3
import (
"fmt"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
awss3 "github.com/aws/aws-sdk-go/service/s3"
"github.com/pulumi/lumi/lib/aws/provider/awsctx"
"github.com/pulumi/lumi/lib/aws/provider/testutil"
"github.com/pulumi/lumi/lib/aws/rpc/s3"
"github.com/pulumi/lumi/pkg/resource"
"github.com/stretchr/testify/assert"
)
func Test(t *testing.T) {
t.Parallel()
prefix := resource.NewUniqueHex("lumitest", 20, 20)
ctx := testutil.CreateContext(t)
defer func() {
buckerr := cleanupBucket(prefix, ctx)
assert.Nil(t, buckerr)
}()
str1 := "<h1>Hello world!</h1>"
str2 := `{"hello": "world"}`
source1 := resource.NewTextAsset(str1)
source2 := resource.NewTextAsset(str2)
resources := map[string]testutil.Resource{
"bucket": {Provider: NewBucketProvider(ctx), Token: BucketToken},
"object": {Provider: NewObjectProvider(ctx), Token: ObjectToken},
}
steps := []testutil.Step{
// Create a bucket and object
{
testutil.ResourceGenerator{
Name: "bucket",
Creator: func(ctx testutil.Context) interface{} {
return &s3.Bucket{
Name: aws.String(prefix),
}
},
},
testutil.ResourceGenerator{
Name: "object",
Creator: func(ctx testutil.Context) interface{} {
return &s3.Object{
Bucket: ctx.GetResourceID("bucket"),
Key: prefix,
Source: &source1,
ContentType: aws.String("text/html"),
}
},
},
},
// Update the object with new `source` content
{
testutil.ResourceGenerator{
Name: "object",
Creator: func(ctx testutil.Context) interface{} {
return &s3.Object{
Bucket: ctx.GetResourceID("bucket"),
Key: prefix,
Source: &source2,
ContentType: aws.String("application/json"),
}
},
},
},
}
props := testutil.ProviderTest(t, resources, steps)
assert.Equal(t, "application/json", props["object"].Fields["contentType"].GetStringValue())
assert.Equal(t, len(str2), int(props["object"].Fields["contentLength"].GetNumberValue()),
"expected object content-length to equal len(%q)", str2)
}
func cleanupBucket(prefix string, ctx *awsctx.Context) error {
fmt.Printf("Cleaning up buckets with name prefix:%v\n", prefix)
list, err := ctx.S3().ListBuckets(&awss3.ListBucketsInput{})
if err != nil {
return err
}
cleaned := 0
for _, buck := range list.Buckets {
if strings.HasPrefix(aws.StringValue(buck.Name), prefix) {
objList, err := ctx.S3().ListObjects(&awss3.ListObjectsInput{
Bucket: buck.Name,
})
if err != nil {
return err
}
for _, obj := range objList.Contents {
_, err = ctx.S3().DeleteObject(&awss3.DeleteObjectInput{
Bucket: buck.Name,
Key: obj.Key,
})
if err != nil {
return err
}
}
ctx.S3().DeleteBucket(&awss3.DeleteBucketInput{
Bucket: buck.Name,
})
if err != nil {
return err
}
cleaned++
}
}
fmt.Printf("Cleaned up %v buckets\n", cleaned)
return nil
}

49
lib/aws/provider/testutil/provider.go
View file

@ -49,13 +49,14 @@ type Step []ResourceGenerator
// be created or updated as needed. After walking through each step, all of the created resources are deleted.
// Check operations are performed on all provided resource inputs during the test.
// performs Check operations on each provided resource.
func ProviderTest(t *testing.T, resources map[string]Resource, steps []Step) {
func ProviderTest(t *testing.T, resources map[string]Resource, steps []Step) map[string]*structpb.Struct {
p := &providerTest{
resources: resources,
namesInCreationOrder: []string{},
ids: map[string]resource.ID{},
props: map[string]*structpb.Struct{},
outProps: map[string]*structpb.Struct{},
}
// For each step, create or update all listed resources
@ -68,20 +69,22 @@ func ProviderTest(t *testing.T, resources map[string]Resource, steps []Step) {
provider := currentResource.Provider
token := currentResource.Token
if id, ok := p.ids[res.Name]; !ok {
id, props := createResource(t, res.Creator(p), provider, token)
id, props, outProps := createResource(t, res.Creator(p), provider, token)
p.ids[res.Name] = resource.ID(id)
p.namesInCreationOrder = append(p.namesInCreationOrder, res.Name)
p.props[res.Name] = props
p.outProps[res.Name] = outProps
if id == "" {
t.Fatal("expected to successfully create resource")
}
} else {
oldProps := p.props[res.Name]
ok, props := updateResource(t, string(id), oldProps, res.Creator(p), provider, token)
ok, props, outProps := updateResource(t, string(id), oldProps, res.Creator(p), provider, token)
if !ok {
t.Fatal("expected to successfully update resource")
}
p.props[res.Name] = props
p.outProps[res.Name] = outProps
}
}
}
@ -96,12 +99,13 @@ func ProviderTest(t *testing.T, resources map[string]Resource, steps []Step) {
t.Fatal("expected to successfully delete resource")
}
}
return p.outProps
}
// ProviderTestSimple takes a resource provider and array of resource steps and performs a Create, as many Updates
// as neeed, and finally a Delete operation on a single resouce of the given type to walk the resource through the
// resource lifecycle. It also performs Check operations on each input state of the resource.
func ProviderTestSimple(t *testing.T, provider lumirpc.ResourceProviderServer, token tokens.Type, steps []interface{}) {
func ProviderTestSimple(t *testing.T, provider lumirpc.ResourceProviderServer, token tokens.Type, steps []interface{}) *structpb.Struct {
resources := map[string]Resource{
"testResource": {
Provider: provider,
@ -120,7 +124,8 @@ func ProviderTestSimple(t *testing.T, provider lumirpc.ResourceProviderServer, t
},
})
}
ProviderTest(t, resources, detailedSteps)
outProps := ProviderTest(t, resources, detailedSteps)
return outProps["testResource"]
}
type providerTest struct {
@ -128,6 +133,7 @@ type providerTest struct {
namesInCreationOrder []string
ids map[string]resource.ID
props map[string]*structpb.Struct
outProps map[string]*structpb.Struct
}
func (p *providerTest) GetResourceID(name string) resource.ID {
@ -140,7 +146,7 @@ func (p *providerTest) GetResourceID(name string) resource.ID {
var _ Context = &providerTest{}
func createResource(t *testing.T, res interface{}, provider lumirpc.ResourceProviderServer,
token tokens.Type) (string, *structpb.Struct) {
token tokens.Type) (string, *structpb.Struct, *structpb.Struct) {
props := plugin.MarshalProperties(nil, resource.NewPropertyMap(res), plugin.MarshalOptions{})
fmt.Printf("[Provider Test]: Checking %v\n", token)
checkResp, err := provider.Check(nil, &lumirpc.CheckRequest{
@ -148,7 +154,7 @@ func createResource(t *testing.T, res interface{}, provider lumirpc.ResourceProv
Properties: props,
})
if !assert.NoError(t, err, "expected no error checking table") {
return "", nil
return "", nil, nil
}
assert.Equal(t, 0, len(checkResp.Failures), "expected no check failures")
fmt.Printf("[Provider Test]: Creating %v\n", token)
@ -157,10 +163,10 @@ func createResource(t *testing.T, res interface{}, provider lumirpc.ResourceProv
Properties: props,
})
if !assert.NoError(t, err, "expected no error creating resource") {
return "", nil
return "", nil, nil
}
if !assert.NotNil(t, resp, "expected a non-nil response") {
return "", nil
return "", nil, nil
}
id := resp.Id
fmt.Printf("[Provider Test]: Getting %v with id %v\n", token, id)
@ -169,16 +175,16 @@ func createResource(t *testing.T, res interface{}, provider lumirpc.ResourceProv
Id: id,
})
if !assert.NoError(t, err, "expected no error reading resource") {
return "", nil
return "", nil, nil
}
if !assert.NotNil(t, getResp, "expected a non-nil response reading the resources") {
return "", nil
return "", nil, nil
}
return id, props
return id, props, getResp.Properties
}
func updateResource(t *testing.T, id string, lastProps *structpb.Struct, res interface{},
provider lumirpc.ResourceProviderServer, token tokens.Type) (bool, *structpb.Struct) {
provider lumirpc.ResourceProviderServer, token tokens.Type) (bool, *structpb.Struct, *structpb.Struct) {
newProps := plugin.MarshalProperties(nil, resource.NewPropertyMap(res), plugin.MarshalOptions{})
fmt.Printf("[Provider Test]: Checking %v\n", token)
checkResp, err := provider.Check(nil, &lumirpc.CheckRequest{
@ -186,7 +192,7 @@ func updateResource(t *testing.T, id string, lastProps *structpb.Struct, res int
Properties: newProps,
})
if !assert.NoError(t, err, "expected no error checking resource") {
return false, nil
return false, nil, nil
}
assert.Equal(t, 0, len(checkResp.Failures), "expected no check failures")
fmt.Printf("[Provider Test]: Updating %v with id %v\n", token, id)
@ -197,9 +203,20 @@ func updateResource(t *testing.T, id string, lastProps *structpb.Struct, res int
News: newProps,
})
if !assert.NoError(t, err, "expected no error creating resource") {
return false, nil
return false, nil, nil
}
return true, newProps
fmt.Printf("[Provider Test]: Getting %v with id %v\n", token, id)
getResp, err := provider.Get(nil, &lumirpc.GetRequest{
Type: string(token),
Id: id,
})
if !assert.NoError(t, err, "expected no error reading resource") {
return false, nil, nil
}
if !assert.NotNil(t, getResp, "expected a non-nil response reading the resources") {
return false, nil, nil
}
return true, newProps, getResp.Properties
}
func deleteResource(t *testing.T, id string, provider lumirpc.ResourceProviderServer, token tokens.Type) bool {

51
lib/aws/rpc/s3/object.go
View file

@ -72,6 +72,42 @@ func (p *ObjectProvider) Check(
resource.NewPropertyError("Object", "source", failure))
}
}
if !unks["contentType"] {
if failure := p.ops.Check(ctx, obj, "contentType"); failure != nil {
failures = append(failures,
resource.NewPropertyError("Object", "contentType", failure))
}
}
if !unks["contentDisposition"] {
if failure := p.ops.Check(ctx, obj, "contentDisposition"); failure != nil {
failures = append(failures,
resource.NewPropertyError("Object", "contentDisposition", failure))
}
}
if !unks["cacheControl"] {
if failure := p.ops.Check(ctx, obj, "cacheControl"); failure != nil {
failures = append(failures,
resource.NewPropertyError("Object", "cacheControl", failure))
}
}
if !unks["contentEncoding"] {
if failure := p.ops.Check(ctx, obj, "contentEncoding"); failure != nil {
failures = append(failures,
resource.NewPropertyError("Object", "contentEncoding", failure))
}
}
if !unks["contentLanguage"] {
if failure := p.ops.Check(ctx, obj, "contentLanguage"); failure != nil {
failures = append(failures,
resource.NewPropertyError("Object", "contentLanguage", failure))
}
}
if !unks["contentLength"] {
if failure := p.ops.Check(ctx, obj, "contentLength"); failure != nil {
failures = append(failures,
resource.NewPropertyError("Object", "contentLength", failure))
}
}
if len(failures) > 0 {
return plugin.NewCheckResponse(resource.NewErrors(failures)), nil
}
@ -138,9 +174,6 @@ func (p *ObjectProvider) InspectChange(
if diff.Changed("bucket") {
replaces = append(replaces, "bucket")
}
if diff.Changed("source") {
replaces = append(replaces, "source")
}
}
more, err := p.ops.InspectChange(ctx, id, old, new, diff)
if err != nil {
@ -194,6 +227,12 @@ type Object struct {
Key string `lumi:"key"`
Bucket resource.ID `lumi:"bucket"`
Source *resource.Asset `lumi:"source,optional"`
ContentType *string `lumi:"contentType,optional"`
ContentDisposition *string `lumi:"contentDisposition,optional"`
CacheControl *string `lumi:"cacheControl,optional"`
ContentEncoding *string `lumi:"contentEncoding,optional"`
ContentLanguage *string `lumi:"contentLanguage,optional"`
ContentLength *float64 `lumi:"contentLength,optional"`
}
// Object's properties have constants to make dealing with diffs and property bags easier.
@ -201,6 +240,12 @@ const (
Object_Key = "key"
Object_Bucket = "bucket"
Object_Source = "source"
Object_ContentType = "contentType"
Object_ContentDisposition = "contentDisposition"
Object_CacheControl = "cacheControl"
Object_ContentEncoding = "contentEncoding"
Object_ContentLanguage = "contentLanguage"
Object_ContentLength = "contentLength"
)