maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
minio/mint/run/core/awscli/test.sh
Harshavardhana 4915433bd2
Support bucket versioning (#9377) 
- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111
2020-06-12 20:04:01 -07:00

1726 lines
56 KiB
Bash
Executable file
Raw Blame History

#!/bin/bash
#
# Mint (C) 2017-2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
HASH_1_KB=$(md5sum "${MINT_DATA_DIR}/datafile-1-kB" | awk '{print $1}')
HASH_65_MB=$(md5sum "${MINT_DATA_DIR}/datafile-65-MB" | awk '{print $1}')
_init() {
AWS="aws --endpoint-url $1"
}
function get_time() {
date +%s%N
}
function get_duration() {
start_time=$1
end_time=$(get_time)
echo $(( (end_time - start_time) / 1000000 ))
}
function log_success() {
function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
printf '{"name": "awscli", "duration": %d, "function": %s, "status": "PASS"}\n' "$1" "$function"
}
function log_failure() {
function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
err=$(echo "$3" | tr -d '\n')
printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "error": "%s"}\n' "$1" "$function" "$err"
}
function log_alert() {
function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
err=$(echo "$4" | tr -d '\n')
printf '{"name": "awscli", "duration": %d, "function": %s, "status": "FAIL", "alert": "%s", "error": "%s"}\n' "$1" "$function" "$3" "$err"
}
function make_bucket() {
# Make bucket
bucket_name="awscli-mint-test-bucket-$RANDOM"
function="${AWS} s3api create-bucket --bucket ${bucket_name}"
# execute the test
out=$($function 2>&1)
rv=$?
# if command is successful print bucket_name or print error
if [ $rv -eq 0 ]; then
echo "${bucket_name}"
else
echo "${out}"
fi
return $rv
}
function delete_bucket() {
# Delete bucket
function="${AWS} s3 rb s3://${1} --force"
out=$($function 2>&1)
rv=$?
# echo the output
echo "${out}"
return $rv
}
# Tests creating, stat and delete on a bucket.
function test_create_bucket() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# save the ref to function being tested, so it can be logged
test_function=${function}
# if make_bucket is successful stat the bucket
if [ $rv -eq 0 ]; then
function="${AWS} s3api head-bucket --bucket ${bucket_name}"
out=$($function 2>&1)
rv=$?
else
# if make bucket failes, $bucket_name has the error output
out="${bucket_name}"
fi
# if stat bucket is successful remove the bucket
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "${bucket_name}")
rv=$?
else
# if make bucket failes, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests creating and deleting an object.
function test_upload_object() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# if upload succeeds download the file
if [ $rv -eq 0 ]; then
function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB /tmp/datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
# calculate the md5 hash of downloaded file
hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}')
fi
# if download succeeds, verify downloaded file
if [ $rv -eq 0 ]; then
if [ "$HASH_1_KB" == "$hash2" ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove download file
rm -f /tmp/datafile-1-kB
else
rv=1
out="Checksum verification failed for uploaded object"
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Test lookup a directory prefix.
function test_lookup_object_prefix() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds create a directory.
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
else
# if make_bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
## Attempt an overwrite of the prefix again and should succeed as well.
function="${AWS} s3api put-object --bucket ${bucket_name} --key prefix/directory/"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# if upload succeeds lookup for the prefix.
if [ $rv -eq 0 ]; then
function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# if directory create succeeds, upload the object.
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key prefix/directory/datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.
if [ $rv -eq 0 ]; then
function="${AWS} s3api delete-object --bucket ${bucket_name} --key prefix/directory/"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# if upload succeeds lookup for the object should succeed.
if [ $rv -eq 0 ]; then
function="${AWS} s3api head-object --bucket ${bucket_name} --key prefix/directory/datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# delete bucket
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
fi
if [ $rv -ne 0 ]; then
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
else
log_success "$(get_duration "$start_time")" "${test_function}"
fi
return $rv
}
# Tests listing objects for both v1 and v2 API.
function test_list_objects() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# if upload objects succeeds, list objects with existing prefix
if [ $rv -eq 0 ]; then
function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB"
test_function=${function}
out=$($function)
rv=$?
key_name=$(echo "$out" | jq -r .Contents[].Key)
if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then
rv=1
# since rv is 0, command passed, but didn't return expected value. In this case set the output
out="list-objects with existing prefix failed"
fi
fi
# if upload objects succeeds, list objects without existing prefix
if [ $rv -eq 0 ]; then
function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix linux"
out=$($function)
rv=$?
key_name=$(echo "$out" | jq -r .Contents[].Key)
if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then
rv=1
out="list-objects without existing prefix failed"
fi
fi
# if upload objects succeeds, list objectsv2 with existing prefix
if [ $rv -eq 0 ]; then
function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB"
out=$($function)
rv=$?
key_name=$(echo "$out" | jq -r .Contents[].Key)
if [ $rv -eq 0 ] && [ "$key_name" != "datafile-1-kB" ]; then
rv=1
out="list-objects-v2 with existing prefix failed"
fi
fi
# if upload objects succeeds, list objectsv2 without existing prefix
if [ $rv -eq 0 ]; then
function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix linux"
out=$($function)
rv=$?
key_name=$(echo "$out" | jq -r .Contents[].Key)
if [ $rv -eq 0 ] && [ "$key_name" != "" ]; then
rv=1
out="list-objects-v2 without existing prefix failed"
fi
fi
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove download file
rm -f /tmp/datafile-1-kB
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
rm -f /tmp/datafile-1-kB
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests multipart API with 0 byte part.
function test_multipart_upload_0byte() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
object_name=${bucket_name}"-object"
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-0-b --bucket ${bucket_name} --key datafile-0-b"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
# create multipart
function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}"
test_function=${function}
out=$($function)
rv=$?
upload_id=$(echo "$out" | jq -r .UploadId)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 1
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-0-b --upload-id ${upload_id} --part-number 1"
out=$($function)
rv=$?
etag1=$(echo "$out" | jq -r .ETag)
fi
if [ $rv -eq 0 ]; then
# Create a multipart struct file for completing multipart transaction
echo "{
\"Parts\": [
{
\"ETag\": ${etag1},
\"PartNumber\": 1
}
]
}" >> /tmp/multipart
fi
if [ $rv -eq 0 ]; then
# Use saved etags to complete the multipart transaction
function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}"
out=$($function)
rv=$?
etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//')
if [ "${etag}" == "" ]; then
rv=1
out="complete-multipart-upload failed"
fi
fi
if [ $rv -eq 0 ]; then
function="${AWS} s3api get-object --bucket ${bucket_name} --key ${object_name} /tmp/datafile-0-b"
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
ret_etag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//')
# match etag
if [ "$etag" != "$ret_etag" ]; then
rv=1
out="Etag mismatch for multipart 0 byte object"
fi
rm -f /tmp/datafile-0-b
fi
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove temp file
rm -f /tmp/multipart
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
rm -f /tmp/multipart
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests multipart API by making each individual calls.
function test_multipart_upload() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
object_name=${bucket_name}"-object"
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
# create multipart
function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}"
test_function=${function}
out=$($function)
rv=$?
upload_id=$(echo "$out" | jq -r .UploadId)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 1
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5-MB --upload-id ${upload_id} --part-number 1"
out=$($function)
rv=$?
etag1=$(echo "$out" | jq -r .ETag)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 2
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-1-kB --upload-id ${upload_id} --part-number 2"
out=$($function)
rv=$?
etag2=$(echo "$out" | jq -r .ETag)
# Create a multipart struct file for completing multipart transaction
echo "{
\"Parts\": [
{
\"ETag\": ${etag1},
\"PartNumber\": 1
},
{
\"ETag\": ${etag2},
\"PartNumber\": 2
}
]
}" >> /tmp/multipart
fi
if [ $rv -eq 0 ]; then
# Use saved etags to complete the multipart transaction
function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}"
out=$($function)
rv=$?
finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//')
if [ "${finalETag}" == "" ]; then
rv=1
out="complete-multipart-upload failed"
fi
fi
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove temp file
rm -f /tmp/multipart
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
rm -f /tmp/multipart
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# List number of objects based on the maxKey
# value set.
function test_max_key_list() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-b --bucket ${bucket_name} --key datafile-1-b"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# copy object server side
if [ $rv -eq 0 ]; then
function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-b-copy --copy-source ${bucket_name}/datafile-1-b"
out=$($function)
rv=$?
fi
if [ $rv -eq 0 ]; then
function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --max-keys 1"
test_function=${function}
out=$($function 2>&1)
rv=$?
if [ $rv -eq 0 ]; then
out=$(echo "$out" | jq '.KeyCount')
rv=$?
fi
fi
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# The command passed, but the delete_bucket failed
out="delete_bucket for test_max_key_list failed"
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Copy object tests for server side copy
# of the object, validates returned md5sum.
function test_copy_object() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# copy object server side
if [ $rv -eq 0 ]; then
function="${AWS} s3api copy-object --bucket ${bucket_name} --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB"
test_function=${function}
out=$($function)
rv=$?
hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//')
if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then
# Verification failed
rv=1
out="Hash mismatch expected $HASH_1_KB, got $hash2"
fi
fi
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Copy object tests for server side copy
# of the object, validates returned md5sum.
# validates change in storage class as well
function test_copy_object_storage_class() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# copy object server side
if [ $rv -eq 0 ]; then
function="${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB-copy --copy-source ${bucket_name}/datafile-1-kB"
test_function=${function}
out=$($function 2>&1)
rv=$?
# if this functionality is not implemented return right away.
if [ $rv -ne 0 ]; then
if echo "$out" | grep -q "NotImplemented"; then
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
return 0
fi
fi
hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//')
if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then
# Verification failed
rv=1
out="Hash mismatch expected $HASH_1_KB, got $hash2"
fi
# if copy succeeds stat the object
if [ $rv -eq 0 ]; then
function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB-copy"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
storageClass=$(echo "$out" | jq -r .StorageClass)
rv=$?
fi
# if head-object succeeds, verify metadata has storage class
if [ $rv -eq 0 ]; then
if [ "${storageClass}" == "null" ]; then
rv=1
out="StorageClass was not applied"
elif [ "${storageClass}" == "STANDARD" ]; then
rv=1
out="StorageClass was applied incorrectly"
fi
fi
fi
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Copy object tests for server side copy
# to itself by changing storage class
function test_copy_object_storage_class_same() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# copy object server side
if [ $rv -eq 0 ]; then
function="${AWS} s3api copy-object --bucket ${bucket_name} --storage-class REDUCED_REDUNDANCY --key datafile-1-kB --copy-source ${bucket_name}/datafile-1-kB"
test_function=${function}
out=$($function 2>&1)
rv=$?
# if this functionality is not implemented return right away.
if [ $rv -ne 0 ]; then
if echo "$out" | grep -q "NotImplemented"; then
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
return 0
fi
fi
hash2=$(echo "$out" | jq -r .CopyObjectResult.ETag | sed -e 's/^"//' -e 's/"$//')
if [ $rv -eq 0 ] && [ "$HASH_1_KB" != "$hash2" ]; then
# Verification failed
rv=1
out="Hash mismatch expected $HASH_1_KB, got $hash2"
fi
# if copy succeeds stat the object
if [ $rv -eq 0 ]; then
function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB"
# save the ref to function being tested, so it can be logged
test_function=${function}
out=$($function 2>&1)
storageClass=$(echo "$out" | jq -r .StorageClass)
rv=$?
fi
# if head-object succeeds, verify metadata has storage class
if [ $rv -eq 0 ]; then
if [ "${storageClass}" == "null" ]; then
rv=1
out="StorageClass was not applied"
elif [ "${storageClass}" == "STANDARD" ]; then
rv=1
out="StorageClass was applied incorrectly"
fi
fi
fi
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests for presigned URL success case, presigned URL
# is correct and accessible - we calculate md5sum of
# the object and validate it against a local files md5sum.
function test_presigned_object() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
function="${AWS} s3 presign s3://${bucket_name}/datafile-1-kB"
test_function=${function}
url=$($function)
rv=$?
curl -sS -X GET "${url}" > /tmp/datafile-1-kB
hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}')
if [ "$HASH_1_KB" == "$hash2" ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove download file
rm -f /tmp/datafile-1-kB
else
rv=1
out="Checksum verification failed for downloaded object"
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests creating and deleting an object - 10MiB
function test_upload_object_10() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-MB --bucket ${bucket_name} --key datafile-10-MB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests multipart API by making each individual calls with 10MiB part size.
function test_multipart_upload_10() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
object_name=${bucket_name}"-object"
rv=$?
if [ $rv -eq 0 ]; then
# create multipart
function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name}"
test_function=${function}
out=$($function)
rv=$?
upload_id=$(echo "$out" | jq -r .UploadId)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 1
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 1"
out=$($function)
rv=$?
etag1=$(echo "$out" | jq -r .ETag)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 2
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-10-MB --upload-id ${upload_id} --part-number 2"
out=$($function)
rv=$?
etag2=$(echo "$out" | jq -r .ETag)
# Create a multipart struct file for completing multipart transaction
echo "{
\"Parts\": [
{
\"ETag\": ${etag1},
\"PartNumber\": 1
},
{
\"ETag\": ${etag2},
\"PartNumber\": 2
}
]
}" >> /tmp/multipart
fi
if [ $rv -eq 0 ]; then
# Use saved etags to complete the multipart transaction
function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}"
out=$($function)
rv=$?
finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//')
if [ "${finalETag}" == "" ]; then
rv=1
out="complete-multipart-upload failed"
fi
fi
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove temp file
rm -f /tmp/multipart
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
rm -f /tmp/multipart
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests `aws s3 cp` by uploading a local file.
function test_aws_s3_cp() {
file_name="${MINT_DATA_DIR}/datafile-65-MB"
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file using cp
if [ $rv -eq 0 ]; then
function="${AWS} s3 cp $file_name s3://${bucket_name}/$(basename "$file_name")"
test_function=${function}
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
function="${AWS} s3 rm s3://${bucket_name}/$(basename "$file_name")"
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
function="${AWS} s3 rb s3://${bucket_name}/"
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# Tests `aws s3 sync` by mirroring all the
# local content to remove bucket.
function test_aws_s3_sync() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds sync all the files in a directory
if [ $rv -eq 0 ]; then
function="${AWS} s3 sync --no-progress $MINT_DATA_DIR s3://${bucket_name}/"
test_function=${function}
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
# remove files recusively
if [ $rv -eq 0 ]; then
function="${AWS} s3 rm --recursive s3://${bucket_name}/"
out=$($function 2>&1)
rv=$?
fi
# delete bucket
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# list objects negative test - tests for following conditions.
# v1 API with max-keys=-1 and max-keys=0
# v2 API with max-keys=-1 and max-keys=0
function test_list_objects_error() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload a file
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
out=$($function 2>&1)
rv=$?
else
# if make bucket fails, $bucket_name has the error output
out="${bucket_name}"
fi
if [ $rv -eq 0 ]; then
# Server replies an error for v1 with max-key=-1
function="${AWS} s3api list-objects --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1"
test_function=${function}
out=$($function 2>&1)
rv=$?
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
fi
if [ $rv -eq 0 ]; then
# Server replies an error for v2 with max-keys=-1
function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=-1"
test_function=${function}
out=$($function 2>&1)
rv=$?
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
fi
if [ $rv -eq 0 ]; then
# Server returns success with no keys when max-keys=0
function="${AWS} s3api list-objects-v2 --bucket ${bucket_name} --prefix datafile-1-kB --max-keys=0"
out=$($function 2>&1)
rv=$?
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# put object negative test - tests for following conditions.
# - invalid object name.
# - invalid Content-Md5
# - invalid Content-Length
function test_put_object_error() {
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# if make bucket succeeds upload an object without content-md5.
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-md5 invalid"
test_function=${function}
out=$($function 2>&1)
rv=$?
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
fi
# upload an object without content-length.
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --content-length -1"
test_function=${function}
out=$($function 2>&1)
rv=$?
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
fi
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# tests server side encryption headers for get and put calls
function test_serverside_encryption() {
#skip server side encryption tests if HTTPS disabled.
if [ "$ENABLE_HTTPS" != "1" ]; then
return 0
fi
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# put object with server side encryption headers
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# now get encrypted object from server
if [ $rv -eq 0 ]; then
etag1=$(echo "$out" | jq -r .ETag)
sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5)
sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm)
function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB"
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
etag2=$(echo "$out" | jq -r .ETag)
sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5)
sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm)
hash2=$(md5sum /tmp/datafile-1-kB | awk '{print $1}')
# match downloaded object's hash to original
if [ "$HASH_1_KB" == "$hash2" ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove download file
rm -f /tmp/datafile-1-kB
else
rv=1
out="Checksum verification failed for downloaded object"
fi
# match etag and SSE headers
if [ "$etag1" != "$etag2" ]; then
rv=1
out="Etag mismatch for object encrypted with server side encryption"
fi
if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then
rv=1
out="sse customer algorithm mismatch"
fi
if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then
rv=1
out="sse customer key mismatch"
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# tests server side encryption headers for multipart put
function test_serverside_encryption_multipart() {
#skip server side encryption tests if HTTPS disabled.
if [ "$ENABLE_HTTPS" != "1" ]; then
return 0
fi
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# put object with server side encryption headers
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-65-MB --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# now get encrypted object from server
if [ $rv -eq 0 ]; then
etag1=$(echo "$out" | jq -r .ETag)
sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5)
sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm)
function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-65-MB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-65-MB"
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
etag2=$(echo "$out" | jq -r .ETag)
sse_customer_key2=$(echo "$out" | jq -r .SSECustomerKeyMD5)
sse_customer_algo2=$(echo "$out" | jq -r .SSECustomerAlgorithm)
hash2=$(md5sum /tmp/datafile-65-MB | awk '{print $1}')
# match downloaded object's hash to original
if [ "$HASH_65_MB" == "$hash2" ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
# remove download file
rm -f /tmp/datafile-65-MB
else
rv=1
out="Checksum verification failed for downloaded object"
fi
# match etag and SSE headers
if [ "$etag1" != "$etag2" ]; then
rv=1
out="Etag mismatch for object encrypted with server side encryption"
fi
if [ "$sse_customer_algo1" != "$sse_customer_algo2" ]; then
rv=1
out="sse customer algorithm mismatch"
fi
if [ "$sse_customer_key1" != "$sse_customer_key2" ]; then
rv=1
out="sse customer key mismatch"
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# tests encrypted copy from multipart encrypted object to
# single part encrypted object. This test in particular checks if copy
# succeeds for the case where encryption overhead for individually
# encrypted parts vs encryption overhead for the original datastream
# differs.
function test_serverside_encryption_multipart_copy() {
#skip server side encryption tests if HTTPS disabled.
if [ "$ENABLE_HTTPS" != "1" ]; then
return 0
fi
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
object_name=${bucket_name}"-object"
rv=$?
if [ $rv -eq 0 ]; then
# create multipart
function="${AWS} s3api create-multipart-upload --bucket ${bucket_name} --key ${object_name} --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
out=$($function)
rv=$?
upload_id=$(echo "$out" | jq -r .UploadId)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 1
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 1 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
out=$($function)
rv=$?
etag1=$(echo "$out" | jq -r .ETag)
fi
if [ $rv -eq 0 ]; then
# Capture etag for part-number 2
function="${AWS} s3api upload-part --bucket ${bucket_name} --key ${object_name} --body ${MINT_DATA_DIR}/datafile-5243880-b --upload-id ${upload_id} --part-number 2 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
out=$($function)
rv=$?
etag2=$(echo "$out" | jq -r .ETag)
# Create a multipart struct file for completing multipart transaction
echo "{
\"Parts\": [
{
\"ETag\": ${etag1},
\"PartNumber\": 1
},
{
\"ETag\": ${etag2},
\"PartNumber\": 2
}
]
}" >> /tmp/multipart
fi
if [ $rv -eq 0 ]; then
# Use saved etags to complete the multipart transaction
function="${AWS} s3api complete-multipart-upload --multipart-upload file:///tmp/multipart --bucket ${bucket_name} --key ${object_name} --upload-id ${upload_id}"
out=$($function)
rv=$?
finalETag=$(echo "$out" | jq -r .ETag | sed -e 's/^"//' -e 's/"$//')
if [ "${finalETag}" == "" ]; then
rv=1
out="complete-multipart-upload failed"
fi
fi
# copy object server side
if [ $rv -eq 0 ]; then
function="${AWS} s3api copy-object --bucket ${bucket_name} --key ${object_name}-copy --copy-source ${bucket_name}/${object_name} --copy-source-sse-customer-algorithm AES256 --copy-source-sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --copy-source-sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
test_function=${function}
out=$($function)
rv=$?
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
rm -f /tmp/multipart
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# tests server side encryption headers for range get calls
function test_serverside_encryption_get_range() {
#skip server side encryption tests if HTTPS disabled.
if [ "$ENABLE_HTTPS" != "1" ]; then
return 0
fi
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# put object with server side encryption headers
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-10-kB --bucket ${bucket_name} --key datafile-10-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# now get encrypted object from server for range 500-999
if [ $rv -eq 0 ]; then
etag1=$(echo "$out" | jq -r .ETag)
sse_customer_key1=$(echo "$out" | jq -r .SSECustomerKeyMD5)
sse_customer_algo1=$(echo "$out" | jq -r .SSECustomerAlgorithm)
function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-10-kB --range bytes=500-999 --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-10-kB"
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -eq 0 ]; then
cnt=$(stat -c%s /tmp/datafile-10-kB)
if [ "$cnt" -ne 500 ]; then
rv=1
fi
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# tests server side encryption error for get and put calls
function test_serverside_encryption_error() {
#skip server side encryption tests if HTTPS disabled.
if [ "$ENABLE_HTTPS" != "1" ]; then
return 0
fi
# log start time
start_time=$(get_time)
function="make_bucket"
bucket_name=$(make_bucket)
rv=$?
# put object with server side encryption headers with MD5Sum mismatch for sse-customer-key-md5 header
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg"
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
# put object with missing server side encryption header sse-customer-algorithm
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
# put object with server side encryption headers successfully
if [ $rv -eq 0 ]; then
function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ= --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg=="
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
# now test get on encrypted object with nonmatching sse-customer-key and sse-customer-md5 headers
if [ $rv -eq 0 ]; then
function="${AWS} s3api get-object --bucket ${bucket_name} --key datafile-1-kB --sse-customer-algorithm AES256 --sse-customer-key MzJieXRlc --sse-customer-key-md5 7PpPLAK26ONlVUGOWlusfg== /tmp/datafile-1-kB"
test_function=${function}
out=$($function 2>&1)
rv=$?
fi
if [ $rv -ne 255 ]; then
rv=1
else
rv=0
fi
# delete bucket
if [ $rv -eq 0 ]; then
function="delete_bucket"
out=$(delete_bucket "$bucket_name")
rv=$?
fi
if [ $rv -eq 0 ]; then
log_success "$(get_duration "$start_time")" "${test_function}"
else
# clean up and log error
${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
log_failure "$(get_duration "$start_time")" "${function}" "${out}"
fi
return $rv
}
# # WORM bucket tests.
# function test_worm_bucket() {
# # log start time
# start_time=$(get_time)
# # Make bucket
# bucket_name="awscli-mint-test-bucket-$RANDOM"
# function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
# # execute the test
# out=$($function 2>&1)
# rv=$?
# if [ $rv -ne 0 ]; then
# # if this functionality is not implemented return right away.
# if echo "$out" | grep -q "NotImplemented"; then
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
# return 0
# fi
# fi
# # if make bucket succeeds set object lock configuration
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api put-object-lock-configuration --bucket ${bucket_name} --object-lock-configuration ObjectLockEnabled=Enabled"
# out=$($function 2>&1)
# rv=$?
# if [ $rv -ne 0 ]; then
# # if this functionality is not implemented return right away.
# if echo "$out" | grep -q "NotImplemented"; then
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
# return 0
# fi
# fi
# else
# # if make bucket fails, $bucket_name has the error output
# out="${bucket_name}"
# fi
# # if setting object lock configuration succeeds, upload a file first time
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
# out=$($function 2>&1)
# rv=$?
# else
# # if make bucket fails, $bucket_name has the error output
# out="${bucket_name}"
# fi
# # second time upload will succeed and there shall be now two versions of the object
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB"
# out=$($function 2>&1)
# rv=$?
# else
# out="First time object upload failed"
# fi
# if [ $rv -eq 0 ]; then
# log_success "$(get_duration "$start_time")" "${test_function}"
# else
# # cleanup is not possible due to one day validity of object lock configurataion
# log_failure "$(get_duration "$start_time")" "${function}" "${out}"
# fi
# return $rv
# }
# # Tests creating and deleting an object with legal hold.
# function test_legal_hold() {
# # log start time
# start_time=$(get_time)
# # Make bucket
# bucket_name="awscli-mint-test-bucket-$RANDOM"
# function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
# # execute the test
# out=$($function 2>&1)
# rv=$?
# if [ $rv -ne 0 ]; then
# # if this functionality is not implemented return right away.
# if echo "$out" | grep -q "NotImplemented"; then
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
# return 0
# fi
# fi
# # if make bucket succeeds upload a file
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON"
# out=$($function 2>&1)
# errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l)
# # skip test for gateways
# if [ "$errcnt" -eq 1 ]; then
# return 0
# fi
# rv=$?
# else
# # if make bucket fails, $bucket_name has the error output
# out="${bucket_name}"
# fi
# # if upload succeeds stat the file
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB"
# # save the ref to function being tested, so it can be logged
# test_function=${function}
# out=$($function 2>&1)
# lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus)
# rv=$?
# fi
# # if head-object succeeds, verify metadata has legal hold status
# if [ $rv -eq 0 ]; then
# if [ "${lhold}" == "" ]; then
# rv=1
# out="Legal hold was not applied"
# fi
# if [ "${lhold}" == "OFF" ]; then
# rv=1
# out="Legal hold was not applied"
# fi
# fi
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF"
# out=$($function 2>&1)
# rv=$?
# else
# # if make bucket fails, $bucket_name has the error output
# out="${bucket_name}"
# fi
# # if upload succeeds download the file
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB"
# # save the ref to function being tested, so it can be logged
# test_function=${function}
# out=$($function 2>&1)
# lhold=$(echo "$out" | jq -r .LegalHold.Status)
# rv=$?
# fi
# # if head-object succeeds, verify metadata has legal hold status
# if [ $rv -eq 0 ]; then
# if [ "${lhold}" == "" ]; then
# rv=1
# out="Legal hold was not applied"
# fi
# if [ "${lhold}" == "ON" ]; then
# rv=1
# out="Legal hold status not turned off"
# fi
# fi
# # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.
# if [ $rv -eq 0 ]; then
# function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB"
# # save the ref to function being tested, so it can be logged
# test_function=${function}
# out=$($function 2>&1)
# rv=$?
# fi
# if [ $rv -eq 0 ]; then
# log_success "$(get_duration "$start_time")" "${test_function}"
# else
# # clean up and log error
# ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
# log_failure "$(get_duration "$start_time")" "${function}" "${out}"
# fi
# return $rv
# }
# main handler for all the tests.
main() {
# Success tests
test_create_bucket && \
test_upload_object && \
test_lookup_object_prefix && \
test_list_objects && \
test_multipart_upload_0byte && \
test_multipart_upload && \
test_max_key_list && \
test_copy_object && \
test_copy_object_storage_class && \
test_copy_object_storage_class_same && \
test_presigned_object && \
test_upload_object_10 && \
test_multipart_upload_10 && \
test_serverside_encryption && \
test_serverside_encryption_get_range && \
test_serverside_encryption_multipart && \
test_serverside_encryption_multipart_copy && \
# Success cli ops.
test_aws_s3_cp && \
test_aws_s3_sync && \
# Error tests
test_list_objects_error && \
test_put_object_error && \
test_serverside_encryption_error
# test_worm_bucket && \
# test_legal_hold
return $?
}
_init "$@" && main
Reference in a new issue View git blame Copy permalink