From 60417950c73ca6ce2c1c81b52b2792d1dd6f863f Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 8 Jul 2020 08:44:43 -0700 Subject: [PATCH] fix: the versioning/object lock documentation appropriately (#9988) - Move the bucket level features into `docs/bucket` directory - fix issue template and simplify some of them --- .github/ISSUE_TEMPLATE.md | 17 ++++++-- .github/ISSUE_TEMPLATE/bug_report.md | 8 ++-- .github/ISSUE_TEMPLATE/config.yml | 4 +- .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/lock.yml | 2 +- .github/stale.yml | 6 +-- docs/bucket/lifecycle/README.md | 57 ++++++++++++++++++++++++++ docs/bucket/retention/README.md | 60 ++++++++++++++++++++++++++++ docs/bucket/versioning/README.md | 1 + docs/lifecycle/README.md | 59 --------------------------- docs/retention/README.md | 56 -------------------------- 11 files changed, 141 insertions(+), 130 deletions(-) create mode 100644 docs/bucket/lifecycle/README.md create mode 100644 docs/bucket/retention/README.md delete mode 100644 docs/lifecycle/README.md delete mode 100644 docs/retention/README.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index e1882afaa..6c86e9e58 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,12 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: community, triage +assignees: '' + +--- + ## Expected Behavior @@ -15,6 +24,8 @@ ## Steps to Reproduce (for bugs) + + 1. 2. 3. @@ -30,8 +41,6 @@ ## Your Environment -* Version used (`minio version`): -* Environment name and version (e.g. nginx 1.9.1): -* Server type and version: +* Version used (`minio --version`): +* Server setup and configuration: * Operating System and version (`uname -a`): -* Link to your project: diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 2e1d28069..6c86e9e58 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,6 +24,8 @@ assignees: '' ## Steps to Reproduce (for bugs) + + 1. 2. 3. @@ -39,8 +41,6 @@ assignees: '' ## Your Environment -* Version used (`minio version`): -* Environment name and version (e.g. nginx 1.9.1): -* Server type and version: +* Version used (`minio --version`): +* Server setup and configuration: * Operating System and version (`uname -a`): -* Link to your project: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index cd58e9cd8..95238236d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,7 +2,7 @@ blank_issues_enabled: false contact_links: - name: MinIO Community Support url: https://slack.min.io - about: Please ask and answer questions here. + about: Join here for Community Support - name: MinIO SUBNET Support url: https://min.io/pricing - about: Join this for Enterprise Support. + about: Join here for Enterprise Support diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 70bb2c6cd..1ac3eb0fc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,4 +16,3 @@ - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here) - [ ] Documentation needed - [ ] Unit tests needed -- [ ] Functional tests needed (If yes, add [mint](https://github.com/minio/mint) PR # here: ) diff --git a/.github/lock.yml b/.github/lock.yml index d06728ebb..f747e2b80 100644 --- a/.github/lock.yml +++ b/.github/lock.yml @@ -11,7 +11,7 @@ skipCreatedBefore: false exemptLabels: [] # Label to add before locking, such as `outdated`. Set to `false` to disable -lockLabel: false +lockLabel: true # Comment to post before locking. Set to `false` to disable lockComment: >- diff --git a/.github/stale.yml b/.github/stale.yml index ffb26d9dd..2a054a9d8 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,11 +1,11 @@ # Configuration for probot-stale - https://github.com/probot/stale # Number of days of inactivity before an Issue or Pull Request becomes stale -daysUntilStale: 90 +daysUntilStale: 30 # Number of days of inactivity before an Issue or Pull Request with the stale label is closed. # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. -daysUntilClose: 30 +daysUntilClose: 15 # Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) onlyLabels: [] @@ -30,7 +30,7 @@ staleLabel: stale # Comment to post when marking as stale. Set to `false` to disable markComment: >- This issue has been automatically marked as stale because it has not had - recent activity. It will be closed after 21 days if no further activity + recent activity. It will be closed after 15 days if no further activity occurs. Thank you for your contributions. # Comment to post when removing the stale label. # unmarkComment: > diff --git a/docs/bucket/lifecycle/README.md b/docs/bucket/lifecycle/README.md new file mode 100644 index 000000000..178f4d14c --- /dev/null +++ b/docs/bucket/lifecycle/README.md @@ -0,0 +1,57 @@ +# Bucket Lifecycle Configuration Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +Enable object lifecycle configuration on buckets to setup automatic deletion of objects after a specified number of days or a specified date. + +## 1. Prerequisites +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide). +- Install `mc` - [mc Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide.html) + +## 2. Enable bucket lifecycle configuration + +- Create a bucket lifecycle configuration which expires the objects under the prefix `old/` on `2020-01-01T00:00:00.000Z` date and the objects under `temp/` after 7 days. +- Enable bucket lifecycle configuration using `mc`: + +```sh +$ mc ilm import play/testbucket +{ + "Rules": [ + { + "Expiration": { + "Date": "2020-01-01T00:00:00.000Z" + }, + "ID": "OldPictures", + "Filter": { + "Prefix": "old/" + }, + "Status": "Enabled" + }, + { + "Expiration": { + "Days": 7 + }, + "ID": "TempUploads", + "Filter": { + "Prefix": "temp/" + }, + "Status": "Enabled" + } + ] +} + +Lifecycle configuration imported successfully to `play/testbucket`. +``` + +- List the current settings +``` +$ mc ilm list play/testbucket + ID | Prefix | Enabled | Expiry | Date/Days | Transition | Date/Days | Storage-Class | Tags +------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------ +OldPictures | old/ | ✓ | ✓ | 1 Jan 2020 | ✗ | | | +------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------ +TempUploads | temp/ | ✓ | ✓ | 7 day(s) | ✗ | | | +------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------ +``` + +## Explore Further +- [MinIO | Golang Client API Reference](https://docs.min.io/docs/golang-client-api-reference.html#SetBucketLifecycle) +- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) diff --git a/docs/bucket/retention/README.md b/docs/bucket/retention/README.md new file mode 100644 index 000000000..66484889e --- /dev/null +++ b/docs/bucket/retention/README.md @@ -0,0 +1,60 @@ +# Object Lock and Immutablity Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO server allows WORM for specific objects or by configuring a bucket with default object lock configuration that applies default retention mode and retention duration to all objects. This makes objects in the bucket immutable i.e. delete of the version are not allowed until an expiry specified in the bucket's object lock configuration or object retention. + +Object locking requires locking to be enabled on a bucket at the time of bucket creation, object locking also automatically enables versioning on the bucket. In addition, a default retention period and retention mode can be configured on a bucket to be applied to objects created in that bucket. + +Independent of retention, an object can also be under legal hold. This effectively disallows all deletes of an object under legal hold until the legal hold is removed by an API call. + +## Get Started + +### 1. Prerequisites + +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide) +- Install `awscli` - [Installing AWS Command Line Interface](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) + +### 2. Set bucket WORM configuration + +WORM on a bucket is enabled by setting object lock configuration. This configuration is applied to existing and new objects in the bucket. Below is an example sets `Governance` mode and one day retention time from object creation time of all objects in `mybucket`. + +```sh +$ awscli s3api put-object-lock-configuration --bucket mybucket --object-lock-configuration 'ObjectLockEnabled=\"Enabled\",Rule={DefaultRetention={Mode=\"GOVERNANCE\",Days=1}}' +``` + +### Set object lock + +PutObject API allows setting per object retention mode and retention duration using `x-amz-object-lock-mode` and `x-amz-object-lock-retain-until-date` headers. This takes precedence over any bucket object lock configuration w.r.t retention. + +```sh +aws s3api put-object --bucket testbucket --key lockme --object-lock-mode GOVERNANCE --object-lock-retain-until-date "2019-11-20" --body /etc/issue +``` + +See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on object locking and permissions required for object retention and governance bypass overrides. + +### Set legal hold on an object + +PutObject API allows setting legal hold using `x-amz-object-lock-legal-hold` header. + +```sh +aws s3api put-object --bucket testbucket --key legalhold --object-lock-legal-hold-status ON --body /etc/issue +``` + +See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on object locking and permissions required for specifying legal hold. + +## Concepts +- If an object is under legal hold, it cannot be deleted unless the legal hold is explicitly removed for the respective version id. DeleteObjectVersion() would fail otherwise. +- In `Compliance` mode, objects cannot be deleted by anyone until retention period is expired for the respective version id. If user has requisite governance bypass permissions, an object's retention date can be extended in `Compliance` mode. +- Once object lock configuration is set to a bucket + - New objects inherit the retention settings of the bucket object lock configuration automatically + - Retention headers can be optionally set when uploading objects + - Explicitly calling PutObjectRetention API call on the object +- *MINIO_NTP_SERVER* environment variable can be set to remote NTP server endpoint if system time is not desired for setting retention dates. +- **Object locking feature is only available in erasure coded and distributed erasrue coded setups**. + +## Explore Further + +- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio) +- [Use `s3cmd` with MinIO Server](https://docs.min.io/docs/s3cmd-with-minio) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide) +- [The MinIO documentation website](https://docs.min.io) diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md index a37a85097..8b50e899f 100644 --- a/docs/bucket/versioning/README.md +++ b/docs/bucket/versioning/README.md @@ -35,3 +35,4 @@ To permanently delete an object you need to specify the version you want to dele - Versioning state applies to all of the objects in the versioning enabled bucket. The first time you enable a bucket for versioning, objects in the bucket are thereafter always versioned and given a unique version ID. - Existing or newer buckets can be created with versioning enabled and eventually can be suspended as well. Existing versions of objects stay as is and can still be accessed using the version ID. - All versions, including delete-markers should be deleted before deleting a bucket. +- **Versioning feature is only available in erasure coded and distributed erasure coded setups**. diff --git a/docs/lifecycle/README.md b/docs/lifecycle/README.md deleted file mode 100644 index 160140eac..000000000 --- a/docs/lifecycle/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Object Lifecycle Configuration Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) - -Enable object lifecycle configuration on buckets to setup automatic deletion of objects after a specified number of days or a specified date. - -## 1. Prerequisites -- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide). -- Install AWS Cli - [Installing AWS Command Line Interface](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) - - -## 2. Enable bucket lifecycle configuration - -1. Create a bucket lifecycle configuration which expires the objects under the prefix `uploads/2015` on `2020-01-01T00:00:00.000Z` date and the objects under `temporary-uploads/` after 7 days. Generate it as shown below: - -```sh -$ cat >bucket-lifecycle.json << EOF -{ - "Rules": [ - { - "Expiration": { - "Date": "2020-01-01T00:00:00.000Z" - }, - "ID": "Delete very old messenger pictures", - "Filter": { - "Prefix": "uploads/2015/" - }, - "Status": "Enabled" - }, - { - "Expiration": { - "Days": 7 - }, - "ID": "Delete temporary uploads", - "Filter": { - "Prefix": "temporary-uploads/" - }, - "Status": "Enabled" - } - ] -} -EOF -``` - -2. Enable bucket lifecycle configuration using `aws-cli`: - -```sh -$ export AWS_ACCESS_KEY_ID="your-access-key" -$ export AWS_SECRET_ACCESS_KEY="your-secret-key" -$ aws s3api put-bucket-lifecycle-configuration --bucket your-bucket --endpoint-url http://minio-server-address:port --lifecycle-configuration file://bucket-lifecycle.json -``` - -3. Verify that the configuration has been added - -```sh -$ aws s3api get-bucket-lifecycle-configuration --bucket your-bucket --endpoint-url http://minio-server-address:port -``` - -## Explore Further -- [MinIO | Golang Client API Reference](https://docs.min.io/docs/golang-client-api-reference.html#SetBucketLifecycle) -- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) diff --git a/docs/retention/README.md b/docs/retention/README.md deleted file mode 100644 index b5a3e9d06..000000000 --- a/docs/retention/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Object Lock and Immutablity [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) - -MinIO server allows selectively specify WORM for specific objects or configuring a bucket with default object lock configuration that applies default retention mode and retention duration to all incoming objects. Essentially, this makes objects in the bucket immutable i.e. delete and overwrite are not allowed till stipulated time specified in the bucket's object lock configuration or object retention. - -Object locking requires locking to be enabled on a bucket at the time of bucket creation. In addition, a default retention period and retention mode can be configured on a bucket to be applied to objects created in that bucket. - -Independently of retention, an object can also be under legal hold. This effectively disallows all deletes and overwrites of an object under legal hold until the hold is lifted. - -## Get Started - -### 1. Prerequisites - -Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide). - -### 2. Set bucket WORM configuration - -WORM on a bucket is enabled by setting object lock configuration. This configuration is applied to existing and new objects in the bucket. Below is an example sets `Governance` mode and one day retention time from object creation time of all objects in `mybucket`. - -```sh -$ awscli s3api put-object-lock-configuration --bucket mybucket --object-lock-configuration 'ObjectLockEnabled=\"Enabled\",Rule={DefaultRetention={Mode=\"GOVERNANCE\",Days=1}}' -``` - -### Set object lock - -PutObject API allows setting per object retention mode and retention duration using `x-amz-object-lock-mode` and `x-amz-object-lock-retain-until-date` headers. This takes precedence over any bucket object lock configuration w.r.t retention. - -```sh -aws s3api put-object --bucket testbucket --key lockme --object-lock-mode GOVERNANCE --object-lock-retain-until-date "2019-11-20" --body /etc/issue -``` - -See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on object locking and permissions required for object retention and governance bypass overrides. - -### Set legal hold on an object - -PutObject API allows setting legal hold using `x-amz-object-lock-legal-hold` header. - -```sh -aws s3api put-object --bucket testbucket --key legalhold --object-lock-legal-hold-status ON --body /etc/issue -``` - -See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on object locking and permissions required for specifying legal hold. - -> NOTE: -> - If an object is under legal hold, it cannot be overwritten unless the legal hold is explicitly removed. -> - In `Compliance` mode, objects cannot be overwritten or deleted by anyone until retention period is expired. If user has requisite governance bypass permissions, an object's retention date can be extended in `Compliance` mode. -> - Currently `Governance` mode does not allow overwriting an existing object as versioning is not available in MinIO. However, if user has requisite `Governance` bypass permissions, an object in `Governance` mode can be overwritten. -> - Once object lock configuration is set to a bucket, new objects inherit the retention settings of the bucket object lock configuration (if set) or the retention headers set in the PUT request or set with PutObjectRetention API call -> - *MINIO_NTP_SERVER* environment variable can be set to remote NTP server endpoint if system time is not desired for setting retention dates. - -## Explore Further - -- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide) -- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio) -- [Use `s3cmd` with MinIO Server](https://docs.min.io/docs/s3cmd-with-minio) -- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide) -- [The MinIO documentation website](https://docs.min.io)