a982baff27
Design: https://gist.github.com/klauspost/025c09b48ed4a1293c917cecfabdf21c Gist of improvements: * Cross-server caching and listing will use the same data across servers and requests. * Lists can be arbitrarily resumed at a constant speed. * Metadata for all files scanned is stored for streaming retrieval. * The existing bloom filters controlled by the crawler is used for validating caches. * Concurrent requests for the same data (or parts of it) will not spawn additional walkers. * Listing a subdirectory of an existing recursive cache will use the cache. * All listing operations are fully streamable so the number of objects in a bucket no longer dictates the amount of memory. * Listings can be handled by any server within the cluster. * Caches are cleaned up when out of date or superseded by a more recent one.
71 lines
1.9 KiB
Go
71 lines
1.9 KiB
Go
/*
|
|
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/minio/minio/cmd/logger"
|
|
)
|
|
|
|
// markerTagVersion is the marker version.
|
|
// Should not need to be updated unless a fundamental change is made to the marker format.
|
|
const markerTagVersion = "v1"
|
|
|
|
// parseMarker will parse a marker possibly encoded with encodeMarker
|
|
func parseMarker(s string) (marker, uuid string) {
|
|
if !strings.Contains(s, "[minio_cache:"+markerTagVersion) {
|
|
return s, ""
|
|
}
|
|
start := strings.LastIndex(s, "[")
|
|
marker = s[:start]
|
|
end := strings.LastIndex(s, "]")
|
|
tag := strings.Trim(s[start:end], "[]")
|
|
tags := strings.Split(tag, ",")
|
|
for _, tag := range tags {
|
|
kv := strings.Split(tag, ":")
|
|
if len(kv) < 2 {
|
|
continue
|
|
}
|
|
switch kv[0] {
|
|
case "minio_cache":
|
|
if kv[1] != markerTagVersion {
|
|
break
|
|
}
|
|
case "id":
|
|
uuid = kv[1]
|
|
default:
|
|
// Ignore unknown
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// encodeMarker will encode a uuid and return it as a marker.
|
|
// uuid cannot contain '[', ':' or ','.
|
|
func encodeMarker(marker, uuid string) string {
|
|
if uuid == "" {
|
|
return marker
|
|
}
|
|
if strings.ContainsAny(uuid, "[:,") {
|
|
logger.LogIf(context.Background(), fmt.Errorf("encodeMarker: uuid %s contained invalid characters", uuid))
|
|
}
|
|
return fmt.Sprintf("%s[minio_cache:%s,id:%s]", marker, markerTagVersion, uuid)
|
|
}
|