mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-09-10 20:22:14 +02:00
fix minio storage iterator path (#24691)
minio storage iterator shows different behavior with local fs iterator. in local fs storage: ``` go s.IterateObjects("prefix", func(path,obj) println(path) // show "prefix/xxx.file" }) ``` in minio storage: ```go s.IterateObjects("prefix", func(path,obj) println(path) // show "xxx.file" }) ``` I think local fs is correct, minio use wrong `basePath` to trim storage path prefix. --------- Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
parent
4810fe55e3
commit
61ad4c607b
6 changed files with 87 additions and 43 deletions
7
.github/workflows/pull-db-tests.yml
vendored
7
.github/workflows/pull-db-tests.yml
vendored
|
@ -102,6 +102,13 @@ jobs:
|
||||||
--health-retries 10
|
--health-retries 10
|
||||||
ports:
|
ports:
|
||||||
- 6379:6379
|
- 6379:6379
|
||||||
|
minio:
|
||||||
|
image: bitnami/minio:2021.3.17
|
||||||
|
env:
|
||||||
|
MINIO_ACCESS_KEY: 123456
|
||||||
|
MINIO_SECRET_KEY: 12345678
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v4
|
- uses: actions/setup-go@v4
|
||||||
|
|
|
@ -133,8 +133,8 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// IterateObjects iterates across the objects in the local storage
|
// IterateObjects iterates across the objects in the local storage
|
||||||
func (l *LocalStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
|
func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
|
||||||
dir := l.buildLocalPath(prefix)
|
dir := l.buildLocalPath(dirName)
|
||||||
return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -4,8 +4,6 @@
|
||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
@ -57,38 +55,5 @@ func TestBuildLocalPath(t *testing.T) {
|
||||||
|
|
||||||
func TestLocalStorageIterator(t *testing.T) {
|
func TestLocalStorageIterator(t *testing.T) {
|
||||||
dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
|
dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
|
||||||
l, err := NewLocalStorage(context.Background(), LocalStorageConfig{Path: dir})
|
testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir})
|
||||||
assert.NoError(t, err)
|
|
||||||
|
|
||||||
testFiles := [][]string{
|
|
||||||
{"a/1.txt", "a1"},
|
|
||||||
{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
|
|
||||||
{"b/1.txt", "b1"},
|
|
||||||
{"b/2.txt", "b2"},
|
|
||||||
{"b/3.txt", "b3"},
|
|
||||||
{"b/x 4.txt", "bx4"},
|
|
||||||
}
|
|
||||||
for _, f := range testFiles {
|
|
||||||
_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
expectedList := map[string][]string{
|
|
||||||
"a": {"a/1.txt"},
|
|
||||||
"b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
|
||||||
"": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
|
||||||
"/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
|
||||||
"a/b/../../a": {"a/1.txt"},
|
|
||||||
}
|
|
||||||
for dir, expected := range expectedList {
|
|
||||||
count := 0
|
|
||||||
err = l.IterateObjects(dir, func(path string, f Object) error {
|
|
||||||
defer f.Close()
|
|
||||||
assert.Contains(t, expected, path)
|
|
||||||
count++
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Len(t, expected, count)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -129,7 +129,11 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *MinioStorage) buildMinioPath(p string) string {
|
func (m *MinioStorage) buildMinioPath(p string) string {
|
||||||
return util.PathJoinRelX(m.basePath, p)
|
p = util.PathJoinRelX(m.basePath, p)
|
||||||
|
if p == "." {
|
||||||
|
p = "" // minio doesn't use dot as relative path
|
||||||
|
}
|
||||||
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens a file
|
// Open opens a file
|
||||||
|
@ -224,14 +228,15 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// IterateObjects iterates across the objects in the miniostorage
|
// IterateObjects iterates across the objects in the miniostorage
|
||||||
func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
|
func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
|
||||||
opts := minio.GetObjectOptions{}
|
opts := minio.GetObjectOptions{}
|
||||||
lobjectCtx, cancel := context.WithCancel(m.ctx)
|
lobjectCtx, cancel := context.WithCancel(m.ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
basePath := m.basePath
|
basePath := m.basePath
|
||||||
if prefix != "" {
|
if dirName != "" {
|
||||||
basePath = m.buildMinioPath(prefix)
|
// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
|
||||||
|
basePath = m.buildMinioPath(dirName) + "/"
|
||||||
}
|
}
|
||||||
|
|
||||||
for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
|
for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
|
||||||
|
@ -244,7 +249,7 @@ func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Ob
|
||||||
}
|
}
|
||||||
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
|
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
|
||||||
defer object.Close()
|
defer object.Close()
|
||||||
return fn(strings.TrimPrefix(mObjInfo.Key, basePath), &minioObject{object})
|
return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
|
||||||
}(object, fn); err != nil {
|
}(object, fn); err != nil {
|
||||||
return convertMinioErr(err)
|
return convertMinioErr(err)
|
||||||
}
|
}
|
||||||
|
|
18
modules/storage/minio_test.go
Normal file
18
modules/storage/minio_test.go
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMinioStorageIterator(t *testing.T) {
|
||||||
|
testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{
|
||||||
|
Endpoint: "127.0.0.1:9000",
|
||||||
|
AccessKeyID: "123456",
|
||||||
|
SecretAccessKey: "12345678",
|
||||||
|
Bucket: "gitea",
|
||||||
|
Location: "us-east-1",
|
||||||
|
})
|
||||||
|
}
|
49
modules/storage/storage_test.go
Normal file
49
modules/storage/storage_test.go
Normal file
|
@ -0,0 +1,49 @@
|
||||||
|
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func testStorageIterator(t *testing.T, typStr string, cfg interface{}) {
|
||||||
|
l, err := NewStorage(typStr, cfg)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
testFiles := [][]string{
|
||||||
|
{"a/1.txt", "a1"},
|
||||||
|
{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
|
||||||
|
{"ab/1.txt", "ab1"},
|
||||||
|
{"b/1.txt", "b1"},
|
||||||
|
{"b/2.txt", "b2"},
|
||||||
|
{"b/3.txt", "b3"},
|
||||||
|
{"b/x 4.txt", "bx4"},
|
||||||
|
}
|
||||||
|
for _, f := range testFiles {
|
||||||
|
_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expectedList := map[string][]string{
|
||||||
|
"a": {"a/1.txt"},
|
||||||
|
"b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
|
||||||
|
"": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||||
|
"/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
|
||||||
|
"a/b/../../a": {"a/1.txt"},
|
||||||
|
}
|
||||||
|
for dir, expected := range expectedList {
|
||||||
|
count := 0
|
||||||
|
err = l.IterateObjects(dir, func(path string, f Object) error {
|
||||||
|
defer f.Close()
|
||||||
|
assert.Contains(t, expected, path)
|
||||||
|
count++
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, expected, count)
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue