mirror of
https://github.com/go-gitea/gitea
synced 2024-12-22 11:14:07 +01:00
Refactor "dump" sub-command (#30240)
Major changes: * Move some functions like "addReader" / "isSubDir" / "addRecursiveExclude" to a separate package, and add tests * Clarify the filename&dump type logic and add tests * Clarify the logger behavior and remove FIXME comments Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
wxiaoguang
GitHub
parent
b28d3a4218
commit
654cfd1dfb
6 changed files with 374 additions and 229 deletions
296
cmd/dump.go
296
cmd/dump.go
|
|
@ -6,14 +6,13 @@ package cmd
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/modules/dump"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
|
@ -25,89 +24,17 @@ import (
|
|||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
func addReader(w archiver.Writer, r io.ReadCloser, info os.FileInfo, customName string, verbose bool) error {
|
||||
if verbose {
|
||||
log.Info("Adding file %s", customName)
|
||||
}
|
||||
|
||||
return w.Write(archiver.File{
|
||||
FileInfo: archiver.FileInfo{
|
||||
FileInfo: info,
|
||||
CustomName: customName,
|
||||
},
|
||||
ReadCloser: r,
|
||||
})
|
||||
}
|
||||
|
||||
func addFile(w archiver.Writer, filePath, absPath string, verbose bool) error {
|
||||
file, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
fileInfo, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, file, fileInfo, filePath, verbose)
|
||||
}
|
||||
|
||||
func isSubdir(upper, lower string) (bool, error) {
|
||||
if relPath, err := filepath.Rel(upper, lower); err != nil {
|
||||
return false, err
|
||||
} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type outputType struct {
|
||||
Enum []string
|
||||
Default string
|
||||
selected string
|
||||
}
|
||||
|
||||
func (o outputType) Join() string {
|
||||
return strings.Join(o.Enum, ", ")
|
||||
}
|
||||
|
||||
func (o *outputType) Set(value string) error {
|
||||
for _, enum := range o.Enum {
|
||||
if enum == value {
|
||||
o.selected = value
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("allowed values are %s", o.Join())
|
||||
}
|
||||
|
||||
func (o outputType) String() string {
|
||||
if o.selected == "" {
|
||||
return o.Default
|
||||
}
|
||||
return o.selected
|
||||
}
|
||||
|
||||
var outputTypeEnum = &outputType{
|
||||
Enum: []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"},
|
||||
Default: "zip",
|
||||
}
|
||||
|
||||
// CmdDump represents the available dump sub-command.
|
||||
var CmdDump = &cli.Command{
|
||||
Name: "dump",
|
||||
Usage: "Dump Gitea files and database",
|
||||
Description: `Dump compresses all related files and database into zip file.
|
||||
It can be used for backup and capture Gitea server image to send to maintainer`,
|
||||
Action: runDump,
|
||||
Name: "dump",
|
||||
Usage: "Dump Gitea files and database",
|
||||
Description: `Dump compresses all related files and database into zip file. It can be used for backup and capture Gitea server image to send to maintainer`,
|
||||
Action: runDump,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "file",
|
||||
Aliases: []string{"f"},
|
||||
Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
|
||||
Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
|
||||
Usage: `Name of the dump file which will be created, default to "gitea-dump-{time}.zip". Supply '-' for stdout. See type for available types.`,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
|
|
@ -160,64 +87,52 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
|
|||
Name: "skip-index",
|
||||
Usage: "Skip bleve index data",
|
||||
},
|
||||
&cli.GenericFlag{
|
||||
&cli.StringFlag{
|
||||
Name: "type",
|
||||
Value: outputTypeEnum,
|
||||
Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
|
||||
Usage: fmt.Sprintf(`Dump output format, default to "zip", supported types: %s`, strings.Join(dump.SupportedOutputTypes, ", ")),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func fatal(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
log.Fatal(format, args...)
|
||||
}
|
||||
|
||||
func runDump(ctx *cli.Context) error {
|
||||
var file *os.File
|
||||
fileName := ctx.String("file")
|
||||
outType := ctx.String("type")
|
||||
if fileName == "-" {
|
||||
file = os.Stdout
|
||||
setupConsoleLogger(log.FATAL, log.CanColorStderr, os.Stderr)
|
||||
} else {
|
||||
for _, suffix := range outputTypeEnum.Enum {
|
||||
if strings.HasSuffix(fileName, "."+suffix) {
|
||||
fileName = strings.TrimSuffix(fileName, "."+suffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
fileName += "." + outType
|
||||
}
|
||||
setting.MustInstalled()
|
||||
|
||||
// make sure we are logging to the console no matter what the configuration tells us do to
|
||||
// FIXME: don't use CfgProvider directly
|
||||
if _, err := setting.CfgProvider.Section("log").NewKey("MODE", "console"); err != nil {
|
||||
fatal("Setting logging mode to console failed: %v", err)
|
||||
}
|
||||
if _, err := setting.CfgProvider.Section("log.console").NewKey("STDERR", "true"); err != nil {
|
||||
fatal("Setting console logger to stderr failed: %v", err)
|
||||
}
|
||||
|
||||
// Set loglevel to Warn if quiet-mode is requested
|
||||
if ctx.Bool("quiet") {
|
||||
if _, err := setting.CfgProvider.Section("log.console").NewKey("LEVEL", "Warn"); err != nil {
|
||||
fatal("Setting console log-level failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if !setting.InstallLock {
|
||||
log.Error("Is '%s' really the right config path?\n", setting.CustomConf)
|
||||
return fmt.Errorf("gitea is not initialized")
|
||||
}
|
||||
setting.LoadSettings() // cannot access session settings otherwise
|
||||
|
||||
quite := ctx.Bool("quiet")
|
||||
verbose := ctx.Bool("verbose")
|
||||
if verbose && ctx.Bool("quiet") {
|
||||
return fmt.Errorf("--quiet and --verbose cannot both be set")
|
||||
if verbose && quite {
|
||||
fatal("Option --quiet and --verbose cannot both be set")
|
||||
}
|
||||
|
||||
// outFileName is either "-" or a file name (will be made absolute)
|
||||
outFileName, outType := dump.PrepareFileNameAndType(ctx.String("file"), ctx.String("type"))
|
||||
if outType == "" {
|
||||
fatal("Invalid output type")
|
||||
}
|
||||
|
||||
outFile := os.Stdout
|
||||
if outFileName != "-" {
|
||||
var err error
|
||||
if outFileName, err = filepath.Abs(outFileName); err != nil {
|
||||
fatal("Unable to get absolute path of dump file: %v", err)
|
||||
}
|
||||
if exist, _ := util.IsExist(outFileName); exist {
|
||||
fatal("Dump file %q exists", outFileName)
|
||||
}
|
||||
if outFile, err = os.Create(outFileName); err != nil {
|
||||
fatal("Unable to create dump file %q: %v", outFileName, err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
}
|
||||
|
||||
setupConsoleLogger(util.Iif(quite, log.WARN, log.INFO), log.CanColorStderr, os.Stderr)
|
||||
|
||||
setting.DisableLoggerInit()
|
||||
setting.LoadSettings() // cannot access session settings otherwise
|
||||
|
||||
stdCtx, cancel := installSignals()
|
||||
defer cancel()
|
||||
|
||||
|
|
@ -226,44 +141,32 @@ func runDump(ctx *cli.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if err := storage.Init(); err != nil {
|
||||
if err = storage.Init(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if file == nil {
|
||||
file, err = os.Create(fileName)
|
||||
if err != nil {
|
||||
fatal("Unable to open %s: %v", fileName, err)
|
||||
}
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
absFileName, err := filepath.Abs(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var iface any
|
||||
if fileName == "-" {
|
||||
iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
|
||||
} else {
|
||||
iface, err = archiver.ByExtension(fileName)
|
||||
}
|
||||
archiverGeneric, err := archiver.ByExtension("." + outType)
|
||||
if err != nil {
|
||||
fatal("Unable to get archiver for extension: %v", err)
|
||||
}
|
||||
|
||||
w, _ := iface.(archiver.Writer)
|
||||
if err := w.Create(file); err != nil {
|
||||
archiverWriter := archiverGeneric.(archiver.Writer)
|
||||
if err := archiverWriter.Create(outFile); err != nil {
|
||||
fatal("Creating archiver.Writer failed: %v", err)
|
||||
}
|
||||
defer w.Close()
|
||||
defer archiverWriter.Close()
|
||||
|
||||
dumper := &dump.Dumper{
|
||||
Writer: archiverWriter,
|
||||
Verbose: verbose,
|
||||
}
|
||||
dumper.GlobalExcludeAbsPath(outFileName)
|
||||
|
||||
if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
|
||||
log.Info("Skip dumping local repositories")
|
||||
} else {
|
||||
log.Info("Dumping local repositories... %s", setting.RepoRootPath)
|
||||
if err := addRecursiveExclude(w, "repos", setting.RepoRootPath, []string{absFileName}, verbose); err != nil {
|
||||
if err := dumper.AddRecursiveExclude("repos", setting.RepoRootPath, nil); err != nil {
|
||||
fatal("Failed to include repositories: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -276,8 +179,7 @@ func runDump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, object, info, path.Join("data", "lfs", objPath), verbose)
|
||||
return dumper.AddReader(object, info, path.Join("data", "lfs", objPath))
|
||||
}); err != nil {
|
||||
fatal("Failed to dump LFS objects: %v", err)
|
||||
}
|
||||
|
|
@ -310,15 +212,13 @@ func runDump(ctx *cli.Context) error {
|
|||
fatal("Failed to dump database: %v", err)
|
||||
}
|
||||
|
||||
if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
|
||||
if err = dumper.AddFile("gitea-db.sql", dbDump.Name()); err != nil {
|
||||
fatal("Failed to include gitea-db.sql: %v", err)
|
||||
}
|
||||
|
||||
if len(setting.CustomConf) > 0 {
|
||||
log.Info("Adding custom configuration file from %s", setting.CustomConf)
|
||||
if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
|
||||
fatal("Failed to include specified app.ini: %v", err)
|
||||
}
|
||||
log.Info("Adding custom configuration file from %s", setting.CustomConf)
|
||||
if err = dumper.AddFile("app.ini", setting.CustomConf); err != nil {
|
||||
fatal("Failed to include specified app.ini: %v", err)
|
||||
}
|
||||
|
||||
if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
|
||||
|
|
@ -326,8 +226,8 @@ func runDump(ctx *cli.Context) error {
|
|||
} else {
|
||||
customDir, err := os.Stat(setting.CustomPath)
|
||||
if err == nil && customDir.IsDir() {
|
||||
if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
|
||||
if err := addRecursiveExclude(w, "custom", setting.CustomPath, []string{absFileName}, verbose); err != nil {
|
||||
if is, _ := dump.IsSubdir(setting.AppDataPath, setting.CustomPath); !is {
|
||||
if err := dumper.AddRecursiveExclude("custom", setting.CustomPath, nil); err != nil {
|
||||
fatal("Failed to include custom: %v", err)
|
||||
}
|
||||
} else {
|
||||
|
|
@ -364,8 +264,7 @@ func runDump(ctx *cli.Context) error {
|
|||
excludes = append(excludes, setting.Attachment.Storage.Path)
|
||||
excludes = append(excludes, setting.Packages.Storage.Path)
|
||||
excludes = append(excludes, setting.Log.RootPath)
|
||||
excludes = append(excludes, absFileName)
|
||||
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
|
||||
if err := dumper.AddRecursiveExclude("data", setting.AppDataPath, excludes); err != nil {
|
||||
fatal("Failed to include data directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -377,8 +276,7 @@ func runDump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, object, info, path.Join("data", "attachments", objPath), verbose)
|
||||
return dumper.AddReader(object, info, path.Join("data", "attachments", objPath))
|
||||
}); err != nil {
|
||||
fatal("Failed to dump attachments: %v", err)
|
||||
}
|
||||
|
|
@ -392,8 +290,7 @@ func runDump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, object, info, path.Join("data", "packages", objPath), verbose)
|
||||
return dumper.AddReader(object, info, path.Join("data", "packages", objPath))
|
||||
}); err != nil {
|
||||
fatal("Failed to dump packages: %v", err)
|
||||
}
|
||||
|
|
@ -409,80 +306,23 @@ func runDump(ctx *cli.Context) error {
|
|||
log.Error("Unable to check if %s exists. Error: %v", setting.Log.RootPath, err)
|
||||
}
|
||||
if isExist {
|
||||
if err := addRecursiveExclude(w, "log", setting.Log.RootPath, []string{absFileName}, verbose); err != nil {
|
||||
if err := dumper.AddRecursiveExclude("log", setting.Log.RootPath, nil); err != nil {
|
||||
fatal("Failed to include log: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fileName != "-" {
|
||||
if err = w.Close(); err != nil {
|
||||
_ = util.Remove(fileName)
|
||||
fatal("Failed to save %s: %v", fileName, err)
|
||||
if outFileName == "-" {
|
||||
log.Info("Finish dumping to stdout")
|
||||
} else {
|
||||
if err = archiverWriter.Close(); err != nil {
|
||||
_ = os.Remove(outFileName)
|
||||
fatal("Failed to save %q: %v", outFileName, err)
|
||||
}
|
||||
|
||||
if err := os.Chmod(fileName, 0o600); err != nil {
|
||||
if err = os.Chmod(outFileName, 0o600); err != nil {
|
||||
log.Info("Can't change file access permissions mask to 0600: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fileName != "-" {
|
||||
log.Info("Finish dumping in file %s", fileName)
|
||||
} else {
|
||||
log.Info("Finish dumping to stdout")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
|
||||
func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
|
||||
absPath, err := filepath.Abs(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
files, err := dir.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, file := range files {
|
||||
currentAbsPath := filepath.Join(absPath, file.Name())
|
||||
currentInsidePath := path.Join(insidePath, file.Name())
|
||||
if file.IsDir() {
|
||||
if !util.SliceContainsString(excludeAbsPath, currentAbsPath) {
|
||||
if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
|
||||
shouldAdd := file.Mode().IsRegular()
|
||||
if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
target, err := filepath.EvalSymlinks(currentAbsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetStat, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shouldAdd = targetStat.Mode().IsRegular()
|
||||
}
|
||||
if shouldAdd {
|
||||
if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Info("Finish dumping in file %s", outFileName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
174
modules/dump/dumper.go
Normal file
174
modules/dump/dumper.go
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"github.com/mholt/archiver/v3"
|
||||
)
|
||||
|
||||
var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"}
|
||||
|
||||
// PrepareFileNameAndType prepares the output file name and type, if the type is not supported, it returns an empty "outType"
|
||||
func PrepareFileNameAndType(argFile, argType string) (outFileName, outType string) {
|
||||
if argFile == "" && argType == "" {
|
||||
outType = SupportedOutputTypes[0]
|
||||
outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
|
||||
} else if argFile == "" {
|
||||
outType = argType
|
||||
outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
|
||||
} else if argType == "" {
|
||||
if filepath.Ext(outFileName) == "" {
|
||||
outType = SupportedOutputTypes[0]
|
||||
outFileName = argFile
|
||||
} else {
|
||||
for _, t := range SupportedOutputTypes {
|
||||
if strings.HasSuffix(argFile, "."+t) {
|
||||
outFileName = argFile
|
||||
outType = t
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
outFileName, outType = argFile, argType
|
||||
}
|
||||
if !slices.Contains(SupportedOutputTypes, outType) {
|
||||
return "", ""
|
||||
}
|
||||
return outFileName, outType
|
||||
}
|
||||
|
||||
func IsSubdir(upper, lower string) (bool, error) {
|
||||
if relPath, err := filepath.Rel(upper, lower); err != nil {
|
||||
return false, err
|
||||
} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type Dumper struct {
|
||||
Writer archiver.Writer
|
||||
Verbose bool
|
||||
|
||||
globalExcludeAbsPaths []string
|
||||
}
|
||||
|
||||
func (dumper *Dumper) AddReader(r io.ReadCloser, info os.FileInfo, customName string) error {
|
||||
if dumper.Verbose {
|
||||
log.Info("Adding file %s", customName)
|
||||
}
|
||||
|
||||
return dumper.Writer.Write(archiver.File{
|
||||
FileInfo: archiver.FileInfo{
|
||||
FileInfo: info,
|
||||
CustomName: customName,
|
||||
},
|
||||
ReadCloser: r,
|
||||
})
|
||||
}
|
||||
|
||||
func (dumper *Dumper) AddFile(filePath, absPath string) error {
|
||||
file, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
fileInfo, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return dumper.AddReader(file, fileInfo, filePath)
|
||||
}
|
||||
|
||||
func (dumper *Dumper) normalizeFilePath(absPath string) string {
|
||||
absPath = filepath.Clean(absPath)
|
||||
if setting.IsWindows {
|
||||
absPath = strings.ToLower(absPath)
|
||||
}
|
||||
return absPath
|
||||
}
|
||||
|
||||
func (dumper *Dumper) GlobalExcludeAbsPath(absPaths ...string) {
|
||||
for _, absPath := range absPaths {
|
||||
dumper.globalExcludeAbsPaths = append(dumper.globalExcludeAbsPaths, dumper.normalizeFilePath(absPath))
|
||||
}
|
||||
}
|
||||
|
||||
func (dumper *Dumper) shouldExclude(absPath string, excludes []string) bool {
|
||||
norm := dumper.normalizeFilePath(absPath)
|
||||
return slices.Contains(dumper.globalExcludeAbsPaths, norm) || slices.Contains(excludes, norm)
|
||||
}
|
||||
|
||||
func (dumper *Dumper) AddRecursiveExclude(insidePath, absPath string, excludes []string) error {
|
||||
excludes = slices.Clone(excludes)
|
||||
for i := range excludes {
|
||||
excludes[i] = dumper.normalizeFilePath(excludes[i])
|
||||
}
|
||||
return dumper.addFileOrDir(insidePath, absPath, excludes)
|
||||
}
|
||||
|
||||
func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string) error {
|
||||
absPath, err := filepath.Abs(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
files, err := dir.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, file := range files {
|
||||
currentAbsPath := filepath.Join(absPath, file.Name())
|
||||
if dumper.shouldExclude(currentAbsPath, excludes) {
|
||||
continue
|
||||
}
|
||||
|
||||
currentInsidePath := path.Join(insidePath, file.Name())
|
||||
if file.IsDir() {
|
||||
if err := dumper.AddFile(currentInsidePath, currentAbsPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
|
||||
shouldAdd := file.Mode().IsRegular()
|
||||
if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
target, err := filepath.EvalSymlinks(currentAbsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetStat, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shouldAdd = targetStat.Mode().IsRegular()
|
||||
}
|
||||
if shouldAdd {
|
||||
if err = dumper.AddFile(currentInsidePath, currentAbsPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
113
modules/dump/dumper_test.go
Normal file
113
modules/dump/dumper_test.go
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
|
||||
"github.com/mholt/archiver/v3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestPrepareFileNameAndType(t *testing.T) {
|
||||
defer timeutil.MockSet(time.Unix(1234, 0))()
|
||||
test := func(argFile, argType, expFile, expType string) {
|
||||
outFile, outType := PrepareFileNameAndType(argFile, argType)
|
||||
assert.Equal(t,
|
||||
fmt.Sprintf("outFile=%s, outType=%s", expFile, expType),
|
||||
fmt.Sprintf("outFile=%s, outType=%s", outFile, outType),
|
||||
fmt.Sprintf("argFile=%s, argType=%s", argFile, argType),
|
||||
)
|
||||
}
|
||||
|
||||
test("", "", "gitea-dump-1234.zip", "zip")
|
||||
test("", "tar.gz", "gitea-dump-1234.tar.gz", "tar.gz")
|
||||
test("", "no-such", "", "")
|
||||
|
||||
test("-", "", "-", "zip")
|
||||
test("-", "tar.gz", "-", "tar.gz")
|
||||
test("-", "no-such", "", "")
|
||||
|
||||
test("a", "", "a", "zip")
|
||||
test("a", "tar.gz", "a", "tar.gz")
|
||||
test("a", "no-such", "", "")
|
||||
|
||||
test("a.zip", "", "a.zip", "zip")
|
||||
test("a.zip", "tar.gz", "a.zip", "tar.gz")
|
||||
test("a.zip", "no-such", "", "")
|
||||
|
||||
test("a.tar.gz", "", "a.tar.gz", "zip")
|
||||
test("a.tar.gz", "tar.gz", "a.tar.gz", "tar.gz")
|
||||
test("a.tar.gz", "no-such", "", "")
|
||||
}
|
||||
|
||||
func TestIsSubDir(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755)
|
||||
|
||||
isSub, err := IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include"))
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, isSub)
|
||||
|
||||
isSub, err = IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include/sub"))
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, isSub)
|
||||
|
||||
isSub, err = IsSubdir(filepath.Join(tmpDir, "include/sub"), filepath.Join(tmpDir, "include"))
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, isSub)
|
||||
}
|
||||
|
||||
type testWriter struct {
|
||||
added []string
|
||||
}
|
||||
|
||||
func (t *testWriter) Create(out io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testWriter) Write(f archiver.File) error {
|
||||
t.added = append(t.added, f.Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDumper(t *testing.T) {
|
||||
sortStrings := func(s []string) []string {
|
||||
sort.Strings(s)
|
||||
return s
|
||||
}
|
||||
tmpDir := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude1"), 0o755)
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude2"), 0o755)
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/a"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/sub/b"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude1/a-1"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude2/a-2"), nil, 0o644)
|
||||
|
||||
tw := &testWriter{}
|
||||
d := &Dumper{Writer: tw}
|
||||
d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1"))
|
||||
err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added))
|
||||
|
||||
tw = &testWriter{}
|
||||
d = &Dumper{Writer: tw}
|
||||
err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added))
|
||||
}
|
||||
|
|
@ -185,8 +185,13 @@ func InitLoggersForTest() {
|
|||
initAllLoggers()
|
||||
}
|
||||
|
||||
var initLoggerDisabled bool
|
||||
|
||||
// initAllLoggers creates all the log services
|
||||
func initAllLoggers() {
|
||||
if initLoggerDisabled {
|
||||
return
|
||||
}
|
||||
initManagedLoggers(log.GetManager(), CfgProvider)
|
||||
|
||||
golog.SetFlags(0)
|
||||
|
|
@ -194,6 +199,10 @@ func initAllLoggers() {
|
|||
golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info))
|
||||
}
|
||||
|
||||
func DisableLoggerInit() {
|
||||
initLoggerDisabled = true
|
||||
}
|
||||
|
||||
func initManagedLoggers(manager *log.LoggerManager, cfg ConfigProvider) {
|
||||
loadLogGlobalFrom(cfg)
|
||||
prepareLoggerConfig(cfg)
|
||||
|
|
|
|||
|
|
@ -21,8 +21,9 @@ var (
|
|||
)
|
||||
|
||||
// MockSet sets the time to a mocked time.Time
|
||||
func MockSet(now time.Time) {
|
||||
func MockSet(now time.Time) func() {
|
||||
mockNow = now
|
||||
return MockUnset
|
||||
}
|
||||
|
||||
// MockUnset will unset the mocked time.Time
|
||||
|
|
|
|||
|
|
@ -213,6 +213,14 @@ func ToPointer[T any](val T) *T {
|
|||
return &val
|
||||
}
|
||||
|
||||
// Iif is an "inline-if", it returns "trueVal" if "condition" is true, otherwise "falseVal"
|
||||
func Iif[T comparable](condition bool, trueVal, falseVal T) T {
|
||||
if condition {
|
||||
return trueVal
|
||||
}
|
||||
return falseVal
|
||||
}
|
||||
|
||||
// IfZero returns "def" if "v" is a zero value, otherwise "v"
|
||||
func IfZero[T comparable](v, def T) T {
|
||||
var zero T
|
||||
|
|
|
|||
Loading…
Reference in a new issue