MirrorHub/forgejo
0
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-10-31 22:58:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

Browse source 
* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type
This commit is contained in:
Lunny Xiao 2020-12-27 11:34:19 +08:00 committed by GitHub
parent 212fa340cf
commit dd08853b10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 1491 additions and 232 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

162
cmd/dump_repo.go Normal file
View file

@ -0,0 +1,162 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"context"
"errors"
"strings"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/urfave/cli"
)
// CmdDumpRepository represents the available dump repository sub-command.
var CmdDumpRepository = cli.Command{
Name: "dump-repo",
Usage: "Dump the repository from git/github/gitea/gitlab",
Description: "This is a command for dumping the repository data.",
Action: runDumpRepository,
Flags: []cli.Flag{
cli.StringFlag{
Name: "git_service",
Value: "",
Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.",
},
cli.StringFlag{
Name: "repo_dir, r",
Value: "./data",
Usage: "Repository dir path to store the data",
},
cli.StringFlag{
Name: "clone_addr",
Value: "",
Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL",
},
cli.StringFlag{
Name: "auth_username",
Value: "",
Usage: "The username to visit the clone_addr",
},
cli.StringFlag{
Name: "auth_password",
Value: "",
Usage: "The password to visit the clone_addr",
},
cli.StringFlag{
Name: "auth_token",
Value: "",
Usage: "The personal token to visit the clone_addr",
},
cli.StringFlag{
Name: "owner_name",
Value: "",
Usage: "The data will be stored on a directory with owner name if not empty",
},
cli.StringFlag{
Name: "repo_name",
Value: "",
Usage: "The data will be stored on a directory with repository name if not empty",
},
cli.StringFlag{
Name: "units",
Value: "",
Usage: `Which items will be migrated, one or more units should be separated as comma.
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
},
},
}
func runDumpRepository(ctx *cli.Context) error {
if err := initDB(); err != nil {
return err
}
log.Trace("AppPath: %s", setting.AppPath)
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath)
setting.InitDBConfig()
var (
serviceType structs.GitServiceType
cloneAddr = ctx.String("clone_addr")
serviceStr = ctx.String("git_service")
)
if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") {
serviceStr = "github"
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") {
serviceStr = "gitlab"
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") {
serviceStr = "gitea"
}
if serviceStr == "" {
return errors.New("git_service missed or clone_addr cannot be recognized")
}
serviceType = convert.ToGitServiceType(serviceStr)
var opts = base.MigrateOptions{
GitServiceType: serviceType,
CloneAddr: cloneAddr,
AuthUsername: ctx.String("auth_username"),
AuthPassword: ctx.String("auth_password"),
AuthToken: ctx.String("auth_token"),
RepoName: ctx.String("repo_name"),
}
if len(ctx.String("units")) == 0 {
opts.Wiki = true
opts.Issues = true
opts.Milestones = true
opts.Labels = true
opts.Releases = true
opts.Comments = true
opts.PullRequests = true
opts.ReleaseAssets = true
} else {
units := strings.Split(ctx.String("units"), ",")
for _, unit := range units {
switch strings.ToLower(unit) {
case "wiki":
opts.Wiki = true
case "issues":
opts.Issues = true
case "milestones":
opts.Milestones = true
case "labels":
opts.Labels = true
case "releases":
opts.Releases = true
case "release_assets":
opts.ReleaseAssets = true
case "comments":
opts.Comments = true
case "pull_requests":
opts.PullRequests = true
}
}
}
if err := migrations.DumpRepository(
context.Background(),
ctx.String("repo_dir"),
ctx.String("owner_name"),
opts,
); err != nil {
log.Fatal("Failed to dump repository: %v", err)
return err
}
log.Trace("Dump finished!!!")
return nil
}

119
cmd/restore_repo.go Normal file
View file

@ -0,0 +1,119 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cmd
import (
"context"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
pull_service "code.gitea.io/gitea/services/pull"
"github.com/urfave/cli"
)
// CmdRestoreRepository represents the available restore a repository sub-command.
var CmdRestoreRepository = cli.Command{
Name: "restore-repo",
Usage: "Restore the repository from disk",
Description: "This is a command for restoring the repository data.",
Action: runRestoreRepository,
Flags: []cli.Flag{
cli.StringFlag{
Name: "repo_dir, r",
Value: "./data",
Usage: "Repository dir path to restore from",
},
cli.StringFlag{
Name: "owner_name",
Value: "",
Usage: "Restore destination owner name",
},
cli.StringFlag{
Name: "repo_name",
Value: "",
Usage: "Restore destination repository name",
},
cli.StringFlag{
Name: "units",
Value: "",
Usage: `Which items will be restored, one or more units should be separated as comma.
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
},
},
}
func runRestoreRepository(ctx *cli.Context) error {
if err := initDB(); err != nil {
return err
}
log.Trace("AppPath: %s", setting.AppPath)
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath)
setting.InitDBConfig()
if err := storage.Init(); err != nil {
return err
}
if err := pull_service.Init(); err != nil {
return err
}
var opts = base.MigrateOptions{
RepoName: ctx.String("repo_name"),
}
if len(ctx.String("units")) == 0 {
opts.Wiki = true
opts.Issues = true
opts.Milestones = true
opts.Labels = true
opts.Releases = true
opts.Comments = true
opts.PullRequests = true
opts.ReleaseAssets = true
} else {
units := strings.Split(ctx.String("units"), ",")
for _, unit := range units {
switch strings.ToLower(unit) {
case "wiki":
opts.Wiki = true
case "issues":
opts.Issues = true
case "milestones":
opts.Milestones = true
case "labels":
opts.Labels = true
case "releases":
opts.Releases = true
case "release_assets":
opts.ReleaseAssets = true
case "comments":
opts.Comments = true
case "pull_requests":
opts.PullRequests = true
}
}
}
if err := migrations.RestoreRepository(
context.Background(),
ctx.String("repo_dir"),
ctx.String("owner_name"),
ctx.String("repo_name"),
); err != nil {
log.Fatal("Failed to restore repository: %v", err)
return err
}
return nil
}

25
docs/content/doc/usage/command-line.en-us.md
View file

@ -441,3 +441,28 @@ Manage running server operations:
- `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
- `--send-to value`, `-s value`: Email address(es) to send to
- `--subject value`, `-S value`: Subject header of sent emails
### dump-repo
Dump-repo dumps repository data from git/github/gitea/gitlab:
- Options:
- `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored.
- `--repo_dir dir`, `-r dir`: Repository dir path to store the data
- `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git
- `--auth_username lunny`: The username to visit the clone_addr
- `--auth_password <password>`: The password to visit the clone_addr
- `--auth_token <token>`: The personal token to visit the clone_addr
- `--owner_name lunny`: The data will be stored on a directory with owner name if not empty
- `--repo_name tango`: The data will be stored on a directory with repository name if not empty
- `--units <units>`: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.
### restore-repo
Restore-repo restore repository data from disk dir:
- Options:
- `--repo_dir dir`, `-r dir`: Repository dir path to restore from
- `--owner_name lunny`: Restore destination owner name
- `--repo_name tango`: Restore destination repository name
- `--units <units>`: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.

2
main.go
View file

@ -72,6 +72,8 @@ arguments - which can alternatively be run by running the subcommand web.`
cmd.Cmdembedded,
cmd.CmdMigrateStorage,
cmd.CmdDocs,
cmd.CmdDumpRepository,
cmd.CmdRestoreRepository,
}
// Now adjust these commands to add our global configuration options

13
models/admin.go
View file

@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error {
Delete(new(Notice))
return err
}
// GetAdminUser returns the first administrator
func GetAdminUser() (*User, error) {
var admin User
has, err := x.Where("is_admin=?", true).Get(&admin)
if err != nil {
return nil, err
} else if !has {
return nil, ErrUserNotExist{}
}
return &admin, nil
}

4
models/task.go
View file

@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error {
if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil {
return err
}
task.Repo.Status = RepositoryReady
if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil {
return err
}
return sess.Commit()
}

8
modules/migrations/base/comment.go
View file

@ -9,10 +9,10 @@ import "time"
// Comment is a standard comment information
type Comment struct {
IssueIndex int64
PosterID int64
PosterName string
PosterEmail string
IssueIndex int64 `yaml:"issue_index"`
PosterID int64 `yaml:"poster_id"`
PosterName string `yaml:"poster_name"`
PosterEmail string `yaml:"poster_email"`
Created time.Time
Updated time.Time
Content string

7
modules/migrations/base/downloader.go
View file

@ -7,20 +7,13 @@ package base
import (
"context"
"io"
"time"
"code.gitea.io/gitea/modules/structs"
)
// AssetDownloader downloads an asset (attachment) for a release
type AssetDownloader interface {
GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
}
// Downloader downloads the site repo informations
type Downloader interface {
AssetDownloader
SetContext(context.Context)
GetRepoInfo() (*Repository, error)
GetTopics() ([]string, error)

8
modules/migrations/base/issue.go
View file

@ -10,15 +10,15 @@ import "time"
// Issue is a standard issue information
type Issue struct {
Number int64
PosterID int64
PosterName string
PosterEmail string
PosterID int64 `yaml:"poster_id"`
PosterName string `yaml:"poster_name"`
PosterEmail string `yaml:"poster_email"`
Title string
Content string
Ref string
Milestone string
State string // closed, open
IsLocked bool
IsLocked bool `yaml:"is_locked"`
Created time.Time
Updated time.Time
Closed *time.Time

1
modules/migrations/base/options.go
View file

@ -31,5 +31,6 @@ type MigrateOptions struct {
Releases bool
Comments bool
PullRequests bool
ReleaseAssets bool
MigrateToRepoID int64
}

22
modules/migrations/base/pullrequest.go
View file

@ -13,11 +13,11 @@ import (
// PullRequest defines a standard pull request information
type PullRequest struct {
Number int64
OriginalNumber int64
OriginalNumber int64 `yaml:"original_number"`
Title string
PosterName string
PosterID int64
PosterEmail string
PosterName string `yaml:"poster_name"`
PosterID int64 `yaml:"poster_id"`
PosterEmail string `yaml:"poster_email"`
Content string
Milestone string
State string
@ -25,14 +25,14 @@ type PullRequest struct {
Updated time.Time
Closed *time.Time
Labels []*Label
PatchURL string
PatchURL string `yaml:"patch_url"`
Merged bool
MergedTime *time.Time
MergeCommitSHA string
MergedTime *time.Time `yaml:"merged_time"`
MergeCommitSHA string `yaml:"merge_commit_sha"`
Head PullRequestBranch
Base PullRequestBranch
Assignees []string
IsLocked bool
IsLocked bool `yaml:"is_locked"`
Reactions []*Reaction
}
@ -43,11 +43,11 @@ func (p *PullRequest) IsForkPullRequest() bool {
// PullRequestBranch represents a pull request branch
type PullRequestBranch struct {
CloneURL string
CloneURL string `yaml:"clone_url"`
Ref string
SHA string
RepoName string
OwnerName string
RepoName string `yaml:"repo_name"`
OwnerName string `yaml:"owner_name"`
}
// RepoPath returns pull request repo path

4
modules/migrations/base/reaction.go
View file

@ -6,7 +6,7 @@ package base
// Reaction represents a reaction to an issue/pr/comment.
type Reaction struct {
UserID int64
UserName string
UserID int64 `yaml:"user_id"`
UserName string `yaml:"user_name"`
Content string
}

25
modules/migrations/base/release.go
View file

@ -4,32 +4,37 @@
package base
import "time"
import (
"io"
"time"
)
// ReleaseAsset represents a release asset
type ReleaseAsset struct {
ID int64
Name string
ContentType *string
ContentType *string `yaml:"content_type"`
Size *int
DownloadCount *int
DownloadCount *int `yaml:"download_count"`
Created time.Time
Updated time.Time
DownloadURL *string
DownloadURL *string `yaml:"download_url"`
// if DownloadURL is nil, the function should be invoked
DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
}
// Release represents a release
type Release struct {
TagName string
TargetCommitish string
TagName string `yaml:"tag_name"`
TargetCommitish string `yaml:"target_commitish"`
Name string
Body string
Draft bool
Prerelease bool
PublisherID int64
PublisherName string
PublisherEmail string
Assets []ReleaseAsset
PublisherID int64 `yaml:"publisher_id"`
PublisherName string `yaml:"publisher_name"`
PublisherEmail string `yaml:"publisher_email"`
Assets []*ReleaseAsset
Created time.Time
Published time.Time
}

8
modules/migrations/base/repo.go
View file

@ -9,10 +9,10 @@ package base
type Repository struct {
Name string
Owner string
IsPrivate bool
IsMirror bool
IsPrivate bool `yaml:"is_private"`
IsMirror bool `yaml:"is_mirror"`
Description string
CloneURL string
OriginalURL string
CloneURL string `yaml:"clone_url"`
OriginalURL string `yaml:"original_url"`
DefaultBranch string
}

26
modules/migrations/base/review.go
View file

@ -17,29 +17,29 @@ const (
// Review is a standard review information
type Review struct {
ID int64
IssueIndex int64
ReviewerID int64
ReviewerName string
IssueIndex int64 `yaml:"issue_index"`
ReviewerID int64 `yaml:"reviewer_id"`
ReviewerName string `yaml:"reviewer_name"`
Official bool
CommitID string
CommitID string `yaml:"commit_id"`
Content string
CreatedAt time.Time
State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
CreatedAt time.Time `yaml:"created_at"`
State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
Comments []*ReviewComment
}
// ReviewComment represents a review comment
type ReviewComment struct {
ID int64
InReplyTo int64
InReplyTo int64 `yaml:"in_reply_to"`
Content string
TreePath string
DiffHunk string
TreePath string `yaml:"tree_path"`
DiffHunk string `yaml:"diff_hunk"`
Position int
Line int
CommitID string
PosterID int64
CommitID string `yaml:"commit_id"`
PosterID int64 `yaml:"poster_id"`
Reactions []*Reaction
CreatedAt time.Time
UpdatedAt time.Time
CreatedAt time.Time `yaml:"created_at"`
UpdatedAt time.Time `yaml:"updated_at"`
}

3
modules/migrations/base/uploader.go
View file

@ -11,7 +11,7 @@ type Uploader interface {
CreateRepo(repo *Repository, opts MigrateOptions) error
CreateTopics(topic ...string) error
CreateMilestones(milestones ...*Milestone) error
CreateReleases(downloader Downloader, releases ...*Release) error
CreateReleases(releases ...*Release) error
SyncTags() error
CreateLabels(labels ...*Label) error
CreateIssues(issues ...*Issue) error
@ -19,5 +19,6 @@ type Uploader interface {
CreatePullRequests(prs ...*PullRequest) error
CreateReviews(reviews ...*Review) error
Rollback() error
Finish() error
Close()
}

591
modules/migrations/dump.go Normal file
View file

@ -0,0 +1,591 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/repository"
"gopkg.in/yaml.v2"
)
var (
_ base.Uploader = &RepositoryDumper{}
)
// RepositoryDumper implements an Uploader to the local directory
type RepositoryDumper struct {
ctx context.Context
baseDir string
repoOwner string
repoName string
opts base.MigrateOptions
milestoneFile *os.File
labelFile *os.File
releaseFile *os.File
issueFile *os.File
commentFiles map[int64]*os.File
pullrequestFile *os.File
reviewFiles map[int64]*os.File
gitRepo *git.Repository
prHeadCache map[string]struct{}
}
// NewRepositoryDumper creates an gitea Uploader
func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) {
baseDir = filepath.Join(baseDir, repoOwner, repoName)
if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
return nil, err
}
return &RepositoryDumper{
ctx: ctx,
opts: opts,
baseDir: baseDir,
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
commentFiles: make(map[int64]*os.File),
reviewFiles: make(map[int64]*os.File),
}, nil
}
// MaxBatchInsertSize returns the table's max batch insert size
func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int {
return 1000
}
func (g *RepositoryDumper) gitPath() string {
return filepath.Join(g.baseDir, "git")
}
func (g *RepositoryDumper) wikiPath() string {
return filepath.Join(g.baseDir, "wiki")
}
func (g *RepositoryDumper) commentDir() string {
return filepath.Join(g.baseDir, "comments")
}
func (g *RepositoryDumper) reviewDir() string {
return filepath.Join(g.baseDir, "reviews")
}
func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
if len(g.opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", g.opts.AuthToken)
}
remoteAddr = u.String()
}
return remoteAddr, nil
}
// CreateRepo creates a repository
func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
if err != nil {
return err
}
defer f.Close()
bs, err := yaml.Marshal(map[string]interface{}{
"name": repo.Name,
"owner": repo.Owner,
"description": repo.Description,
"clone_addr": opts.CloneAddr,
"original_url": repo.OriginalURL,
"is_private": opts.Private,
"service_type": opts.GitServiceType,
"wiki": opts.Wiki,
"issues": opts.Issues,
"milestones": opts.Milestones,
"labels": opts.Labels,
"releases": opts.Releases,
"comments": opts.Comments,
"pulls": opts.PullRequests,
"assets": opts.ReleaseAssets,
})
if err != nil {
return err
}
if _, err := f.Write(bs); err != nil {
return err
}
repoPath := g.gitPath()
if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
return err
}
migrateTimeout := 2 * time.Hour
remoteAddr, err := g.setURLToken(repo.CloneURL)
if err != nil {
return err
}
err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
})
if err != nil {
return fmt.Errorf("Clone: %v", err)
}
if opts.Wiki {
wikiPath := g.wikiPath()
wikiRemotePath := repository.WikiRemoteURL(remoteAddr)
if len(wikiRemotePath) > 0 {
if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
Branch: "master",
}); err != nil {
log.Warn("Clone wiki: %v", err)
if err := os.RemoveAll(wikiPath); err != nil {
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
}
}
}
g.gitRepo, err = git.OpenRepository(g.gitPath())
return err
}
// Close closes this uploader
func (g *RepositoryDumper) Close() {
if g.gitRepo != nil {
g.gitRepo.Close()
}
if g.milestoneFile != nil {
g.milestoneFile.Close()
}
if g.labelFile != nil {
g.labelFile.Close()
}
if g.releaseFile != nil {
g.releaseFile.Close()
}
if g.issueFile != nil {
g.issueFile.Close()
}
for _, f := range g.commentFiles {
f.Close()
}
if g.pullrequestFile != nil {
g.pullrequestFile.Close()
}
for _, f := range g.reviewFiles {
f.Close()
}
}
// CreateTopics creates topics
func (g *RepositoryDumper) CreateTopics(topics ...string) error {
f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
if err != nil {
return err
}
defer f.Close()
bs, err := yaml.Marshal(map[string]interface{}{
"topics": topics,
})
if err != nil {
return err
}
if _, err := f.Write(bs); err != nil {
return err
}
return nil
}
// CreateMilestones creates milestones
func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error {
var err error
if g.milestoneFile == nil {
g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(milestones)
if err != nil {
return err
}
if _, err := g.milestoneFile.Write(bs); err != nil {
return err
}
return nil
}
// CreateLabels creates labels
func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
var err error
if g.labelFile == nil {
g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(labels)
if err != nil {
return err
}
if _, err := g.labelFile.Write(bs); err != nil {
return err
}
return nil
}
// CreateReleases creates releases
func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
if g.opts.ReleaseAssets {
for _, release := range releases {
attachDir := filepath.Join("release_assets", release.TagName)
if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
return err
}
for _, asset := range release.Assets {
attachLocalPath := filepath.Join(attachDir, asset.Name)
// download attachment
err := func(attachPath string) error {
var rc io.ReadCloser
var err error
if asset.DownloadURL == nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else {
resp, err := http.Get(*asset.DownloadURL)
if err != nil {
return err
}
rc = resp.Body
}
defer rc.Close()
fw, err := os.Create(attachPath)
if err != nil {
return fmt.Errorf("Create: %v", err)
}
defer fw.Close()
_, err = io.Copy(fw, rc)
return err
}(filepath.Join(g.baseDir, attachLocalPath))
if err != nil {
return err
}
asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
}
}
}
var err error
if g.releaseFile == nil {
g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(releases)
if err != nil {
return err
}
if _, err := g.releaseFile.Write(bs); err != nil {
return err
}
return nil
}
// SyncTags syncs releases with tags in the database
func (g *RepositoryDumper) SyncTags() error {
return nil
}
// CreateIssues creates issues
func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error {
var err error
if g.issueFile == nil {
g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(issues)
if err != nil {
return err
}
if _, err := g.issueFile.Write(bs); err != nil {
return err
}
return nil
}
func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error {
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return err
}
for number, items := range itemsMap {
var err error
itemFile := itemFiles[number]
if itemFile == nil {
itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
if err != nil {
return err
}
itemFiles[number] = itemFile
}
bs, err := yaml.Marshal(items)
if err != nil {
return err
}
if _, err := itemFile.Write(bs); err != nil {
return err
}
}
return nil
}
// CreateComments creates comments of issues
func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
var commentsMap = make(map[int64][]interface{}, len(comments))
for _, comment := range comments {
commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
}
return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
}
// CreatePullRequests creates pull requests
func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
for _, pr := range prs {
// download patch file
err := func() error {
u, err := g.setURLToken(pr.PatchURL)
if err != nil {
return err
}
resp, err := http.Get(u)
if err != nil {
return err
}
defer resp.Body.Close()
pullDir := filepath.Join(g.gitPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
}
fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
f, err := os.Create(fPath)
if err != nil {
return err
}
defer f.Close()
if _, err = io.Copy(f, resp.Body); err != nil {
return err
}
pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
return nil
}()
if err != nil {
return err
}
// set head information
pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
return err
}
p, err := os.Create(filepath.Join(pullHead, "head"))
if err != nil {
return err
}
_, err = p.WriteString(pr.Head.SHA)
p.Close()
if err != nil {
return err
}
if pr.IsForkPullRequest() && pr.State != "closed" {
if pr.Head.OwnerName != "" {
remote := pr.Head.OwnerName
_, ok := g.prHeadCache[remote]
if !ok {
// git remote add
// TODO: how to handle private CloneURL?
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
if err != nil {
log.Error("AddRemote failed: %s", err)
} else {
g.prHeadCache[remote] = struct{}{}
ok = true
}
}
if ok {
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath())
if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
} else {
headBranch := filepath.Join(g.gitPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
return err
}
b, err := os.Create(headBranch)
if err != nil {
return err
}
_, err = b.WriteString(pr.Head.SHA)
b.Close()
if err != nil {
return err
}
}
}
}
}
}
var err error
if g.pullrequestFile == nil {
if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
return err
}
g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(prs)
if err != nil {
return err
}
if _, err := g.pullrequestFile.Write(bs); err != nil {
return err
}
return nil
}
// CreateReviews create pull request reviews
func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error {
var reviewsMap = make(map[int64][]interface{}, len(reviews))
for _, review := range reviews {
reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
}
return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap)
}
// Rollback when migrating failed, this will rollback all the changes.
func (g *RepositoryDumper) Rollback() error {
g.Close()
return os.RemoveAll(g.baseDir)
}
// Finish when migrating succeed, this will update something.
func (g *RepositoryDumper) Finish() error {
return nil
}
// DumpRepository dump repository according MigrateOptions to a local directory
func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
downloader, err := newDownloader(ctx, ownerName, opts)
if err != nil {
return err
}
uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)
if err != nil {
return err
}
if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
return err
}
return nil
}
// RestoreRepository restore a repository from the disk directory
func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error {
doer, err := models.GetAdminUser()
if err != nil {
return err
}
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName)
downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName)
if err != nil {
return err
}
if err = migrateRepository(downloader, uploader, base.MigrateOptions{
Wiki: true,
Issues: true,
Milestones: true,
Labels: true,
Releases: true,
Comments: true,
PullRequests: true,
ReleaseAssets: true,
}); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
return err
}
return nil
}

3
modules/migrations/error.go
View file

@ -14,6 +14,9 @@ import (
var (
// ErrNotSupported returns the error not supported
ErrNotSupported = errors.New("not supported")
// ErrRepoNotCreated returns the error that repository not created
ErrRepoNotCreated = errors.New("repository is not created yet")
)
// IsRateLimitError returns true if the err is github.RateLimitError

6
modules/migrations/git.go
View file

@ -6,7 +6,6 @@ package migrations
import (
"context"
"io"
"code.gitea.io/gitea/modules/migrations/base"
)
@ -65,11 +64,6 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
// GetAsset returns an asset
func (g *PlainGitDownloader) GetAsset(_ string, _, _ int64) (io.ReadCloser, error) {
return nil, ErrNotSupported
}
// GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, false, ErrNotSupported

31
modules/migrations/gitea_downloader.go
View file

@ -268,13 +268,27 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
for _, asset := range rel.Attachments {
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
r.Assets = append(r.Assets, base.ReleaseAsset{
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: asset.ID,
Name: asset.Name,
Size: &size,
DownloadCount: &dlCount,
Created: asset.Created,
DownloadURL: &asset.DownloadURL,
DownloadFunc: func() (io.ReadCloser, error) {
asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
if err != nil {
return nil, err
}
// FIXME: for a private download?
resp, err := http.Get(asset.DownloadURL)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
},
})
}
return r
@ -310,21 +324,6 @@ func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
return releases, nil
}
// GetAsset returns an asset
func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, error) {
asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, relID, id)
if err != nil {
return nil, err
}
resp, err := http.Get(asset.DownloadURL)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
}
func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
var reactions []*base.Reaction
if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {

60
modules/migrations/gitea_uploader.go
View file

@ -10,7 +10,6 @@ import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
@ -28,6 +27,7 @@ import (
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/uri"
"code.gitea.io/gitea/services/pull"
gouuid "github.com/google/uuid"
@ -86,6 +86,22 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
return 10
}
func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
var fullRemoteAddr = remoteAddr
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
fullRemoteAddr = u.String()
}
return fullRemoteAddr, nil
}
// CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
owner, err := models.GetUserByName(g.repoOwner)
@ -93,19 +109,10 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err
}
var remoteAddr = repo.CloneURL
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(repo.CloneURL)
if err != nil {
return err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
remoteAddr = u.String()
remoteAddr, err := fullURL(opts, repo.CloneURL)
if err != nil {
return err
}
var r *models.Repository
if opts.MigrateToRepoID <= 0 {
r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
@ -224,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
}
// CreateReleases creates releases
func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
var rels = make([]*models.Release, 0, len(releases))
for _, release := range releases {
var rel = models.Release{
@ -283,25 +290,27 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases
// download attachment
err = func() error {
// asset.DownloadURL maybe a local file
var rc io.ReadCloser
if asset.DownloadURL == nil {
rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID)
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else {
resp, err := http.Get(*asset.DownloadURL)
rc, err = uri.Open(*asset.DownloadURL)
if err != nil {
return err
}
rc = resp.Body
}
defer rc.Close()
_, err = storage.Attachments.Save(attach.RelativePath(), rc)
return err
}()
if err != nil {
return err
}
rel.Attachments = append(rel.Attachments, &attach)
}
@ -559,11 +568,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
// download patch file
err := func() error {
resp, err := http.Get(pr.PatchURL)
// pr.PatchURL maybe a local file
ret, err := uri.Open(pr.PatchURL)
if err != nil {
return err
}
defer resp.Body.Close()
defer ret.Close()
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
@ -573,7 +583,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
return err
}
defer f.Close()
_, err = io.Copy(f, resp.Body)
_, err = io.Copy(f, ret)
return err
}()
if err != nil {
@ -859,3 +869,13 @@ func (g *GiteaLocalUploader) Rollback() error {
}
return nil
}
// Finish when migrating success, this will do some status update things.
func (g *GiteaLocalUploader) Finish() error {
if g.repo == nil || g.repo.ID <= 0 {
return ErrRepoNotCreated
}
g.repo.Status = models.RepositoryReady
return models.UpdateRepositoryCols(g.repo, "status")
}

1
modules/migrations/gitea_uploader_test.go
View file

@ -52,6 +52,7 @@ func TestGiteaUploadRepo(t *testing.T) {
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
assert.True(t, repo.HasWiki())
assert.EqualValues(t, models.RepositoryReady, repo.Status)
milestones, err := models.GetMilestones(models.GetMilestonesOption{
RepoID: repo.ID,

25
modules/migrations/github.go
View file

@ -291,7 +291,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
}
for _, asset := range rel.Assets {
r.Assets = append(r.Assets, base.ReleaseAsset{
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: *asset.ID,
Name: *asset.Name,
ContentType: asset.ContentType,
@ -299,6 +299,16 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
DownloadCount: asset.DownloadCount,
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
DownloadFunc: func() (io.ReadCloser, error) {
asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, *asset.ID, http.DefaultClient)
if err != nil {
return nil, err
}
if asset == nil {
return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
}
return asset, nil
},
})
}
return r
@ -330,18 +340,6 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
return releases, nil
}
// GetAsset returns an asset
func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, error) {
asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient)
if err != nil {
return nil, err
}
if asset == nil {
return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
}
return asset, nil
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
@ -363,6 +361,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
g.rate = &resp.Rate
for _, issue := range issues {
if issue.IsPullRequest() {

44
modules/migrations/gitlab.go
View file

@ -295,12 +295,32 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
}
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, base.ReleaseAsset{
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: int64(asset.ID),
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
Size: &zero,
DownloadCount: &zero,
DownloadFunc: func() (io.ReadCloser, error) {
link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", link.URL, nil)
if err != nil {
return nil, err
}
req = req.WithContext(g.ctx)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
},
})
}
return r
@ -329,28 +349,6 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
return releases, nil
}
// GetAsset returns an asset
func (g *GitlabDownloader) GetAsset(tag string, _, id int64) (io.ReadCloser, error) {
link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id), gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", link.URL, nil)
if err != nil {
return nil, err
}
req = req.WithContext(g.ctx)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
}
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {

182
modules/migrations/migrate.go
View file

@ -73,10 +73,30 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
if err != nil {
return nil, err
}
downloader, err := newDownloader(ctx, ownerName, opts)
if err != nil {
return nil, err
}
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
uploader.gitServiceType = opts.GitServiceType
if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
log.Error("create respotiry notice failed: ", err2)
}
return nil, err
}
return uploader.repo, nil
}
func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
var (
downloader base.Downloader
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
err error
)
for _, factory := range factories {
@ -101,24 +121,10 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
log.Trace("Will migrate from git: %s", opts.OriginalURL)
}
uploader.gitServiceType = opts.GitServiceType
if setting.Migrations.MaxAttempts > 1 {
downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
}
if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
log.Error("create repository notice failed: ", err2)
}
return nil, err
}
return uploader.repo, nil
return downloader, nil
}
// migrateRepository will download information and then upload it to Uploader, this is a simple
@ -204,7 +210,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
relBatchSize = len(releases)
}
if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil {
if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
return err
}
releases = releases[relBatchSize:]
@ -235,31 +241,30 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
if !opts.Comments {
continue
}
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, issue := range issues {
comments, err := downloader.GetComments(issue.Number)
if err != nil {
return err
}
allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize {
if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
if opts.Comments {
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, issue := range issues {
log.Trace("migrating issue %d's comments", issue.Number)
comments, err := downloader.GetComments(issue.Number)
if err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
allComments = append(allComments, comments...)
if len(allComments) > 0 {
if err := uploader.CreateComments(allComments...); err != nil {
return err
if len(allComments) >= commentBatchSize {
if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
if len(allComments) > 0 {
if err := uploader.CreateComments(allComments...); err != nil {
return err
}
}
}
@ -282,65 +287,64 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
if !opts.Comments {
continue
}
// plain comments
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, pr := range prs {
comments, err := downloader.GetComments(pr.Number)
if err != nil {
return err
}
allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize {
if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
if opts.Comments {
// plain comments
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, pr := range prs {
log.Trace("migrating pull request %d's comments", pr.Number)
comments, err := downloader.GetComments(pr.Number)
if err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
if len(allComments) > 0 {
if err := uploader.CreateComments(allComments...); err != nil {
return err
}
}
// migrate reviews
var allReviews = make([]*base.Review, 0, reviewBatchSize)
for _, pr := range prs {
number := pr.Number
allComments = append(allComments, comments...)
// on gitlab migrations pull number change
if pr.OriginalNumber > 0 {
number = pr.OriginalNumber
}
reviews, err := downloader.GetReviews(number)
if pr.OriginalNumber > 0 {
for i := range reviews {
reviews[i].IssueIndex = pr.Number
if len(allComments) >= commentBatchSize {
if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
if err != nil {
return err
}
allReviews = append(allReviews, reviews...)
if len(allReviews) >= reviewBatchSize {
if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
if len(allComments) > 0 {
if err := uploader.CreateComments(allComments...); err != nil {
return err
}
allReviews = allReviews[reviewBatchSize:]
}
}
if len(allReviews) > 0 {
if err := uploader.CreateReviews(allReviews...); err != nil {
return err
// migrate reviews
var allReviews = make([]*base.Review, 0, reviewBatchSize)
for _, pr := range prs {
number := pr.Number
// on gitlab migrations pull number change
if pr.OriginalNumber > 0 {
number = pr.OriginalNumber
}
reviews, err := downloader.GetReviews(number)
if pr.OriginalNumber > 0 {
for i := range reviews {
reviews[i].IssueIndex = pr.Number
}
}
if err != nil {
return err
}
allReviews = append(allReviews, reviews...)
if len(allReviews) >= reviewBatchSize {
if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
return err
}
allReviews = allReviews[reviewBatchSize:]
}
}
if len(allReviews) > 0 {
if err := uploader.CreateReviews(allReviews...); err != nil {
return err
}
}
}
@ -350,7 +354,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
}
return nil
return uploader.Finish()
}
// Init migrations service

276
modules/migrations/restore.go Normal file
View file

@ -0,0 +1,276 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"code.gitea.io/gitea/modules/migrations/base"
"gopkg.in/yaml.v2"
)
// RepositoryRestorer implements an Downloader from the local directory
type RepositoryRestorer struct {
ctx context.Context
baseDir string
repoOwner string
repoName string
}
// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
baseDir, err := filepath.Abs(baseDir)
if err != nil {
return nil, err
}
return &RepositoryRestorer{
ctx: ctx,
baseDir: baseDir,
repoOwner: owner,
repoName: repoName,
}, nil
}
func (r *RepositoryRestorer) commentDir() string {
return filepath.Join(r.baseDir, "comments")
}
func (r *RepositoryRestorer) reviewDir() string {
return filepath.Join(r.baseDir, "reviews")
}
// SetContext set context
func (r *RepositoryRestorer) SetContext(ctx context.Context) {
r.ctx = ctx
}
// GetRepoInfo returns a repository information
func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
p := filepath.Join(r.baseDir, "repo.yml")
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
var opts = make(map[string]string)
err = yaml.Unmarshal(bs, &opts)
if err != nil {
return nil, err
}
isPrivate, _ := strconv.ParseBool(opts["is_private"])
return &base.Repository{
Owner: r.repoOwner,
Name: r.repoName,
IsPrivate: isPrivate,
Description: opts["description"],
OriginalURL: opts["original_url"],
CloneURL: opts["clone_addr"],
DefaultBranch: opts["default_branch"],
}, nil
}
// GetTopics return github topics
func (r *RepositoryRestorer) GetTopics() ([]string, error) {
p := filepath.Join(r.baseDir, "topic.yml")
var topics = struct {
Topics []string `yaml:"topics"`
}{}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &topics)
if err != nil {
return nil, err
}
return topics.Topics, nil
}
// GetMilestones returns milestones
func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
var milestones = make([]*base.Milestone, 0, 10)
p := filepath.Join(r.baseDir, "milestone.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &milestones)
if err != nil {
return nil, err
}
return milestones, nil
}
// GetReleases returns releases
func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
var releases = make([]*base.Release, 0, 10)
p := filepath.Join(r.baseDir, "release.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &releases)
if err != nil {
return nil, err
}
for _, rel := range releases {
for _, asset := range rel.Assets {
*asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
}
}
return releases, nil
}
// GetLabels returns labels
func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
var labels = make([]*base.Label, 0, 10)
p := filepath.Join(r.baseDir, "label.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &labels)
if err != nil {
return nil, err
}
return labels, nil
}
// GetIssues returns issues according start and limit
func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
var issues = make([]*base.Issue, 0, 10)
p := filepath.Join(r.baseDir, "issue.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, true, nil
}
return nil, false, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, false, err
}
err = yaml.Unmarshal(bs, &issues)
if err != nil {
return nil, false, err
}
return issues, true, nil
}
// GetComments returns comments according issueNumber
func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) {
var comments = make([]*base.Comment, 0, 10)
p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber))
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &comments)
if err != nil {
return nil, err
}
return comments, nil
}
// GetPullRequests returns pull requests according page and perPage
func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
var pulls = make([]*base.PullRequest, 0, 10)
p := filepath.Join(r.baseDir, "pull_request.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, true, nil
}
return nil, false, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, false, err
}
err = yaml.Unmarshal(bs, &pulls)
if err != nil {
return nil, false, err
}
for _, pr := range pulls {
pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
}
return pulls, true, nil
}
// GetReviews returns pull requests review
func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
var reviews = make([]*base.Review, 0, 10)
p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber))
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := ioutil.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &reviews)
if err != nil {
return nil, err
}
return reviews, nil
}

40
modules/uri/uri.go Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uri
import (
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
)
// ErrURISchemeNotSupported represents a scheme error
type ErrURISchemeNotSupported struct {
Scheme string
}
func (e ErrURISchemeNotSupported) Error() string {
return fmt.Sprintf("Unsupported scheme: %v", e.Scheme)
}
// Open open a local file or a remote file
func Open(uriStr string) (io.ReadCloser, error) {
u, err := url.Parse(uriStr)
if err != nil {
return nil, err
}
switch strings.ToLower(u.Scheme) {
case "http", "https":
f, err := http.Get(uriStr)
return f.Body, err
case "file":
return os.Open(u.Path)
default:
return nil, ErrURISchemeNotSupported{Scheme: u.Scheme}
}
}

20
modules/uri/uri_test.go Normal file
View file

@ -0,0 +1,20 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uri
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestReadURI(t *testing.T) {
p, err := filepath.Abs("./uri.go")
assert.NoError(t, err)
f, err := Open("file://" + p)
assert.NoError(t, err)
defer f.Close()
}

7
routers/api/v1/repo/migrate.go
View file

@ -176,11 +176,8 @@ func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) {
}
if err == nil {
repo.Status = models.RepositoryReady
if err := models.UpdateRepositoryCols(repo, "status"); err == nil {
notification.NotifyMigrateRepository(ctx.User, repoOwner, repo)
return
}
notification.NotifyMigrateRepository(ctx.User, repoOwner, repo)
return
}
if repo != nil {