From 4334fe754e4a338a13073f649707acd26d1f4a12 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 11 Mar 2019 11:44:58 +0800 Subject: [PATCH] update git vendor to fix wrong release commit id and add migrations (#6224) * update git vendor to fix wrong release commit id and add migrations * fix count * fix migration release * fix tests --- Gopkg.lock | 4 +- models/migrations/migrations.go | 2 + models/migrations/v82.go | 87 +++++++++++++++++++++++++ models/repo_branch.go | 9 +-- routers/repo/view.go | 2 +- vendor/code.gitea.io/git/cache.go | 11 ++++ vendor/code.gitea.io/git/commit_info.go | 20 ++++-- vendor/code.gitea.io/git/repo_commit.go | 9 ++- vendor/code.gitea.io/git/repo_tag.go | 4 +- 9 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 models/migrations/v82.go create mode 100644 vendor/code.gitea.io/git/cache.go diff --git a/Gopkg.lock b/Gopkg.lock index c9b70766f7..dae99b340f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,11 +3,11 @@ [[projects]] branch = "master" - digest = "1:0a001725d6e1b35faccf15cbc4f782b67a0d77f4bbf56e51a4b244f92e7c60ca" + digest = "1:0f0ada42a7b1bd64794bf8fea917c2cd626d6b0539173e3704cd764b93eb5312" name = "code.gitea.io/git" packages = ["."] pruneopts = "NUT" - revision = "0aea7f12d36ed49bcac560b61301cff88e478e5c" + revision = "8983773ac6fef49203e7ee8cdbfde3e118bc3421" [[projects]] branch = "master" diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 1fde096fbe..2807de78b1 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -217,6 +217,8 @@ var migrations = []Migration{ NewMigration("add is locked to issues", addIsLockedToIssues), // v81 -> v82 NewMigration("update U2F counter type", changeU2FCounterType), + // v82 -> v83 + NewMigration("hot fix for wrong release sha1 on release table", fixReleaseSha1OnReleaseTable), } // Migrate database to current version diff --git a/models/migrations/v82.go b/models/migrations/v82.go new file mode 100644 index 0000000000..6c62a8e24a --- /dev/null +++ b/models/migrations/v82.go @@ -0,0 +1,87 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/git" + "code.gitea.io/gitea/models" + "github.com/go-xorm/xorm" +) + +func fixReleaseSha1OnReleaseTable(x *xorm.Engine) error { + type Release struct { + ID int64 + RepoID int64 + Sha1 string + TagName string + } + + // Update release sha1 + const batchSize = 100 + sess := x.NewSession() + defer sess.Close() + + var ( + err error + count int + gitRepoCache = make(map[int64]*git.Repository) + repoCache = make(map[int64]*models.Repository) + ) + + if err = sess.Begin(); err != nil { + return err + } + + for start := 0; ; start += batchSize { + releases := make([]*Release, 0, batchSize) + if err = sess.Limit(batchSize, start).Asc("id").Where("is_tag=?", false).Find(&releases); err != nil { + return err + } + if len(releases) == 0 { + break + } + + for _, release := range releases { + gitRepo, ok := gitRepoCache[release.RepoID] + if !ok { + repo, ok := repoCache[release.RepoID] + if !ok { + repo, err = models.GetRepositoryByID(release.RepoID) + if err != nil { + return err + } + repoCache[release.RepoID] = repo + } + + gitRepo, err = git.OpenRepository(repo.RepoPath()) + if err != nil { + return err + } + gitRepoCache[release.RepoID] = gitRepo + } + + release.Sha1, err = gitRepo.GetTagCommitID(release.TagName) + if err != nil { + return err + } + + if _, err = sess.ID(release.ID).Cols("sha1").Update(release); err != nil { + return err + } + + count++ + if count >= 1000 { + if err = sess.Commit(); err != nil { + return err + } + if err = sess.Begin(); err != nil { + return err + } + count = 0 + } + } + } + return sess.Commit() +} diff --git a/models/repo_branch.go b/models/repo_branch.go index 88417cbd36..c4013dfb82 100644 --- a/models/repo_branch.go +++ b/models/repo_branch.go @@ -110,10 +110,6 @@ func (repo *Repository) CheckBranchName(name string) error { return err } - if _, err := gitRepo.GetTag(name); err == nil { - return ErrTagAlreadyExists{name} - } - branches, err := repo.GetBranches() if err != nil { return err @@ -127,6 +123,11 @@ func (repo *Repository) CheckBranchName(name string) error { return ErrBranchNameConflict{branch.Name} } } + + if _, err := gitRepo.GetTag(name); err == nil { + return ErrTagAlreadyExists{name} + } + return nil } diff --git a/routers/repo/view.go b/routers/repo/view.go index c14537bf97..786cfcbc56 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -49,7 +49,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { } entries.CustomSort(base.NaturalSortLess) - ctx.Data["Files"], err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath) + ctx.Data["Files"], err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil) if err != nil { ctx.ServerError("GetCommitsInfo", err) return diff --git a/vendor/code.gitea.io/git/cache.go b/vendor/code.gitea.io/git/cache.go new file mode 100644 index 0000000000..dbbbafae4c --- /dev/null +++ b/vendor/code.gitea.io/git/cache.go @@ -0,0 +1,11 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +// LastCommitCache cache +type LastCommitCache interface { + Get(repoPath, ref, entryPath string) (*Commit, error) + Put(repoPath, ref, entryPath string, commit *Commit) error +} diff --git a/vendor/code.gitea.io/git/commit_info.go b/vendor/code.gitea.io/git/commit_info.go index 6b42b57c90..971082be1f 100644 --- a/vendor/code.gitea.io/git/commit_info.go +++ b/vendor/code.gitea.io/git/commit_info.go @@ -72,13 +72,20 @@ func (state *getCommitsInfoState) getTargetedEntryPath() string { } // repeatedly perform targeted searches for unpopulated entries -func targetedSearch(state *getCommitsInfoState, done chan error) { +func targetedSearch(state *getCommitsInfoState, done chan error, cache LastCommitCache) { for { entryPath := state.getTargetedEntryPath() if len(entryPath) == 0 { done <- nil return } + if cache != nil { + commit, err := cache.Get(state.headCommit.repo.Path, state.headCommit.ID.String(), entryPath) + if err == nil && commit != nil { + state.update(entryPath, commit) + continue + } + } command := NewCommand("rev-list", "-1", state.headCommit.ID.String(), "--", entryPath) output, err := command.RunInDir(state.headCommit.repo.Path) if err != nil { @@ -96,6 +103,9 @@ func targetedSearch(state *getCommitsInfoState, done chan error) { return } state.update(entryPath, commit) + if cache != nil { + cache.Put(state.headCommit.repo.Path, state.headCommit.ID.String(), entryPath, commit) + } } } @@ -118,9 +128,9 @@ func initGetCommitInfoState(entries Entries, headCommit *Commit, treePath string } // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(commit *Commit, treePath string) ([][]interface{}, error) { +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, error) { state := initGetCommitInfoState(tes, commit, treePath) - if err := getCommitsInfo(state); err != nil { + if err := getCommitsInfo(state, cache); err != nil { return nil, err } if len(state.commits) < len(state.entryPaths) { @@ -188,7 +198,7 @@ func (state *getCommitsInfoState) update(entryPath string, commit *Commit) bool const getCommitsInfoPretty = "--pretty=format:%H %ct %s" -func getCommitsInfo(state *getCommitsInfoState) error { +func getCommitsInfo(state *getCommitsInfoState, cache LastCommitCache) error { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() @@ -215,7 +225,7 @@ func getCommitsInfo(state *getCommitsInfoState) error { numThreads := runtime.NumCPU() done := make(chan error, numThreads) for i := 0; i < numThreads; i++ { - go targetedSearch(state, done) + go targetedSearch(state, done, cache) } scanner := bufio.NewScanner(readCloser) diff --git a/vendor/code.gitea.io/git/repo_commit.go b/vendor/code.gitea.io/git/repo_commit.go index bfbf5c6dae..fbb8f97a1d 100644 --- a/vendor/code.gitea.io/git/repo_commit.go +++ b/vendor/code.gitea.io/git/repo_commit.go @@ -32,7 +32,14 @@ func (repo *Repository) GetBranchCommitID(name string) (string, error) { // GetTagCommitID returns last commit ID string of given tag. func (repo *Repository) GetTagCommitID(name string) (string, error) { - return repo.GetRefCommitID(TagPrefix + name) + stdout, err := NewCommand("rev-list", "-n", "1", name).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "unknown revision or path") { + return "", ErrNotExist{name, ""} + } + return "", err + } + return strings.TrimSpace(stdout), nil } // parseCommitData parses commit information from the (uncompressed) raw diff --git a/vendor/code.gitea.io/git/repo_tag.go b/vendor/code.gitea.io/git/repo_tag.go index 77867f46c1..84825d7dc3 100644 --- a/vendor/code.gitea.io/git/repo_tag.go +++ b/vendor/code.gitea.io/git/repo_tag.go @@ -76,12 +76,12 @@ func (repo *Repository) getTag(id SHA1) (*Tag, error) { // GetTag returns a Git tag by given name. func (repo *Repository) GetTag(name string) (*Tag, error) { - stdout, err := NewCommand("show-ref", "--tags", name).RunInDir(repo.Path) + idStr, err := repo.GetTagCommitID(name) if err != nil { return nil, err } - id, err := NewIDFromString(strings.Split(stdout, " ")[0]) + id, err := NewIDFromString(idStr) if err != nil { return nil, err }