MirrorHub/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2024-12-23 01:44:22 +01:00
Code Issues Releases Wiki Activity

Refactor editor upload, update and delete to use git plumbing and add LFS support (#5702)

Browse source 
* Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFile

* Use git plumbing for upload: #5621 repo_editor.go: GetDiffPreview

* Use git plumbing for upload: #5621 repo_editor.go: DeleteRepoFile

* Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFiles

* Move branch checkout functions out of repo_editor.go as they are no longer used there

* BUGFIX: The default permissions should be 100644

    This is a change from the previous code but is more in keeping
    with the default behaviour of git.

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Standardise cleanUploadFilename to more closely match git

See verify_path in: 7f4e641693/read-cache.c (L951)

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Redirect on bad paths

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Refactor to move the uploading functions out to a module

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add LFS support

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Update upload.go attribution header

Upload.go is essentially the remnants of repo_editor.go. The remaining code is essentially unchanged from the Gogs code, hence the Gogs attribution.

* Delete upload files after session committed

* Ensure that GIT_AUTHOR_NAME etc. are valid for git

see #5774

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Add in test cases per @lafriks comment

* Add space between gitea and github imports

Signed-off-by: Andrew Thornton <art27@cantab.net>

* more examples in TestCleanUploadName

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix formatting

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Set the SSH_ORIGINAL_COMMAND to ensure hooks are run

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Switch off SSH_ORIGINAL_COMMAND

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2019-02-12 13:07:31 +00:00 committed by Lauris BH
parent fc038caa69
commit 296814e887
11 changed files with 1135 additions and 598 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
models/lfs.go
View file

@ -1,7 +1,11 @@
package models
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"code.gitea.io/gitea/modules/util"
)
@ -16,6 +20,11 @@ type LFSMetaObject struct {
CreatedUnix util.TimeStamp `xorm:"created"`
}
// Pointer returns the string representation of an LFS pointer file
func (m *LFSMetaObject) Pointer() string {
return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
}
// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
// This structure is fetched via SSH and passed by the Git LFS client to the server
// endpoint for authorization.
@ -67,6 +76,16 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
return m, sess.Commit()
}
// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content
func GenerateLFSOid(content io.Reader) (string, error) {
h := sha256.New()
if _, err := io.Copy(h, content); err != nil {
return "", err
}
sum := h.Sum(nil)
return hex.EncodeToString(sum), nil
}
// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
// the returned pointer is a valid LFSMetaObject.

40
models/repo_branch.go
View file

@ -14,6 +14,46 @@ import (
"github.com/Unknwon/com"
)
// discardLocalRepoBranchChanges discards local commits/changes of
// given branch to make sure it is even to remote branch.
func discardLocalRepoBranchChanges(localPath, branch string) error {
if !com.IsExist(localPath) {
return nil
}
// No need to check if nothing in the repository.
if !git.IsBranchExist(localPath, branch) {
return nil
}
refName := "origin/" + branch
if err := git.ResetHEAD(localPath, true, refName); err != nil {
return fmt.Errorf("git reset --hard %s: %v", refName, err)
}
return nil
}
// DiscardLocalRepoBranchChanges discards the local repository branch changes
func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error {
return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch)
}
// checkoutNewBranch checks out to a new branch from the a branch name.
func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error {
if err := git.Checkout(localPath, git.CheckoutOptions{
Timeout: time.Duration(setting.Git.Timeout.Pull) * time.Second,
Branch: newBranch,
OldBranch: oldBranch,
}); err != nil {
return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err)
}
return nil
}
// CheckoutNewBranch checks out a new branch
func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error {
return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch)
}
// Branch holds the branch information
type Branch struct {
Path string

572
models/repo_editor.go
View file

@ -1,572 +0,0 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"os"
"os/exec"
"path"
"path/filepath"
"time"
"github.com/Unknwon/com"
gouuid "github.com/satori/go.uuid"
"code.gitea.io/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
)
// ___________ .___.__ __ ___________.__.__
// \_ _____/ __| _/|__|/ |_ \_ _____/|__| | ____
// | __)_ / __ | | \ __\ | __) | | | _/ __ \
// | \/ /_/ | | || | | \ | | |_\ ___/
// /_______ /\____ | |__||__| \___ / |__|____/\___ >
// \/ \/ \/ \/
// discardLocalRepoBranchChanges discards local commits/changes of
// given branch to make sure it is even to remote branch.
func discardLocalRepoBranchChanges(localPath, branch string) error {
if !com.IsExist(localPath) {
return nil
}
// No need to check if nothing in the repository.
if !git.IsBranchExist(localPath, branch) {
return nil
}
refName := "origin/" + branch
if err := git.ResetHEAD(localPath, true, refName); err != nil {
return fmt.Errorf("git reset --hard %s: %v", refName, err)
}
return nil
}
// DiscardLocalRepoBranchChanges discards the local repository branch changes
func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error {
return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch)
}
// checkoutNewBranch checks out to a new branch from the a branch name.
func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error {
if err := git.Checkout(localPath, git.CheckoutOptions{
Timeout: time.Duration(setting.Git.Timeout.Pull) * time.Second,
Branch: newBranch,
OldBranch: oldBranch,
}); err != nil {
return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err)
}
return nil
}
// CheckoutNewBranch checks out a new branch
func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error {
return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch)
}
// UpdateRepoFileOptions holds the repository file update options
type UpdateRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
OldTreeName string
NewTreeName string
Message string
Content string
IsNewFile bool
}
// UpdateRepoFile adds or updates a file in repository.
func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (err error) {
repoWorkingPool.CheckIn(com.ToStr(repo.ID))
defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil {
return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err)
} else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil {
return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err)
}
if opts.OldBranch != opts.NewBranch {
if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil {
return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err)
}
}
localPath := repo.LocalCopyPath()
oldFilePath := path.Join(localPath, opts.OldTreeName)
filePath := path.Join(localPath, opts.NewTreeName)
dir := path.Dir(filePath)
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return fmt.Errorf("Failed to create dir %s: %v", dir, err)
}
// If it's meant to be a new file, make sure it doesn't exist.
if opts.IsNewFile {
if com.IsExist(filePath) {
return ErrRepoFileAlreadyExist{filePath}
}
}
// Ignore move step if it's a new file under a directory.
// Otherwise, move the file when name changed.
if com.IsFile(oldFilePath) && opts.OldTreeName != opts.NewTreeName {
if err = git.MoveFile(localPath, opts.OldTreeName, opts.NewTreeName); err != nil {
return fmt.Errorf("git mv %s %s: %v", opts.OldTreeName, opts.NewTreeName, err)
}
}
if err = ioutil.WriteFile(filePath, []byte(opts.Content), 0666); err != nil {
return fmt.Errorf("WriteFile: %v", err)
}
if err = git.AddChanges(localPath, true); err != nil {
return fmt.Errorf("git add --all: %v", err)
} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
Committer: doer.NewGitSig(),
Message: opts.Message,
}); err != nil {
return fmt.Errorf("CommitChanges: %v", err)
} else if err = git.Push(localPath, git.PushOptions{
Remote: "origin",
Branch: opts.NewBranch,
}); err != nil {
return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
}
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
log.Error(4, "OpenRepository: %v", err)
return nil
}
commit, err := gitRepo.GetBranchCommit(opts.NewBranch)
if err != nil {
log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err)
return nil
}
// Simulate push event.
oldCommitID := opts.LastCommitID
if opts.NewBranch != opts.OldBranch {
oldCommitID = git.EmptySHA
}
if err = repo.GetOwner(); err != nil {
return fmt.Errorf("GetOwner: %v", err)
}
err = PushUpdate(
opts.NewBranch,
PushUpdateOptions{
PusherID: doer.ID,
PusherName: doer.Name,
RepoUserName: repo.Owner.Name,
RepoName: repo.Name,
RefFullName: git.BranchPrefix + opts.NewBranch,
OldCommitID: oldCommitID,
NewCommitID: commit.ID.String(),
},
)
if err != nil {
return fmt.Errorf("PushUpdate: %v", err)
}
UpdateRepoIndexer(repo)
return nil
}
// GetDiffPreview produces and returns diff result of a file which is not yet committed.
func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *Diff, err error) {
repoWorkingPool.CheckIn(com.ToStr(repo.ID))
defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
if err = repo.DiscardLocalRepoBranchChanges(branch); err != nil {
return nil, fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", branch, err)
} else if err = repo.UpdateLocalCopyBranch(branch); err != nil {
return nil, fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", branch, err)
}
localPath := repo.LocalCopyPath()
filePath := path.Join(localPath, treePath)
dir := filepath.Dir(filePath)
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return nil, fmt.Errorf("Failed to create dir %s: %v", dir, err)
}
if err = ioutil.WriteFile(filePath, []byte(content), 0666); err != nil {
return nil, fmt.Errorf("WriteFile: %v", err)
}
cmd := exec.Command("git", "diff", treePath)
cmd.Dir = localPath
cmd.Stderr = os.Stderr
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("StdoutPipe: %v", err)
}
if err = cmd.Start(); err != nil {
return nil, fmt.Errorf("Start: %v", err)
}
pid := process.GetManager().Add(fmt.Sprintf("GetDiffPreview [repo_path: %s]", repo.RepoPath()), cmd)
defer process.GetManager().Remove(pid)
diff, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
if err != nil {
return nil, fmt.Errorf("ParsePatch: %v", err)
}
if err = cmd.Wait(); err != nil {
return nil, fmt.Errorf("Wait: %v", err)
}
return diff, nil
}
// ________ .__ __ ___________.__.__
// \______ \ ____ | | _____/ |_ ____ \_ _____/|__| | ____
// | | \_/ __ \| | _/ __ \ __\/ __ \ | __) | | | _/ __ \
// | ` \ ___/| |_\ ___/| | \ ___/ | \ | | |_\ ___/
// /_______ /\___ >____/\___ >__| \___ > \___ / |__|____/\___ >
// \/ \/ \/ \/ \/ \/
//
// DeleteRepoFileOptions holds the repository delete file options
type DeleteRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
TreePath string
Message string
}
// DeleteRepoFile deletes a repository file
func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) (err error) {
repoWorkingPool.CheckIn(com.ToStr(repo.ID))
defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil {
return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err)
} else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil {
return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err)
}
if opts.OldBranch != opts.NewBranch {
if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil {
return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err)
}
}
localPath := repo.LocalCopyPath()
if err = os.Remove(path.Join(localPath, opts.TreePath)); err != nil {
return fmt.Errorf("Remove: %v", err)
}
if err = git.AddChanges(localPath, true); err != nil {
return fmt.Errorf("git add --all: %v", err)
} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
Committer: doer.NewGitSig(),
Message: opts.Message,
}); err != nil {
return fmt.Errorf("CommitChanges: %v", err)
} else if err = git.Push(localPath, git.PushOptions{
Remote: "origin",
Branch: opts.NewBranch,
}); err != nil {
return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
}
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
log.Error(4, "OpenRepository: %v", err)
return nil
}
commit, err := gitRepo.GetBranchCommit(opts.NewBranch)
if err != nil {
log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err)
return nil
}
// Simulate push event.
oldCommitID := opts.LastCommitID
if opts.NewBranch != opts.OldBranch {
oldCommitID = git.EmptySHA
}
if err = repo.GetOwner(); err != nil {
return fmt.Errorf("GetOwner: %v", err)
}
err = PushUpdate(
opts.NewBranch,
PushUpdateOptions{
PusherID: doer.ID,
PusherName: doer.Name,
RepoUserName: repo.Owner.Name,
RepoName: repo.Name,
RefFullName: git.BranchPrefix + opts.NewBranch,
OldCommitID: oldCommitID,
NewCommitID: commit.ID.String(),
},
)
if err != nil {
return fmt.Errorf("PushUpdate: %v", err)
}
return nil
}
// ____ ___ .__ .___ ___________.___.__
// | | \______ | | _________ __| _/ \_ _____/| | | ____ ______
// | | /\____ \| | / _ \__ \ / __ | | __) | | | _/ __ \ / ___/
// | | / | |_> > |_( <_> ) __ \_/ /_/ | | \ | | |_\ ___/ \___ \
// |______/ | __/|____/\____(____ /\____ | \___ / |___|____/\___ >____ >
// |__| \/ \/ \/ \/ \/
//
// Upload represent a uploaded file to a repo to be deleted when moved
type Upload struct {
ID int64 `xorm:"pk autoincr"`
UUID string `xorm:"uuid UNIQUE"`
Name string
}
// UploadLocalPath returns where uploads is stored in local file system based on given UUID.
func UploadLocalPath(uuid string) string {
return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid)
}
// LocalPath returns where uploads are temporarily stored in local file system.
func (upload *Upload) LocalPath() string {
return UploadLocalPath(upload.UUID)
}
// NewUpload creates a new upload object.
func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) {
upload := &Upload{
UUID: gouuid.NewV4().String(),
Name: name,
}
localPath := upload.LocalPath()
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
return nil, fmt.Errorf("MkdirAll: %v", err)
}
fw, err := os.Create(localPath)
if err != nil {
return nil, fmt.Errorf("Create: %v", err)
}
defer fw.Close()
if _, err = fw.Write(buf); err != nil {
return nil, fmt.Errorf("Write: %v", err)
} else if _, err = io.Copy(fw, file); err != nil {
return nil, fmt.Errorf("Copy: %v", err)
}
if _, err := x.Insert(upload); err != nil {
return nil, err
}
return upload, nil
}
// GetUploadByUUID returns the Upload by UUID
func GetUploadByUUID(uuid string) (*Upload, error) {
upload := &Upload{UUID: uuid}
has, err := x.Get(upload)
if err != nil {
return nil, err
} else if !has {
return nil, ErrUploadNotExist{0, uuid}
}
return upload, nil
}
// GetUploadsByUUIDs returns multiple uploads by UUIDS
func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) {
if len(uuids) == 0 {
return []*Upload{}, nil
}
// Silently drop invalid uuids.
uploads := make([]*Upload, 0, len(uuids))
return uploads, x.In("uuid", uuids).Find(&uploads)
}
// DeleteUploads deletes multiple uploads
func DeleteUploads(uploads ...*Upload) (err error) {
if len(uploads) == 0 {
return nil
}
sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
return err
}
ids := make([]int64, len(uploads))
for i := 0; i < len(uploads); i++ {
ids[i] = uploads[i].ID
}
if _, err = sess.
In("id", ids).
Delete(new(Upload)); err != nil {
return fmt.Errorf("delete uploads: %v", err)
}
for _, upload := range uploads {
localPath := upload.LocalPath()
if !com.IsFile(localPath) {
continue
}
if err := os.Remove(localPath); err != nil {
return fmt.Errorf("remove upload: %v", err)
}
}
return sess.Commit()
}
// DeleteUpload delete a upload
func DeleteUpload(u *Upload) error {
return DeleteUploads(u)
}
// DeleteUploadByUUID deletes a upload by UUID
func DeleteUploadByUUID(uuid string) error {
upload, err := GetUploadByUUID(uuid)
if err != nil {
if IsErrUploadNotExist(err) {
return nil
}
return fmt.Errorf("GetUploadByUUID: %v", err)
}
if err := DeleteUpload(upload); err != nil {
return fmt.Errorf("DeleteUpload: %v", err)
}
return nil
}
// UploadRepoFileOptions contains the uploaded repository file options
type UploadRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
TreePath string
Message string
Files []string // In UUID format.
}
// UploadRepoFiles uploads files to a repository
func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) (err error) {
if len(opts.Files) == 0 {
return nil
}
uploads, err := GetUploadsByUUIDs(opts.Files)
if err != nil {
return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err)
}
repoWorkingPool.CheckIn(com.ToStr(repo.ID))
defer repoWorkingPool.CheckOut(com.ToStr(repo.ID))
if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil {
return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err)
} else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil {
return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err)
}
if opts.OldBranch != opts.NewBranch {
if err = repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil {
return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err)
}
}
localPath := repo.LocalCopyPath()
dirPath := path.Join(localPath, opts.TreePath)
if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
return fmt.Errorf("Failed to create dir %s: %v", dirPath, err)
}
// Copy uploaded files into repository.
for _, upload := range uploads {
tmpPath := upload.LocalPath()
targetPath := path.Join(dirPath, upload.Name)
if !com.IsFile(tmpPath) {
continue
}
if err = com.Copy(tmpPath, targetPath); err != nil {
return fmt.Errorf("Copy: %v", err)
}
}
if err = git.AddChanges(localPath, true); err != nil {
return fmt.Errorf("git add --all: %v", err)
} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
Committer: doer.NewGitSig(),
Message: opts.Message,
}); err != nil {
return fmt.Errorf("CommitChanges: %v", err)
} else if err = git.Push(localPath, git.PushOptions{
Remote: "origin",
Branch: opts.NewBranch,
}); err != nil {
return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
}
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
log.Error(4, "OpenRepository: %v", err)
return nil
}
commit, err := gitRepo.GetBranchCommit(opts.NewBranch)
if err != nil {
log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err)
return nil
}
// Simulate push event.
oldCommitID := opts.LastCommitID
if opts.NewBranch != opts.OldBranch {
oldCommitID = git.EmptySHA
}
if err = repo.GetOwner(); err != nil {
return fmt.Errorf("GetOwner: %v", err)
}
err = PushUpdate(
opts.NewBranch,
PushUpdateOptions{
PusherID: doer.ID,
PusherName: doer.Name,
RepoUserName: repo.Owner.Name,
RepoName: repo.Name,
RefFullName: git.BranchPrefix + opts.NewBranch,
OldCommitID: oldCommitID,
NewCommitID: commit.ID.String(),
},
)
if err != nil {
return fmt.Errorf("PushUpdate: %v", err)
}
return DeleteUploads(uploads...)
}

155
models/upload.go Normal file
View file

@ -0,0 +1,155 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
"fmt"
"io"
"mime/multipart"
"os"
"path"
"github.com/Unknwon/com"
gouuid "github.com/satori/go.uuid"
"code.gitea.io/gitea/modules/setting"
)
// ____ ___ .__ .___ ___________.___.__
// | | \______ | | _________ __| _/ \_ _____/| | | ____ ______
// | | /\____ \| | / _ \__ \ / __ | | __) | | | _/ __ \ / ___/
// | | / | |_> > |_( <_> ) __ \_/ /_/ | | \ | | |_\ ___/ \___ \
// |______/ | __/|____/\____(____ /\____ | \___ / |___|____/\___ >____ >
// |__| \/ \/ \/ \/ \/
//
// Upload represent a uploaded file to a repo to be deleted when moved
type Upload struct {
ID int64 `xorm:"pk autoincr"`
UUID string `xorm:"uuid UNIQUE"`
Name string
}
// UploadLocalPath returns where uploads is stored in local file system based on given UUID.
func UploadLocalPath(uuid string) string {
return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid)
}
// LocalPath returns where uploads are temporarily stored in local file system.
func (upload *Upload) LocalPath() string {
return UploadLocalPath(upload.UUID)
}
// NewUpload creates a new upload object.
func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) {
upload := &Upload{
UUID: gouuid.NewV4().String(),
Name: name,
}
localPath := upload.LocalPath()
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
return nil, fmt.Errorf("MkdirAll: %v", err)
}
fw, err := os.Create(localPath)
if err != nil {
return nil, fmt.Errorf("Create: %v", err)
}
defer fw.Close()
if _, err = fw.Write(buf); err != nil {
return nil, fmt.Errorf("Write: %v", err)
} else if _, err = io.Copy(fw, file); err != nil {
return nil, fmt.Errorf("Copy: %v", err)
}
if _, err := x.Insert(upload); err != nil {
return nil, err
}
return upload, nil
}
// GetUploadByUUID returns the Upload by UUID
func GetUploadByUUID(uuid string) (*Upload, error) {
upload := &Upload{UUID: uuid}
has, err := x.Get(upload)
if err != nil {
return nil, err
} else if !has {
return nil, ErrUploadNotExist{0, uuid}
}
return upload, nil
}
// GetUploadsByUUIDs returns multiple uploads by UUIDS
func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) {
if len(uuids) == 0 {
return []*Upload{}, nil
}
// Silently drop invalid uuids.
uploads := make([]*Upload, 0, len(uuids))
return uploads, x.In("uuid", uuids).Find(&uploads)
}
// DeleteUploads deletes multiple uploads
func DeleteUploads(uploads ...*Upload) (err error) {
if len(uploads) == 0 {
return nil
}
sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
return err
}
ids := make([]int64, len(uploads))
for i := 0; i < len(uploads); i++ {
ids[i] = uploads[i].ID
}
if _, err = sess.
In("id", ids).
Delete(new(Upload)); err != nil {
return fmt.Errorf("delete uploads: %v", err)
}
if err = sess.Commit(); err != nil {
return err
}
for _, upload := range uploads {
localPath := upload.LocalPath()
if !com.IsFile(localPath) {
continue
}
if err := os.Remove(localPath); err != nil {
return fmt.Errorf("remove upload: %v", err)
}
}
return nil
}
// DeleteUploadByUUID deletes a upload by UUID
func DeleteUploadByUUID(uuid string) error {
upload, err := GetUploadByUUID(uuid)
if err != nil {
if IsErrUploadNotExist(err) {
return nil
}
return fmt.Errorf("GetUploadByUUID: %v", err)
}
if err := DeleteUploads(upload); err != nil {
return fmt.Errorf("DeleteUpload: %v", err)
}
return nil
}

100
modules/uploader/delete.go Normal file
View file

@ -0,0 +1,100 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uploader
import (
"fmt"
"code.gitea.io/git"
"code.gitea.io/gitea/models"
)
// DeleteRepoFileOptions holds the repository delete file options
type DeleteRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
TreePath string
Message string
}
// DeleteRepoFile deletes a file in the given repository
func DeleteRepoFile(repo *models.Repository, doer *models.User, opts *DeleteRepoFileOptions) error {
t, err := NewTemporaryUploadRepository(repo)
defer t.Close()
if err != nil {
return err
}
if err := t.Clone(opts.OldBranch); err != nil {
return err
}
if err := t.SetDefaultIndex(); err != nil {
return err
}
filesInIndex, err := t.LsFiles(opts.TreePath)
if err != nil {
return fmt.Errorf("UpdateRepoFile: %v", err)
}
inFilelist := false
for _, file := range filesInIndex {
if file == opts.TreePath {
inFilelist = true
}
}
if !inFilelist {
return git.ErrNotExist{RelPath: opts.TreePath}
}
if err := t.RemoveFilesFromIndex(opts.TreePath); err != nil {
return err
}
// Now write the tree
treeHash, err := t.WriteTree()
if err != nil {
return err
}
// Now commit the tree
commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
if err != nil {
return err
}
// Then push this tree to NewBranch
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
return err
}
// Simulate push event.
oldCommitID := opts.LastCommitID
if opts.NewBranch != opts.OldBranch {
oldCommitID = git.EmptySHA
}
if err = repo.GetOwner(); err != nil {
return fmt.Errorf("GetOwner: %v", err)
}
err = models.PushUpdate(
opts.NewBranch,
models.PushUpdateOptions{
PusherID: doer.ID,
PusherName: doer.Name,
RepoUserName: repo.Owner.Name,
RepoName: repo.Name,
RefFullName: git.BranchPrefix + opts.NewBranch,
OldCommitID: oldCommitID,
NewCommitID: commitHash,
},
)
if err != nil {
return fmt.Errorf("PushUpdate: %v", err)
}
// FIXME: Should we UpdateRepoIndexer(repo) here?
return nil
}

38
modules/uploader/diff.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uploader
import (
"strings"
"code.gitea.io/gitea/models"
)
// GetDiffPreview produces and returns diff result of a file which is not yet committed.
func GetDiffPreview(repo *models.Repository, branch, treePath, content string) (*models.Diff, error) {
t, err := NewTemporaryUploadRepository(repo)
defer t.Close()
if err != nil {
return nil, err
}
if err := t.Clone(branch); err != nil {
return nil, err
}
if err := t.SetDefaultIndex(); err != nil {
return nil, err
}
// Add the object to the database
objectHash, err := t.HashObject(strings.NewReader(content))
if err != nil {
return nil, err
}
// Add the object to the index
if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil {
return nil, err
}
return t.DiffIndex()
}

359
modules/uploader/repo.go Normal file
View file

@ -0,0 +1,359 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uploader
import (
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
"github.com/Unknwon/com"
)
// TemporaryUploadRepository is a type to wrap our upload repositories
type TemporaryUploadRepository struct {
repo *models.Repository
basePath string
}
// NewTemporaryUploadRepository creates a new temporary upload repository
func NewTemporaryUploadRepository(repo *models.Repository) (*TemporaryUploadRepository, error) {
timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE
basePath := path.Join(models.LocalCopyPath(), "upload-"+timeStr+".git")
if err := os.MkdirAll(path.Dir(basePath), os.ModePerm); err != nil {
return nil, fmt.Errorf("Failed to create dir %s: %v", basePath, err)
}
t := &TemporaryUploadRepository{repo: repo, basePath: basePath}
return t, nil
}
// Close the repository cleaning up all files
func (t *TemporaryUploadRepository) Close() {
if _, err := os.Stat(t.basePath); !os.IsNotExist(err) {
os.RemoveAll(t.basePath)
}
}
// Clone the base repository to our path and set branch as the HEAD
func (t *TemporaryUploadRepository) Clone(branch string) error {
if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute,
fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath),
"git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil {
return fmt.Errorf("Clone: %v %s", err, stderr)
}
return nil
}
// SetDefaultIndex sets the git index to our HEAD
func (t *TemporaryUploadRepository) SetDefaultIndex() error {
if _, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath),
"git", "read-tree", "HEAD"); err != nil {
return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr)
}
return nil
}
// LsFiles checks if the given filename arguments are in the index
func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) {
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"ls-files", "-z", "--"}
for _, arg := range filenames {
if arg != "" {
cmdArgs = append(cmdArgs, arg)
}
}
cmd := exec.CommandContext(ctx, "git", cmdArgs...)
desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
return nil, err
}
filelist := make([]string, len(filenames))
for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) {
filelist = append(filelist, string(line))
}
return filelist, err
}
// RemoveFilesFromIndex removes the given files from the index
func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error {
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
stdIn := new(bytes.Buffer)
for _, file := range filenames {
if file != "" {
stdIn.WriteString("0 0000000000000000000000000000000000000000\t")
stdIn.WriteString(file)
stdIn.WriteByte('\000')
}
}
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"}
cmd := exec.CommandContext(ctx, "git", cmdArgs...)
desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
cmd.Stdin = bytes.NewReader(stdIn.Bytes())
if err := cmd.Start(); err != nil {
return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
}
return err
}
// HashObject writes the provided content to the object db and returns its hash
func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) {
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin")
hashCmd.Dir = t.basePath
hashCmd.Stdin = content
stdOutBuffer := new(bytes.Buffer)
stdErrBuffer := new(bytes.Buffer)
hashCmd.Stdout = stdOutBuffer
hashCmd.Stderr = stdErrBuffer
desc := fmt.Sprintf("hashObject: (git hash-object)")
if err := hashCmd.Start(); err != nil {
return "", fmt.Errorf("git hash-object: %s", err)
}
pid := process.GetManager().Add(desc, hashCmd)
err := hashCmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer)
return "", err
}
return strings.TrimSpace(stdOutBuffer.String()), nil
}
// AddObjectToIndex adds the provided object hash to the index with the provided mode and path
func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error {
if _, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath),
"git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil {
return fmt.Errorf("git update-index: %s", stderr)
}
return nil
}
// WriteTree writes the current index as a tree to the object db and returns its hash
func (t *TemporaryUploadRepository) WriteTree() (string, error) {
treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath),
"git", "write-tree")
if err != nil {
return "", fmt.Errorf("git write-tree: %s", stderr)
}
return strings.TrimSpace(treeHash), nil
}
// CommitTree creates a commit from a given tree for the user with provided message
func (t *TemporaryUploadRepository) CommitTree(doer *models.User, treeHash string, message string) (string, error) {
commitTimeStr := time.Now().Format(time.UnixDate)
sig := doer.NewGitSig()
// FIXME: Should we add SSH_ORIGINAL_COMMAND to this
// Because this may call hooks we should pass in the environment
env := append(os.Environ(),
"GIT_AUTHOR_NAME="+sig.Name,
"GIT_AUTHOR_EMAIL="+sig.Email,
"GIT_AUTHOR_DATE="+commitTimeStr,
"GIT_COMMITTER_NAME="+sig.Name,
"GIT_COMMITTER_EMAIL="+sig.Email,
"GIT_COMMITTER_DATE="+commitTimeStr,
)
commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute,
t.basePath,
fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath),
env,
"git", "commit-tree", treeHash, "-p", "HEAD", "-m", message)
if err != nil {
return "", fmt.Errorf("git commit-tree: %s", stderr)
}
return strings.TrimSpace(commitHash), nil
}
// Push the provided commitHash to the repository branch by the provided user
func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, branch string) error {
isWiki := "false"
if strings.HasSuffix(t.repo.Name, ".wiki") {
isWiki = "true"
}
sig := doer.NewGitSig()
// FIXME: Should we add SSH_ORIGINAL_COMMAND to this
// Because calls hooks we need to pass in the environment
env := append(os.Environ(),
"GIT_AUTHOR_NAME="+sig.Name,
"GIT_AUTHOR_EMAIL="+sig.Email,
"GIT_COMMITTER_NAME="+sig.Name,
"GIT_COMMITTER_EMAIL="+sig.Email,
models.EnvRepoName+"="+t.repo.Name,
models.EnvRepoUsername+"="+t.repo.OwnerName,
models.EnvRepoIsWiki+"="+isWiki,
models.EnvPusherName+"="+doer.Name,
models.EnvPusherID+"="+fmt.Sprintf("%d", doer.ID),
models.ProtectedBranchRepoID+"="+fmt.Sprintf("%d", t.repo.ID),
)
if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute,
t.basePath,
fmt.Sprintf("actuallyPush (git push): %s", t.basePath),
env,
"git", "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil {
return fmt.Errorf("git push: %s", stderr)
}
return nil
}
// DiffIndex returns a Diff of the current index to the head
func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) {
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
stdErr := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD")
cmd.Dir = t.basePath
cmd.Stderr = stdErr
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String())
}
if err = cmd.Start(); err != nil {
return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String())
}
pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd)
defer process.GetManager().Remove(pid)
diff, err = models.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
if err != nil {
return nil, fmt.Errorf("ParsePatch: %v", err)
}
if err = cmd.Wait(); err != nil {
return nil, fmt.Errorf("Wait: %v", err)
}
return diff, nil
}
// CheckAttribute checks the given attribute of the provided files
func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) {
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"}
for _, arg := range args {
if arg != "" {
cmdArgs = append(cmdArgs, arg)
}
}
cmd := exec.CommandContext(ctx, "git", cmdArgs...)
desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
return nil, err
}
fields := bytes.Split(stdOut.Bytes(), []byte{'\000'})
if len(fields)%3 != 1 {
return nil, fmt.Errorf("Wrong number of fields in return from check-attr")
}
var name2attribute2info = make(map[string]map[string]string)
for i := 0; i < (len(fields) / 3); i++ {
filename := string(fields[3*i])
attribute := string(fields[3*i+1])
info := string(fields[3*i+2])
attribute2info := name2attribute2info[filename]
if attribute2info == nil {
attribute2info = make(map[string]string)
}
attribute2info[attribute] = info
name2attribute2info[filename] = attribute2info
}
return name2attribute2info, err
}

159
modules/uploader/update.go Normal file
View file

@ -0,0 +1,159 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uploader
import (
"fmt"
"strings"
"code.gitea.io/git"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
)
// UpdateRepoFileOptions holds the repository file update options
type UpdateRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
OldTreeName string
NewTreeName string
Message string
Content string
IsNewFile bool
}
// UpdateRepoFile adds or updates a file in the given repository
func UpdateRepoFile(repo *models.Repository, doer *models.User, opts *UpdateRepoFileOptions) error {
t, err := NewTemporaryUploadRepository(repo)
defer t.Close()
if err != nil {
return err
}
if err := t.Clone(opts.OldBranch); err != nil {
return err
}
if err := t.SetDefaultIndex(); err != nil {
return err
}
filesInIndex, err := t.LsFiles(opts.NewTreeName, opts.OldTreeName)
if err != nil {
return fmt.Errorf("UpdateRepoFile: %v", err)
}
if opts.IsNewFile {
for _, file := range filesInIndex {
if file == opts.NewTreeName {
return models.ErrRepoFileAlreadyExist{FileName: opts.NewTreeName}
}
}
}
//var stdout string
if opts.OldTreeName != opts.NewTreeName && len(filesInIndex) > 0 {
for _, file := range filesInIndex {
if file == opts.OldTreeName {
if err := t.RemoveFilesFromIndex(opts.OldTreeName); err != nil {
return err
}
}
}
}
// Check there is no way this can return multiple infos
filename2attribute2info, err := t.CheckAttribute("filter", opts.NewTreeName)
if err != nil {
return err
}
content := opts.Content
var lfsMetaObject *models.LFSMetaObject
if filename2attribute2info[opts.NewTreeName] != nil && filename2attribute2info[opts.NewTreeName]["filter"] == "lfs" {
// OK so we are supposed to LFS this data!
oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content))
if err != nil {
return err
}
lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID}
content = lfsMetaObject.Pointer()
}
// Add the object to the database
objectHash, err := t.HashObject(strings.NewReader(content))
if err != nil {
return err
}
// Add the object to the index
if err := t.AddObjectToIndex("100644", objectHash, opts.NewTreeName); err != nil {
return err
}
// Now write the tree
treeHash, err := t.WriteTree()
if err != nil {
return err
}
// Now commit the tree
commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
if err != nil {
return err
}
if lfsMetaObject != nil {
// We have an LFS object - create it
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
if err != nil {
return err
}
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
if !contentStore.Exists(lfsMetaObject) {
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
}
return err
}
}
}
// Then push this tree to NewBranch
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
return err
}
// Simulate push event.
oldCommitID := opts.LastCommitID
if opts.NewBranch != opts.OldBranch {
oldCommitID = git.EmptySHA
}
if err = repo.GetOwner(); err != nil {
return fmt.Errorf("GetOwner: %v", err)
}
err = models.PushUpdate(
opts.NewBranch,
models.PushUpdateOptions{
PusherID: doer.ID,
PusherName: doer.Name,
RepoUserName: repo.Owner.Name,
RepoName: repo.Name,
RefFullName: git.BranchPrefix + opts.NewBranch,
OldCommitID: oldCommitID,
NewCommitID: commitHash,
},
)
if err != nil {
return fmt.Errorf("PushUpdate: %v", err)
}
models.UpdateRepoIndexer(repo)
return nil
}

206
modules/uploader/upload.go Normal file
View file

@ -0,0 +1,206 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package uploader
import (
"fmt"
"os"
"path"
"strings"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/git"
"code.gitea.io/gitea/models"
)
// UploadRepoFileOptions contains the uploaded repository file options
type UploadRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
TreePath string
Message string
Files []string // In UUID format.
}
type uploadInfo struct {
upload *models.Upload
lfsMetaObject *models.LFSMetaObject
}
func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error {
for _, info := range *infos {
if info.lfsMetaObject == nil {
continue
}
if !info.lfsMetaObject.Existing {
if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil {
original = fmt.Errorf("%v, %v", original, err)
}
}
}
return original
}
// UploadRepoFiles uploads files to the given repository
func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error {
if len(opts.Files) == 0 {
return nil
}
uploads, err := models.GetUploadsByUUIDs(opts.Files)
if err != nil {
return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err)
}
t, err := NewTemporaryUploadRepository(repo)
defer t.Close()
if err != nil {
return err
}
if err := t.Clone(opts.OldBranch); err != nil {
return err
}
if err := t.SetDefaultIndex(); err != nil {
return err
}
names := make([]string, len(uploads))
infos := make([]uploadInfo, len(uploads))
for i, upload := range uploads {
names[i] = upload.Name
infos[i] = uploadInfo{upload: upload}
}
filename2attribute2info, err := t.CheckAttribute("filter", names...)
if err != nil {
return err
}
// Copy uploaded files into repository.
for i, uploadInfo := range infos {
file, err := os.Open(uploadInfo.upload.LocalPath())
if err != nil {
return err
}
defer file.Close()
var objectHash string
if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" {
// Handle LFS
// FIXME: Inefficient! this should probably happen in models.Upload
oid, err := models.GenerateLFSOid(file)
if err != nil {
return err
}
fileInfo, err := file.Stat()
if err != nil {
return err
}
uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID}
if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil {
return err
}
infos[i] = uploadInfo
} else {
if objectHash, err = t.HashObject(file); err != nil {
return err
}
}
// Add the object to the index
if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil {
return err
}
}
// Now write the tree
treeHash, err := t.WriteTree()
if err != nil {
return err
}
// Now commit the tree
commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
if err != nil {
return err
}
// Now deal with LFS objects
for _, uploadInfo := range infos {
if uploadInfo.lfsMetaObject == nil {
continue
}
uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject)
if err != nil {
// OK Now we need to cleanup
return cleanUpAfterFailure(&infos, t, err)
}
// Don't move the files yet - we need to ensure that
// everything can be inserted first
}
// OK now we can insert the data into the store - there's no way to clean up the store
// once it's in there, it's in there.
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
for _, uploadInfo := range infos {
if uploadInfo.lfsMetaObject == nil {
continue
}
if !contentStore.Exists(uploadInfo.lfsMetaObject) {
file, err := os.Open(uploadInfo.upload.LocalPath())
if err != nil {
return cleanUpAfterFailure(&infos, t, err)
}
defer file.Close()
// FIXME: Put regenerates the hash and copies the file over.
// I guess this strictly ensures the soundness of the store but this is inefficient.
if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil {
// OK Now we need to cleanup
// Can't clean up the store, once uploaded there they're there.
return cleanUpAfterFailure(&infos, t, err)
}
}
}
// Then push this tree to NewBranch
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
return err
}
// Simulate push event.
oldCommitID := opts.LastCommitID
if opts.NewBranch != opts.OldBranch {
oldCommitID = git.EmptySHA
}
if err = repo.GetOwner(); err != nil {
return fmt.Errorf("GetOwner: %v", err)
}
err = models.PushUpdate(
opts.NewBranch,
models.PushUpdateOptions{
PusherID: doer.ID,
PusherName: doer.Name,
RepoUserName: repo.Owner.Name,
RepoName: repo.Name,
RefFullName: git.BranchPrefix + opts.NewBranch,
OldCommitID: oldCommitID,
NewCommitID: commitHash,
},
)
if err != nil {
return fmt.Errorf("PushUpdate: %v", err)
}
// FIXME: Should we models.UpdateRepoIndexer(repo) here?
return models.DeleteUploads(uploads...)
}

59
routers/repo/editor.go
View file

@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/templates"
"code.gitea.io/gitea/modules/uploader"
)
const (
@ -62,6 +63,16 @@ func editFile(ctx *context.Context, isNewFile bool) {
ctx.Data["RequireSimpleMDE"] = true
canCommit := renderCommitRights(ctx)
treePath := cleanUploadFileName(ctx.Repo.TreePath)
if treePath != ctx.Repo.TreePath {
if isNewFile {
ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_new", ctx.Repo.BranchName, treePath))
} else {
ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_edit", ctx.Repo.BranchName, treePath))
}
return
}
treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath)
if !isNewFile {
@ -155,7 +166,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo
oldBranchName := ctx.Repo.BranchName
branchName := oldBranchName
oldTreePath := ctx.Repo.TreePath
oldTreePath := cleanUploadFileName(ctx.Repo.TreePath)
lastCommit := form.LastCommit
form.LastCommit = ctx.Repo.Commit.ID.String()
@ -298,7 +309,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo
message += "\n\n" + form.CommitMessage
}
if err := ctx.Repo.Repository.UpdateRepoFile(ctx.User, models.UpdateRepoFileOptions{
if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, &uploader.UpdateRepoFileOptions{
LastCommitID: lastCommit,
OldBranch: oldBranchName,
NewBranch: branchName,
@ -328,7 +339,11 @@ func NewFilePost(ctx *context.Context, form auth.EditRepoFileForm) {
// DiffPreviewPost render preview diff page
func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) {
treePath := ctx.Repo.TreePath
treePath := cleanUploadFileName(ctx.Repo.TreePath)
if len(treePath) == 0 {
ctx.Error(500, "file name to diff is invalid")
return
}
entry, err := ctx.Repo.Commit.GetTreeEntryByPath(treePath)
if err != nil {
@ -339,7 +354,7 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) {
return
}
diff, err := ctx.Repo.Repository.GetDiffPreview(ctx.Repo.BranchName, treePath, form.Content)
diff, err := uploader.GetDiffPreview(ctx.Repo.Repository, ctx.Repo.BranchName, treePath, form.Content)
if err != nil {
ctx.Error(500, "GetDiffPreview: "+err.Error())
return
@ -358,7 +373,14 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) {
func DeleteFile(ctx *context.Context) {
ctx.Data["PageIsDelete"] = true
ctx.Data["BranchLink"] = ctx.Repo.RepoLink + "/src/" + ctx.Repo.BranchNameSubURL()
ctx.Data["TreePath"] = ctx.Repo.TreePath
treePath := cleanUploadFileName(ctx.Repo.TreePath)
if treePath != ctx.Repo.TreePath {
ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_delete", ctx.Repo.BranchName, treePath))
return
}
ctx.Data["TreePath"] = treePath
canCommit := renderCommitRights(ctx)
ctx.Data["commit_summary"] = ""
@ -426,7 +448,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) {
message += "\n\n" + form.CommitMessage
}
if err := ctx.Repo.Repository.DeleteRepoFile(ctx.User, models.DeleteRepoFileOptions{
if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, &uploader.DeleteRepoFileOptions{
LastCommitID: ctx.Repo.CommitID,
OldBranch: oldBranchName,
NewBranch: branchName,
@ -453,6 +475,12 @@ func UploadFile(ctx *context.Context) {
ctx.Data["PageIsUpload"] = true
renderUploadSettings(ctx)
canCommit := renderCommitRights(ctx)
treePath := cleanUploadFileName(ctx.Repo.TreePath)
if treePath != ctx.Repo.TreePath {
ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_upload", ctx.Repo.BranchName, treePath))
return
}
ctx.Repo.TreePath = treePath
treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath)
if len(treeNames) == 0 {
@ -489,10 +517,6 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
}
form.TreePath = cleanUploadFileName(form.TreePath)
if len(form.TreePath) == 0 {
ctx.Error(500, "Upload file name is invalid")
return
}
treeNames, treePaths := getParentTreeFields(form.TreePath)
if len(treeNames) == 0 {
@ -559,7 +583,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
message += "\n\n" + form.CommitMessage
}
if err := ctx.Repo.Repository.UploadRepoFiles(ctx.User, models.UploadRepoFileOptions{
if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, &uploader.UploadRepoFileOptions{
LastCommitID: ctx.Repo.CommitID,
OldBranch: oldBranchName,
NewBranch: branchName,
@ -576,12 +600,13 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
}
func cleanUploadFileName(name string) string {
name = strings.TrimLeft(name, "./\\")
name = strings.Replace(name, "../", "", -1)
name = strings.Replace(name, "..\\", "", -1)
name = strings.TrimPrefix(path.Clean(name), ".git/")
if name == ".git" {
return ""
// Rebase the filename
name = strings.Trim(path.Clean("/"+name), " /")
// Git disallows any filenames to have a .git directory in them.
for _, part := range strings.Split(name, "/") {
if strings.ToLower(part) == ".git" {
return ""
}
}
return name
}

26
routers/repo/editor_test.go
View file

@ -15,16 +15,24 @@ func TestCleanUploadName(t *testing.T) {
models.PrepareTestEnv(t)
var kases = map[string]string{
".git/refs/master": "git/refs/master",
"/root/abc": "root/abc",
"./../../abc": "abc",
"a/../.git": "a/.git",
"a/../../../abc": "a/abc",
"../../../acd": "acd",
"../../.git/abc": "git/abc",
"..\\..\\.git/abc": "git/abc",
".git/refs/master": "",
"/root/abc": "root/abc",
"./../../abc": "abc",
"a/../.git": "",
"a/../../../abc": "abc",
"../../../acd": "acd",
"../../.git/abc": "",
"..\\..\\.git/abc": "..\\..\\.git/abc",
"..\\../.git/abc": "",
"..\\../.git": "",
"abc/../def": "def",
".drone.yml": ".drone.yml",
".abc/def/.drone.yml": ".abc/def/.drone.yml",
"..drone.yml.": "..drone.yml.",
"..a.dotty...name...": "..a.dotty...name...",
"..a.dotty../.folder../.name...": "..a.dotty../.folder../.name...",
}
for k, v := range kases {
assert.EqualValues(t, v, cleanUploadFileName(k))
assert.EqualValues(t, cleanUploadFileName(k), v)
}
}