forgejo/models/repo/mirror.go

185 lines
4.6 KiB
Go
Raw Normal View History

// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"context"
"errors"
"fmt"
"time"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
2019-10-17 11:26:49 +02:00
"xorm.io/xorm"
)
// ErrMirrorNotExist mirror does not exist error
var ErrMirrorNotExist = errors.New("Mirror does not exist")
// RemoteMirrorer defines base methods for pull/push mirrors.
type RemoteMirrorer interface {
GetRepository() *Repository
GetRemoteName() string
}
// Mirror represents mirror information of a repository.
type Mirror struct {
2017-01-06 16:14:33 +01:00
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
Repo *Repository `xorm:"-"`
Interval time.Duration
EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"`
NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"`
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-09 00:25:57 +02:00
LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"`
LFSEndpoint string `xorm:"lfs_endpoint TEXT"`
Address string `xorm:"-"`
}
func init() {
db.RegisterModel(new(Mirror))
}
2016-11-26 01:30:21 +01:00
// BeforeInsert will be invoked by XORM before inserting a record
func (m *Mirror) BeforeInsert() {
if m != nil {
m.UpdatedUnix = timeutil.TimeStampNow()
m.NextUpdateUnix = timeutil.TimeStampNow()
}
}
// AfterLoad is invoked from XORM after setting the values of all fields of this object.
func (m *Mirror) AfterLoad(session *xorm.Session) {
if m == nil {
return
}
var err error
m.Repo, err = getRepositoryByID(session, m.RepoID)
if err != nil {
Better logging (#6038) (#6095) * Panic don't fatal on create new logger Fixes #5854 Signed-off-by: Andrew Thornton <art27@cantab.net> * partial broken * Update the logging infrastrcture Signed-off-by: Andrew Thornton <art27@cantab.net> * Reset the skip levels for Fatal and Error Signed-off-by: Andrew Thornton <art27@cantab.net> * broken ncsa * More log.Error fixes Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove nal * set log-levels to lowercase * Make console_test test all levels * switch to lowercased levels * OK now working * Fix vetting issues * Fix lint * Fix tests * change default logging to match current gitea * Improve log testing Signed-off-by: Andrew Thornton <art27@cantab.net> * reset error skip levels to 0 * Update documentation and access logger configuration * Redirect the router log back to gitea if redirect macaron log but also allow setting the log level - i.e. TRACE * Fix broken level caching * Refactor the router log * Add Router logger * Add colorizing options * Adjust router colors * Only create logger if they will be used * update app.ini.sample * rename Attribute ColorAttribute * Change from white to green for function * Set fatal/error levels * Restore initial trace logger * Fix Trace arguments in modules/auth/auth.go * Properly handle XORMLogger * Improve admin/config page * fix fmt * Add auto-compression of old logs * Update error log levels * Remove the unnecessary skip argument from Error, Fatal and Critical * Add stacktrace support * Fix tests * Remove x/sync from vendors? * Add stderr option to console logger * Use filepath.ToSlash to protect against Windows in tests * Remove prefixed underscores from names in colors.go * Remove not implemented database logger This was removed from Gogs on 4 Mar 2016 but left in the configuration since then. * Ensure that log paths are relative to ROOT_PATH * use path.Join * rename jsonConfig to logConfig * Rename "config" to "jsonConfig" to make it clearer * Requested changes * Requested changes: XormLogger * Try to color the windows terminal If successful default to colorizing the console logs * fixup * Colorize initially too * update vendor * Colorize logs on default and remove if this is not a colorizing logger * Fix documentation * fix test * Use go-isatty to detect if on windows we are on msys or cygwin * Fix spelling mistake * Add missing vendors * More changes * Rationalise the ANSI writer protection * Adjust colors on advice from @0x5c * Make Flags a comma separated list * Move to use the windows constant for ENABLE_VIRTUAL_TERMINAL_PROCESSING * Ensure matching is done on the non-colored message - to simpify EXPRESSION
2019-04-02 09:48:31 +02:00
log.Error("getRepositoryByID[%d]: %v", m.ID, err)
}
}
// GetRepository returns the repository.
func (m *Mirror) GetRepository() *Repository {
return m.Repo
}
// GetRemoteName returns the name of the remote.
func (m *Mirror) GetRemoteName() string {
return "origin"
}
// ScheduleNextUpdate calculates and sets next update time.
func (m *Mirror) ScheduleNextUpdate() {
if m.Interval != 0 {
m.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(m.Interval)
} else {
m.NextUpdateUnix = 0
}
}
func getMirrorByRepoID(e db.Engine, repoID int64) (*Mirror, error) {
m := &Mirror{RepoID: repoID}
has, err := e.Get(m)
if err != nil {
return nil, err
} else if !has {
return nil, ErrMirrorNotExist
}
return m, nil
}
// GetMirrorByRepoID returns mirror information of a repository.
func GetMirrorByRepoID(repoID int64) (*Mirror, error) {
return getMirrorByRepoID(db.GetEngine(db.DefaultContext), repoID)
}
func updateMirror(e db.Engine, m *Mirror) error {
_, err := e.ID(m.ID).AllCols().Update(m)
return err
}
2016-11-26 01:30:21 +01:00
// UpdateMirror updates the mirror
func UpdateMirror(m *Mirror) error {
return updateMirror(db.GetEngine(db.DefaultContext), m)
}
// TouchMirror updates the mirror updatedUnix
func TouchMirror(ctx context.Context, m *Mirror) error {
m.UpdatedUnix = timeutil.TimeStampNow()
_, err := db.GetEngine(ctx).ID(m.ID).Cols("updated_unix").Update(m)
return err
}
2016-11-26 01:30:21 +01:00
// DeleteMirrorByRepoID deletes a mirror by repoID
func DeleteMirrorByRepoID(repoID int64) error {
_, err := db.GetEngine(db.DefaultContext).Delete(&Mirror{RepoID: repoID})
return err
}
// MirrorsIterate iterates all mirror repositories.
func MirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
2016-11-10 16:16:32 +01:00
Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0").
OrderBy("updated_unix ASC").
Limit(limit).
Iterate(new(Mirror), f)
}
// InsertMirror inserts a mirror to database
func InsertMirror(mirror *Mirror) error {
_, err := db.GetEngine(db.DefaultContext).Insert(mirror)
return err
}
// MirrorRepositoryList contains the mirror repositories
type MirrorRepositoryList []*Repository
func (repos MirrorRepositoryList) loadAttributes(e db.Engine) error {
if len(repos) == 0 {
return nil
}
// Load mirrors.
repoIDs := make([]int64, 0, len(repos))
for i := range repos {
if !repos[i].IsMirror {
continue
}
repoIDs = append(repoIDs, repos[i].ID)
}
mirrors := make([]*Mirror, 0, len(repoIDs))
if err := e.
Where("id > 0").
In("repo_id", repoIDs).
Find(&mirrors); err != nil {
return fmt.Errorf("find mirrors: %v", err)
}
set := make(map[int64]*Mirror)
for i := range mirrors {
set[mirrors[i].RepoID] = mirrors[i]
}
for i := range repos {
repos[i].Mirror = set[repos[i].ID]
}
return nil
}
// LoadAttributes loads the attributes for the given MirrorRepositoryList
func (repos MirrorRepositoryList) LoadAttributes() error {
return repos.loadAttributes(db.GetEngine(db.DefaultContext))
}