Split lfs size from repository size (#22900)

releated to #21820

- Split `Size` in repository table as two new colunms, one is `GitSize`
for git size, the other is `LFSSize` for lfs data. still store full size
in `Size` colunm.
- Show full size on ui, but show each of them by a `title`;  example:

![image](https://user-images.githubusercontent.com/25342410/218636251-e200f085-d7e7-4a25-9ff1-b586a63e07a9.png)

- Return full size in api response.

---------

Signed-off-by: a1012112796 <1012112796@qq.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com>
Co-authored-by: Giteabot <teabot@gitea.io>

models/db/search.go

@@ -20,6 +20,10 @@ const (
 	SearchOrderByNewest                SearchOrderBy = "created_unix DESC"
 	SearchOrderBySize                  SearchOrderBy = "size ASC"
 	SearchOrderBySizeReverse           SearchOrderBy = "size DESC"
+	SearchOrderByGitSize               SearchOrderBy = "git_size ASC"
+	SearchOrderByGitSizeReverse        SearchOrderBy = "git_size DESC"
+	SearchOrderByLFSSize               SearchOrderBy = "lfs_size ASC"
+	SearchOrderByLFSSizeReverse        SearchOrderBy = "lfs_size DESC"
 	SearchOrderByID                    SearchOrderBy = "id ASC"
 	SearchOrderByIDReverse             SearchOrderBy = "id DESC"
 	SearchOrderByStars                 SearchOrderBy = "num_stars ASC"

models/migrations/migrations.go

@@ -507,6 +507,8 @@ var migrations = []Migration{
 	NewMigration("Add variable table", v1_21.CreateVariableTable),
 	// v262 -> v263
 	NewMigration("Add TriggerEvent to action_run table", v1_21.AddTriggerEventToActionRun),
+	// v263 -> v264
+	NewMigration("Add git_size and lfs_size columns to repository table", v1_21.AddGitSizeAndLFSSizeToRepositoryTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_21/v263.go

@@ -0,0 +1,41 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_21 //nolint
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+// AddGitSizeAndLFSSizeToRepositoryTable: add GitSize and LFSSize columns to Repository
+func AddGitSizeAndLFSSizeToRepositoryTable(x *xorm.Engine) error {
+	type Repository struct {
+		GitSize int64 `xorm:"NOT NULL DEFAULT 0"`
+		LFSSize int64 `xorm:"NOT NULL DEFAULT 0"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(Repository)); err != nil {
+		return fmt.Errorf("Sync2: %w", err)
+	}
+
+	_, err := sess.Exec(`UPDATE repository SET lfs_size=(SELECT SUM(size) FROM lfs_meta_object WHERE lfs_meta_object.repository_id=repository.ID) WHERE EXISTS (SELECT 1 FROM lfs_meta_object WHERE lfs_meta_object.repository_id=repository.ID)`)
+	if err != nil {
+		return err
+	}
+
+	_, err = sess.Exec(`UPDATE repository SET git_size = size - lfs_size`)
+	if err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/repo/repo.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
@@ -163,6 +164,8 @@ type Repository struct {
 	IsTemplate                      bool               `xorm:"INDEX NOT NULL DEFAULT false"`
 	TemplateID                      int64              `xorm:"INDEX"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
+	GitSize                         int64              `xorm:"NOT NULL DEFAULT 0"`
+	LFSSize                         int64              `xorm:"NOT NULL DEFAULT 0"`
 	CodeIndexerStatus               *RepoIndexerStatus `xorm:"-"`
 	StatsIndexerStatus              *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
@@ -196,6 +199,42 @@ func (repo *Repository) SanitizedOriginalURL() string {
 	return u.String()
 }
 
+// text representations to be returned in SizeDetail.Name
+const (
+	SizeDetailNameGit = "git"
+	SizeDetailNameLFS = "lfs"
+)
+
+type SizeDetail struct {
+	Name string
+	Size int64
+}
+
+// SizeDetails forms a struct with various size details about repository
+func (repo *Repository) SizeDetails() []SizeDetail {
+	sizeDetails := []SizeDetail{
+		{
+			Name: SizeDetailNameGit,
+			Size: repo.GitSize,
+		},
+		{
+			Name: SizeDetailNameLFS,
+			Size: repo.LFSSize,
+		},
+	}
+	return sizeDetails
+}
+
+// SizeDetailsString returns a concatenation of all repository size details as a string
+func (repo *Repository) SizeDetailsString() string {
+	var str strings.Builder
+	sizeDetails := repo.SizeDetails()
+	for _, detail := range sizeDetails {
+		str.WriteString(fmt.Sprintf("%s: %s, ", detail.Name, base.FileSize(detail.Size)))
+	}
+	return strings.TrimSuffix(str.String(), ", ")
+}
+
 func (repo *Repository) LogString() string {
 	if repo == nil {
 		return "<Repository nil>"

models/repo/update.go

@@ -185,9 +185,11 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s
 }
 
 // UpdateRepoSize updates the repository size, calculating it using getDirectorySize
-func UpdateRepoSize(ctx context.Context, repoID, size int64) error {
+func UpdateRepoSize(ctx context.Context, repoID, gitSize, lfsSize int64) error {
-	_, err := db.GetEngine(ctx).ID(repoID).Cols("size").NoAutoTime().Update(&Repository{
+	_, err := db.GetEngine(ctx).ID(repoID).Cols("size", "git_size", "lfs_size").NoAutoTime().Update(&Repository{
-		Size: size,
+		Size:    gitSize + lfsSize,
+		GitSize: gitSize,
+		LFSSize: lfsSize,
 	})
 	return err
 }

modules/repository/create.go

@@ -330,7 +330,7 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error {
 		return fmt.Errorf("updateSize: GetLFSMetaObjects: %w", err)
 	}
 
-	return repo_model.UpdateRepoSize(ctx, repo.ID, size+lfsSize)
+	return repo_model.UpdateRepoSize(ctx, repo.ID, size, lfsSize)
 }
 
 // CheckDaemonExportOK creates/removes git-daemon-export-ok for git-daemon...

options/locale/locale_en-US.ini

@@ -2797,6 +2797,7 @@ repos.stars = Stars
 repos.forks = Forks
 repos.issues = Issues
 repos.size = Size
+repos.lfs_size = LFS Size
 
 packages.package_manage_panel = Package Management
 packages.total_size = Total Size: %s

routers/web/explore/repo.go

@@ -73,6 +73,14 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
 		orderBy = db.SearchOrderBySizeReverse
 	case "size":
 		orderBy = db.SearchOrderBySize
+	case "reversegitsize":
+		orderBy = db.SearchOrderByGitSizeReverse
+	case "gitsize":
+		orderBy = db.SearchOrderByGitSize
+	case "reverselfssize":
+		orderBy = db.SearchOrderByLFSSizeReverse
+	case "lfssize":
+		orderBy = db.SearchOrderByLFSSize
 	case "moststars":
 		orderBy = db.SearchOrderByStarsReverse
 	case "feweststars":

templates/admin/repo/list.tmpl

@@ -29,9 +29,13 @@
 							{{SortArrow "mostforks" "fewestforks" $.SortType false}}
 						</th>
 						<th>{{.locale.Tr "admin.repos.issues"}}</th>
-						<th  data-sortt-asc="size" data-sortt-desc="reversesize">
+						<th  data-sortt-asc="gitsize" data-sortt-desc="reversegitsize">
 							{{.locale.Tr "admin.repos.size"}}
-							{{SortArrow "size" "reversesize" $.SortType false}}
+							{{SortArrow "gitsize" "reversegitsize" $.SortType false}}
+						</th>
+						<th  data-sortt-asc="lfssize" data-sortt-desc="reverselfssize">
+							{{.locale.Tr "admin.repos.lfs_size"}}
+							{{SortArrow "lfssize" "reverselfssize" $.SortType false}}
 						</th>
 						<th>{{.locale.Tr "admin.auths.updated"}}</th>
 						<th>{{.locale.Tr "admin.users.created"}}</th>
@@ -80,7 +84,8 @@
 							<td>{{.NumStars}}</td>
 							<td>{{.NumForks}}</td>
 							<td>{{.NumIssues}}</td>
-							<td>{{FileSize .Size}}</td>
+							<td>{{FileSize .GitSize}}</td>
+							<td>{{FileSize .LFSSize}}</td>
 							<td>{{DateTime "short" .UpdatedUnix}}</td>
 							<td>{{DateTime "short" .CreatedUnix}}</td>
 							<td><a class="delete-button" href="" data-url="{{$.Link}}/delete?page={{$.Page.Paginater.Current}}&sort={{$.SortType}}" data-id="{{.ID}}" data-name="{{.Name}}">{{svg "octicon-trash"}}</a></td>

templates/repo/settings/options.tmpl

@@ -14,7 +14,7 @@
 				</div>
 				<div class="inline field">
 					<label>{{.locale.Tr "repo.repo_size"}}</label>
-					<span>{{FileSize .Repository.Size}}</span>
+					<span {{if not (eq .Repository.Size 0)}} data-tooltip-content="{{.Repository.SizeDetailsString}}"{{end}}>{{FileSize .Repository.Size}}</span>
 				</div>
 				<div class="inline field">
 					<label>{{.locale.Tr "repo.template"}}</label>

templates/repo/sub_menu.tmpl

@@ -14,7 +14,7 @@
 						<a href="{{.RepoLink}}/tags">{{svg "octicon-tag"}} <b>{{.NumTags}}</b> {{.locale.TrN .NumTags "repo.tag" "repo.tags"}}</a>
 					</div>
 				{{end}}
-				<div class="item">
+				<div class="item" {{if not (eq .Repository.Size 0)}}data-tooltip-content="{{.Repository.SizeDetailsString}}"{{end}}>
 					{{$fileSizeFormatted := FileSize .Repository.Size}}{{/* the formatted string is always "{val} {unit}" */}}
 					{{$fileSizeFields := StringUtils.Split $fileSizeFormatted " "}}
 					<span>{{svg "octicon-database"}} <b>{{.locale.PrettyNumber (index $fileSizeFields 0)}}</b> {{index $fileSizeFields 1}}</span>

templates/user/settings/repos.tmpl

@@ -24,7 +24,7 @@
 											<span class="icon">{{svg "octicon-repo"}}</span>
 										{{end}}
 										<a class="muted name" href="{{$repo.Link}}">{{$repo.OwnerName}}/{{$repo.Name}}</a>
-										<span class="text light-3">{{FileSize $repo.Size}}</span>
+										<span class="text light-3" {{if not (eq $repo.Size 0)}} data-tooltip-content="{{$repo.SizeDetailsString}}"{{end}}>{{FileSize $repo.Size}}</span>
 										{{if $repo.IsFork}}
 											{{$.locale.Tr "repo.forked_from"}}
 											<span><a href="{{$repo.BaseRepo.Link}}">{{$repo.BaseRepo.OwnerName}}/{{$repo.BaseRepo.Name}}</a></span>