forgejo/models/db/engine.go

358 lines
10 KiB
Go
Raw Normal View History

2014-02-12 18:49:46 +01:00
// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
2014-02-12 18:49:46 +01:00
package db
2014-02-18 23:48:02 +01:00
import (
"context"
2014-10-19 07:35:24 +02:00
"database/sql"
"errors"
2014-02-18 23:48:02 +01:00
"fmt"
"io"
"reflect"
"strings"
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
"time"
2014-02-18 23:48:02 +01:00
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
2019-10-17 11:26:49 +02:00
"xorm.io/xorm"
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
"xorm.io/xorm/contexts"
"xorm.io/xorm/names"
"xorm.io/xorm/schemas"
2016-11-26 01:20:18 +01:00
_ "github.com/denisenkom/go-mssqldb" // Needed for the MSSQL driver
_ "github.com/go-sql-driver/mysql" // Needed for the MySQL driver
_ "github.com/lib/pq" // Needed for the Postgresql driver
2014-02-18 23:48:02 +01:00
)
var (
x *xorm.Engine
tables []any
initFuncs []func() error
)
2014-10-19 07:35:24 +02:00
// Engine represents a xorm engine or session.
type Engine interface {
Table(tableNameOrBean any) *xorm.Session
Count(...any) (int64, error)
Decr(column string, arg ...any) *xorm.Session
Delete(...any) (int64, error)
Truncate(...any) (int64, error)
Exec(...any) (sql.Result, error)
Find(any, ...any) error
Get(beans ...any) (bool, error)
ID(any) *xorm.Session
In(string, ...any) *xorm.Session
Incr(column string, arg ...any) *xorm.Session
Insert(...any) (int64, error)
Iterate(any, xorm.IterFunc) error
[SEMVER] store SemVer in ForgejoSemVer after a database upgrade (cherry picked from commit b7fe7cf401f4bddd6455efc651f7ac054f3fe1cf) (cherry picked from commit cf339eed4f4851b18448dbdd83df32d00bc6f45b) (cherry picked from commit 4f3a16168bbeced519a60c32e10e2895d9367238) (cherry picked from commit 6f5bbc53fcebd614f5ee8627c7d6e3c637ffa694) (cherry picked from commit aca42b422e76668387769c15868ef77b073bb7db) (cherry picked from commit 5a7f7580e525694d1f27e12329c8532bd29273f8) (cherry picked from commit 06c383c807ad49e1e35b429a10c6a11c65aeebe5) (cherry picked from commit fe831dcb53b81b6cc632be751bfcbfb9bc00efd3) (cherry picked from commit cd12cd0dbce47c6117ea579ae5019c182155b3a9) (cherry picked from commit cc79163703ce31706c86b88c38fb8a20ed745e20) (cherry picked from commit 0102a5715ea6a03b560cc2f0b6cbe2b2576c255e) (cherry picked from commit 403f7520b3056eace36eae505afbab6a05f597b7) (cherry picked from commit a3b61510a246f61c174ddc3c288e556522a6aab9) (cherry picked from commit f83f0f9feb76a8b62ca4d74dc2785c713fbec282) (cherry picked from commit fd1c3a6d09a057070844cd955d0920518e60f408) (cherry picked from commit f7cdc3d6f1ad1ff01bb1814207fcb41210db80e8) (cherry picked from commit 060121b644e0515a6b673cdd514d52e6fa1e6ec3) (cherry picked from commit 62c847ff0235196e73e860d0d1658f0734985270) (cherry picked from commit 4d051b51c2813828b978ef84b6b72c0ba051741b) (cherry picked from commit 86e6981a936b1937065605892083b2488424bf5e) (cherry picked from commit c1fc9e441b1467033911f3848f82a89aaacf98a9) (cherry picked from commit 8bb2f0871a507d881acf74acf8fb90fd5ebff567) (cherry picked from commit 0cd9fe52511ee06f1b8849325d196f60d8ccd151) (cherry picked from commit b0b44778b4d3e51852d0669dfe6d9a6c334b90e4) (cherry picked from commit 7c2f4f749f1c8986875eebf95254a3db151e5248)
2023-08-08 23:55:25 +02:00
IsTableExist(any) (bool, error)
Join(joinOperator string, tablename, condition any, args ...any) *xorm.Session
SQL(any, ...any) *xorm.Session
Where(any, ...any) *xorm.Session
2019-06-12 21:41:28 +02:00
Asc(colNames ...string) *xorm.Session
Desc(colNames ...string) *xorm.Session
API add/generalize pagination (#9452) * paginate results * fixed deadlock * prevented breaking change * updated swagger * go fmt * fixed find topic * go mod tidy * go mod vendor with go1.13.5 * fixed repo find topics * fixed unit test * added Limit method to Engine struct; use engine variable when provided; fixed gitignore * use ItemsPerPage for default pagesize; fix GetWatchers, getOrgUsersByOrgID and GetStargazers; fix GetAllCommits headers; reverted some changed behaviors * set Page value on Home route * improved memory allocations * fixed response headers * removed logfiles * fixed import order * import order * improved swagger * added function to get models.ListOptions from context * removed pagesize diff on unit test * fixed imports * removed unnecessary struct field * fixed go fmt * scoped PR * code improvements * code improvements * go mod tidy * fixed import order * fixed commit statuses session * fixed files headers * fixed headers; added pagination for notifications * go mod tidy * go fmt * removed Private from user search options; added setting.UI.IssuePagingNum as default valeu on repo's issues list * Apply suggestions from code review Co-Authored-By: 6543 <6543@obermui.de> Co-Authored-By: zeripath <art27@cantab.net> * fixed build error * CI.restart() * fixed merge conflicts resolve * fixed conflicts resolve * improved FindTrackedTimesOptions.ToOptions() method * added backwards compatibility on ListReleases request; fixed issue tracked time ToSession * fixed build error; fixed swagger template * fixed swagger template * fixed ListReleases backwards compatibility * added page to user search route Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath <art27@cantab.net>
2020-01-24 20:00:29 +01:00
Limit(limit int, start ...int) *xorm.Session
NoAutoTime() *xorm.Session
SumInt(bean any, columnName string) (res int64, err error)
Sync(...any) error
Select(string) *xorm.Session
NotIn(string, ...any) *xorm.Session
OrderBy(any, ...any) *xorm.Session
Exist(...any) (bool, error)
Distinct(...string) *xorm.Session
Query(...any) ([]map[string][]byte, error)
Cols(...string) *xorm.Session
Context(ctx context.Context) *xorm.Session
Ping() error
2014-10-19 07:35:24 +02:00
}
// TableInfo returns table's information via an object
func TableInfo(v any) (*schemas.Table, error) {
return x.TableInfo(v)
}
// DumpTables dump tables information
func DumpTables(tables []*schemas.Table, w io.Writer, tp ...schemas.DBType) error {
return x.DumpTables(tables, w, tp...)
}
2016-11-26 01:20:18 +01:00
// RegisterModel registers model, if initfunc provided, it will be invoked after data model sync
func RegisterModel(bean any, initFunc ...func() error) {
tables = append(tables, bean)
if len(initFuncs) > 0 && initFunc[0] != nil {
initFuncs = append(initFuncs, initFunc[0])
}
}
2014-03-21 06:48:10 +01:00
2014-04-05 16:46:32 +02:00
func init() {
2016-11-26 01:20:18 +01:00
gonicNames := []string{"SSL", "UID"}
2015-08-27 17:06:14 +02:00
for _, name := range gonicNames {
names.LintGonicMapper[name] = true
2015-08-27 17:06:14 +02:00
}
2014-04-05 16:46:32 +02:00
}
// newXORMEngine returns a new XORM engine from the configuration
func newXORMEngine() (*xorm.Engine, error) {
connStr, err := setting.DBConnStr()
if err != nil {
return nil, err
2014-03-30 16:47:08 +02:00
}
var engine *xorm.Engine
if setting.Database.Type.IsPostgreSQL() && len(setting.Database.Schema) > 0 {
// OK whilst we sort out our schema issues - create a schema aware postgres
registerPostgresSchemaDriver()
engine, err = xorm.NewEngine("postgresschema", connStr)
} else {
engine, err = xorm.NewEngine(setting.Database.Type.String(), connStr)
}
if err != nil {
return nil, err
}
if setting.Database.Type == "mysql" {
engine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"})
} else if setting.Database.Type == "mssql" {
engine.Dialect().SetParams(map[string]string{"DEFAULT_VARCHAR": "nvarchar"})
}
engine.SetSchema(setting.Database.Schema)
return engine, nil
}
// SyncAllTables sync the schemas of all tables, is required by unit test code
func SyncAllTables() error {
Fix wrong warn messages in migration steps (#25475) The recent change on xorm for `Sync` is it will not warn when database have columns which is not listed on struct. So we just need this warn logs when `Sync` the whole database but not in the migrations Sync. This PR will remove almost unnecessary warning logs on migrations. Now below logs in CI will disappear. ```log 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column creator_id but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column is_closed but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column board_type but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column type but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column closed_date_unix but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column created_unix but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column updated_unix but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column card_type but struct has not related field ```
2023-06-24 10:20:08 +02:00
_, err := x.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{
WarnIfDatabaseColumnMissed: true,
}, tables...)
return err
}
// InitEngine initializes the xorm.Engine and sets it as db.DefaultContext
func InitEngine(ctx context.Context) error {
xormEngine, err := newXORMEngine()
2014-02-18 23:48:02 +01:00
if err != nil {
return fmt.Errorf("failed to connect to database: %w", err)
2014-02-18 23:48:02 +01:00
}
xormEngine.SetMapper(names.GonicMapper{})
2014-12-07 02:22:48 +01:00
// WARNING: for serv command, MUST remove the output to os.stdout,
2014-03-20 21:04:56 +01:00
// so use log file to instead print to stdout.
xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL))
xormEngine.ShowSQL(setting.Database.LogSQL)
xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns)
xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns)
xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
xormEngine.SetConnMaxIdleTime(setting.Database.ConnMaxIdleTime)
xormEngine.SetDefaultContext(ctx)
if setting.Database.SlowQueryThreshold > 0 {
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
xormEngine.AddHook(&SlowQueryHook{
Treshold: setting.Database.SlowQueryThreshold,
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
Logger: log.GetLogger("xorm"),
})
}
errorLogger := log.GetLogger("xorm")
if setting.IsInTesting {
errorLogger = log.GetLogger(log.DEFAULT)
}
xormEngine.AddHook(&ErrorQueryHook{
Logger: errorLogger,
})
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
SetDefaultEngine(ctx, xormEngine)
return nil
}
// SetDefaultEngine sets the default engine for db
func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) {
x = eng
DefaultContext = &Context{
Context: ctx,
e: x,
}
2014-02-18 23:48:02 +01:00
}
// UnsetDefaultEngine closes and unsets the default engine
// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now,
// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `x` and DefaultContext without close
// Global database engine related functions are all racy and there is no graceful close right now.
func UnsetDefaultEngine() {
if x != nil {
_ = x.Close()
x = nil
}
DefaultContext = nil
}
// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext
// This function must never call .Sync() if the provided migration function fails.
// When called from the "doctor" command, the migration function is a version check
// that prevents the doctor from fixing anything in the database if the migration level
// is different from the expected value.
func InitEngineWithMigration(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
if err = InitEngine(ctx); err != nil {
return err
2014-04-05 16:46:32 +02:00
}
2015-01-22 13:49:52 +01:00
if err = x.Ping(); err != nil {
return err
}
preprocessDatabaseCollation(x)
// We have to run migrateFunc here in case the user is re-running installation on a previously created DB.
// If we do not then table schemas will be changed and there will be conflicts when the migrations run properly.
//
// Installation should only be being re-run if users want to recover an old database.
// However, we should think carefully about should we support re-install on an installed instance,
// as there may be other problems due to secret reinitialization.
if err = migrateFunc(x); err != nil {
return fmt.Errorf("migrate: %w", err)
2015-01-22 13:49:52 +01:00
}
if err = SyncAllTables(); err != nil {
return fmt.Errorf("sync database struct error: %w", err)
2014-02-19 10:50:53 +01:00
}
2015-01-23 08:54:16 +01:00
for _, initFunc := range initFuncs {
if err := initFunc(); err != nil {
return fmt.Errorf("initFunc failed: %w", err)
}
}
return nil
2014-02-18 23:48:02 +01:00
}
2014-03-20 21:04:56 +01:00
// NamesToBean return a list of beans or an error
func NamesToBean(names ...string) ([]any, error) {
beans := []any{}
if len(names) == 0 {
beans = append(beans, tables...)
return beans, nil
}
// Need to map provided names to beans...
beanMap := make(map[string]any)
for _, bean := range tables {
beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean
beanMap[strings.ToLower(x.TableName(bean))] = bean
beanMap[strings.ToLower(x.TableName(bean, true))] = bean
}
gotBean := make(map[any]bool)
for _, name := range names {
bean, ok := beanMap[strings.ToLower(strings.TrimSpace(name))]
if !ok {
return nil, fmt.Errorf("no table found that matches: %s", name)
}
if !gotBean[bean] {
beans = append(beans, bean)
gotBean[bean] = true
}
}
return beans, nil
}
// DumpDatabase dumps all data from database according the special database SQL syntax to file system.
func DumpDatabase(filePath, dbType string) error {
var tbs []*schemas.Table
for _, t := range tables {
t, err := x.TableInfo(t)
if err != nil {
return err
}
tbs = append(tbs, t)
}
type Version struct {
ID int64 `xorm:"pk autoincr"`
Version int64
}
t, err := x.TableInfo(&Version{})
if err != nil {
return err
}
tbs = append(tbs, t)
if len(dbType) > 0 {
return x.DumpTablesToFile(tbs, filePath, schemas.DBType(dbType))
}
return x.DumpTablesToFile(tbs, filePath)
2014-05-05 06:55:17 +02:00
}
// MaxBatchInsertSize returns the table's max batch insert size
func MaxBatchInsertSize(bean any) int {
t, err := x.TableInfo(bean)
if err != nil {
return 50
}
return 999 / len(t.ColumnsSeq())
}
// IsTableNotEmpty returns true if table has at least one record
func IsTableNotEmpty(tableName string) (bool, error) {
return x.Table(tableName).Exist()
}
// DeleteAllRecords will delete all the records of this table
func DeleteAllRecords(tableName string) error {
_, err := x.Exec(fmt.Sprintf("DELETE FROM %s", tableName))
return err
}
// GetMaxID will return max id of the table
func GetMaxID(beanOrTableName any) (maxID int64, err error) {
_, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID)
return maxID, err
}
func SetLogSQL(ctx context.Context, on bool) {
e := GetEngine(ctx)
if x, ok := e.(*xorm.Engine); ok {
x.ShowSQL(on)
} else if sess, ok := e.(*xorm.Session); ok {
sess.Engine().ShowSQL(on)
}
}
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996) (cherry picked from commit e636e9f4beca7273dd8622baedb2f0c01db30449) (cherry picked from commit bf04ae86037f5cb5a81d02750aead2742b040367) (cherry picked from commit 93b19e3568169bd1cf9b8b78c1751c3d2d65a1b6) (cherry picked from commit 83f91363ad071675c73a1f636271cc043bf69707) (cherry picked from commit e34a05bc7319072b70d387975342d617b8136655) (cherry picked from commit 68569aeee9805aaa8e98c6e7fe8058095e290061)
2023-08-18 04:39:23 +02:00
type SlowQueryHook struct {
Treshold time.Duration
Logger log.Logger
}
var _ contexts.Hook = &SlowQueryHook{}
func (SlowQueryHook) BeforeProcess(c *contexts.ContextHook) (context.Context, error) {
return c.Ctx, nil
}
func (h *SlowQueryHook) AfterProcess(c *contexts.ContextHook) error {
if c.ExecuteTime >= h.Treshold {
h.Logger.Log(8, log.WARN, "[Slow SQL Query] %s %v - %v", c.SQL, c.Args, c.ExecuteTime)
}
return nil
}
type ErrorQueryHook struct {
Logger log.Logger
}
var _ contexts.Hook = &ErrorQueryHook{}
func (ErrorQueryHook) BeforeProcess(c *contexts.ContextHook) (context.Context, error) {
return c.Ctx, nil
}
func (h *ErrorQueryHook) AfterProcess(c *contexts.ContextHook) error {
if c.Err != nil && !errors.Is(c.Err, context.Canceled) {
h.Logger.Log(8, log.ERROR, "[Error SQL Query] %s %v - %v", c.SQL, c.Args, c.Err)
}
return nil
}