mirror of
https://github.com/go-gitea/gitea
synced 2024-12-23 06:24:28 +01:00
Fix postgres ID sequences broken by recreate-table (#15015)
* Fix postgres ID sequences broken by recreate-table Unfortunately there is a subtle problem with recreatetable on postgres which leads to the sequences not being renamed and not being left at 0. Fix #14725 Signed-off-by: Andrew Thornton <art27@cantab.net> * let us try information_schema instead Signed-off-by: Andrew Thornton <art27@cantab.net> * try again Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io> Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
parent
71aca93dec
commit
070c57867d
4 changed files with 198 additions and 1 deletions
|
@ -5,10 +5,13 @@
|
||||||
package models
|
package models
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"xorm.io/builder"
|
"xorm.io/builder"
|
||||||
)
|
)
|
||||||
|
@ -315,3 +318,61 @@ func CountCommentTypeLabelWithEmptyLabel() (int64, error) {
|
||||||
func FixCommentTypeLabelWithEmptyLabel() (int64, error) {
|
func FixCommentTypeLabelWithEmptyLabel() (int64, error) {
|
||||||
return x.Where(builder.Eq{"type": CommentTypeLabel, "label_id": 0}).Delete(new(Comment))
|
return x.Where(builder.Eq{"type": CommentTypeLabel, "label_id": 0}).Delete(new(Comment))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CountBadSequences looks for broken sequences from recreate-table mistakes
|
||||||
|
func CountBadSequences() (int64, error) {
|
||||||
|
if !setting.Database.UsePostgreSQL {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sess := x.NewSession()
|
||||||
|
defer sess.Close()
|
||||||
|
|
||||||
|
var sequences []string
|
||||||
|
schema := sess.Engine().Dialect().URI().Schema
|
||||||
|
|
||||||
|
sess.Engine().SetSchema("")
|
||||||
|
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
sess.Engine().SetSchema(schema)
|
||||||
|
|
||||||
|
return int64(len(sequences)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FixBadSequences fixes for broken sequences from recreate-table mistakes
|
||||||
|
func FixBadSequences() error {
|
||||||
|
if !setting.Database.UsePostgreSQL {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sess := x.NewSession()
|
||||||
|
defer sess.Close()
|
||||||
|
if err := sess.Begin(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var sequences []string
|
||||||
|
schema := sess.Engine().Dialect().URI().Schema
|
||||||
|
|
||||||
|
sess.Engine().SetSchema("")
|
||||||
|
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sess.Engine().SetSchema(schema)
|
||||||
|
|
||||||
|
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
|
||||||
|
|
||||||
|
for _, sequence := range sequences {
|
||||||
|
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
|
||||||
|
newSequenceName := tableName + "_id_seq"
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sess.Commit()
|
||||||
|
}
|
||||||
|
|
|
@ -296,6 +296,8 @@ var migrations = []Migration{
|
||||||
NewMigration("Add time_id column to Comment", addTimeIDCommentColumn),
|
NewMigration("Add time_id column to Comment", addTimeIDCommentColumn),
|
||||||
// v174 -> v175
|
// v174 -> v175
|
||||||
NewMigration("create repo transfer table", addRepoTransfer),
|
NewMigration("create repo transfer table", addRepoTransfer),
|
||||||
|
// v175 -> v176
|
||||||
|
NewMigration("Fix Postgres ID Sequences broken by recreate-table", fixPostgresIDSequences),
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCurrentDBVersion returns the current db version
|
// GetCurrentDBVersion returns the current db version
|
||||||
|
@ -565,6 +567,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case setting.Database.UsePostgreSQL:
|
case setting.Database.UsePostgreSQL:
|
||||||
|
var originalSequences []string
|
||||||
|
type sequenceData struct {
|
||||||
|
LastValue int `xorm:"'last_value'"`
|
||||||
|
IsCalled bool `xorm:"'is_called'"`
|
||||||
|
}
|
||||||
|
sequenceMap := map[string]sequenceData{}
|
||||||
|
|
||||||
|
schema := sess.Engine().Dialect().URI().Schema
|
||||||
|
sess.Engine().SetSchema("")
|
||||||
|
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
|
||||||
|
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sess.Engine().SetSchema(schema)
|
||||||
|
|
||||||
|
for _, sequence := range originalSequences {
|
||||||
|
sequenceData := sequenceData{}
|
||||||
|
if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
|
||||||
|
log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sequenceMap[sequence] = sequenceData
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
// CASCADE causes postgres to drop all the constraints on the old table
|
// CASCADE causes postgres to drop all the constraints on the old table
|
||||||
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
|
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
|
||||||
log.Error("Unable to drop old table %s. Error: %v", tableName, err)
|
log.Error("Unable to drop old table %s. Error: %v", tableName, err)
|
||||||
|
@ -578,7 +605,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
var indices []string
|
var indices []string
|
||||||
schema := sess.Engine().Dialect().URI().Schema
|
|
||||||
sess.Engine().SetSchema("")
|
sess.Engine().SetSchema("")
|
||||||
if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
|
if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
|
||||||
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
|
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
|
||||||
|
@ -594,6 +620,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var sequences []string
|
||||||
|
sess.Engine().SetSchema("")
|
||||||
|
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
|
||||||
|
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sess.Engine().SetSchema(schema)
|
||||||
|
|
||||||
|
for _, sequence := range sequences {
|
||||||
|
newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
|
||||||
|
log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
val, ok := sequenceMap[newSequenceName]
|
||||||
|
if newSequenceName == tableName+"_id_seq" {
|
||||||
|
if ok && val.LastValue != 0 {
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
|
||||||
|
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// We're going to try to guess this
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
|
||||||
|
log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if ok {
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
|
||||||
|
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
case setting.Database.UseMSSQL:
|
case setting.Database.UseMSSQL:
|
||||||
// MSSQL will drop all the constraints on the old table
|
// MSSQL will drop all the constraints on the old table
|
||||||
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
|
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
|
||||||
|
|
53
models/migrations/v175.go
Normal file
53
models/migrations/v175.go
Normal file
|
@ -0,0 +1,53 @@
|
||||||
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package migrations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"xorm.io/xorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fixPostgresIDSequences(x *xorm.Engine) error {
|
||||||
|
if !setting.Database.UsePostgreSQL {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sess := x.NewSession()
|
||||||
|
defer sess.Close()
|
||||||
|
if err := sess.Begin(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var sequences []string
|
||||||
|
schema := sess.Engine().Dialect().URI().Schema
|
||||||
|
|
||||||
|
sess.Engine().SetSchema("")
|
||||||
|
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
|
||||||
|
log.Error("Unable to find sequences: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sess.Engine().SetSchema(schema)
|
||||||
|
|
||||||
|
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
|
||||||
|
|
||||||
|
for _, sequence := range sequences {
|
||||||
|
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
|
||||||
|
newSequenceName := tableName + "_id_seq"
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
|
||||||
|
log.Error("Unable to rename %s to %s. Error: %v", sequence, newSequenceName, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
|
||||||
|
log.Error("Unable to reset sequence %s for %s. Error: %v", newSequenceName, tableName, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sess.Commit()
|
||||||
|
}
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/models/migrations"
|
"code.gitea.io/gitea/models/migrations"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
)
|
)
|
||||||
|
|
||||||
func checkDBConsistency(logger log.Logger, autofix bool) error {
|
func checkDBConsistency(logger log.Logger, autofix bool) error {
|
||||||
|
@ -131,6 +132,25 @@ func checkDBConsistency(logger log.Logger, autofix bool) error {
|
||||||
}
|
}
|
||||||
// TODO: function to recalc all counters
|
// TODO: function to recalc all counters
|
||||||
|
|
||||||
|
if setting.Database.UsePostgreSQL {
|
||||||
|
count, err = models.CountBadSequences()
|
||||||
|
if err != nil {
|
||||||
|
logger.Critical("Error: %v whilst checking sequence values")
|
||||||
|
}
|
||||||
|
if count > 0 {
|
||||||
|
if autofix {
|
||||||
|
err := models.FixBadSequences()
|
||||||
|
if err != nil {
|
||||||
|
logger.Critical("Error: %v whilst attempting to fix sequences")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logger.Info("%d sequences updated", count)
|
||||||
|
} else {
|
||||||
|
logger.Warn("%d sequences with incorrect values", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue