Make signal store retry if postgres dies

database/database.go

@@ -18,14 +18,18 @@ package database
 
 import (
 	"database/sql"
+	"errors"
 	"fmt"
+	"net"
 	"time"
 
 	"github.com/lib/pq"
 	_ "github.com/mattn/go-sqlite3"
-	"go.mau.fi/whatsmeow/store/sqlstore"
 	log "maunium.net/go/maulogger/v2"
 
+	"go.mau.fi/whatsmeow/store"
+	"go.mau.fi/whatsmeow/store/sqlstore"
+
 	"maunium.net/go/mautrix-whatsapp/config"
 	"maunium.net/go/mautrix-whatsapp/database/upgrades"
 )
@@ -120,3 +124,28 @@ func (db *Database) Init() error {
 type Scannable interface {
 	Scan(...interface{}) error
 }
+
+func isRetryableError(err error) bool {
+	if pqError := (&pq.Error{}); errors.As(err, &pqError) {
+		switch pqError.Code.Class() {
+		case "08", // Connection Exception
+			"53", // Insufficient Resources (e.g. too many connections)
+			"57": // Operator Intervention (e.g. server restart)
+			return true
+		}
+	} else if netError := (&net.OpError{}); errors.As(err, &netError) {
+		return true
+	}
+	return false
+}
+
+func (db *Database) HandleSignalStoreError(device *store.Device, action string, attemptIndex int, err error) (retry bool) {
+	if db.dialect != "sqlite" && isRetryableError(err) {
+		sleepTime := time.Duration(attemptIndex*2) * time.Second
+		device.Log.Warnf("Failed to %s (attempt #%d): %v - retrying in %v", action, attemptIndex+1, err, sleepTime)
+		time.Sleep(sleepTime)
+		return true
+	}
+	device.Log.Errorf("Failed to %s: %v", action, err)
+	return false
+}

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/prometheus/client_golang v1.11.1
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/tidwall/gjson v1.14.1
-	go.mau.fi/whatsmeow v0.0.0-20220428170557-dafd80811aec
+	go.mau.fi/whatsmeow v0.0.0-20220429151409-d4e97cef00e0
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
 	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
 	google.golang.org/protobuf v1.28.0

go.sum

@@ -120,8 +120,8 @@ github.com/yuin/goldmark v1.4.12 h1:6hffw6vALvEDqJ19dOJvJKOoAOKe4NDaTqvd2sktGN0=
 github.com/yuin/goldmark v1.4.12/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mau.fi/libsignal v0.0.0-20220425070825-c40c839ee6a0 h1:3IQF2bgAyibdo77hTejwuJe4jlypj9QaE4xCQuxrThM=
 go.mau.fi/libsignal v0.0.0-20220425070825-c40c839ee6a0/go.mod h1:kBOXTvYyDG/q1Ihgvd4J6WenGPh7wtEGvPKF6vmf5ak=
-go.mau.fi/whatsmeow v0.0.0-20220428170557-dafd80811aec h1:uiWnHE3Vv8FiyRyOEMH21k8azkw8u8UHgadfmEWWCo0=
+go.mau.fi/whatsmeow v0.0.0-20220429151409-d4e97cef00e0 h1:++OC/3EYrPhxWSD8bo2varzXIx2UjRS9dR9Lo+yH2X0=
-go.mau.fi/whatsmeow v0.0.0-20220428170557-dafd80811aec/go.mod h1:iUBgOLNaqShLrR17u0kIiRptIGFH+nbT1tRhaWBEX/c=
+go.mau.fi/whatsmeow v0.0.0-20220429151409-d4e97cef00e0/go.mod h1:iUBgOLNaqShLrR17u0kIiRptIGFH+nbT1tRhaWBEX/c=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

main.go

@@ -264,6 +264,7 @@ func (bridge *Bridge) Init() {
 	bridge.AS.StateStore = bridge.StateStore
 
 	bridge.WAContainer = sqlstore.NewWithDB(bridge.DB.DB, bridge.Config.AppService.Database.Type, nil)
+	bridge.WAContainer.DatabaseErrorHandler = bridge.DB.HandleSignalStoreError
 
 	ss := bridge.Config.AppService.Provisioning.SharedSecret
 	if len(ss) > 0 && ss != "disable" {

user.go

@@ -170,6 +170,7 @@ func (bridge *Bridge) loadDBUser(dbUser *database.User, mxid *id.UserID) *User {
 			user.JID = types.EmptyJID
 			user.Update()
 		} else {
+			user.Session.Log = &waLogger{user.log.Sub("Session")}
 			bridge.usersByUsername[user.JID.User] = user
 		}
 	}