LordMZTE/mautrix-whatsapp
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move Matrix state store to main database. Fixes #64

Browse source
This commit is contained in:
Tulir Asokan 2019-08-25 16:29:35 +03:00
parent 15b0798cdd
commit ab91d326fd
9 changed files with 370 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
database/database.go
View file

@ -29,7 +29,8 @@ import (
type Database struct {
*sql.DB
log log.Logger
log log.Logger
dialect string
User *UserQuery
Portal *PortalQuery
@ -48,8 +49,9 @@ func New(dbType string, uri string) (*Database, error) {
}
db := &Database{
DB: conn,
log: log.Sub("Database"),
DB: conn,
log: log.Sub("Database"),
dialect: dbType,
}
db.User = &UserQuery{
db: db,
@ -70,8 +72,8 @@ func New(dbType string, uri string) (*Database, error) {
return db, nil
}
func (db *Database) Init(dialectName string) error {
return upgrades.Run(db.log.Sub("Upgrade"), dialectName, db.DB)
func (db *Database) Init() error {
return upgrades.Run(db.log.Sub("Upgrade"), db.dialect, db.DB)
}
type Scannable interface {

240
database/statestore.go Normal file
View file

@ -0,0 +1,240 @@
// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
// Copyright (C) 2019 Tulir Asokan
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package database
import (
"encoding/json"
"fmt"
"sync"
log "maunium.net/go/maulogger/v2"
"maunium.net/go/mautrix"
"maunium.net/go/mautrix-appservice"
)
type SQLStateStore struct {
*appservice.TypingStateStore
db *Database
log log.Logger
Typing map[string]map[string]int64
typingLock sync.RWMutex
}
func NewSQLStateStore(db *Database) *SQLStateStore {
return &SQLStateStore{
TypingStateStore: appservice.NewTypingStateStore(),
db: db,
log: log.Sub("StateStore"),
}
}
func (store *SQLStateStore) IsRegistered(userID string) bool {
row := store.db.QueryRow("SELECT EXISTS(SELECT 1 FROM mx_registrations WHERE user_id=$1)", userID)
var isRegistered bool
err := row.Scan(&isRegistered)
if err != nil {
store.log.Warnfln("Failed to scan registration existence for %s: %v", userID, err)
}
return isRegistered
}
func (store *SQLStateStore) MarkRegistered(userID string) {
var err error
if store.db.dialect == "postgres" {
_, err = store.db.Exec("INSERT INTO mx_registrations (user_id) VALUES ($1) ON CONFLICT (user_id) DO NOTHING", userID)
} else if store.db.dialect == "sqlite3" {
_, err = store.db.Exec("INSERT OR REPLACE INTO mx_registrations (user_id) VALUES ($1)", userID)
} else {
err = fmt.Errorf("unsupported dialect %s", store.db.dialect)
}
if err != nil {
store.log.Warnfln("Failed to mark %s as registered: %v", userID, err)
}
}
func (store *SQLStateStore) GetRoomMemberships(roomID string) map[string]mautrix.Membership {
memberships := make(map[string]mautrix.Membership)
rows, err := store.db.Query("SELECT user_id, membership FROM mx_user_profile WHERE room_id=$1", roomID)
if err != nil {
return memberships
}
var userID string
var membership mautrix.Membership
for rows.Next() {
err := rows.Scan(&userID, &membership)
if err != nil {
store.log.Warnfln("Failed to scan membership in %s: %v", roomID, err)
} else {
memberships[userID] = membership
}
}
return memberships
}
func (store *SQLStateStore) GetMembership(roomID, userID string) mautrix.Membership {
row := store.db.QueryRow("SELECT membership FROM mx_user_profile WHERE room_id=$1 AND user_id=$2", roomID, userID)
membership := mautrix.MembershipLeave
if row != nil && row.Scan(&membership) != nil {
store.log.Warnln("Failed to scan membership of %s in %s: %v", userID, roomID, membership)
}
return membership
}
func (store *SQLStateStore) IsInRoom(roomID, userID string) bool {
return store.IsMembership(roomID, userID, "join")
}
func (store *SQLStateStore) IsInvited(roomID, userID string) bool {
return store.IsMembership(roomID, userID, "join", "invite")
}
func (store *SQLStateStore) IsMembership(roomID, userID string, allowedMemberships ...mautrix.Membership) bool {
membership := store.GetMembership(roomID, userID)
for _, allowedMembership := range allowedMemberships {
if allowedMembership == membership {
return true
}
}
return false
}
func (store *SQLStateStore) SetMembership(roomID, userID string, membership mautrix.Membership) {
var err error
if store.db.dialect == "postgres" {
_, err = store.db.Exec(`INSERT INTO mx_user_profile (room_id, user_id, membership) VALUES ($1, $2, $3)
ON CONFLICT (room_id, user_id) DO UPDATE SET membership=$3`, roomID, userID, membership)
} else if store.db.dialect == "sqlite3" {
_, err = store.db.Exec("INSERT OR REPLACE INTO mx_user_profile (room_id, user_id, membership) VALUES ($1, $2, $3)", roomID, userID, membership)
} else {
err = fmt.Errorf("unsupported dialect %s", store.db.dialect)
}
if err != nil {
store.log.Warnfln("Failed to set membership of %s in %s to %s: %v", userID, roomID, membership, err)
}
}
func (store *SQLStateStore) SetPowerLevels(roomID string, levels *mautrix.PowerLevels) {
levelsBytes, err := json.Marshal(levels)
if err != nil {
store.log.Errorfln("Failed to marshal power levels of %s: %v", roomID, err)
return
}
if store.db.dialect == "postgres" {
_, err = store.db.Exec(`INSERT INTO mx_room_state (room_id, power_levels) VALUES ($1, $2)
ON CONFLICT (room_id) DO UPDATE SET power_levels=$2`, roomID, levelsBytes)
} else if store.db.dialect == "sqlite3" {
_, err = store.db.Exec("INSERT OR REPLACE INTO mx_room_state (room_id, power_levels) VALUES ($1, $2)", roomID, levelsBytes)
} else {
err = fmt.Errorf("unsupported dialect %s", store.db.dialect)
}
if err != nil {
store.log.Warnfln("Failed to store power levels of %s: %v", roomID, err)
}
}
func (store *SQLStateStore) GetPowerLevels(roomID string) (levels *mautrix.PowerLevels) {
row := store.db.QueryRow("SELECT power_levels FROM mx_room_state WHERE room_id=$1", roomID)
if row == nil {
return
}
var data []byte
err := row.Scan(&data)
if err != nil {
store.log.Errorln("Failed to scan power levels of %s: %v", roomID, err)
return
}
levels = &mautrix.PowerLevels{}
err = json.Unmarshal(data, levels)
if err != nil {
store.log.Errorln("Failed to parse power levels of %s: %v", roomID, err)
return nil
}
return
}
func (store *SQLStateStore) GetPowerLevel(roomID, userID string) int {
if store.db.dialect == "postgres" {
row := store.db.QueryRow(`SELECT
COALESCE((power_levels->'users'->$2)::int, (power_levels->'users_default')::int, 0)
FROM mx_room_state WHERE room_id=$1`, roomID, userID)
if row == nil {
// Power levels not in db
return 0
}
var powerLevel int
err := row.Scan(&powerLevel)
if err != nil {
store.log.Errorln("Failed to scan power level of %s in %s: %v", userID, roomID, err)
}
return powerLevel
}
return store.GetPowerLevels(roomID).GetUserLevel(userID)
}
func (store *SQLStateStore) GetPowerLevelRequirement(roomID string, eventType mautrix.EventType) int {
if store.db.dialect == "postgres" {
defaultType := "events_default"
defaultValue := 0
if eventType.IsState() {
defaultType = "state_default"
defaultValue = 50
}
row := store.db.QueryRow(`SELECT
COALESCE((power_levels->'events'->$2)::int, (power_levels->'$3')::int, $4)
FROM mx_room_state WHERE room_id=$1`, roomID, eventType.Type, defaultType, defaultValue)
if row == nil {
// Power levels not in db
return defaultValue
}
var powerLevel int
err := row.Scan(&powerLevel)
if err != nil {
store.log.Errorln("Failed to scan power level for %s in %s: %v", eventType, roomID, err)
}
return powerLevel
}
return store.GetPowerLevels(roomID).GetEventLevel(eventType)
}
func (store *SQLStateStore) HasPowerLevel(roomID, userID string, eventType mautrix.EventType) bool {
if store.db.dialect == "postgres" {
defaultType := "events_default"
defaultValue := 0
if eventType.IsState() {
defaultType = "state_default"
defaultValue = 50
}
row := store.db.QueryRow(`SELECT
COALESCE((power_levels->'users'->$2)::int, (power_levels->'users_default')::int, 0)
>=
COALESCE((power_levels->'events'->$3)::int, (power_levels->'$4')::int, $5)
FROM mx_room_state WHERE room_id=$1`, roomID, userID, eventType.Type, defaultType, defaultValue)
if row == nil {
// Power levels not in db
return defaultValue == 0
}
var hasPower bool
err := row.Scan(&hasPower)
if err != nil {
store.log.Errorln("Failed to scan power level for %s in %s: %v", eventType, roomID, err)
}
return hasPower
}
return store.GetPowerLevel(roomID, userID) >= store.GetPowerLevelRequirement(roomID, eventType)
}

107
database/upgrades/2019-08-25-move-state-store-to-db.go Normal file
View file

@ -0,0 +1,107 @@
package upgrades
import (
"database/sql"
"encoding/json"
"fmt"
"io/ioutil"
"strings"
"maunium.net/go/mautrix"
"maunium.net/go/mautrix-appservice"
)
func init() {
migrateRegistrations := func(tx *sql.Tx, registrations map[string]bool) error {
values := make([]interface{}, 0, len(registrations))
valueStrings := make([]string, 0, len(registrations))
i := 1
for userID, registered := range registrations {
if registered {
values = append(values, userID)
valueStrings = append(valueStrings, fmt.Sprintf("($%d)", i))
i++
}
}
valueString := strings.Join(valueStrings, ",")
_, err := tx.Exec("INSERT INTO mx_registrations (user_id) VALUES "+valueString, values...)
return err
}
migrateMemberships := func(tx *sql.Tx, rooms map[string]map[string]mautrix.Membership) error {
var values []interface{}
var valueStrings []string
i := 1
for roomID, members := range rooms {
for userID, membership := range members {
values = append(values, roomID, userID, membership)
valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", i, i+1, i+2))
i += 3
}
}
valueString := strings.Join(valueStrings, ",")
_, err := tx.Exec("INSERT INTO mx_user_profile (room_id, user_id, membership) VALUES "+valueString, values...)
return err
}
migratePowerLevels := func(tx *sql.Tx, rooms map[string]*mautrix.PowerLevels) error {
var values []interface{}
var valueStrings []string
i := 1
for roomID, powerLevels := range rooms {
powerLevelBytes, err := json.Marshal(powerLevels)
if err != nil {
return err
}
values = append(values, roomID, powerLevelBytes)
valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d)", i, i+1))
i += 2
}
valueString := strings.Join(valueStrings, ",")
_, err := tx.Exec("INSERT INTO mx_room_state (room_id, power_levels) VALUES "+valueString, values...)
return err
}
userProfileTable := `CREATE TABLE mx_user_profile (
room_id VARCHAR(255),
user_id VARCHAR(255),
membership VARCHAR(15) NOT NULL,
PRIMARY KEY (room_id, user_id)
)`
roomStateTable := `CREATE TABLE mx_room_state (
room_id VARCHAR(255) PRIMARY KEY,
power_levels TEXT
)`
registrationsTable := `CREATE TABLE mx_registrations (
user_id VARCHAR(255) PRIMARY KEY
)`
upgrades[9] = upgrade{"Move state store to main DB", func(dialect Dialect, tx *sql.Tx, db *sql.DB) error {
store := appservice.NewBasicStateStore().(*appservice.BasicStateStore)
if dialect == Postgres {
roomStateTable = strings.Replace(roomStateTable, "TEXT", "JSONB", 1)
}
if data, err := ioutil.ReadFile("mx-state.json"); err != nil {
return err
} else if err = json.Unmarshal(data, &store); err != nil {
return err
} else if _, err := tx.Exec(userProfileTable); err != nil {
return err
} else if _, err = tx.Exec(roomStateTable); err != nil {
return err
} else if _, err = tx.Exec(registrationsTable); err != nil {
return err
} else if err = migrateRegistrations(tx, store.Registrations); err != nil {
return err
} else if err = migrateMemberships(tx, store.Memberships); err != nil {
return err
} else if err = migratePowerLevels(tx, store.PowerLevels); err != nil {
return err
}
return nil
}}
}

2
database/upgrades/upgrades.go
View file

@ -22,7 +22,7 @@ type upgrade struct {
fn upgradeFunc
}
const NumberOfUpgrades = 9
const NumberOfUpgrades = 10
var upgrades [NumberOfUpgrades]upgrade

3
example-config.yaml
View file

@ -27,9 +27,6 @@ appservice:
max_open_conns: 20
max_idle_conns: 2
# Path to the Matrix room state store.
state_store_path: ./mx-state.json
# The unique ID of this appservice.
id: whatsapp
# Appservice bot details.

6
go.mod
View file

@ -11,11 +11,11 @@ require (
gopkg.in/yaml.v2 v2.2.2
maunium.net/go/mauflag v1.0.0
maunium.net/go/maulogger/v2 v2.0.0
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190622085722-6406f15cb8e3
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190824195529-3507711d75aa
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190825132810-9d870654e9d2
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190825132837-ffcc1e3c7ed7
)
replace (
github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.0.2-0.20190824212315-7c97464c8c54
github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.0.2-0.20190825095058-bb85c7370264
gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1
)

6
go.sum
View file

@ -28,6 +28,8 @@ github.com/tulir/go-whatsapp v0.0.2-0.20190824193805-f9a7dc8e6a5f h1:a6GstbsMbzd
github.com/tulir/go-whatsapp v0.0.2-0.20190824193805-f9a7dc8e6a5f/go.mod h1:u3Hdptbz3iB5y/NEoSKgsp9hBzUlm0A5OrLMVdENAX8=
github.com/tulir/go-whatsapp v0.0.2-0.20190824212315-7c97464c8c54 h1:Pe+DJtXEbTe+FhmOxrY74EbqfqGjZTxxaU3iLMqo0CA=
github.com/tulir/go-whatsapp v0.0.2-0.20190824212315-7c97464c8c54/go.mod h1:u3Hdptbz3iB5y/NEoSKgsp9hBzUlm0A5OrLMVdENAX8=
github.com/tulir/go-whatsapp v0.0.2-0.20190825095058-bb85c7370264 h1:tzi3IcrgOUdC+eSE5J5GHoeMeVK5uTT+YR21m+EPgzY=
github.com/tulir/go-whatsapp v0.0.2-0.20190825095058-bb85c7370264/go.mod h1:u3Hdptbz3iB5y/NEoSKgsp9hBzUlm0A5OrLMVdENAX8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -50,9 +52,13 @@ maunium.net/go/maulogger/v2 v2.0.0/go.mod h1:Hbbkq3NV6jvJodByZu1mgEF3fpT7Kz9z0Mj
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190606153009-ca5d9535b6cc/go.mod h1:O+QWJP3H7BZEzIBSrECKpnpRnEKBwaoWVEu/yZwVwxg=
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190622085722-6406f15cb8e3 h1:oVabjOi2rV7qb9IHUm/SVKjbTcLyCVGw2FCsvulDsZg=
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190622085722-6406f15cb8e3/go.mod h1:O+QWJP3H7BZEzIBSrECKpnpRnEKBwaoWVEu/yZwVwxg=
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190825132810-9d870654e9d2 h1:0iVxLLAOSBqtJqhIjW9EbblMsaSYoCJRo5mHPZnytUk=
maunium.net/go/mautrix v0.1.0-alpha.3.0.20190825132810-9d870654e9d2/go.mod h1:O+QWJP3H7BZEzIBSrECKpnpRnEKBwaoWVEu/yZwVwxg=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190618052224-6e6c9bb47548 h1:ni1nqs+2AOO+g1ND6f2W0pMcb6sIDVqzerXosO+pI2g=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190618052224-6e6c9bb47548/go.mod h1:yVWU0gvIHIXClgyVnShiufiDksFbFrBqHG9lDAYcmGI=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190822210104-3e49344e186b h1:/03X0PPgtk4pqXcdH86xMzOl891whG5A1hFXQ+xXons=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190822210104-3e49344e186b/go.mod h1:yVWU0gvIHIXClgyVnShiufiDksFbFrBqHG9lDAYcmGI=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190824195529-3507711d75aa h1:saXgLXgTM5wbiY4+GMbEfaN+boCdGUBcDTIle6BemlI=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190824195529-3507711d75aa/go.mod h1:PZTnFsEUt4YALO/OGGyWXeQulBVGEPjI8FQ88XCbu/w=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190825132837-ffcc1e3c7ed7 h1:0B3zGyfNVxdCcoFDgvDh8CW9U0gkCDvwKbP0a0UIUEQ=
maunium.net/go/mautrix-appservice v0.1.0-alpha.3.0.20190825132837-ffcc1e3c7ed7/go.mod h1:FJRRpH5+p3wCfEt6u/3kMeu9aGX/pk2PqtvjRDRW74w=

21
main.go
View file

@ -74,7 +74,7 @@ type Bridge struct {
Config *config.Config
DB *database.Database
Log log.Logger
StateStore *AutosavingStateStore
StateStore *database.SQLStateStore
Bot *appservice.IntentAPI
Formatter *Formatter
@ -157,15 +157,6 @@ func (bridge *Bridge) Init() {
}
bridge.AS.Log = log.Sub("Matrix")
bridge.Log.Debugln("Initializing state store")
bridge.StateStore = NewAutosavingStateStore(bridge.Config.AppService.StateStore)
err = bridge.StateStore.Load()
if err != nil {
bridge.Log.Fatalln("Failed to load state store:", err)
os.Exit(13)
}
bridge.AS.StateStore = bridge.StateStore
bridge.Log.Debugln("Initializing database")
bridge.DB, err = database.New(bridge.Config.AppService.Database.Type, bridge.Config.AppService.Database.URI)
if err != nil && (err != upgrades.UnsupportedDatabaseVersion || !*ignoreUnsupportedDatabase) {
@ -173,6 +164,10 @@ func (bridge *Bridge) Init() {
os.Exit(14)
}
bridge.Log.Debugln("Initializing state store")
bridge.StateStore = database.NewSQLStateStore(bridge.DB)
bridge.AS.StateStore = bridge.StateStore
bridge.DB.SetMaxOpenConns(bridge.Config.AppService.Database.MaxOpenConns)
bridge.DB.SetMaxIdleConns(bridge.Config.AppService.Database.MaxIdleConns)
@ -184,7 +179,7 @@ func (bridge *Bridge) Init() {
}
func (bridge *Bridge) Start() {
err := bridge.DB.Init(bridge.Config.AppService.Database.Type)
err := bridge.DB.Init()
if err != nil {
bridge.Log.Fatalln("Failed to initialize database:", err)
os.Exit(15)
@ -255,10 +250,6 @@ func (bridge *Bridge) Stop() {
user.SetSession(&sess)
}
}
err := bridge.StateStore.Save()
if err != nil {
bridge.Log.Warnln("Failed to save state store:", err)
}
}
func (bridge *Bridge) Main() {

78
statestore.go
View file

@ -1,78 +0,0 @@
// mautrix-whatsapp - A Matrix-WhatsApp puppeting bridge.
// Copyright (C) 2019 Tulir Asokan
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package main
import (
"encoding/json"
"io/ioutil"
"os"
"maunium.net/go/mautrix-appservice"
"maunium.net/go/mautrix"
)
type AutosavingStateStore struct {
appservice.StateStore
Path string
}
func NewAutosavingStateStore(path string) *AutosavingStateStore {
return &AutosavingStateStore{
StateStore: appservice.NewBasicStateStore(),
Path: path,
}
}
func (store *AutosavingStateStore) Save() error {
store.RLock()
defer store.RUnlock()
data, err := json.Marshal(store.StateStore)
if err != nil {
return err
}
return ioutil.WriteFile(store.Path, data, 0600)
}
func (store *AutosavingStateStore) Load() error {
store.Lock()
defer store.Unlock()
data, err := ioutil.ReadFile(store.Path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
return json.Unmarshal(data, store.StateStore)
}
func (store *AutosavingStateStore) MarkRegistered(userID string) {
store.StateStore.MarkRegistered(userID)
store.Save()
}
func (store *AutosavingStateStore) SetMembership(roomID, userID string, membership mautrix.Membership) {
store.StateStore.SetMembership(roomID, userID, membership)
store.Save()
}
func (store *AutosavingStateStore) SetPowerLevels(roomID string, levels *mautrix.PowerLevels) {
store.StateStore.SetPowerLevels(roomID, levels)
store.Save()
}