mirror of
https://github.com/matrix-org/dendrite
synced 2024-12-24 05:33:47 +01:00
5caae6f3a0
* bugfix: fix panic on new invite events from sytest I'm unsure why the previous code didn't work, but it's clearer, quicker and easier to read the `LastInsertID()` way. Previously, the code would panic as the SELECT would fail to find the last inserted row ID. * sqlite: Fix UNIQUE violations and close more cursors - Add missing `defer rows.Close()` - Do not have the state block NID as a PRIMARY KEY else it breaks for blocks with >1 state event in them. Instead, rejig the queries so we can still have monotonically increasing integers without using AUTOINCREMENT (which mandates PRIMARY KEY). * sqlite: Add missing variadic function * Use LastInsertId because empirically it works over the SELECT form (though I don't know why that is) * sqlite: Fix invite table by using the global stream pos rather than one specific to invites If we don't use the global, clients don't get notified about any invites because the position is too low. * linting: shadowing * sqlite: do not use last rowid, we already know the stream pos! * sqlite: Fix account data table in syncapi by commiting insert txns! * sqlite: Fix failing federation invite Was failing with 'database is locked' due to multiple write txns being taken out. * sqlite: Ensure we return exactly the number of events found in the database Previously we would return exactly the number of *requested* events, which meant that several zero-initialised events would bubble through the system, failing at JSON serialisation time. * sqlite: let's just ignore the problem for now.... * linting
284 lines
8.2 KiB
Go
284 lines
8.2 KiB
Go
// Copyright 2017 Vector Creations Ltd
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package consumers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
"github.com/matrix-org/dendrite/common"
|
|
"github.com/matrix-org/dendrite/common/config"
|
|
"github.com/matrix-org/dendrite/roomserver/api"
|
|
"github.com/matrix-org/dendrite/syncapi/storage"
|
|
"github.com/matrix-org/dendrite/syncapi/sync"
|
|
"github.com/matrix-org/dendrite/syncapi/types"
|
|
"github.com/matrix-org/gomatrixserverlib"
|
|
log "github.com/sirupsen/logrus"
|
|
sarama "gopkg.in/Shopify/sarama.v1"
|
|
)
|
|
|
|
// OutputRoomEventConsumer consumes events that originated in the room server.
|
|
type OutputRoomEventConsumer struct {
|
|
roomServerConsumer *common.ContinualConsumer
|
|
db storage.Database
|
|
notifier *sync.Notifier
|
|
query api.RoomserverQueryAPI
|
|
}
|
|
|
|
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
|
|
func NewOutputRoomEventConsumer(
|
|
cfg *config.Dendrite,
|
|
kafkaConsumer sarama.Consumer,
|
|
n *sync.Notifier,
|
|
store storage.Database,
|
|
queryAPI api.RoomserverQueryAPI,
|
|
) *OutputRoomEventConsumer {
|
|
|
|
consumer := common.ContinualConsumer{
|
|
Topic: string(cfg.Kafka.Topics.OutputRoomEvent),
|
|
Consumer: kafkaConsumer,
|
|
PartitionStore: store,
|
|
}
|
|
s := &OutputRoomEventConsumer{
|
|
roomServerConsumer: &consumer,
|
|
db: store,
|
|
notifier: n,
|
|
query: queryAPI,
|
|
}
|
|
consumer.ProcessMessage = s.onMessage
|
|
|
|
return s
|
|
}
|
|
|
|
// Start consuming from room servers
|
|
func (s *OutputRoomEventConsumer) Start() error {
|
|
return s.roomServerConsumer.Start()
|
|
}
|
|
|
|
// onMessage is called when the sync server receives a new event from the room server output log.
|
|
// It is not safe for this function to be called from multiple goroutines, or else the
|
|
// sync stream position may race and be incorrectly calculated.
|
|
func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
|
|
// Parse out the event JSON
|
|
var output api.OutputEvent
|
|
if err := json.Unmarshal(msg.Value, &output); err != nil {
|
|
// If the message was invalid, log it and move on to the next message in the stream
|
|
log.WithError(err).Errorf("roomserver output log: message parse failure")
|
|
return nil
|
|
}
|
|
|
|
switch output.Type {
|
|
case api.OutputTypeNewRoomEvent:
|
|
return s.onNewRoomEvent(context.TODO(), *output.NewRoomEvent)
|
|
case api.OutputTypeNewInviteEvent:
|
|
return s.onNewInviteEvent(context.TODO(), *output.NewInviteEvent)
|
|
case api.OutputTypeRetireInviteEvent:
|
|
return s.onRetireInviteEvent(context.TODO(), *output.RetireInviteEvent)
|
|
default:
|
|
log.WithField("type", output.Type).Debug(
|
|
"roomserver output log: ignoring unknown output type",
|
|
)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func (s *OutputRoomEventConsumer) onNewRoomEvent(
|
|
ctx context.Context, msg api.OutputNewRoomEvent,
|
|
) error {
|
|
ev := msg.Event
|
|
log.WithFields(log.Fields{
|
|
"event_id": ev.EventID(),
|
|
"room_id": ev.RoomID(),
|
|
}).Info("received event from roomserver")
|
|
|
|
addsStateEvents, err := s.lookupStateEvents(msg.AddsStateEventIDs, ev)
|
|
if err != nil {
|
|
log.WithFields(log.Fields{
|
|
"event": string(ev.JSON()),
|
|
log.ErrorKey: err,
|
|
"add": msg.AddsStateEventIDs,
|
|
"del": msg.RemovesStateEventIDs,
|
|
}).Panicf("roomserver output log: state event lookup failure")
|
|
}
|
|
|
|
ev, err = s.updateStateEvent(ev)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for i := range addsStateEvents {
|
|
addsStateEvents[i], err = s.updateStateEvent(addsStateEvents[i])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
pduPos, err := s.db.WriteEvent(
|
|
ctx,
|
|
&ev,
|
|
addsStateEvents,
|
|
msg.AddsStateEventIDs,
|
|
msg.RemovesStateEventIDs,
|
|
msg.TransactionID,
|
|
false,
|
|
)
|
|
if err != nil {
|
|
// panic rather than continue with an inconsistent database
|
|
log.WithFields(log.Fields{
|
|
"event": string(ev.JSON()),
|
|
log.ErrorKey: err,
|
|
"add": msg.AddsStateEventIDs,
|
|
"del": msg.RemovesStateEventIDs,
|
|
}).Panicf("roomserver output log: write event failure")
|
|
return nil
|
|
}
|
|
s.notifier.OnNewEvent(&ev, "", nil, types.PaginationToken{PDUPosition: pduPos})
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *OutputRoomEventConsumer) onNewInviteEvent(
|
|
ctx context.Context, msg api.OutputNewInviteEvent,
|
|
) error {
|
|
pduPos, err := s.db.AddInviteEvent(ctx, msg.Event)
|
|
if err != nil {
|
|
// panic rather than continue with an inconsistent database
|
|
log.WithFields(log.Fields{
|
|
"event": string(msg.Event.JSON()),
|
|
"pdupos": pduPos,
|
|
log.ErrorKey: err,
|
|
}).Panicf("roomserver output log: write invite failure")
|
|
return nil
|
|
}
|
|
s.notifier.OnNewEvent(&msg.Event, "", nil, types.PaginationToken{PDUPosition: pduPos})
|
|
return nil
|
|
}
|
|
|
|
func (s *OutputRoomEventConsumer) onRetireInviteEvent(
|
|
ctx context.Context, msg api.OutputRetireInviteEvent,
|
|
) error {
|
|
err := s.db.RetireInviteEvent(ctx, msg.EventID)
|
|
if err != nil {
|
|
// panic rather than continue with an inconsistent database
|
|
log.WithFields(log.Fields{
|
|
"event_id": msg.EventID,
|
|
log.ErrorKey: err,
|
|
}).Panicf("roomserver output log: remove invite failure")
|
|
return nil
|
|
}
|
|
// TODO: Notify any active sync requests that the invite has been retired.
|
|
// s.notifier.OnNewEvent(nil, msg.TargetUserID, syncStreamPos)
|
|
return nil
|
|
}
|
|
|
|
// lookupStateEvents looks up the state events that are added by a new event.
|
|
func (s *OutputRoomEventConsumer) lookupStateEvents(
|
|
addsStateEventIDs []string, event gomatrixserverlib.Event,
|
|
) ([]gomatrixserverlib.Event, error) {
|
|
// Fast path if there aren't any new state events.
|
|
if len(addsStateEventIDs) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
// Fast path if the only state event added is the event itself.
|
|
if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
|
|
return []gomatrixserverlib.Event{event}, nil
|
|
}
|
|
|
|
// Check if this is re-adding a state events that we previously processed
|
|
// If we have previously received a state event it may still be in
|
|
// our event database.
|
|
result, err := s.db.Events(context.TODO(), addsStateEventIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
missing := missingEventsFrom(result, addsStateEventIDs)
|
|
|
|
// Check if event itself is being added.
|
|
for _, eventID := range missing {
|
|
if eventID == event.EventID() {
|
|
result = append(result, event)
|
|
break
|
|
}
|
|
}
|
|
missing = missingEventsFrom(result, addsStateEventIDs)
|
|
|
|
if len(missing) == 0 {
|
|
return result, nil
|
|
}
|
|
|
|
// At this point the missing events are neither the event itself nor are
|
|
// they present in our local database. Our only option is to fetch them
|
|
// from the roomserver using the query API.
|
|
eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
|
|
var eventResp api.QueryEventsByIDResponse
|
|
if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
result = append(result, eventResp.Events...)
|
|
missing = missingEventsFrom(result, addsStateEventIDs)
|
|
|
|
if len(missing) != 0 {
|
|
return nil, fmt.Errorf(
|
|
"missing %d state events IDs at event %q", len(missing), event.EventID(),
|
|
)
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event) (gomatrixserverlib.Event, error) {
|
|
var stateKey string
|
|
if event.StateKey() == nil {
|
|
stateKey = ""
|
|
} else {
|
|
stateKey = *event.StateKey()
|
|
}
|
|
|
|
prevEvent, err := s.db.GetStateEvent(
|
|
context.TODO(), event.Type(), event.RoomID(), stateKey,
|
|
)
|
|
if err != nil {
|
|
return event, err
|
|
}
|
|
|
|
if prevEvent == nil {
|
|
return event, nil
|
|
}
|
|
|
|
prev := types.PrevEventRef{
|
|
PrevContent: prevEvent.Content(),
|
|
ReplacesState: prevEvent.EventID(),
|
|
PrevSender: prevEvent.Sender(),
|
|
}
|
|
|
|
return event.SetUnsigned(prev)
|
|
}
|
|
|
|
func missingEventsFrom(events []gomatrixserverlib.Event, required []string) []string {
|
|
have := map[string]bool{}
|
|
for _, event := range events {
|
|
have[event.EventID()] = true
|
|
}
|
|
var missing []string
|
|
for _, eventID := range required {
|
|
if !have[eventID] {
|
|
missing = append(missing, eventID)
|
|
}
|
|
}
|
|
return missing
|
|
}
|