0
0
Fork 0
mirror of https://github.com/matrix-org/dendrite synced 2024-11-18 15:50:52 +01:00
dendrite/internal/eventutil/events.go
Neil Alexander 3ea21273bc
Ristretto cache (#2563)
* Try Ristretto cache

* Tweak

* It's beautiful

* Update GMSL

* More strict keyable interface

* Fix that some more

* Make less panicky

* Don't enforce mutability checks for now

* Determine mutability using deep equality

* Tweaks

* Namespace keys

* Make federation caches mutable

* Update cost estimation, add metric

* Update GMSL

* Estimate cost for metrics better

* Reduce counters a bit

* Try caching events

* Some guards

* Try again

* Try this

* Use separate caches for hopefully better hash distribution

* Fix bug with admitting events into cache

* Try to fix bugs

* Check nil

* Try that again

* Preserve order jeezo this is messy

* thanks VS Code for doing exactly the wrong thing

* Try this again

* Be more specific

* aaaaargh

* One more time

* That might be better

* Stronger sorting

* Cache expiries, async publishing of EDUs

* Put it back

* Use a shared cache again

* Cost estimation fixes

* Update ristretto

* Reduce counters a bit

* Clean up a bit

* Update GMSL

* 1GB

* Configurable cache sizees

* Tweaks

* Add `config.DataUnit` for specifying friendly cache sizes

* Various tweaks

* Update GMSL

* Add back some lazy loading caching

* Include key in cost

* Include key in cost

* Tweak max age handling, config key name

* Only register prometheus metrics if requested

* Review comments @S7evinK

* Don't return errors when creating caches (it is better just to crash since otherwise we'll `nil`-pointer exception everywhere)

* Review comments

* Update sample configs

* Update GHA Workflow

* Update Complement images to Go 1.18

* Remove the cache test from the federation API as we no longer guarantee immediate cache admission

* Don't check the caches in the renewal test

* Possibly fix the upgrade tests

* Update to matrix-org/gomatrixserverlib#322

* Update documentation to refer to Go 1.18
2022-07-11 14:31:31 +01:00

187 lines
6.2 KiB
Go

// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package eventutil
import (
"context"
"errors"
"fmt"
"time"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
// ErrRoomNoExists is returned when trying to lookup the state of a room that
// doesn't exist
var ErrRoomNoExists = errors.New("room does not exist")
// QueryAndBuildEvent builds a Matrix event using the event builder and roomserver query
// API client provided. If also fills roomserver query API response (if provided)
// in case the function calling FillBuilder needs to use it.
// Returns ErrRoomNoExists if the state of the room could not be retrieved because
// the room doesn't exist
// Returns an error if something else went wrong
func QueryAndBuildEvent(
ctx context.Context,
builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time,
rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse,
) (*gomatrixserverlib.HeaderedEvent, error) {
if queryRes == nil {
queryRes = &api.QueryLatestEventsAndStateResponse{}
}
eventsNeeded, err := queryRequiredEventsForBuilder(ctx, builder, rsAPI, queryRes)
if err != nil {
// This can pass through a ErrRoomNoExists to the caller
return nil, err
}
return BuildEvent(ctx, builder, cfg, evTime, eventsNeeded, queryRes)
}
// BuildEvent builds a Matrix event from the builder and QueryLatestEventsAndStateResponse
// provided.
func BuildEvent(
ctx context.Context,
builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time,
eventsNeeded *gomatrixserverlib.StateNeeded, queryRes *api.QueryLatestEventsAndStateResponse,
) (*gomatrixserverlib.HeaderedEvent, error) {
err := addPrevEventsToEvent(builder, eventsNeeded, queryRes)
if err != nil {
return nil, err
}
event, err := builder.Build(
evTime, cfg.ServerName, cfg.KeyID,
cfg.PrivateKey, queryRes.RoomVersion,
)
if err != nil {
return nil, err
}
return event.Headered(queryRes.RoomVersion), nil
}
// queryRequiredEventsForBuilder queries the roomserver for auth/prev events needed for this builder.
func queryRequiredEventsForBuilder(
ctx context.Context,
builder *gomatrixserverlib.EventBuilder,
rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse,
) (*gomatrixserverlib.StateNeeded, error) {
eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
if err != nil {
return nil, fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err)
}
if len(eventsNeeded.Tuples()) == 0 {
return nil, errors.New("expecting state tuples for event builder, got none")
}
// Ask the roomserver for information about this room
queryReq := api.QueryLatestEventsAndStateRequest{
RoomID: builder.RoomID,
StateToFetch: eventsNeeded.Tuples(),
}
return &eventsNeeded, rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes)
}
// addPrevEventsToEvent fills out the prev_events and auth_events fields in builder
func addPrevEventsToEvent(
builder *gomatrixserverlib.EventBuilder,
eventsNeeded *gomatrixserverlib.StateNeeded,
queryRes *api.QueryLatestEventsAndStateResponse,
) error {
if !queryRes.RoomExists {
return ErrRoomNoExists
}
eventFormat, err := queryRes.RoomVersion.EventFormat()
if err != nil {
return fmt.Errorf("queryRes.RoomVersion.EventFormat: %w", err)
}
builder.Depth = queryRes.Depth
authEvents := gomatrixserverlib.NewAuthEvents(nil)
for i := range queryRes.StateEvents {
err = authEvents.AddEvent(queryRes.StateEvents[i].Event)
if err != nil {
return fmt.Errorf("authEvents.AddEvent: %w", err)
}
}
refs, err := eventsNeeded.AuthEventReferences(&authEvents)
if err != nil {
return fmt.Errorf("eventsNeeded.AuthEventReferences: %w", err)
}
truncAuth, truncPrev := truncateAuthAndPrevEvents(refs, queryRes.LatestEvents)
switch eventFormat {
case gomatrixserverlib.EventFormatV1:
builder.AuthEvents = truncAuth
builder.PrevEvents = truncPrev
case gomatrixserverlib.EventFormatV2:
v2AuthRefs, v2PrevRefs := []string{}, []string{}
for _, ref := range truncAuth {
v2AuthRefs = append(v2AuthRefs, ref.EventID)
}
for _, ref := range truncPrev {
v2PrevRefs = append(v2PrevRefs, ref.EventID)
}
builder.AuthEvents = v2AuthRefs
builder.PrevEvents = v2PrevRefs
}
return nil
}
// truncateAuthAndPrevEvents limits the number of events we add into
// an event as prev_events or auth_events.
// NOTSPEC: The limits here feel a bit arbitrary but they are currently
// here because of https://github.com/matrix-org/matrix-doc/issues/2307
// and because Synapse will just drop events that don't comply.
func truncateAuthAndPrevEvents(auth, prev []gomatrixserverlib.EventReference) (
truncAuth, truncPrev []gomatrixserverlib.EventReference,
) {
truncAuth, truncPrev = auth, prev
if len(truncAuth) > 10 {
truncAuth = truncAuth[:10]
}
if len(truncPrev) > 20 {
truncPrev = truncPrev[:20]
}
return
}
// RedactEvent redacts the given event and sets the unsigned field appropriately. This should be used by
// downstream components to the roomserver when an OutputTypeRedactedEvent occurs.
func RedactEvent(redactionEvent, redactedEvent *gomatrixserverlib.Event) error {
// sanity check
if redactionEvent.Type() != gomatrixserverlib.MRoomRedaction {
return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type())
}
redactedEvent.Redact()
if err := redactedEvent.SetUnsignedField("redacted_because", redactionEvent); err != nil {
return err
}
// NOTSPEC: sytest relies on this unspecced field existing :(
if err := redactedEvent.SetUnsignedField("redacted_by", redactionEvent.EventID()); err != nil {
return err
}
return nil
}