diff --git a/config/config.go b/config/config.go
index f7f6b1f..e793a33 100644
--- a/config/config.go
+++ b/config/config.go
@@ -123,6 +123,7 @@ func (config *Config) MakeAppService() (*appservice.AppService, error) {
 	as.HomeserverURL = config.Homeserver.Address
 	as.Host.Hostname = config.AppService.Hostname
 	as.Host.Port = config.AppService.Port
+	as.DefaultHTTPRetries = 4
 	var err error
 	as.Registration, err = config.GetRegistration()
 	return as, err
diff --git a/crypto.go b/crypto.go
index a0f7f20..4e41266 100644
--- a/crypto.go
+++ b/crypto.go
@@ -122,6 +122,7 @@ func (helper *CryptoHelper) loginBot() (*mautrix.Client, error) {
 	}
 	client.Logger = helper.baseLog.Sub("Bot")
 	client.Client = helper.bridge.AS.HTTPClient
+	client.DefaultHTTPRetries = helper.bridge.AS.DefaultHTTPRetries
 	flows, err := client.GetLoginFlows()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get supported login flows: %w", err)
diff --git a/custompuppet.go b/custompuppet.go
index 83404e4..3476cb5 100644
--- a/custompuppet.go
+++ b/custompuppet.go
@@ -90,6 +90,7 @@ func (puppet *Puppet) newCustomIntent() (*appservice.IntentAPI, error) {
 	}
 	client.Logger = puppet.bridge.AS.Log.Sub(string(puppet.CustomMXID))
 	client.Client = puppet.bridge.AS.HTTPClient
+	client.DefaultHTTPRetries = puppet.bridge.AS.DefaultHTTPRetries
 	client.Syncer = puppet
 	client.Store = puppet
 
diff --git a/go.mod b/go.mod
index f78bc95..8170448 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	gopkg.in/yaml.v2 v2.3.0
 	maunium.net/go/mauflag v1.0.0
 	maunium.net/go/maulogger/v2 v2.2.4
-	maunium.net/go/mautrix v0.9.4
+	maunium.net/go/mautrix v0.9.6
 )
 
 replace github.com/Rhymen/go-whatsapp => github.com/tulir/go-whatsapp v0.4.3
diff --git a/go.sum b/go.sum
index 4a27778..c25db96 100644
--- a/go.sum
+++ b/go.sum
@@ -518,5 +518,7 @@ maunium.net/go/mautrix v0.9.3 h1:v7tcbsD5xhCGqLv4V/CU0eqIgmGk3PlB8Ca7r1J85dA=
 maunium.net/go/mautrix v0.9.3/go.mod h1:mckyHSKKyI0PQF2K9MgWMMDUWH1meCNggE28ILTLuMg=
 maunium.net/go/mautrix v0.9.4 h1:OE4IBQuN2DyrgHFAWjGerQk5Fs16kvP4gunHVJNxhRE=
 maunium.net/go/mautrix v0.9.4/go.mod h1:3NVsKW7MZbzIA3DOn6oni1IUvTTeDcdp5OnNmL5yfwM=
+maunium.net/go/mautrix v0.9.6 h1:X/AoDLpy9NH8AiKuEZd8+Dd4BFJ59wLuVJCfDg3YmQA=
+maunium.net/go/mautrix v0.9.6/go.mod h1:mckyHSKKyI0PQF2K9MgWMMDUWH1meCNggE28ILTLuMg=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
diff --git a/portal.go b/portal.go
index 4d7bb38..36b97e1 100644
--- a/portal.go
+++ b/portal.go
@@ -1290,35 +1290,7 @@ func (portal *Portal) sendMainIntentMessage(content interface{}) (*mautrix.RespS
 	return portal.sendMessage(portal.MainIntent(), event.EventMessage, content, 0)
 }
 
-const MessageSendRetries = 5
-const MediaUploadRetries = 5
-const BadGatewaySleep = 5 * time.Second
-
 func (portal *Portal) sendMessage(intent *appservice.IntentAPI, eventType event.Type, content interface{}, timestamp int64) (*mautrix.RespSendEvent, error) {
-	return portal.sendMessageWithRetry(intent, eventType, content, timestamp, MessageSendRetries)
-}
-
-func isGatewayError(err error) bool {
-	if err == nil {
-		return false
-	}
-	var httpErr mautrix.HTTPError
-	return errors.As(err, &httpErr) && (httpErr.IsStatus(http.StatusBadGateway) || httpErr.IsStatus(http.StatusGatewayTimeout))
-}
-
-func (portal *Portal) sendMessageWithRetry(intent *appservice.IntentAPI, eventType event.Type, content interface{}, timestamp int64, retries int) (*mautrix.RespSendEvent, error) {
-	for ; ; retries-- {
-		resp, err := portal.sendMessageDirect(intent, eventType, content, timestamp)
-		if retries > 0 && isGatewayError(err) {
-			portal.log.Warnfln("Got gateway error trying to send message, retrying in %d seconds", int(BadGatewaySleep.Seconds()))
-			time.Sleep(BadGatewaySleep)
-		} else {
-			return resp, err
-		}
-	}
-}
-
-func (portal *Portal) sendMessageDirect(intent *appservice.IntentAPI, eventType event.Type, content interface{}, timestamp int64) (*mautrix.RespSendEvent, error) {
 	wrappedContent := event.Content{Parsed: content}
 	if timestamp != 0 && intent.IsCustomPuppet {
 		wrappedContent.Raw = map[string]interface{}{
@@ -1648,18 +1620,6 @@ type mediaMessage struct {
 	sendAsSticker bool
 }
 
-func (portal *Portal) uploadWithRetry(intent *appservice.IntentAPI, data []byte, mimeType string, retries int) (*mautrix.RespMediaUpload, error) {
-	for ; ; retries-- {
-		uploaded, err := intent.UploadBytes(data, mimeType)
-		if isGatewayError(err) {
-			portal.log.Warnfln("Got gateway error trying to upload media, retrying in %d seconds", int(BadGatewaySleep.Seconds()))
-			time.Sleep(BadGatewaySleep)
-		} else {
-			return uploaded, err
-		}
-	}
-}
-
 func (portal *Portal) HandleMediaMessage(source *User, msg mediaMessage) {
 	intent := portal.startHandling(source, msg.info)
 	if intent == nil {
@@ -1692,7 +1652,7 @@ func (portal *Portal) HandleMediaMessage(source *User, msg mediaMessage) {
 
 	data, uploadMimeType, file := portal.encryptFile(data, msg.mimeType)
 
-	uploaded, err := portal.uploadWithRetry(intent, data, uploadMimeType, MediaUploadRetries)
+	uploaded, err := intent.UploadBytes(data, uploadMimeType)
 	if err != nil {
 		if errors.Is(err, mautrix.MTooLarge) {
 			portal.sendMediaBridgeFailure(source, intent, msg.info, errors.New("homeserver rejected too large file"))