diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go
index cfbb05327..f767615c8 100644
--- a/syncapi/consumers/roomserver.go
+++ b/syncapi/consumers/roomserver.go
@@ -428,6 +428,13 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent(
 		return
 	}
 
+	// Only notify clients about retired invite events, if the user didn't accept the invite.
+	// The PDU stream will also receive an event about accepting the invitation, so there should
+	// be a "smooth" transition from invite -> join, and not invite -> leave -> join
+	if msg.Membership == gomatrixserverlib.Join {
+		return
+	}
+
 	// Notify any active sync requests that the invite has been retired.
 	s.inviteStream.Advance(pduPos)
 	s.notifier.OnNewInvite(types.StreamingToken{InvitePosition: pduPos}, msg.TargetUserID)
diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go
index 7875ffa35..700f25c10 100644
--- a/syncapi/streams/stream_invite.go
+++ b/syncapi/streams/stream_invite.go
@@ -74,21 +74,26 @@ func (p *InviteStreamProvider) IncrementalSync(
 		return to
 	}
 	for roomID := range retiredInvites {
-		if _, ok := req.Response.Rooms.Join[roomID]; !ok {
-			lr := types.NewLeaveResponse()
-			h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
-			lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
-				// fake event ID which muxes in the to position
-				EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
-				OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
-				RoomID:         roomID,
-				Sender:         req.Device.UserID,
-				StateKey:       &req.Device.UserID,
-				Type:           "m.room.member",
-				Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
-			})
-			req.Response.Rooms.Leave[roomID] = lr
+		if _, ok := req.Response.Rooms.Invite[roomID]; ok {
+			continue
 		}
+		if _, ok := req.Response.Rooms.Join[roomID]; ok {
+			continue
+		}
+		lr := types.NewLeaveResponse()
+		h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
+		lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
+			// fake event ID which muxes in the to position
+			EventID:        "$" + base64.RawURLEncoding.EncodeToString(h[:]),
+			OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+			RoomID:         roomID,
+			Sender:         req.Device.UserID,
+			StateKey:       &req.Device.UserID,
+			Type:           "m.room.member",
+			Content:        gomatrixserverlib.RawJSON(`{"membership":"leave"}`),
+		})
+		req.Response.Rooms.Leave[roomID] = lr
+
 	}
 
 	return maxID