diff --git a/database/backfillqueue.go b/database/backfillqueue.go
index 4da6123..b39604e 100644
--- a/database/backfillqueue.go
+++ b/database/backfillqueue.go
@@ -106,6 +106,16 @@ func (bq *BackfillQuery) DeleteAll(userID id.UserID) error {
 	return err
 }
 
+func (bq *BackfillQuery) DeleteAllForPortal(userID id.UserID, portalKey PortalKey) error {
+	_, err := bq.db.Exec(`
+		DELETE FROM backfill_queue
+		WHERE user_mxid=$1
+			AND portal_jid=$2
+			AND portal_receiver=$3
+	`, userID, portalKey.JID, portalKey.Receiver)
+	return err
+}
+
 type Backfill struct {
 	db  *Database
 	log log.Logger
diff --git a/database/historysync.go b/database/historysync.go
index d2c166a..eba4f8d 100644
--- a/database/historysync.go
+++ b/database/historysync.go
@@ -314,3 +314,13 @@ func (hsq *HistorySyncQuery) DeleteAllMessages(userID id.UserID) error {
 	_, err := hsq.db.Exec("DELETE FROM history_sync_message WHERE user_mxid=$1", userID)
 	return err
 }
+
+func (hsq *HistorySyncQuery) DeleteAllMessagesForPortal(userID id.UserID, portalKey PortalKey) error {
+	_, err := hsq.db.Exec(`
+		DELETE FROM history_sync_message
+		WHERE user_mxid=$1
+			AND portal_jid=$2
+			AND portal_receiver=$3
+	`, userID, portalKey.JID, portalKey.Receiver)
+	return err
+}
diff --git a/portal.go b/portal.go
index 87913b0..11ad723 100644
--- a/portal.go
+++ b/portal.go
@@ -1221,7 +1221,15 @@ func (portal *Portal) CreateMatrixRoom(user *User, groupInfo *types.GroupInfo, i
 	} else {
 		if groupInfo == nil || !isFullInfo {
 			foundInfo, err := user.Client.GetGroupInfo(portal.Key.JID)
-			if err != nil {
+
+			// Ensure that the user is actually a participant in the conversation
+			// before creating the matrix room
+			if errors.Is(err, whatsmeow.ErrNotInGroup) {
+				user.log.Debugfln("Skipping creating matrix room for %s because the user is not a participant", portal.Key.JID)
+				user.bridge.DB.BackfillQuery.DeleteAllForPortal(user.MXID, portal.Key)
+				user.bridge.DB.HistorySyncQuery.DeleteAllMessagesForPortal(user.MXID, portal.Key)
+				return err
+			} else if err != nil {
 				portal.log.Warnfln("Failed to get group info through %s: %v", user.JID, err)
 			} else {
 				groupInfo = foundInfo