diff --git a/historysync.go b/historysync.go
index 8790b9c..31978fe 100644
--- a/historysync.go
+++ b/historysync.go
@@ -43,30 +43,32 @@ type wrappedInfo struct {
 }
 
 func (user *User) handleHistorySyncsLoop() {
-	reCheckQueue := make(chan bool, 1)
-	if user.bridge.Config.Bridge.HistorySync.Backfill {
-		// Start the backfill queue.
-		user.BackfillQueue = &BackfillQueue{
-			BackfillQuery:             user.bridge.DB.BackfillQuery,
-			ImmediateBackfillRequests: make(chan *database.Backfill, 1),
-			DeferredBackfillRequests:  make(chan *database.Backfill, 1),
-			ReCheckQueue:              make(chan bool, 1),
-			log:                       user.log.Sub("BackfillQueue"),
-		}
-		reCheckQueue = user.BackfillQueue.ReCheckQueue
-
-		// Immediate backfills can be done in parallel
-		for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ {
-			go user.handleBackfillRequestsLoop(user.BackfillQueue.ImmediateBackfillRequests)
-		}
-
-		// Deferred backfills should be handled synchronously so as not to
-		// overload the homeserver. Users can configure their backfill stages
-		// to be more or less aggressive with backfilling at this stage.
-		go user.handleBackfillRequestsLoop(user.BackfillQueue.DeferredBackfillRequests)
-		go user.BackfillQueue.RunLoops(user)
+	if !user.bridge.Config.Bridge.HistorySync.Backfill {
+		return
 	}
 
+	reCheckQueue := make(chan bool, 1)
+	// Start the backfill queue.
+	user.BackfillQueue = &BackfillQueue{
+		BackfillQuery:             user.bridge.DB.BackfillQuery,
+		ImmediateBackfillRequests: make(chan *database.Backfill, 1),
+		DeferredBackfillRequests:  make(chan *database.Backfill, 1),
+		ReCheckQueue:              make(chan bool, 1),
+		log:                       user.log.Sub("BackfillQueue"),
+	}
+	reCheckQueue = user.BackfillQueue.ReCheckQueue
+
+	// Immediate backfills can be done in parallel
+	for i := 0; i < user.bridge.Config.Bridge.HistorySync.Immediate.WorkerCount; i++ {
+		go user.handleBackfillRequestsLoop(user.BackfillQueue.ImmediateBackfillRequests)
+	}
+
+	// Deferred backfills should be handled synchronously so as not to
+	// overload the homeserver. Users can configure their backfill stages
+	// to be more or less aggressive with backfilling at this stage.
+	go user.handleBackfillRequestsLoop(user.BackfillQueue.DeferredBackfillRequests)
+	go user.BackfillQueue.RunLoops(user)
+
 	// Always save the history syncs for the user. If they want to enable
 	// backfilling in the future, we will have it in the database.
 	for evt := range user.historySyncs {
diff --git a/user.go b/user.go
index 854535d..46dbefa 100644
--- a/user.go
+++ b/user.go
@@ -188,7 +188,9 @@ func (bridge *Bridge) NewUser(dbUser *database.User) *User {
 	user.RelayWhitelisted = user.bridge.Config.Bridge.Permissions.IsRelayWhitelisted(user.MXID)
 	user.Whitelisted = user.bridge.Config.Bridge.Permissions.IsWhitelisted(user.MXID)
 	user.Admin = user.bridge.Config.Bridge.Permissions.IsAdmin(user.MXID)
-	go user.handleHistorySyncsLoop()
+	if user.bridge.Config.Bridge.HistorySync.Backfill {
+		go user.handleHistorySyncsLoop()
+	}
 	return user
 }
 
@@ -692,7 +694,9 @@ func (user *User) HandleEvent(event interface{}) {
 		portal := user.GetPortalByMessageSource(v.Info.MessageSource)
 		portal.messages <- PortalMessage{undecryptable: v, source: user}
 	case *events.HistorySync:
-		user.historySyncs <- v
+		if user.bridge.Config.Bridge.HistorySync.Backfill {
+			user.historySyncs <- v
+		}
 	case *events.Mute:
 		portal := user.GetPortalByJID(v.JID)
 		if portal != nil {