diff --git a/matrix.go b/matrix.go
index aa63e57..adb76d4 100644
--- a/matrix.go
+++ b/matrix.go
@@ -50,6 +50,11 @@ func NewMatrixHandler(bridge *Bridge) *MatrixHandler {
 func (mx *MatrixHandler) HandleBotInvite(evt *gomatrix.Event) {
 	intent := mx.as.BotIntent()
 
+	user := mx.bridge.GetUser(evt.Sender)
+	if user == nil {
+		return
+	}
+
 	resp, err := intent.JoinRoom(evt.RoomID, "", nil)
 	if err != nil {
 		mx.log.Debugln("Failed to join room", evt.RoomID, "with invite from", evt.Sender)
@@ -69,6 +74,13 @@ func (mx *MatrixHandler) HandleBotInvite(evt *gomatrix.Event) {
 		return
 	}
 
+	if !user.Whitelisted {
+		intent.SendNotice(resp.RoomID, "You are not whitelisted to use this bridge.\n"+
+			"If you're the owner of this bridge, see the bridge.permissions section in your config file.")
+		intent.LeaveRoom(resp.RoomID)
+		return
+	}
+
 	hasPuppets := false
 	for mxid, _ := range members.Joined {
 		if mxid == intent.UserID || mxid == evt.Sender {
@@ -92,7 +104,6 @@ func (mx *MatrixHandler) HandleBotInvite(evt *gomatrix.Event) {
 }
 
 func (mx *MatrixHandler) HandleMembership(evt *gomatrix.Event) {
-	mx.log.Debugln(evt.Content, evt.Content.Membership, evt.GetStateKey())
 	if evt.Content.Membership == "invite" && evt.GetStateKey() == mx.as.BotMXID() {
 		mx.HandleBotInvite(evt)
 	}
@@ -100,7 +111,7 @@ func (mx *MatrixHandler) HandleMembership(evt *gomatrix.Event) {
 
 func (mx *MatrixHandler) HandleRoomMetadata(evt *gomatrix.Event) {
 	user := mx.bridge.GetUser(types.MatrixUserID(evt.Sender))
-	if user == nil {
+	if user == nil || !user.Whitelisted {
 		return
 	}
 
@@ -131,6 +142,10 @@ func (mx *MatrixHandler) HandleMessage(evt *gomatrix.Event) {
 	roomID := types.MatrixRoomID(evt.RoomID)
 	user := mx.bridge.GetUser(types.MatrixUserID(evt.Sender))
 
+	if !user.Whitelisted {
+		return
+	}
+
 	if evt.Content.MsgType == gomatrix.MsgText {
 		commandPrefix := mx.bridge.Config.Bridge.CommandPrefix
 		hasCommandPrefix := strings.HasPrefix(evt.Content.Body, commandPrefix)
diff --git a/user.go b/user.go
index 6503271..e29442d 100644
--- a/user.go
+++ b/user.go
@@ -38,6 +38,9 @@ type User struct {
 	bridge *Bridge
 	log    log.Logger
 
+	Admin       bool
+	Whitelisted bool
+
 	portalsByMXID map[types.MatrixRoomID]*Portal
 	portalsByJID  map[types.WhatsAppID]*Portal
 	portalsLock   sync.Mutex
@@ -94,6 +97,8 @@ func (bridge *Bridge) NewUser(dbUser *database.User) *User {
 		portalsByJID:  make(map[types.WhatsAppID]*Portal),
 		puppets:       make(map[types.WhatsAppID]*Puppet),
 	}
+	user.Whitelisted = user.bridge.Config.Bridge.Permissions.IsWhitelisted(user.ID)
+	user.Admin = user.bridge.Config.Bridge.Permissions.IsAdmin(user.ID)
 	user.htmlParser = user.newHTMLParser()
 	user.waReplString, user.waReplFunc = user.newWhatsAppFormatMaps()
 	return user