diff --git a/src/s_user.c b/src/s_user.c
index 01a21e879..319e933f7 100644
--- a/src/s_user.c
+++ b/src/s_user.c
@@ -965,6 +965,8 @@ user_mode(struct Client *client_p, struct Client *source_p, int parc, const char
 					source_p->localClient->opername = NULL;
 
 					rb_dlinkFindDestroy(source_p, &local_oper_list);
+					privilegeset_unref(source_p->localClient->privset);
+					source_p->localClient->privset = NULL;
 				}
 
 				rb_dlinkFindDestroy(source_p, &oper_list);