diff --git a/authd/provider.c b/authd/provider.c
index c83e5761a..51ee9ee0b 100644
--- a/authd/provider.c
+++ b/authd/provider.c
@@ -174,7 +174,7 @@ void notice_client(struct auth_client *auth, const char *notice)
 }
 
 /* Begin authenticating user */
-static void start_auth(const char *cid, const char *l_ip, const char *l_port, const char *c_addr, const char *c_port)
+static void start_auth(const char *cid, const char *l_ip, const char *l_port, const char *c_ip, const char *c_port)
 {
 	rb_dlink_node *ptr;
 	struct auth_provider *provider;
@@ -191,11 +191,22 @@ static void start_auth(const char *cid, const char *l_ip, const char *l_port, co
 
 	auth->cid = (uint16_t)lcid;
 
-	(void)rb_inet_pton_sock(l_ip, (struct sockaddr *)&auth->l_ip);
-	auth->l_port = (uint16_t)atoi(l_port); /* Safe cast, port shouldn't exceed 16 bits  */
+	(void)rb_inet_pton_sock(l_ip, (struct sockaddr *)&auth->l_addr);
+#ifdef RB_IPV6
+	if(GET_SS_FAMILY(&auth->l_addr) == AF_INET6)
+		((struct sockaddr_in6 *)&auth->l_addr)->sin6_port = htons(atoi(l_ip));
+	else
+#endif
+		((struct sockaddr_in *)&auth->l_addr)->sin_port = htons(atoi(l_ip));
+
+	(void)rb_inet_pton_sock(c_ip, (struct sockaddr *)&auth->c_addr);
+#ifdef RB_IPV6
+	if(GET_SS_FAMILY(&auth->c_addr) == AF_INET6)
+		((struct sockaddr_in6 *)&auth->c_addr)->sin6_port = htons(atoi(c_ip));
+	else
+#endif
+		((struct sockaddr_in *)&auth->c_addr)->sin_port = htons(atoi(c_ip));
 
-	(void)rb_inet_pton_sock(c_addr, (struct sockaddr *)&auth->c_addr);
-	auth->c_port = (uint16_t)atoi(c_port);
 
 	RB_DLINK_FOREACH(ptr, auth_providers.head)
 	{
diff --git a/authd/provider.h b/authd/provider.h
index 8bd4d18ff..a7a7905a6 100644
--- a/authd/provider.h
+++ b/authd/provider.h
@@ -39,11 +39,8 @@ struct auth_client
 {
 	uint16_t cid;				/* Client ID */
 
-	struct rb_sockaddr_storage l_ip;	/* Listener IP address */
-	uint16_t l_port;			/* Listener port */
-
+	struct rb_sockaddr_storage l_addr;	/* Listener IP address */
 	struct rb_sockaddr_storage c_addr;	/* Client IP address */
-	uint16_t c_port;			/* Client port */
 
 	char hostname[HOSTLEN + 1];		/* Used for DNS lookup */
 	char username[USERLEN + 1];		/* Used for ident lookup */