diff --git a/src/session_impl.cpp b/src/session_impl.cpp
index 4b9f58b03..ac7b98f28 100644
--- a/src/session_impl.cpp
+++ b/src/session_impl.cpp
@@ -2057,6 +2057,9 @@ retry:
 			if (listen_port_retries > 0)
 			{
 				m_listen_interface.port(m_listen_interface.port() + 1);
+				// update the actual port m_listen_interface was derived from also
+				if (!m_listen_interfaces.empty())
+					m_listen_interfaces[0].second += 1;
 				--listen_port_retries;
 				goto retry;
 			}
@@ -2130,6 +2133,9 @@ retry:
 			if (listen_port_retries > 0)
 			{
 				m_listen_interface.port(m_listen_interface.port() + 1);
+				// update the actual port m_listen_interface was derived from also
+				if (!m_listen_interfaces.empty())
+					m_listen_interfaces[0].second += 1;
 				--listen_port_retries;
 				goto retry;
 			}