diff --git a/src/socket.c b/src/socket.c
index 950c956..dbd781c 100644
--- a/src/socket.c
+++ b/src/socket.c
@@ -680,8 +680,6 @@ do_read( conn_t *sock, char *buf, int len )
 	int n;
 
 	assert( sock->fd >= 0 );
-	if (pending_wakeup( &sock->fd_timeout ))
-		conf_wakeup( &sock->fd_timeout, sock->conf->timeout );
 #ifdef HAVE_LIBSSL
 	if (sock->ssl) {
 		if ((n = ssl_return( "read from", sock, SSL_read( sock->ssl, buf, len ) )) <= 0)
@@ -1055,6 +1053,9 @@ socket_fd_cb( int events, void *aux )
 	if (events & POLLOUT)
 		conf_notifier( &conn->notify, POLLIN, 0 );
 
+	if (pending_wakeup( &conn->fd_timeout ))
+		conf_wakeup( &conn->fd_timeout, conn->conf->timeout );
+
 #ifdef HAVE_LIBSSL
 	if (conn->state == SCK_STARTTLS) {
 		start_tls_p2( conn );