diff --git a/apps/wolfssh/wolfssh.c b/apps/wolfssh/wolfssh.c index 9951acf49..1952b36a6 100644 --- a/apps/wolfssh/wolfssh.c +++ b/apps/wolfssh/wolfssh.c @@ -1171,6 +1171,7 @@ static THREAD_RETURN WOLFSSH_THREAD wolfSSH_Client(void* args) socklen_t clientAddrSz = sizeof(clientAddr); int ret = 0; int ioErr = 0; + int waitRet = WS_SUCCESS; byte keepOpen; #ifdef USE_WINDOWS_API byte rawMode = 0; @@ -1389,51 +1390,40 @@ static THREAD_RETURN WOLFSSH_THREAD wolfSSH_Client(void* args) if (ret == WS_FATAL_ERROR) { ret = wolfSSH_get_error(ssh); } + if (ret != WS_SUCCESS && ret != WS_WANT_WRITE && ret != WS_WANT_READ && + ret != WS_CHANNEL_CLOSED && ret != WS_EOF && + ret != WS_SOCKET_ERROR_E && ret != WS_DISCONNECT) { + WLOG(WS_LOG_DEBUG, "Sending the shutdown messages failed."); + } /* do not continue on with shutdown process if peer already disconnected */ if (ret != WS_SOCKET_ERROR_E && wolfSSH_get_error(ssh) != WS_SOCKET_ERROR_E) { -#ifndef WOLFSSL_NUCLEUS - if (ret == WS_WANT_WRITE) { - /* The close messages are queued and the threads are done, no - * one else is going to send them. */ - ret = FlushQueuedSend(ssh, NULL); - if (ret == WS_WANT_WRITE) { - /* The peer stopped reading. The socket closes next, there is - * nothing left to push the messages out with. */ - ret = WS_SUCCESS; - } - } -#endif - - if (ret == WS_SUCCESS) { - ret = wolfSSH_worker(ssh, NULL); - if (ret == WS_FATAL_ERROR) { - ret = wolfSSH_get_error(ssh); - } - if (ret == WS_WANT_WRITE) { - /* The close messages are already out, whatever the drain - * still wants to send is a reply to the peer. */ - ret = WS_SUCCESS; - } - } - else if (ret != WS_CHANNEL_CLOSED && ret != WS_WANT_READ - && ret != WS_EOF) { - WLOG(WS_LOG_DEBUG, "Sending the shutdown messages failed."); + /* The exit status read below comes ahead of the peer's close, and + * the drain would discard it. A peer that disconnected, hung up or + * never closed still ended the session; any other error broke it. */ + if (ret != WS_CHANNEL_CLOSED) { + waitRet = WaitForChannelClose(ssh, sockFd); + if (waitRet == WS_WANT_READ || waitRet == WS_DISCONNECT || + waitRet == WS_SOCKET_ERROR_E) + waitRet = WS_SUCCESS; } - if (ret == WS_CHANNEL_CLOSED || ret == WS_WANT_READ - || ret == WS_EOF) { - /* Shutting down. The channel closing or the peer's EOF isn't a - * fail, and neither is the peer having nothing ready on this - * non-blocking socket; either way there is nothing left to wait - * for. */ + /* The close messages a short send left queued go out with the + * disconnect: the threads are done, no one else is going to send + * them. */ + ret = SendDisconnectAndDrain(ssh, sockFd); + if (ret == WS_WANT_READ || ret == WS_WANT_WRITE) { + /* The socket closes next, so there is nothing left to push a + * queued disconnect out with, and a peer that never hung up is + * not a fail. */ ret = WS_SUCCESS; } else if (ret != WS_SUCCESS) { - WLOG(WS_LOG_DEBUG, - "Failed to listen for close messages from the peer."); + WLOG(WS_LOG_DEBUG, "Sending the disconnect failed."); } + if (ret == WS_SUCCESS) + ret = waitRet; } WCLOSESOCKET(sockFd); diff --git a/apps/wolfsshd/wolfsshd.c b/apps/wolfsshd/wolfsshd.c index cd79628b6..55ad3507b 100644 --- a/apps/wolfsshd/wolfsshd.c +++ b/apps/wolfsshd/wolfsshd.c @@ -3792,6 +3792,7 @@ static void* HandleConnection(void* arg) { int ret = WS_SUCCESS; int error; + int graceExpired = 0; #ifdef _WIN32 byte threaded = 0; #endif @@ -3894,6 +3895,7 @@ static void* HandleConnection(void* arg) ret != WS_SFTP_COMPLETE && ret != WS_SCP_INIT) { wolfSSH_Log(WS_LOG_ERROR, "[SSHD] Failed login within grace period"); + graceExpired = 1; } #ifdef _WIN32 @@ -4080,63 +4082,28 @@ static void* HandleConnection(void* arg) if (error != WS_SOCKET_ERROR_E && error != WS_FATAL_ERROR) { wolfSSH_Log(WS_LOG_INFO, "[SSHD] Attempting to close down connection"); ret = wolfSSH_shutdown(ssh); + error = wolfSSH_get_error(ssh); /* peer hung up, stop shutdown */ - if (ret == WS_SOCKET_ERROR_E) { + if (ret == WS_SOCKET_ERROR_E || error == WS_SOCKET_ERROR_E) { ret = 0; } + } - error = wolfSSH_get_error(ssh); - if (error != WS_SOCKET_ERROR_E && - (error == WS_WANT_READ || error == WS_WANT_WRITE)) { - int maxAttempt = 10; /* make 10 attempts max before giving up */ - int attempt; - - for (attempt = 0; attempt < maxAttempt; attempt++) { - ret = wolfSSH_worker(ssh, NULL); - error = wolfSSH_get_error(ssh); - - /* peer successfully closed down gracefully */ - if (ret == WS_CHANNEL_CLOSED || ret == WS_EOF) { - ret = 0; - break; - } - - /* peer hung up, stop shutdown */ - if (ret == WS_SOCKET_ERROR_E) { - ret = 0; - break; - } - - if (ret == WS_FATAL_ERROR && - (error != WS_WANT_READ && - error != WS_WANT_WRITE)) { - break; - } - #ifdef _WIN32 - Sleep(1); - #else - usleep(100000); - #endif - } - - if (attempt == maxAttempt) { - wolfSSH_Log(WS_LOG_INFO, - "[SSHD] Gave up on graceful shutdown, closing the socket"); - } + if (conn != NULL) { + /* The close exchange ended the session, so the half close alone + * says the daemon is done: OpenSSH's client exits 255 on a + * disconnect that lands before it is finished. A peer that let the + * grace time run out is not waited on a second time. */ + if (error != WS_SOCKET_ERROR_E && !graceExpired && + HalfCloseAndDrain(ssh, conn->fd) == WS_WANT_READ) { + wolfSSH_Log(WS_LOG_INFO, + "[SSHD] Gave up waiting for the peer to hang up, " + "closing the socket"); } } - - /* check if there is a response to the shutdown */ wolfSSH_free(ssh); if (conn != NULL) { - byte sc[1024]; - shutdown(conn->fd, 1); - /* Spin until socket closes. */ - do { - ret = (int)recv(conn->fd, (char*)sc, 1024, 0); - } while (ret > 0); - WCLOSESOCKET(conn->fd); } wolfSSH_Log(WS_LOG_INFO, "[SSHD] Return from closing connection = %d", ret); diff --git a/configure.ac b/configure.ac index fcc8d9553..d3337bb9d 100644 --- a/configure.ac +++ b/configure.ac @@ -87,7 +87,7 @@ AC_ARG_WITH(wolfssl, ) AC_CHECK_LIB([wolfssl],[wolfCrypt_Init],,[AC_MSG_ERROR([libwolfssl is required for ${PACKAGE}. It can be obtained from https://www.wolfssl.com/download.html/ .])]) -AC_CHECK_FUNCS([gethostbyname getaddrinfo gettimeofday inet_ntoa memset socket wc_ecc_set_rng]) +AC_CHECK_FUNCS([gethostbyname getaddrinfo gettimeofday inet_ntoa memset socket socketpair wc_ecc_set_rng]) # futimens()/utimensat() declarations are feature-test-macro gated, so a plain # AC_CHECK_FUNCS link test can pass while the prototype stays hidden at compile diff --git a/examples/client/client.c b/examples/client/client.c index 951566795..2ded5a415 100644 --- a/examples/client/client.c +++ b/examples/client/client.c @@ -762,6 +762,8 @@ THREAD_RETURN WOLFSSH_THREAD client_test(void* args) socklen_t clientAddrSz = sizeof(clientAddr); char rxBuf[80]; int ret = 0; + int error = 0; + int waitRet = WS_SUCCESS; int ch; int userEcc = 0; word16 port = wolfSshPort; @@ -1250,26 +1252,41 @@ THREAD_RETURN WOLFSSH_THREAD client_test(void* args) #endif } ret = wolfSSH_shutdown(ssh); - /* do not continue on with shutdown process if peer already disconnected. - * A peer EOF is not a disconnect: the channel is still open and its close - * is still owed, so the drain below is exactly what is wanted. */ - if (ret != WS_SOCKET_ERROR_E && wolfSSH_get_error(ssh) != WS_SOCKET_ERROR_E - && wolfSSH_get_error(ssh) != WS_CHANNEL_CLOSED) { - if (ret != WS_SUCCESS && ret != WS_WANT_WRITE) { - ClientFreeBuffers(pubKeyName, privKeyName, NULL); - wolfSSH_free(ssh); - wolfSSH_CTX_free(ctx); - err_sys("Sending the shutdown messages failed."); - } - ret = wolfSSH_worker(ssh, NULL); - if (ret != WS_SUCCESS && ret != WS_SOCKET_ERROR_E && - ret != WS_CHANNEL_CLOSED && ret != WS_EOF) { - ClientFreeBuffers(pubKeyName, privKeyName, NULL); - wolfSSH_free(ssh); - wolfSSH_CTX_free(ctx); - err_sys("Failed to listen for close messages from the peer."); + error = wolfSSH_get_error(ssh); + /* WS_FATAL_ERROR only says to go look, the session has the detail. The + * teardown sends are the point of the call, so an unexpected result is + * worth a line, but the drain below still runs: the peer's close is + * still owed. */ + if (ret == WS_FATAL_ERROR) + ret = error; + if (ret != WS_SUCCESS && ret != WS_WANT_WRITE && ret != WS_WANT_READ && + ret != WS_CHANNEL_CLOSED && ret != WS_EOF && + ret != WS_SOCKET_ERROR_E && ret != WS_DISCONNECT) + printf("Sending the shutdown messages failed, error %d\n", ret); + /* The exit status read below comes ahead of the peer's close, and the + * drain would discard it. A peer that disconnected, hung up or never + * closed still ended the session; any other error broke it. */ + if (ret != WS_CHANNEL_CLOSED && error != WS_SOCKET_ERROR_E) { + waitRet = WaitForChannelClose(ssh, sockFd); + if (waitRet == WS_WANT_READ || waitRet == WS_DISCONNECT || + waitRet == WS_SOCKET_ERROR_E) + waitRet = WS_SUCCESS; + } + ret = WS_SUCCESS; + + /* peer already hung up, just close. A peer EOF is not a hang up: the + * channel is still open and its close is still owed, so the drain is + * exactly what is wanted there. */ + if (error != WS_SOCKET_ERROR_E) { + ret = SendDisconnectAndDrain(ssh, sockFd); + if (ret == WS_WANT_READ || ret == WS_WANT_WRITE) { + printf("Gave up on a graceful shutdown, " + "closing the socket\n"); + ret = WS_SUCCESS; } } + if (ret == WS_SUCCESS) + ret = waitRet; WCLOSESOCKET(sockFd); #if defined(WOLFSSH_TERM) || defined(WOLFSSH_SHELL) diff --git a/examples/echoserver/echoserver.c b/examples/echoserver/echoserver.c index 18af1085c..ca45977b0 100644 --- a/examples/echoserver/echoserver.c +++ b/examples/echoserver/echoserver.c @@ -2352,55 +2352,23 @@ static THREAD_RETURN WOLFSSH_THREAD server_worker(void* vArgs) } if (error != WS_SOCKET_ERROR_E && error != WS_FATAL_ERROR) { - ret = wolfSSH_shutdown(threadCtx->ssh); + (void)wolfSSH_shutdown(threadCtx->ssh); + error = wolfSSH_get_error(threadCtx->ssh); /* peer hung up, stop shutdown */ - if (ret == WS_SOCKET_ERROR_E) { + if (error == WS_SOCKET_ERROR_E) { ret = 0; } - - /* The peer's close already retired the channel: a completed - * shutdown, not a failure. Left non-zero it sets quit, taking the - * server down after one session. */ - if (ret == WS_CHANNEL_CLOSED) { + else { + /* The close exchange ended the session, and OpenSSH's client + * exits 255 on a disconnect that lands before it is done, so + * the half close alone says we are finished. */ + ret = HalfCloseAndDrain(threadCtx->ssh, threadCtx->fd); + if (ret == WS_WANT_READ) + printf("Gave up waiting for the peer to hang up, " + "closing the socket\n"); ret = 0; } - - error = wolfSSH_get_error(threadCtx->ssh); - if (error != WS_SOCKET_ERROR_E && - (error == WS_WANT_READ || error == WS_WANT_WRITE)) { - int maxAttempt = 10; /* make 10 attempts max before giving up */ - int attempt; - - for (attempt = 0; attempt < maxAttempt; attempt++) { - ret = wolfSSH_worker(threadCtx->ssh, NULL); - error = wolfSSH_get_error(threadCtx->ssh); - - /* peer successfully closed down gracefully */ - if (ret == WS_CHANNEL_CLOSED || ret == WS_EOF) { - ret = 0; - break; - } - - /* peer hung up, stop shutdown */ - if (ret == WS_SOCKET_ERROR_E) { - ret = 0; - break; - } - - if (error == WS_WANT_READ || error == WS_WANT_WRITE) { - /* Wanting read or wanting write. Clear ret. */ - ret = 0; - } - else { - break; - } - } - - if (attempt == maxAttempt) { - printf("Gave up on graceful shutdown, closing the socket\n"); - } - } } if (threadCtx->fd != -1) { diff --git a/examples/portfwd/portfwd.c b/examples/portfwd/portfwd.c index 953bcf074..add9aafcd 100644 --- a/examples/portfwd/portfwd.c +++ b/examples/portfwd/portfwd.c @@ -934,11 +934,22 @@ THREAD_RETURN WOLFSSH_THREAD portfwd_worker(void* args) } ret = wolfSSH_shutdown(ssh); - /* The socket closes next, so a queued write and a retired channel are - * both done as far as this teardown is concerned. */ + /* A queued write and a retired channel are both done as far as this + * teardown is concerned: the disconnect carries the write out. */ if (ret != WS_SUCCESS && ret != WS_WANT_WRITE && ret != WS_CHANNEL_CLOSED) err_sys("Closing port forward stream failed."); + /* peer already hung up, just close */ + if (wolfSSH_get_error(ssh) != WS_SOCKET_ERROR_E) { + ret = SendDisconnectAndDrain(ssh, sshFd); + if (ret == WS_WANT_READ || ret == WS_WANT_WRITE) { + printf("Gave up on a graceful shutdown, closing the socket\n"); + ret = WS_SUCCESS; + } + else if (ret != WS_SUCCESS) + err_sys("Sending the disconnect failed."); + } + WCLOSESOCKET(sshFd); if (listenFd != (SOCKET_T)-1) WCLOSESOCKET(listenFd); diff --git a/examples/scpclient/scpclient.c b/examples/scpclient/scpclient.c index 0872f30de..ab1e71efd 100644 --- a/examples/scpclient/scpclient.c +++ b/examples/scpclient/scpclient.c @@ -317,22 +317,23 @@ THREAD_RETURN WOLFSSH_THREAD scp_client(void* args) } ret = wolfSSH_shutdown(ssh); - /* do not continue on with shutdown process if peer already disconnected. - * A peer EOF is not a disconnect: the channel is still open and its close - * is still owed, so the drain below is exactly what is wanted. */ - if (ret != WS_CHANNEL_CLOSED && ret != WS_SOCKET_ERROR_E && - wolfSSH_get_error(ssh) != WS_SOCKET_ERROR_E && - wolfSSH_get_error(ssh) != WS_CHANNEL_CLOSED) { - if (ret != WS_SUCCESS && ret != WS_WANT_WRITE) { - WLOG(WS_LOG_DEBUG, "Sending the shutdown messages failed."); - } - else { - ret = wolfSSH_worker(ssh, NULL); - if (ret != WS_SUCCESS && ret != WS_CHANNEL_CLOSED - && ret != WS_EOF) { - WLOG(WS_LOG_DEBUG, - "Failed to listen for close messages from the peer."); - } + /* WS_FATAL_ERROR only says to go look, the session has the detail. */ + if (ret == WS_FATAL_ERROR) + ret = wolfSSH_get_error(ssh); + if (ret != WS_SUCCESS && ret != WS_WANT_WRITE && ret != WS_WANT_READ && + ret != WS_CHANNEL_CLOSED && ret != WS_EOF && + ret != WS_SOCKET_ERROR_E && ret != WS_DISCONNECT) { + WLOG(WS_LOG_DEBUG, "Sending the shutdown messages failed."); + } + ret = WS_SUCCESS; + + /* peer already hung up, just close */ + if (wolfSSH_get_error(ssh) != WS_SOCKET_ERROR_E) { + ret = SendDisconnectAndDrain(ssh, sockFd); + if (ret == WS_WANT_READ || ret == WS_WANT_WRITE) { + WLOG(WS_LOG_DEBUG, + "Gave up on a graceful shutdown, closing the socket"); + ret = WS_SUCCESS; } } WCLOSESOCKET(sockFd); diff --git a/examples/sftpclient/sftpclient.c b/examples/sftpclient/sftpclient.c index 5ccfbe908..097f70b27 100644 --- a/examples/sftpclient/sftpclient.c +++ b/examples/sftpclient/sftpclient.c @@ -1940,51 +1940,18 @@ THREAD_RETURN WOLFSSH_THREAD sftpclient_test(void* args) WFREE(workingDir, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (ret == WS_SUCCESS) { - int err; - ret = wolfSSH_shutdown(ssh); - - /* peer hung up or channel already closed, stop trying. - * wolfSSH_shutdown() folds a peer EOF into WS_SUCCESS itself, so - * there is no WS_EOF to test for here. */ - if (ret == WS_SOCKET_ERROR_E || ret == WS_ERROR || - ret == WS_CHANNEL_CLOSED) { - ret = 0; - } - - err = wolfSSH_get_error(ssh); - if (err != WS_SOCKET_ERROR_E && - (err == WS_WANT_READ || err == WS_WANT_WRITE)) { - int maxAttempt = 10; /* make 10 attempts max before giving up */ - int attempt; - - for (attempt = 0; attempt < maxAttempt; attempt++) { - ret = wolfSSH_worker(ssh, NULL); - err = wolfSSH_get_error(ssh); - - /* peer successfully closed down gracefully */ - if (ret == WS_CHANNEL_CLOSED || ret == WS_EOF) { - ret = 0; - break; - } + int error; - /* peer hung up, stop shutdown */ - if (ret == WS_SOCKET_ERROR_E || ret == WS_ERROR) { - ret = 0; - break; - } - - if (err == WS_WANT_READ || err == WS_WANT_WRITE) { - /* Wanting read or wanting write. Clear ret. */ - ret = 0; - } - else { - break; - } - } + (void)wolfSSH_shutdown(ssh); + error = wolfSSH_get_error(ssh); - if (attempt == maxAttempt) { - printf("SFTP client gave up on graceful shutdown," + /* peer already hung up, just close */ + if (error != WS_SOCKET_ERROR_E) { + ret = SendDisconnectAndDrain(ssh, sockFd); + if (ret == WS_WANT_READ || ret == WS_WANT_WRITE) { + printf("SFTP client gave up on a graceful shutdown, " "closing the socket\n"); + ret = WS_SUCCESS; } } } diff --git a/tests/unit.c b/tests/unit.c index cbd00701f..c0fd583d9 100644 --- a/tests/unit.c +++ b/tests/unit.c @@ -47,6 +47,11 @@ #include #define WOLFSSH_TEST_HEX2BIN +/* The socket helpers, for the drain tests. A socketpair() found by configure + * says the platform has the rest of the sockets API they use, too. */ +#ifdef HAVE_SOCKETPAIR + #define WOLFSSH_TEST_CLIENT +#endif #include #include "unit.h" @@ -9256,6 +9261,303 @@ static int test_SendEofAfterDisconnect(void) } #endif /* NO_WOLFSSH_CLIENT */ +#if !defined(NO_WOLFSSH_CLIENT) && defined(HAVE_SOCKETPAIR) && \ + defined(WSHUTDOWN) +/* The drain helpers run over a real socket, so these use a socket pair with + * the session on one end and the test playing the peer on the other. The + * send callback optionally reports one would-block first. */ +static int s_drainWantWrites = 0; + +/* A send to a peer that is gone has to fail, not raise SIGPIPE. */ +#ifdef MSG_NOSIGNAL + #define DRAIN_SEND_FLAGS MSG_NOSIGNAL +#else + #define DRAIN_SEND_FLAGS 0 +#endif + +static int DrainSocketIoSend(WOLFSSH* ssh, void* buf, word32 sz, void* ctx) +{ + ssize_t n; + + (void)ssh; + if (s_drainWantWrites > 0) { + s_drainWantWrites--; + return WS_CBIO_ERR_WANT_WRITE; + } + n = send(*(SOCKET_T*)ctx, buf, sz, DRAIN_SEND_FLAGS); + return (n < 0) ? WS_CBIO_ERR_GENERAL : (int)n; +} + +enum { + DRAIN_PEER_HANGS_UP, + DRAIN_SHORT_SEND, + DRAIN_ALREADY_DISCONNECTED, + DRAIN_HALF_CLOSE_ONLY, + DRAIN_QUEUED_DISCONNECT, + DRAIN_PEER_GONE +}; + +/* Returns 0 when the helper returned want and the peer saw the disconnect + * on the wire, or saw nothing where nothing was meant to be sent. */ +static int RunDrainCase(int mode, int want, int base) +{ + WOLFSSH_CTX* ctx = NULL; + WOLFSSH* ssh = NULL; + SOCKET_T sv[2] = { -1, -1 }; + byte buf[256]; + int result = 0; + int ret; + ssize_t n; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) + return base; +#ifdef SO_NOSIGPIPE + { + int on = 1; + (void)setsockopt(sv[0], SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on)); + } +#endif + + ctx = wolfSSH_CTX_new(WOLFSSH_ENDPOINT_CLIENT, NULL); + if (ctx == NULL) { result = base - 1; goto done; } + wolfSSH_SetIOSend(ctx, DrainSocketIoSend); + + ssh = wolfSSH_new(ctx); + if (ssh == NULL) { result = base - 2; goto done; } + ssh->connectState = CONNECT_SERVER_USERAUTH_ACCEPT_DONE; + wolfSSH_SetIOWriteCtx(ssh, &sv[0]); + + (void)shutdown(sv[1], SHUT_WR); /* the peer hangs up */ + if (mode == DRAIN_PEER_GONE) { + /* and is gone before the disconnect goes out */ + close(sv[1]); + sv[1] = -1; + } + if (mode == DRAIN_SHORT_SEND) + s_drainWantWrites = 1; + if (mode == DRAIN_ALREADY_DISCONNECTED) + ssh->disconnected = 1; /* the peer's disconnect came first */ + if (mode == DRAIN_QUEUED_DISCONNECT) { + /* a short send leaves our own disconnect queued */ + s_drainWantWrites = 1; + ret = wolfSSH_SendDisconnect(ssh, WOLFSSH_DISCONNECT_BY_APPLICATION); + if (ret != WS_WANT_WRITE || !wolfSSH_OutputPending(ssh)) { + result = base - 7; + goto done; + } + } + + if (mode == DRAIN_HALF_CLOSE_ONLY || mode == DRAIN_QUEUED_DISCONNECT) + ret = HalfCloseAndDrain(ssh, sv[0]); + else + ret = SendDisconnectAndDrain(ssh, sv[0]); + /* the would-block was consumed, so the retry loop really ran */ + if (s_drainWantWrites != 0) { result = base - 6; goto done; } + if (ret != want) { result = base - 3; goto done; } + + /* The helper half closed its end, so the peer reads what was sent + * and then EOF. */ + if (mode == DRAIN_PEER_GONE) + goto done; + n = recv(sv[1], buf, sizeof(buf), MSG_DONTWAIT); + if (mode == DRAIN_ALREADY_DISCONNECTED || mode == DRAIN_HALF_CLOSE_ONLY) { + if (n != 0) { result = base - 4; goto done; } + } + else if (n <= 0) { + result = base - 5; + } + +done: + wolfSSH_free(ssh); + wolfSSH_CTX_free(ctx); + if (sv[0] != -1) + close(sv[0]); + if (sv[1] != -1) + close(sv[1]); + return result; +} + +/* The disconnect goes out and the peer's hang up ends the drain. */ +static int test_DrainPeerHangsUp(void) +{ + return RunDrainCase(DRAIN_PEER_HANGS_UP, WS_SUCCESS, -2100); +} + +/* A would-block on the first send leaves the disconnect queued; the retry + * flushes it and the drain still completes. */ +static int test_DrainRetriesShortSend(void) +{ + return RunDrainCase(DRAIN_SHORT_SEND, WS_SUCCESS, -2110); +} + +/* The peer disconnected first, so nothing is sent but its hang up is still + * waited for and reported as success. */ +static int test_DrainPeerAlreadyDisconnected(void) +{ + return RunDrainCase(DRAIN_ALREADY_DISCONNECTED, WS_SUCCESS, -2120); +} + +/* The server's path: nothing goes out, the half close alone ends the + * session and the peer's hang up is still seen. */ +static int test_DrainHalfCloseOnly(void) +{ + return RunDrainCase(DRAIN_HALF_CLOSE_ONLY, WS_SUCCESS, -2130); +} + +/* A disconnect of our own that a short send left queued still has to go + * out, though the session is already disconnected and the worker refuses + * to send on it. */ +static int test_DrainFlushesQueuedDisconnect(void) +{ + return RunDrainCase(DRAIN_QUEUED_DISCONNECT, WS_SUCCESS, -2140); +} + +/* The peer closed the connection after the close exchange, so the + * disconnect send fails, and that is the peer having hung up. */ +static int test_DrainPeerGone(void) +{ + return RunDrainCase(DRAIN_PEER_GONE, WS_SUCCESS, -2150); +} + +/* A session an error already ended has no close coming, so the wait reports + * that error at once instead of sitting out its deadline on a peer that is + * still connected. */ +static int test_WaitForChannelCloseSessionOver(void) +{ + static const int errs[] = { WS_OVERFLOW_E, WS_DISCONNECT }; + WOLFSSH_CTX* ctx = NULL; + WOLFSSH* ssh = NULL; + SOCKET_T sv[2] = { -1, -1 }; + time_t start; + int result = 0; + int ret; + word32 i; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) + return -2170; + + ctx = wolfSSH_CTX_new(WOLFSSH_ENDPOINT_CLIENT, NULL); + if (ctx == NULL) { result = -2171; goto done; } + ssh = wolfSSH_new(ctx); + if (ssh == NULL) { result = -2172; goto done; } + ssh->connectState = CONNECT_SERVER_USERAUTH_ACCEPT_DONE; + wolfSSH_set_fd(ssh, sv[0]); + + for (i = 0; i < sizeof(errs) / sizeof(errs[0]); i++) { + ssh->error = errs[i]; + start = WTIME(NULL); + ret = WaitForChannelClose(ssh, sv[0]); + if (ret != errs[i]) { result = -2173; goto done; } + if (WTIME(NULL) - start > 1) { result = -2174; goto done; } + } + +done: + wolfSSH_free(ssh); + wolfSSH_CTX_free(ctx); + close(sv[0]); + close(sv[1]); + return result; +} + +#if defined(WOLFSSH_TEST_INTERNAL) && \ + (defined(WOLFSSH_TERM) || defined(WOLFSSH_SHELL)) +/* Builds a plaintext CHANNEL_REQUEST exit-status packet into pkt (needs 40 + * bytes), returning its size. */ +static word32 BuildExitStatusPacket(byte* pkt, word32 channelId, + word32 status) +{ + word32 i = 0; + + /* packet_length = padLen(1) + payload(25) + padding(10) = 36. */ + PutU32BE(pkt + i, 36); i += 4; + pkt[i++] = 10; + pkt[i++] = MSGID_CHANNEL_REQUEST; + PutU32BE(pkt + i, channelId); i += 4; + PutU32BE(pkt + i, 11); i += 4; + WMEMCPY(pkt + i, "exit-status", 11); i += 11; + pkt[i++] = 0; /* want reply */ + PutU32BE(pkt + i, status); i += 4; + WMEMSET(pkt + i, 0, 10); i += 10; + + return i; /* 40 */ +} + +/* Builds a plaintext empty IGNORE packet into pkt (needs 16 bytes), + * returning its size. */ +static word32 BuildIgnorePacket(byte* pkt) +{ + word32 i = 0; + + /* packet_length = padLen(1) + msgid(1) + string(4) + padding(6) = 12. */ + PutU32BE(pkt + i, 12); i += 4; + pkt[i++] = 6; + pkt[i++] = MSGID_IGNORE; + PutU32BE(pkt + i, 0); i += 4; + WMEMSET(pkt + i, 0, 6); i += 6; + + return i; /* 16 */ +} + +/* The client read loop stops at the peer's EOF, and the peer sends one more + * message before its exit status. The worker pass inside wolfSSH_shutdown() + * takes that one, so the exit status is parsed only if the close is waited + * for ahead of the drain, which discards what it reads. */ +static int test_WaitForChannelCloseKeepsExitStatus(void) +{ + WOLFSSH_CTX* ctx = NULL; + WOLFSSH* ssh = NULL; + WOLFSSH_CHANNEL* ch; + SOCKET_T sv[2] = { -1, -1 }; + byte pkt[128]; + word32 sz = 0; + int result = 0; + int ret; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) + return -2160; + + ctx = wolfSSH_CTX_new(WOLFSSH_ENDPOINT_CLIENT, NULL); + if (ctx == NULL) { result = -2161; goto done; } + ssh = wolfSSH_new(ctx); + if (ssh == NULL) { result = -2162; goto done; } + ssh->connectState = CONNECT_SERVER_USERAUTH_ACCEPT_DONE; + wolfSSH_set_fd(ssh, sv[0]); + + ch = ChannelNew(ssh, ID_CHANTYPE_SESSION, 1024, 1024); + if (ch == NULL) { result = -2163; goto done; } + if (ChannelAppend(ssh, ch) != WS_SUCCESS) { + ChannelDelete(ch, ssh->ctx->heap); + result = -2164; + goto done; + } + + sz += BuildChannelEofPacket(pkt + sz, ch->channel); + sz += BuildIgnorePacket(pkt + sz); + sz += BuildExitStatusPacket(pkt + sz, ch->channel, 42); + sz += BuildChannelClosePacket(pkt + sz, ch->channel); + if (send(sv[1], pkt, sz, 0) != (ssize_t)sz) { result = -2165; goto done; } + + (void)wolfSSH_worker(ssh, NULL); /* the read loop's EOF */ + ret = wolfSSH_shutdown(ssh); + if (ret == WS_CHANNEL_CLOSED) { result = -2166; goto done; } + + ret = WaitForChannelClose(ssh, sv[0]); + if (ret != WS_SUCCESS) { result = -2167; goto done; } + if (wolfSSH_GetExitStatus(ssh) != 42) { result = -2168; goto done; } + +done: + wolfSSH_free(ssh); + wolfSSH_CTX_free(ctx); + if (sv[0] != -1) + close(sv[0]); + if (sv[1] != -1) + close(sv[1]); + return result; +} +#endif /* WOLFSSH_TEST_INTERNAL && (WOLFSSH_TERM || WOLFSSH_SHELL) */ +#endif /* !NO_WOLFSSH_CLIENT && HAVE_SOCKETPAIR && WSHUTDOWN */ + + #ifndef NO_WOLFSSH_CLIENT /* SendChannelEof() addresses the channel by peer id, and peerChannel is 0 * until the peer confirms the open. wolfSSH_ChannelSendEof() on a channel @@ -23456,6 +23758,45 @@ int wolfSSH_UnitTest(int argc, char** argv) testResult = testResult || unitResult; #endif /* NO_WOLFSSH_CLIENT */ +#if !defined(NO_WOLFSSH_CLIENT) && defined(HAVE_SOCKETPAIR) && \ + defined(WSHUTDOWN) + unitResult = test_DrainPeerHangsUp(); + printf("DrainPeerHangsUp: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; + unitResult = test_DrainRetriesShortSend(); + printf("DrainRetriesShortSend: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; + unitResult = test_DrainPeerAlreadyDisconnected(); + printf("DrainPeerAlreadyDisconnected: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; + unitResult = test_DrainHalfCloseOnly(); + printf("DrainHalfCloseOnly: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; + unitResult = test_DrainFlushesQueuedDisconnect(); + printf("DrainFlushesQueuedDisconnect: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; + unitResult = test_DrainPeerGone(); + printf("DrainPeerGone: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; + unitResult = test_WaitForChannelCloseSessionOver(); + printf("WaitForChannelCloseSessionOver: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; +#if defined(WOLFSSH_TEST_INTERNAL) && \ + (defined(WOLFSSH_TERM) || defined(WOLFSSH_SHELL)) + unitResult = test_WaitForChannelCloseKeepsExitStatus(); + printf("WaitForChannelCloseKeepsExitStatus: %s\n", + (unitResult == 0 ? "SUCCESS" : "FAILED")); + testResult = testResult || unitResult; +#endif +#endif + #ifndef NO_WOLFSSH_CLIENT unitResult = test_SendChannelEofWantWrite(); printf("SendChannelEofWantWrite: %s\n", diff --git a/wolfssh/test.h b/wolfssh/test.h index 8ed3d24de..9cf012dbf 100644 --- a/wolfssh/test.h +++ b/wolfssh/test.h @@ -40,6 +40,10 @@ #include /*#include */ #endif +#include +#include + +#include #ifdef WOLFSSH_SCP /* for WS_CallbackScpSend used in func_args; test.h is included by sources @@ -241,8 +245,11 @@ #endif +/* WSHUTDOWN half closes: stop sending, keep receiving until peer EOF. Left + * undefined where the stack has no half close. */ #ifdef USE_WINDOWS_API #define WCLOSESOCKET(s) closesocket(s) + #define WSHUTDOWN(s) (void)shutdown((s), SD_SEND) #define WSTARTTCP() do { WSADATA wsd; (void)WSAStartup(0x0002, &wsd); } while(0) #elif defined(MICROCHIP_TCPIP) || defined(MICROCHIP_MPLAB_HARMONY) #ifdef MICROCHIP_MPLAB_HARMONY @@ -254,8 +261,13 @@ #elif defined(WOLFSSL_NUCLEUS) #define WCLOSESOCKET(s) NU_Close_Socket((s)) #define WSTARTTCP() +#elif defined(FREESCALE_MQX) + /* RTCS shutdown() takes its own flags, not SHUT_WR */ + #define WCLOSESOCKET(s) close(s) + #define WSTARTTCP() #else #define WCLOSESOCKET(s) close(s) + #define WSHUTDOWN(s) (void)shutdown((s), SHUT_WR) #define WSTARTTCP() #endif @@ -825,6 +837,263 @@ static INLINE int tcp_select_write(SOCKET_T socketfd, int to_sec) return WS_SELECT_FAIL; } + +/* Push out what a short send left queued, half close the socket, then read + * until the peer hangs up. The library reads nothing once the session is + * over, so the drain reads the socket directly and discards what it gets, + * any protocol message still in flight included: collect the exit status + * and the like before calling. Flush and drain together give up ten + * seconds after the call, plus the one-second select() wait in progress. + * Caller still owns and closes the socket. Returns WS_SUCCESS once the + * peer has hung up, WS_WANT_READ if it never did, or WS_WANT_WRITE if the + * queued output never went out. */ +static INLINE int HalfCloseAndDrain(WOLFSSH* ssh, SOCKET_T fd) +{ + time_t deadline = WTIME(NULL) + 10; + int ret; + int sel; + int fails = 0; +#ifdef WSHUTDOWN + char buf[1024]; + int reads = 0; + int rcv; +#endif + + /* The worker flushes after its receive pass, and any of its normal + * events is a pass that made progress: wolfSSH_OutputPending() says + * when the flush is done. A session that is over or a socket that is + * gone ends the flush, and the drain still runs: either way the peer's + * hang up is what is left to see. */ + while (wolfSSH_OutputPending(ssh)) { + if (WTIME(NULL) >= deadline) + return WS_WANT_WRITE; + sel = tcp_select_write(fd, 1); + if (sel == WS_SELECT_SEND_READY) { + ret = wolfSSH_worker(ssh, NULL); + if (ret == WS_FATAL_ERROR) + ret = wolfSSH_get_error(ssh); + if (ret == WS_DISCONNECT) { + /* The worker sends nothing once the session is disconnected. + * A disconnect of our own that a short send left queued is + * the one thing still owed, and this flush-only path sends + * it; it refuses after the peer's disconnect, and then the + * queued output is moot. */ + ret = wolfSSH_SendDisconnect(ssh, + WOLFSSH_DISCONNECT_BY_APPLICATION); + if (ret == WS_FATAL_ERROR) + break; + } + if (ret != WS_SUCCESS && ret != WS_WANT_READ && + ret != WS_WANT_WRITE && ret != WS_CHAN_RXD && + ret != WS_EXTDATA && ret != WS_EOF && + ret != WS_REKEYING && ret != WS_CHANNEL_CLOSED) + break; + } + else if (sel == WS_SELECT_FAIL) { + /* EINTR is a failure too, so a few are waited out, but a + * select() that keeps failing must not spin to the deadline */ + if (++fails >= 10) + return WS_WANT_WRITE; + } + else if (sel != WS_SELECT_TIMEOUT) { + /* an exception alone is urgent data, not room to send */ + return WS_WANT_WRITE; + } + } + +#ifdef WSHUTDOWN + WSHUTDOWN(fd); + fails = 0; + /* The read cap bounds the bytes a streaming peer costs. It covers two + * default windows, so a peer that was mid-echo is not reported as never + * hanging up. */ + while (reads < 256 && WTIME(NULL) < deadline) { + sel = tcp_select(fd, 1); + if (sel == WS_SELECT_TIMEOUT) + continue; + if (sel == WS_SELECT_FAIL) { + if (++fails >= 10) + break; + continue; + } + /* an exception alone is urgent data, which recv() would block on */ + if (sel != WS_SELECT_RECV_READY) + break; + + reads++; + rcv = (int)recv(fd, buf, (int)sizeof(buf), 0); + if (rcv == 0) + return WS_SUCCESS; + if (rcv < 0) { + #ifdef USE_WINDOWS_API + int err = WSAGetLastError(); + if (err == WSAEWOULDBLOCK || err == WSAEINTR) + continue; + #else + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) + continue; + #endif + /* a reset, the peer is gone */ + return WS_SUCCESS; + } + } + return WS_WANT_READ; +#else + (void)fd; + return WS_SUCCESS; +#endif +} + + +/* Run the session until the peer's channel close, so what the peer sends + * ahead of it, the exit status included, is parsed rather than drained. For + * a client whose wolfSSH_shutdown() did not already see the close. Gives up + * ten seconds after the call. Returns WS_SUCCESS once the channel is + * closed, WS_WANT_READ if the close never came, or the error that ended + * the session, at once if it already had. */ +static INLINE int WaitForChannelClose(WOLFSSH* ssh, SOCKET_T fd) +{ + time_t deadline = WTIME(NULL) + 10; + int ret; + int sel; + int ready; + int fails = 0; + + /* A session that is already over has no close coming. */ + ret = wolfSSH_get_error(ssh); + if (ret == WS_CHANNEL_CLOSED) + return WS_SUCCESS; + if (ret != WS_SUCCESS && ret != WS_WANT_READ && + ret != WS_WANT_WRITE && ret != WS_CHAN_RXD && + ret != WS_EXTDATA && ret != WS_EOF && ret != WS_REKEYING) + return ret; + + while (WTIME(NULL) < deadline) { + /* The close the peer is to answer may still be queued, and the + * worker flushes it. */ + if (wolfSSH_OutputPending(ssh)) { + sel = tcp_select_write(fd, 1); + ready = (sel == WS_SELECT_SEND_READY); + } + else { + sel = tcp_select(fd, 1); + ready = (sel == WS_SELECT_RECV_READY); + } + if (sel == WS_SELECT_TIMEOUT) + continue; + if (sel == WS_SELECT_FAIL) { + if (++fails >= 10) + break; + continue; + } + /* an exception alone is urgent data, which the worker would block + * on */ + if (!ready) + break; + + ret = wolfSSH_worker(ssh, NULL); + if (ret == WS_FATAL_ERROR) + ret = wolfSSH_get_error(ssh); + if (ret == WS_CHANNEL_CLOSED) + return WS_SUCCESS; + if (ret != WS_SUCCESS && ret != WS_WANT_READ && + ret != WS_WANT_WRITE && ret != WS_CHAN_RXD && + ret != WS_EXTDATA && ret != WS_EOF && ret != WS_REKEYING) + return ret; + } + return WS_WANT_READ; +} + + +/* Send MSG_DISCONNECT, then HalfCloseAndDrain(). A server ending a session + * normally wants the drain alone: the close exchange already ended the + * session, and OpenSSH's client fails on a disconnect that lands before it + * is done. The send retry gives up ten seconds in, so with the drain a peer + * that never hangs up costs up to twenty seconds. Returns what + * the drain returns, WS_SUCCESS too if the send found the peer gone, + * WS_WANT_WRITE if the disconnect is still queued, or the error that failed + * the send. The disconnect carries out whatever a short send left queued + * before it. */ +static INLINE int SendDisconnectAndDrain(WOLFSSH* ssh, SOCKET_T fd) +{ + time_t deadline = WTIME(NULL) + 10; + int ret; + int sel; + int timeouts = 0; + + /* A retry flushes the disconnect a short send left queued. A socket + * that is ready while the send still is not counts against the + * deadline, not the timeouts. */ + ret = wolfSSH_SendDisconnect(ssh, WOLFSSH_DISCONNECT_BY_APPLICATION); + while (ret == WS_WANT_WRITE) { + if (WTIME(NULL) >= deadline) + return WS_WANT_WRITE; + sel = tcp_select_write(fd, 1); + if (sel == WS_SELECT_SEND_READY) { + ret = wolfSSH_SendDisconnect(ssh, + WOLFSSH_DISCONNECT_BY_APPLICATION); + } + else if (sel == WS_SELECT_TIMEOUT || sel == WS_SELECT_FAIL) { + /* select() fails on EINTR too, so a failure gets the same + * bounded retry as a timeout. */ + if (++timeouts >= 10) + return WS_WANT_WRITE; + } + else { + /* an exception alone is urgent data, not room to send */ + return WS_WANT_WRITE; + } + } + + /* The peer's disconnect ended the session first, so there is nothing + * to send, but its hang up is still worth waiting for. */ + if (ret == WS_FATAL_ERROR && wolfSSH_get_error(ssh) == WS_DISCONNECT) + ret = WS_SUCCESS; + /* The peer reset or closed the connection after the close exchange, so + * it already hung up, as a reset during the drain says too. */ + else if (ret == WS_SOCKET_ERROR_E) + return WS_SUCCESS; + if (ret != WS_SUCCESS) + return ret; + + return HalfCloseAndDrain(ssh, fd); +} + +#elif defined(WOLFSSH_TEST_SERVER) || defined(WOLFSSH_TEST_CLIENT) + +/* MQX has no half close, so the drain is one flushing worker pass, and no + * select helpers to retry with, so the disconnect gets one send. */ +static INLINE int HalfCloseAndDrain(WOLFSSH* ssh, SOCKET_T fd) +{ + (void)fd; + if (wolfSSH_OutputPending(ssh)) + (void)wolfSSH_worker(ssh, NULL); + return WS_SUCCESS; +} + +/* Nor a select() to bound a longer wait with, so one worker pass. */ +static INLINE int WaitForChannelClose(WOLFSSH* ssh, SOCKET_T fd) +{ + (void)fd; + if (wolfSSH_worker(ssh, NULL) == WS_CHANNEL_CLOSED) + return WS_SUCCESS; + return WS_WANT_READ; +} + +static INLINE int SendDisconnectAndDrain(WOLFSSH* ssh, SOCKET_T fd) +{ + int ret; + + ret = wolfSSH_SendDisconnect(ssh, WOLFSSH_DISCONNECT_BY_APPLICATION); + if (ret == WS_FATAL_ERROR && wolfSSH_get_error(ssh) == WS_DISCONNECT) + ret = WS_SUCCESS; + else if (ret == WS_SOCKET_ERROR_E) + return WS_SUCCESS; + if (ret != WS_SUCCESS) + return ret; + return HalfCloseAndDrain(ssh, fd); +} + #endif /* WOLFSSH_TEST_SERVER || WOLFSSH_TEST_CLIENT */