Mysterious 1 second wait for requests that exceed backlog

Asked 2 years ago, Updated 2 years ago, 367 views

When I was experimenting with writing a C++ client/server code on a local host that repeatedly connects and disconnects TCP, I encountered a mysterious one-second pause in processing requests that exceed the backlog of the listen argument.Please explain this phenomenon and tell me how to deal with it.

Lubuntu 20.04 x86_64
gcc 9.4.0
C++17

#include<iostream>
# include <cstring>
# include <chrono>
# include <sstream>
# include <iomanip>
# include <arpa/inet.h>
# include<sys/socket.h>
# include <unistd.h>
using namespace std;
const string&appname(const string&s=string()){
    static string_appname;
    if(!s.empty())_appname=s;
    return_appname;
}
using namespace std::chrono;
US>string timestr() {
    system_clock::time_point scp=system_clock::now();
    time_tt=system_clock::to_time_t(scp);
    tmlt;
    US>stringstream s;
    ss<put_time(localtime_r(&t,&lt), "%Y/%m/%e%H:%M:%S")<", "<setw(3)<setfill('0')<duration_cast<millisecond_(:glock_time)))
    returns.str();
}
void log(ostream&out, const string&s) {out<<timestr()<"<appname()<":"<s<endl;}
void log(const string&s) {log(cout,s);}
void log_error(const string&s) {log(cerr,s+":"+strror(errno)));}
#define LOG_ERROR()log_error(string(__FILE__)+":"+to_string(__LINE__))
US>structure Sock {
    Sock(intsock):_sock{sock}{}
    ~Sock() { if(_sock>=0)::close(_sock);}
    operator int() {return_sock;}
    void close() { if(_sock>=0)::close(_sock);_sock=-1;}
private:
    int_sock;
    Sock(){}Sock(const Sock&){}Sock&operator=(const Sock&){return*this;}
};
# define IF_ERROR_RETURN(cond) if(cond) {LOG_ERROR(); return;}
int client(const int PORT=8000, const size_t N=0x100000){
    auto func=[](auto id, int port){
        US>//#string idhead {to_string(id)+":"};
        auto log_with_id = [&] (autos) {log(idhead+s);};
        Socksock=socket(AF_INET, SOCK_STREAM, 0);
        IF_ERROR_RETURN(sock<0);
        structure sockaddr_inaddr;
        addr.sin_family=AF_INET;
        addr.sin_port = htons(port);
        IF_ERROR_RETURN(inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr)<=0);
        IF_ERROR_RETURN(connect(sock, (struct sockaddr*)&addr,sizeof(addr))<0);
        log_with_id("connection established");
        structure sockaddr_in local_address;
        socklen_taddr_size=sizeof(local_address);
        IF_ERROR_RETURN (getsockname(sock, (sockaddr*) (void*)&local_address,&addr_size)!=0);
        log_with_id("localport:"+to_string(ntohs(local_address.sin_port)));
        shutdown (sock, SHUT_WR);
        log_with_id("write shutdown");
        shutdown (sock, SHUT_RD);
        log_with_id("recv shutdown");
        sock.close();
    };
    for(inti=0;i<N;++i){
        strings {to_string(i)};
        log(s+":started");
        func(i,PORT);
        log(s+":finished");
    }
    return 0;
}
#define IF_ERROR_BREAK(cond) if(cond) {LOG_ERROR();break;}
void server(const int PORT=8000) {
    auto log_server=[](const strings) { log(string("server:")+s);};
    structure sockaddr_inaddr;
    structure sockaddr_inclient;
    intsock;
    Sock serverSock=socket(AF_INET, SOCK_STREAM, 0);
    IF_ERROR_RETURN (serverSock<0);
    addr.sin_family=AF_INET;
    addr.sin_port = htons(PORT);
    addr.sin_addr.s_addr=INADDR_ANY;
    IF_ERROR_RETURN(bind(serverSock, (struct sockaddr*)&addr,sizeof(addr))<0);
    IF_ERROR_RETURN (listen(serverSock, 5)<0);
    while(true){
        socklen_tlen=sizeof(client);
        Socksock=accept(serverSock, (struct sockaddr*)&client,&len);
        IF_ERROR_BREAK(sock<0);
        log_server("remoteport:"+to_string(ntohs(client.sin_port)));
        char buff [4096];
        ssize_treadlen=read(sock, buff, sizeof(buff)));
        IF_ERROR_BREAK(readlen<0);
        if(readlen!=0)log_server("read!!!not empty!!!");
        sock.close();
        log_server("closed");
    }
    serverSock.close();
}
int main(intargc, char*argv[], char*envp[]){
    appname(argv[0]);
    if(argc>1&string(argv[1])=="--server"){server();return0;}
    else {return client();}
}

Run the following script to log

trap'pkill-P$$'INT
./maxbacklog_time --server&
server=$!
./maxbacklog_time
kill$server

The client (225) has been waiting for about a second for some reason, as shown below.

...
October 27, 2022 18:44:31,859./maxbacklog_time:server:remoteport:35178
...
2022/10/27 18:44:31,859./maxbacklog_time:221:localport:35200
...
2022/10/27 18:44:31,865./maxbacklog_time:222:localport:35206
...
2022/10/27 18:44:31,865./maxbacklog_time:223:localport:35222
...
2022/10/27 18:44:31,865./maxbacklog_time:224:localport:35232
...
October 27, 2022 18:44:31,865./maxbacklog_time:224:finished
2022/10/27 18:44:31,865./maxbacklog_time:225:started
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
October 27, 2022 18:44:31,867./maxbacklog_time:server:remoteport:35186
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
October 27, 2022 18:44:31,867./maxbacklog_time:server:remoteport:35196
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
October 27, 2022 18:44:31,867./maxbacklog_time:server:remoteport:35200
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
October 27, 2022 18:44:31,867./maxbacklog_time:server:remoteport:35206
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
October 27, 2022 18:44:31,867./maxbacklog_time:server:remoteport:35222
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
October 27, 2022 18:44:31,867./maxbacklog_time:server:remoteport:35232
2022/10/27 18:44:31,867./maxbacklog_time:server:closed
2022/10/27 18:44:32,879./maxbacklog_time:225:connection established
...

Please explain the symptom and let me know if there is a way to avoid a mysterious second without changing synchronous I/O & single thread & (application layer) data transmission and reception.

c++ network tcp

2022-10-29 09:03

1 Answers

Packet Log

 18:44:31.865734 IP localhost.35238>localhost.8000: Flags [S], seq2050667274, win65495, options [mss65495, sackOK, TSval2726005063ecr0, nop, wscale7], length 0
18:44:31.867175 IP localhost.8000>localhost.35186: Flags [F.], seq1,ack2,win512,options [nop,nop,TSval2726005064ecr2726005056],length0
18:44:31.867183 IP localhost.35186>localhost.8000: Flags[.],ack2,win512,options[nop,nop,TSval2726005064ecr2726005064],length0
18:44:31.867284 IP localhost.8000>localhost.35196: Flags [F.], seq1,ack2,win512,options [nop,nop,TSval2726005064ecr2726005056],length0
18:44:31.867290 IP localhost.35196>localhost.8000: Flags[.],ack2,win512,options[nop,nop,TSval2726005064ecr2726005064],length0
18:44:31.867383 IP localhost.8000>localhost.35200: Flags [F.], seq1,ack2,win512,options [nop,nop,TSval2726005064ecr2726005062],length0
18:44:31.867393 IP localhost.35200>localhost.8000: Flags[.],ack2,win512,options[nop,nop,TSval2726005064ecr2726005064],length0
18:44:31.867455 IP localhost.8000>localhost.35206: Flags [F.], seq1,ack2,win512,options [nop,nop,TSval2726005064ecr2726005063],length0
18:44:31.867479 IP localhost.35206>localhost.8000: Flags[.],ack2,win512,options[nop,nop,TSval2726005064ecr2726005064],length0
18:44:31.867541 IP localhost.8000>localhost.35222: Flags [F.], seq1,ack2,win512,options [nop,nop,TSval2726005064ecr2726005063],length0
18:44:31.867566 IP localhost.35222>localhost.8000: Flags[.],ack2,win512,options[nop,nop,TSval2726005064ecr2726005064],length0
18:44:31.867628 IP localhost.8000>localhost.35232: Flags [F.], seq1,ack2,win512,options [nop,nop,TSval2726005065ecr2726005063],length0
18:44:31.867651 IP localhost.35232>localhost.8000: Flags[.],ack2,win512,options[nop,nop,TSval2726005065ecr2726005065],length0
18:44:32.879272 IP localhost.35238>localhost.8000: Flags [S], seq2050667274, win65495, options [mss65495, sackOK, TSval2726006076ecr0, nop, wscale7], length 0

In a nutshell, a mysterious second is Linux's SYN packet retransmission time.

Why doesn't the server and client alternate in the log

The reason why servers and clients do not alternate between reading and writing using synchronous I/O and single thread without checking communication status is that when the server receives SYN packets that appear when the client requests a connection, the TCP stack returns SYN-ACK before accept returns.

http://arturchiao.art/blog/tcp-listen-a-tale-of-two-queues/

When the client receives the SYN-ACK, the client returns the ACK and the connect returns, so it can disconnect and start the next connection process.Therefore, depending on the timing, multiple connectivity/disconnect operations can be performed before the server processes accept.

What is a mysterious second?

If the server stores the accept queue for backlog minutes, the SYN packet is discarded, so the client does not receive a SYN-ACK.Therefore, the client will start retransmitting the SYN packet after 1 second.This means that it takes a second for the client to detect that the SYN packet has been dropped.

As long as the SYN-ACK is returned before accept, there is no guarantee that the server and client will move alternately, and this will prevent them from accumulating in the accept queue for the backlog.Unless you change the data transferred to and from the application layer, it seems impossible.

Server → One byte of communication to the client will cause the server and client to alternate and no SYN retransmissions will occur.

@@-51,6+51,10@@int client(const int PORT=8000,const
         socklen_taddr_size=sizeof(local_address);
         IF_ERROR_RETURN (getsockname(sock, (sockaddr*) (void*)&local_address,&addr_size)!=0);
         log_with_id("localport:"+to_string(ntohs(local_address.sin_port)));
+        char buffer;
+        ssize_tlen=read(sock,&buffer,sizeof(buffer)));
+        IF_ERROR_RETURN(len<0);
+        if(len==0) {log_with_id("unexpected EOF!!!");}
         shutdown (sock, SHUT_WR);
         log_with_id("write shutdown");
         shutdown (sock, SHUT_RD);
@@ -83,7+87,9@@void server(const int PORT=8000){
         Socksock=accept(serverSock, (struct sockaddr*)&client,&len);
         IF_ERROR_BREAK(sock<0);
         log_server("remoteport:"+to_string(ntohs(client.sin_port)));
-        char buff [4096];
+        char buff [4096]; buff [0] = 0;
+        IF_ERROR_BREAK (write(sock, buff, 1)<0);
+        log_server("wrotte1byte");
         ssize_treadlen=read(sock, buff, sizeof(buff)));
         IF_ERROR_BREAK(readlen<0);
         if(readlen!=0)log_server("read!!!not empty!!!");


2022-10-29 09:03

If you have any answers or tips


© 2024 OneMinuteCode. All rights reserved.