Threaded Apache and lingering_close

[Jim Jagielski <ji...@jaguNET.com>]

The last version of RSTs threaded apache included some handling
of a lingering_close... below I've condense the info:


enum fd_state { fd_closed = 0, fd_listener, fd_active, fd_keepalive,
		fd_linger, };

typedef struct fd_record {
    enum fd_state state;
    int fd;
    struct timeval lru;
    struct sockaddr addr;
    protocol_rec *protocol;
} fd_record;

static void bump_fd_count (enum fd_state state, int incr) {
    switch (state) {
    case fd_listener: num_fds_listening += incr; break;
    case fd_active: num_fds_active += incr; break;
    case fd_keepalive:
    case fd_linger: num_fds_keptalive += incr; break;
    case fd_closed: num_fds_closed += incr; break;
    default: assert (!"illegal fd state");
    }
}

static void set_fd_state (enum fd_state new_state,
			  int fd, struct sockaddr *addr,
			  protocol_rec *protocol)
{
    int i;
    fd_record *rec = NULL;
    
    /* First look for active record on this fd --- */
    
    for (i = 0; i < fd_table_size; ++i)
	if (fd_table[i].fd == fd && fd_table[i].state != fd_closed) {
	    rec = &fd_table[i];
	    break;
	}

    /* None? --- then opening; allocate new one */

    if (rec == NULL) {
#ifndef NASTY_RACE_CONDITION_FIXED_REALLY_THIS_TIME
	if (new_state == fd_closed)
	    return;
#endif	
	assert (new_state != fd_closed);
	
	if (fd > max_table_fd) max_table_fd = fd;
	
	for (i = 0; i < fd_table_size; ++i)
	    if (fd_table[i].state == fd_closed) {
		rec = &fd_table[i];
		break;
	    }
    } 
	
    /* Whatever we found, it is leaving its old state...
     * update it and keep the counts in sync.
     */
    
    assert (rec != NULL);
    
    bump_fd_count (rec->state, -1);
    bump_fd_count (new_state, 1);

    rec->fd = fd;
    rec->state = new_state;
    if (addr != NULL) rec->addr = *addr;
    if (protocol != NULL) rec->protocol = protocol;
    gettimeofday (&rec->lru, NULL);
}

/* Finding least-recently-used ketpalive (or lingering) fd,
 * so we can bounce it.
 */

static int find_lru_keepalive () {
    int i;
    fd_record *rec = NULL;

    assert (num_fds_keptalive > 0);

    /* Find first */

    for (i = 0; i < fd_table_size; ++i)
	if (fd_table[i].state == fd_keepalive
	    || fd_table[i].state == fd_linger)
	{
	    rec = &fd_table[i];
	    break;
	}

    /* Find best --- usecs of no account here... */

    for (; i < fd_table_size; ++i)
	if ((fd_table[i].state == fd_keepalive
	     || fd_table[i].state == fd_linger)
	    && fd_table[i].lru.tv_sec < rec->lru.tv_sec)
	{
	    rec = &fd_table[i];
	}

    return rec->fd;
}

/*****************************************************************
 *
 * Timeout handling.  With multiple transactions in a single address
 * space, global vars are out the window for this.  With multiple
 * threads involved in servicing a given transaction, as for mux,
 * per-thread data also gets ugly...
 */

static void timeout_or_abort (conn_rec *c)
{
    char errstr[MAX_STRING_LEN];
    void *dirconf = c->server->lookup_defaults; /* Sigh... */
    int is_timeout = (errno == ETIMEDOUT);

    /* Common code for timeout and io_error */

    if (!is_timeout) {
        sprintf(errstr,"%s lost connection to client %s",
	    c->timeout_name ? c->timeout_name : "request",
	    get_remote_host(c, dirconf, REMOTE_NAME));
    } else {
        sprintf(errstr,"%s timed out for %s",
	    c->timeout_name ? c->timeout_name : "request",
	    get_remote_host(c, dirconf, REMOTE_NAME));
    }
    
    if (!c->keptalive)
	log_error (errstr, c->server);

    abort_connection (c);
}

void hard_timeout (char *name, request_rec *r) {
    r->connection->timeout_name = name;
}

void soft_timeout (char *name, request_rec *r) {
    r->connection->timeout_name = name;
}

void kill_timeout (request_rec *r) {
    r->connection->timeout_name = NULL;
}

void reset_timeout (request_rec *r) {
}

static int transaction_thread (void *ainfo)
{
    struct new_conn_info *info = (struct new_conn_info *)ainfo;
    pool *ptrans = info->p;
    struct sockaddr sa_client = info->sa_client;
    protocol_rec *protocol = info->protocol;
    int clen;
    struct sockaddr sa_server;
    conn_rec *current_conn;
    int got_req;
    int keepalive, aborted;
    
    clen = sizeof(sa_server);
    if (getsockname(info->csd, &sa_server, &clen) < 0) {
	log_unixerr("getsockname", NULL, NULL, server_conf);
	destroy_pool (ptrans);
	set_fd_state (fd_closed, info->csd, NULL, NULL);
	return 0;
    }
	
    current_conn = raw_connection (ptrans, server_conf, protocol, info->csd,
				   (struct sockaddr_in *)&sa_client,
				   (struct sockaddr_in *)&sa_server);
	
    while ((got_req = handle_connection_protocol (current_conn))
	   && current_conn->keepalive && !current_conn->aborted)
    {
	/* If the client has another transaction coming immediately,
	 * no need to thrash the memory pool; keep on processing it in
	 * this thread.  Maintaining idle threads does have a cost,
	 * though, and if a thread would go idle for *too* long, then
	 * we don't want to waste RAM on its stack.  So, wait a bit,
	 * and then tear down the thread if we haven't heard from the
	 * guy (but leave the connection open in case he comes back).
	 */
	current_conn->timeout_secs = current_conn->server->keep_alive_timeout;
    }
	
    THR_set_thread_name ("Thread shutting down");

    /* Whichever happened, we're finished here; close the FILE* *without*
     * closing the socket... yet.  Annoying how we have to trick sfio like
     * this, but them's the breaks.  (What's more annoying is the need to
     * keep another thread from spawning off a child while we're doing this
     * if we ever go preemptive, to keep the descriptor we're messing with
     * from getting passed to that child).
     */
    
    keepalive = current_conn->keepalive;
    aborted = current_conn->aborted;
    
    destroy_pool (ptrans);	/* Closes FILE *'s */
    
    /* OK, this thread is done... have to arrange for disposition of
     * the client socket.  If keeping alive, arrange for a new thread
     * to be started if the client ever does send another request.
     */

    if (keepalive && !aborted) {
	set_fd_state (fd_keepalive, info->csd, NULL, NULL);
	return 0;
    }

    /* No more requests on this connection.
     * Close forcibly if we timed out, otherwise try to arrange a
     * lingering close (assuming a decently working client ---
     * client_is_macintrash used to be one of the triggers for
     * a hard close, while I was using that kludge).
     */

    if (aborted || !got_req) {
	close (info->csd);
	set_fd_state (fd_closed, info->csd, NULL, NULL);
    } else {
	shutdown (info->csd, 1);
	set_fd_state (fd_linger, info->csd, NULL, NULL);
    }

    return 0;
}


/* State-transition function for the connection-handling state machine. */

static void handle_readable_fd (pool *pconf, fd_record *rec,
				THR_attr *thread_parms)
{
    long numrd;
    int csd, clen;
    struct sockaddr sa_client;
    
    switch (rec->state) {
    case fd_linger:
	/* Sometimes clients chase a genuine request with a few trash
	 * newlines... be prepared for that.  (NB we couldn't treat it
	 * as a request even if we wanted to --- we've already shut down
	 * our half of the connection as the first step in a lingering close).
	 */

	if (ioctl (rec->fd, FIONREAD, (void *)&numrd) >= 0
	    && numrd > 0)
	{
	    /* Client sent extra junk.  Get rid of it.
	     * On error, lose the connection; we don't need the trouble
	     * (especially since the client is clearly deranged to begin with).
	     */
	    char junkbuf[256];
	    if (read (rec->fd, junkbuf, sizeof(junkbuf)) <= 0) {
		log_unixerr("read from half-closed connection",
			    NULL, NULL, server_conf);
		shutdown (rec->fd, 2);
		close (rec->fd);
		set_fd_state (fd_closed, rec->fd, NULL, NULL);
	    }
	}
	else {
	    /* Got client ack, have EOF.  Can close safely now. */
	    close (rec->fd);
	    set_fd_state (fd_closed, rec->fd, NULL, NULL);
	}
	break;

    case fd_keepalive:
	/* If the client has closed the socket on us, close the connection */

	if (ioctl (rec->fd, FIONREAD, (void *)&numrd) < 0 || numrd == 0) {
	    shutdown (rec->fd, 2); close (rec->fd);
	    set_fd_state (fd_closed, rec->fd, NULL, NULL);
	}
	else {
	    /* OK, this is for real.  Start a new thread... */
	    set_fd_state (fd_active, rec->fd, NULL, NULL);
	    start_transaction_thread (pconf, rec->fd, &rec->addr,
				      rec->protocol, thread_parms);
	}
	break;

    case fd_listener:

	if ((csd = try_accept (rec, &sa_client, &clen)) != -1) 
	    start_transaction_thread (pconf, csd, &sa_client,
				      rec->protocol, thread_parms);
	
	break;
	
    default:
	break;			/* We don't care */
    }
}

-- 
====================================================================
      Jim Jagielski            |       jaguNET Access Services
     jim@jaguNET.com           |       http://www.jaguNET.com/
                  "Not the Craw... the CRAW!"

Re: Threaded Apache and lingering_close

[Marc Slemko <ma...@znep.com>]

On Fri, 31 Jan 1997, Jim Jagielski wrote:

> The last version of RSTs threaded apache included some handling
> of a lingering_close... below I've condense the info:
[...]

>     
>     switch (rec->state) {
>     case fd_linger:
> 	/* Sometimes clients chase a genuine request with a few trash
> 	 * newlines... be prepared for that.  (NB we couldn't treat it
> 	 * as a request even if we wanted to --- we've already shut down
> 	 * our half of the connection as the first step in a lingering close).
> 	 */
> 
> 	if (ioctl (rec->fd, FIONREAD, (void *)&numrd) >= 0
> 	    && numrd > 0)
> 	{
> 	    /* Client sent extra junk.  Get rid of it.
> 	     * On error, lose the connection; we don't need the trouble
> 	     * (especially since the client is clearly deranged to begin with).
> 	     */
> 	    char junkbuf[256];
> 	    if (read (rec->fd, junkbuf, sizeof(junkbuf)) <= 0) {
> 		log_unixerr("read from half-closed connection",
> 			    NULL, NULL, server_conf);
> 		shutdown (rec->fd, 2);
> 		close (rec->fd);
> 		set_fd_state (fd_closed, rec->fd, NULL, NULL);
> 	    }
> 	}
> 	else {
> 	    /* Got client ack, have EOF.  Can close safely now. */
> 	    close (rec->fd);
> 	    set_fd_state (fd_closed, rec->fd, NULL, NULL);
> 	}
> 	break;

It looks like the only major thing we are doing differently (well,
aside from completely different request handling) is it does an
ioctl to check if anything is waiting for read, we loop in a select().
The reasons for using an ioctl in the above loop are fairly obvious,
although I guess a select could have done a poll.