You are viewing a plain text version of this content. The canonical link for it is here.
Posted to modperl@perl.apache.org by jo...@sunstarsys.com on 2000/09/02 08:58:52 UTC
Re: 1.3.x mod_proxy and HTTP/1.1
Here's a patch to mod_proxy that enables keepalives on the browser connection.
It's not well tested, so please report any problems you find.
The patch enables a new directive,
ProxyPostMax <maxfilesize>[KM][B]
Setting the ProxyPostMax directive makes mod_proxy store
the request's body to a tempfile on the proxy BEFORE opening the server conection.
This should lighten the load on a 'heavy' mod_perl backend server, if you're
processing a lot of POST data with it. (In theory, anyway.)
If this 'store-and-forward' feature isn't something that interests you,
don't bother testing this patch. I understand Apache 1.3.13 will include a
full HTTP/1.1 proxy.
Chuck Murcko <ch...@topsail.org> writes:
> Hi Joe. There's actually already a complete HTTP/1.1 patch for 1.3.13,
> but we'll only be making this available for user integration, as no
> further development is happening for 1.3.x Apache other than bug fixes.
> If you want to contribute this patch, I can put it into the contrib area
> at www.apache.org for the web server project.
>
> joe@sunstarsys.com wrote:
> >
> > Chuck,
> >
> > Are you still working on mod_proxy for
> > Apache 1.3.x?
> >
> > I've been putting together a patch to mod_proxy
> > that enables HTTP/1.1 responses (in particular
> > keep-alive connections) on the browser-side.
> > It also adds a new configuration directive
> > "ProxyPostMax".
> >
> > Setting this directive, e.g.
> >
> > ProxyPostMax 8MB ,
> >
> > would make mod_proxy behave as a
> > store-and-forward proxy on the upload side.
> > Basically, this means the origin server's connection isn't
> > opened until the request body is uploaded to a tempfile
> > on the proxy. Lots of us mod_perl folks have been
> > looking for such a feature.
> >
> > This is just a hack- it's not an attempt to implement a
> > fully compliant HTTP/1.1 proxy. If you're interested,
> > I'll send you the patch to 1.3.12 once I've tested got it
> > working with the cache enabled.
> >
> > --
> > Joe Schaefer
> > joe@sunstarsys.com
> >
> > SunStar Systems, Inc.
>
> --
> Chuck
> Chuck Murcko
> Topsail Group
> chuck@topsail.org
>
Here's the patch. You should cd to apache's src/modules
subdir before applying it.
diff -ur proxy/mod_proxy.c /usr/src/apache_1.3.12/src/modules/proxy/mod_proxy.c
--- proxy/mod_proxy.c Wed Mar 15 00:25:29 2000
+++ /usr/src/apache_1.3.12/src/modules/proxy/mod_proxy.c Fri Sep 1 23:20:07 2000
@@ -361,7 +361,7 @@
ap_psprintf(r->pool, "%d", (maxfwd > 0) ? maxfwd-1 : 0));
}
- if ((rc = ap_setup_client_block(r, REQUEST_CHUNKED_ERROR)))
+ if (rc = ap_setup_client_block(r, REQUEST_CHUNKED_ERROR))
return rc;
url = r->filename + 6;
@@ -487,7 +487,7 @@
ps->req_set = 0;
ps->recv_buffer_size = 0; /* this default was left unset for some reason */
ps->recv_buffer_size_set = 0;
-
+ ps->postmax = DEFAULT_POST_MAX;
ps->cache.root = NULL;
ps->cache.space = DEFAULT_CACHE_SPACE;
ps->cache.space_set = 0;
@@ -530,6 +530,7 @@
ps->viaopt = (overrides->viaopt_set == 0) ? base->viaopt : overrides->viaopt;
ps->req = (overrides->req_set == 0) ? base->req : overrides->req;
ps->recv_buffer_size = (overrides->recv_buffer_size_set == 0) ? base->recv_buffer_size : overrides->recv_buffer_size;
+ ps->postmax = (overrides->postmax <= 0) ? base->postmax : overrides->postmax;
ps->cache.root = (overrides->cache.root == NULL) ? base->cache.root : overrides->cache.root;
ps->cache.space = (overrides->cache.space_set == 0) ? base->cache.space : overrides->cache.space;
@@ -743,6 +744,16 @@
return NULL;
}
+static const char *
+ set_proxy_postmax(cmd_parms *parms, void *dummy, long int flag)
+{
+ proxy_server_conf *psf =
+ ap_get_module_config(parms->server->module_config, &proxy_module);
+
+ psf->postmax = flag;
+ return NULL;
+}
+
static const char *
set_cache_size(cmd_parms *parms, char *struct_ptr, char *arg)
@@ -947,6 +958,33 @@
return NULL;
}
+static const char*
+ set_post_max(cmd_parms *parms, void *dummy, char *arg)
+{
+ long int size;
+ int scan;
+ char a;
+ proxy_server_conf *psf =
+ ap_get_module_config(parms->server->module_config, &proxy_module);
+ scan = sscanf(arg, "%ld %[MmKk]", &size, &a);
+ if (scan <= 0)
+ return "ProxyPostMax not a number in B, KB, or MB";
+ if (scan == 2) {
+ switch (a) {
+ case 'M':
+ case 'm':
+ size *= 1024;
+ /* fall through */
+ case 'K':
+ case 'k':
+ size *= 1024;
+ }
+ }
+ psf->postmax = size;
+
+ return NULL;
+}
+
static const handler_rec proxy_handlers[] =
{
{"proxy-server", proxy_handler},
@@ -995,6 +1033,8 @@
"Force a http cache completion after this percentage is loaded"},
{"ProxyVia", set_via_opt, NULL, RSRC_CONF, TAKE1,
"Configure Via: proxy header header to one of: on | off | block | full"},
+ {"ProxyPostMax", set_post_max, NULL, RSRC_CONF, TAKE1,
+ "Maximum size of request body"},
{NULL}
};
diff -ur proxy/mod_proxy.h /usr/src/apache_1.3.12/src/modules/proxy/mod_proxy.h
--- proxy/mod_proxy.h Tue Jan 11 09:13:44 2000
+++ /usr/src/apache_1.3.12/src/modules/proxy/mod_proxy.h Sat Sep 2 02:26:35 2000
@@ -139,7 +139,7 @@
#define DEFAULT_HTTPS_PORT 443
#define DEFAULT_SNEWS_PORT 563
#define DEFAULT_PROSPERO_PORT 1525 /* WARNING: conflict w/Oracle */
-
+#define DEFAULT_POST_MAX (-1)
/* Some WWW schemes and their default ports; this is basically /etc/services */
struct proxy_services {
const char *scheme;
@@ -222,6 +222,7 @@
via_full
} viaopt; /* how to deal with proxy Via: headers */
char viaopt_set;
+ long int postmax;
size_t recv_buffer_size;
char recv_buffer_size_set;
} proxy_server_conf;
diff -ur proxy/proxy_cache.c /usr/src/apache_1.3.12/src/modules/proxy/proxy_cache.c
--- proxy/proxy_cache.c Wed Dec 8 18:02:43 1999
+++ /usr/src/apache_1.3.12/src/modules/proxy/proxy_cache.c Sat Sep 2 01:34:56 2000
@@ -769,6 +769,11 @@
}
ap_pclosef(r->pool, ap_bfileno(cachefp, B_WR));
Explain0("Use local copy, cached file hasn't changed");
+/*JS*/
+ r->status = 304;
+ ap_proxy_send_headers(r, c->resp_line, c->hdrs);
+ r->sent_bodyct = 1;
+/*end JS*/
return HTTP_NOT_MODIFIED;
}
@@ -999,6 +1004,11 @@
ap_pclosef(r->pool, ap_bfileno(c->fp, B_WR));
Explain0("Remote document not modified, use local copy");
/* CHECKME: Is this right? Shouldn't we check IMS again here? */
+/*JS*/
+ r->status = 304;
+ ap_proxy_send_headers(r, c->resp_line, c->hdrs);
+ r->sent_bodyct = 1;
+/*end JS*/
return HTTP_NOT_MODIFIED;
}
else {
diff -ur proxy/proxy_http.c /usr/src/apache_1.3.12/src/modules/proxy/proxy_http.c
--- proxy/proxy_http.c Wed Mar 15 00:25:29 2000
+++ /usr/src/apache_1.3.12/src/modules/proxy/proxy_http.c Fri Sep 1 23:01:29 2000
@@ -63,6 +63,13 @@
#include "http_core.h"
#include "util_date.h"
+long int delta (long int new, long int *old)
+{
+ long int result;
+ result = new - *old;
+ *old = new;
+ return result;
+}
/*
* Canonicalise http-like URLs.
* scheme is the scheme for the URL
@@ -172,7 +179,7 @@
const char *strp;
char *strp2;
const char *err, *desthost;
- int i, j, sock, len, backasswards;
+ int i, j, sock, len, backasswards, rc;
array_header *reqhdrs_arr;
table *resp_hdrs;
table_entry *reqhdrs;
@@ -180,6 +187,7 @@
struct in_addr destaddr;
struct hostent server_hp;
BUFF *f;
+ FILE *infile = NULL;
char buffer[HUGE_STRING_LEN];
char portstr[32];
pool *p = r->pool;
@@ -266,6 +274,60 @@
#endif
}
+ /* JS: get client data before connecting to host! */
+ /* content-length limitations ? (add new directive)*/
+ /* setup_client_block is caller's responsibility */
+
+ if (conf->postmax >= 0 && ap_should_client_block(r)) {
+ long int len, s, t, dt, da, a = 0, b = 0, eta, rate;
+ int upload = 0;
+ ap_log_rerror(APLOG_MARK, APLOG_ERR, r,
+ "GOTCHA: %d", conf->postmax);
+ len = r->remaining;
+ s = time(&t); /* start time */
+
+ if (len > conf->postmax)
+ return ap_proxyerror(r, HTTP_REQUEST_ENTITY_TOO_LARGE, ap_pstrcat(r->pool,
+ "[proxy] request body (%d bytes )too large: %d bytes MAX",
+ r->remaining, conf->postmax, NULL));
+
+ if (! (infile = tmpfile()) )
+ return ap_proxyerror(r, HTTP_BAD_GATEWAY, ap_pstrcat(r->pool,
+ "couldn't make temp file", NULL));
+ ap_note_cleanups_for_file(p, infile);
+
+/* prep fifo, use ticket hash for filename
+ if (len > conf->postmin) {
+
+ upload = 1;
+ }
+*/
+ ap_hard_timeout("[proxy] reading data from client", r);
+
+ while ((i = ap_get_client_block(r, buffer, sizeof buffer)) > 0) {
+ fwrite(buffer, 1, i, infile);
+ ap_reset_timeout(r);
+ a += i;
+ /* test for upload fifo */
+ if (upload && time(NULL) - t > 1) {
+ da = delta(a, &b);
+ dt = delta(time(NULL), &t);
+ eta = r->remaining * ( t - s ) / a;
+ rate = da / dt;
+
+ }
+ }
+ /*JS:necessary? fflush(infile); */
+
+ ap_kill_timeout(r);
+ fseek(infile, 0, 0);
+ if (i<0)
+ return ap_proxyerror(r, HTTP_PARTIAL_CONTENT,
+ "error reading dataset from client");
+
+ }
+
+
sock = ap_psocket(p, PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sock == -1) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, r,
@@ -315,7 +377,8 @@
strerror(errno), NULL));
}
- clear_connection(r->pool, r->headers_in); /* Strip connection-based headers */
+ /* JS: this affects keep-alives !!
+ clear_connection(r->pool, r->headers_in);*/ /* Strip connection-based headers */
f = ap_bcreate(p, B_RDWR | B_SOCKET);
ap_bpushfd(f, sock, sock);
@@ -335,6 +398,7 @@
ap_hard_timeout("proxy send", r);
ap_bvputs(f, r->method, " ", proxyhost ? url : urlptr, " HTTP/1.0" CRLF,
NULL);
+ ap_bvputs(f, "Connection: close", CRLF, NULL);
#ifdef EAPI
{
int rc = DECLINED;
@@ -392,22 +456,41 @@
* suppressed if THIS server requested the authentication,
* not when a frontend proxy requested it!
*/
+ || !strcasecmp(reqhdrs[i].key, "Connection")
+ || !strcasecmp(reqhdrs[i].key, "Keep-Alive")
|| !strcasecmp(reqhdrs[i].key, "Proxy-Authorization"))
continue;
ap_bvputs(f, reqhdrs[i].key, ": ", reqhdrs[i].val, CRLF, NULL);
}
ap_bputs(CRLF, f);
+ ap_kill_timeout(r);
+
/* send the request data, if any. */
+ ap_hard_timeout("[proxy] transferring data from client to server", r);
+
+ if (conf->postmax < 0 && ap_should_client_block(r)) {
+
+ while ((i = ap_get_client_block(r, buffer, sizeof buffer)) > 0) {
+ ap_bwrite(f,buffer,i);
+ ap_reset_timeout(r);
+ }
+ if (i<0)
+ return ap_proxyerror(r, HTTP_PARTIAL_CONTENT,
+ "error reading dataset from client");
- if (ap_should_client_block(r)) {
- while ((i = ap_get_client_block(r, buffer, sizeof buffer)) > 0)
+ } else if (infile) {
+ while (( i = fread(buffer, 1, sizeof buffer, infile)) > 0 ) {
ap_bwrite(f, buffer, i);
+ ap_reset_timeout(r);
+ }
+ fclose(infile);
}
+
ap_bflush(f);
ap_kill_timeout(r);
- ap_hard_timeout("proxy receive", r);
+ ap_hard_timeout("[proxy] parsing http headers from server", r);
len = ap_bgets(buffer, sizeof buffer - 1, f);
if (len == -1) {
@@ -479,7 +562,7 @@
ap_get_server_name(r), portstr)
);
}
-
+ /* JS: This one makes sense! */
clear_connection(p, resp_hdrs); /* Strip Connection hdrs */
}
else {
@@ -493,8 +576,7 @@
}
c->hdrs = resp_hdrs;
-
- ap_kill_timeout(r);
+ ap_kill_timeout(r); /*JS parsing http headers from server */
/*
* HTTP/1.0 requires us to accept 3 types of dates, but only generate
@@ -525,13 +607,14 @@
return i;
}
- ap_hard_timeout("proxy receive", r);
+ ap_hard_timeout("proxy receive content", r);
/* write status line */
if (!r->assbackwards)
- ap_rvputs(r, "HTTP/1.0 ", r->status_line, CRLF, NULL);
+ ap_proxy_send_headers(r, "IGNORED", resp_hdrs);
+
if (c != NULL && c->fp != NULL &&
- ap_bvputs(c->fp, "HTTP/1.0 ", r->status_line, CRLF, NULL) == -1) {
+ ap_bvputs(c->fp, "HTTP/1.1 ", r->status_line, CRLF, NULL) == -1) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, c->req,
"proxy: error writing status line to %s", c->tempfile);
c = ap_proxy_cache_error(c);
@@ -543,7 +626,7 @@
ap_table_do(ap_proxy_send_hdr_line, &tdo, resp_hdrs, NULL);
if (!r->assbackwards)
- ap_rputs(CRLF, r);
+/* ap_rputs(CRLF, r); */
if (c != NULL && c->fp != NULL && ap_bputs(CRLF, c->fp) == -1) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, c->req,
"proxy: error writing CRLF to %s", c->tempfile);
@@ -552,6 +635,7 @@
ap_bsetopt(r->connection->client, BO_BYTECT, &zero);
r->sent_bodyct = 1;
+
/* Is it an HTTP/0.9 respose? If so, send the extra data */
if (backasswards) {
ap_bwrite(r->connection->client, buffer, len);
diff -ur proxy/proxy_util.c /usr/src/apache_1.3.12/src/modules/proxy/proxy_util.c
--- proxy/proxy_util.c Mon Feb 7 19:34:40 2000
+++ /usr/src/apache_1.3.12/src/modules/proxy/proxy_util.c Mon Aug 28 02:48:14 2000
@@ -499,8 +499,9 @@
int alternate_timeouts = 1; /* 1 if we alternate between soft & hard timeouts */
total_bytes_rcvd = 0;
- if (c != NULL)
- c->written = 0;
+ if (c != NULL) {
+ c->written = 0;
+ }
#ifdef CHARSET_EBCDIC
/* The cache copy is ASCII, not EBCDIC, even for text/html) */
@@ -542,7 +543,7 @@
*/
for (ok = 1; ok; ) {
if (alternate_timeouts)
- ap_hard_timeout("proxy recv body from upstream server", r);
+ ap_hard_timeout("[proxy] recv body from server", r);
/* Read block from server */
n = ap_bread(f, buf, IOBUFSIZE);
@@ -555,40 +556,42 @@
if (n == -1) { /* input error */
if (c != NULL) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, c->req,
- "proxy: error reading from %s", c->url);
+ "[proxy] error reading from %s", c->url);
c = ap_proxy_cache_error(c);
}
break;
}
if (n == 0)
break; /* EOF */
- o = 0;
+ o=0;
total_bytes_rcvd += n;
/* Write to cache first. */
/*@@@ XXX FIXME: Assuming that writing the cache file won't time out?!!? */
- if (c != NULL && c->fp != NULL) {
- if (ap_bwrite(c->fp, &buf[0], n) != n) {
- ap_log_rerror(APLOG_MARK, APLOG_ERR, c->req,
- "proxy: error writing to %s", c->tempfile);
+
+ if (c != NULL && c->fp != NULL) {
+ if (ap_bwrite(c->fp, buf, n) != n) {
+ ap_log_rerror(APLOG_MARK, APLOG_ERR, c->req,
+ "[proxy] error writing to %s", c->tempfile);
c = ap_proxy_cache_error(c);
- } else {
- c->written += n;
- }
- }
+ } else {
+ c->written += n;
+ }
- /* Write the block to the client, detect aborted transfers */
- while (!con->aborted && n > 0) {
- if (alternate_timeouts)
- ap_soft_timeout("proxy send body", r);
-
- w = ap_bwrite(con->client, &buf[o], n);
-
- if (alternate_timeouts)
- ap_kill_timeout(r);
- else
- ap_reset_timeout(r);
+ }
+ while(!con->aborted && n > 0) {
+
+ if (alternate_timeouts)
+ ap_soft_timeout("proxy send body", r);
+
+ w = ap_bwrite(con->client, buf+o, n);
+
+ if (alternate_timeouts)
+ ap_kill_timeout(r);
+ else
+ ap_reset_timeout(r);
+
if (w <= 0) {
if (c != NULL && c->fp != NULL) {
/* when a send failure occurs, we need to decide
@@ -603,14 +606,18 @@
ap_pclosef(c->req->pool, ap_bfileno(c->fp, B_WR));
c->fp = NULL;
unlink(c->tempfile);
- c = NULL;
+ c = NULL;
}
}
con->aborted = 1;
break;
}
+
+/* if (buf[n] == EOF)
+ break; */
+
n -= w;
- o += w;
+ o += w;
} /* while client alive and more data to send */
} /* loop and ap_bread while "ok" */
@@ -621,6 +628,7 @@
return total_bytes_rcvd;
}
+
/*
* Sends response line and headers. Uses the client fd and the
* headers_out array from the passed request_rec to talk to the client
@@ -634,16 +642,19 @@
BUFF *fp = r->connection->client;
table_entry *elts = (table_entry *) ap_table_elts(t)->elts;
- ap_bvputs(fp, respline, CRLF, NULL);
+/*JS ap_bvputs(fp, respline, CRLF, NULL); */
for (i = 0; i < ap_table_elts(t)->nelts; ++i) {
if (elts[i].key != NULL) {
- ap_bvputs(fp, elts[i].key, ": ", elts[i].val, CRLF, NULL);
+/*JS ap_bvputs(fp, elts[i].key, ": ", elts[i].val, CRLF, NULL); */
ap_table_addn(r->headers_out, elts[i].key, elts[i].val);
}
}
-
- ap_bputs(CRLF, fp);
+ r->proxyreq = NOT_PROXY;
+ r->content_type = ap_table_get(t, "Content-Type");
+ ap_send_http_header(r);
+ r->proxyreq = 1;
+/*JS ap_bputs(CRLF, fp); */
}
@@ -1270,8 +1281,8 @@
if (key == NULL || value == NULL || value[0] == '\0')
return 1;
- if (!parm->req->assbackwards)
- ap_rvputs(parm->req, key, ": ", value, CRLF, NULL);
+/*JS if (!parm->req->assbackwards)
+ ap_rvputs(parm->req, key, ": ", value, CRLF, NULL); */
if (parm->cache != NULL && parm->cache->fp != NULL &&
ap_bvputs(parm->cache->fp, key, ": ", value, CRLF, NULL) == -1) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, parm->cache->req,
Joe Schaefer
joe@sunstarsys.com
SunStar Systems, Inc.