You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@httpd.apache.org by mi...@apache.org on 2006/10/27 15:28:57 UTC
svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Author: minfrin
Date: Fri Oct 27 06:28:56 2006
New Revision: 468373
URL: http://svn.apache.org/viewvc?view=rev&rev=468373
Log:
mod_cache: Pass the output filter stack through the store_body()
hook, giving each cache backend the ability to make a better
decision as to how it will allocate the tasks of writing to the
cache and writing to the network. Previously the write to the
cache task needed to be complete before the same brigade was
written to the network, and this caused timing and memory issues
on large cached files. This fix replaces the previous fix for
PR39380.
Modified:
httpd/httpd/trunk/CHANGES
httpd/httpd/trunk/modules/cache/mod_cache.c
httpd/httpd/trunk/modules/cache/mod_cache.h
httpd/httpd/trunk/modules/cache/mod_disk_cache.c
httpd/httpd/trunk/modules/cache/mod_disk_cache.h
httpd/httpd/trunk/modules/cache/mod_mem_cache.c
Modified: httpd/httpd/trunk/CHANGES
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/CHANGES?view=diff&rev=468373&r1=468372&r2=468373
==============================================================================
--- httpd/httpd/trunk/CHANGES [utf-8] (original)
+++ httpd/httpd/trunk/CHANGES [utf-8] Fri Oct 27 06:28:56 2006
@@ -2,6 +2,15 @@
Changes with Apache 2.3.0
[Remove entries to the current 2.0 and 2.2 section below, when backported]
+ *) mod_cache: Pass the output filter stack through the store_body()
+ hook, giving each cache backend the ability to make a better
+ decision as to how it will allocate the tasks of writing to the
+ cache and writing to the network. Previously the write to the
+ cache task needed to be complete before the same brigade was
+ written to the network, and this caused timing and memory issues
+ on large cached files. This fix replaces the previous fix for this
+ PR below. PR39380 [Graham Leggett]
+
*) Fix issue which could cause error messages to be written to access logs
on Win32. PR 40476. [Tom Donovan <Tom.Donovan acm.org>]
Modified: httpd/httpd/trunk/modules/cache/mod_cache.c
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/cache/mod_cache.c?view=diff&rev=468373&r1=468372&r2=468373
==============================================================================
--- httpd/httpd/trunk/modules/cache/mod_cache.c (original)
+++ httpd/httpd/trunk/modules/cache/mod_cache.c Fri Oct 27 06:28:56 2006
@@ -366,13 +366,7 @@
/* pass the brigades into the cache, then pass them
* up the filter stack
*/
- rv = cache->provider->store_body(cache->handle, r, in);
- if (rv != APR_SUCCESS) {
- ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, r->server,
- "cache: Cache provider's store_body failed!");
- ap_remove_output_filter(f);
- }
- return ap_pass_brigade(f->next, in);
+ return cache->provider->store_body(cache->handle, f, in);
}
/*
@@ -829,14 +823,8 @@
return ap_pass_brigade(f->next, in);
}
- rv = cache->provider->store_body(cache->handle, r, in);
- if (rv != APR_SUCCESS) {
- ap_log_error(APLOG_MARK, APLOG_DEBUG, rv, r->server,
- "cache: store_body failed");
- ap_remove_output_filter(f);
- }
+ return cache->provider->store_body(cache->handle, f, in);
- return ap_pass_brigade(f->next, in);
}
/*
Modified: httpd/httpd/trunk/modules/cache/mod_cache.h
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/cache/mod_cache.h?view=diff&rev=468373&r1=468372&r2=468373
==============================================================================
--- httpd/httpd/trunk/modules/cache/mod_cache.h (original)
+++ httpd/httpd/trunk/modules/cache/mod_cache.h Fri Oct 27 06:28:56 2006
@@ -210,7 +210,7 @@
typedef struct {
int (*remove_entity) (cache_handle_t *h);
apr_status_t (*store_headers)(cache_handle_t *h, request_rec *r, cache_info *i);
- apr_status_t (*store_body)(cache_handle_t *h, request_rec *r, apr_bucket_brigade *b);
+ apr_status_t (*store_body)(cache_handle_t *h, ap_filter_t *f, apr_bucket_brigade *b);
apr_status_t (*recall_headers) (cache_handle_t *h, request_rec *r);
apr_status_t (*recall_body) (cache_handle_t *h, apr_pool_t *p, apr_bucket_brigade *bb);
int (*create_entity) (cache_handle_t *h, request_rec *r,
Modified: httpd/httpd/trunk/modules/cache/mod_disk_cache.c
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/cache/mod_disk_cache.c?view=diff&rev=468373&r1=468372&r2=468373
==============================================================================
--- httpd/httpd/trunk/modules/cache/mod_disk_cache.c (original)
+++ httpd/httpd/trunk/modules/cache/mod_disk_cache.c Fri Oct 27 06:28:56 2006
@@ -58,7 +58,7 @@
/* Forward declarations */
static int remove_entity(cache_handle_t *h);
static apr_status_t store_headers(cache_handle_t *h, request_rec *r, cache_info *i);
-static apr_status_t store_body(cache_handle_t *h, request_rec *r, apr_bucket_brigade *b);
+static apr_status_t store_body(cache_handle_t *h, ap_filter_t *f, apr_bucket_brigade *b);
static apr_status_t recall_headers(cache_handle_t *h, request_rec *r);
static apr_status_t recall_body(cache_handle_t *h, apr_pool_t *p, apr_bucket_brigade *bb);
static apr_status_t read_array(request_rec *r, apr_array_header_t* arr,
@@ -527,6 +527,7 @@
dobj->initial_size = len;
dobj->file_size = -1;
dobj->updtimeout = conf->updtimeout;
+ dobj->frv = APR_SUCCESS;
return OK;
}
@@ -716,8 +717,6 @@
disk_cache_object_t *dobj)
{
apr_off_t off;
- core_dir_config *pdconf = ap_get_module_config(r->per_dir_config,
- &core_module);
apr_time_t starttime = apr_time_now();
int flags;
apr_status_t rc;
@@ -835,6 +834,13 @@
return DECLINED;
}
+ /* TODO: We have the ability to serve partially cached requests,
+ * however in order to avoid some sticky what if conditions
+ * should the content turn out to be too large to be cached,
+ * we must only allow partial cache serving if the cached
+ * entry has a content length known in advance.
+ */
+
info->status = dobj->disk_info.status;
info->date = dobj->disk_info.date;
info->expire = dobj->disk_info.expire;
@@ -1286,9 +1292,15 @@
static apr_status_t open_new_file(request_rec *r, const char *filename,
apr_file_t **fd, disk_cache_conf *conf)
{
- int flags = APR_CREATE | APR_WRITE | APR_BINARY | APR_BUFFERED | APR_EXCL;
+ int flags;
apr_status_t rv;
+ flags = APR_CREATE | APR_WRITE | APR_READ | APR_BINARY | APR_BUFFERED | APR_EXCL | APR_TRUNCATE;
+#if APR_HAS_SENDFILE
+ flags |= ((pdconf->enable_sendfile == ENABLE_SENDFILE_OFF)
+ ? 0 : APR_SENDFILE_ENABLED);
+#endif
+
while(1) {
rv = apr_file_open(fd, filename, flags,
APR_FPROT_UREAD | APR_FPROT_UWRITE, r->pool);
@@ -1611,150 +1623,50 @@
return APR_SUCCESS;
}
-
-static apr_status_t copy_body(apr_pool_t *p,
- apr_file_t *srcfd, apr_off_t srcoff,
- apr_file_t *destfd, apr_off_t destoff,
- apr_off_t len)
-{
- apr_status_t rc;
- apr_size_t size;
- apr_finfo_t finfo;
- apr_time_t starttime = apr_time_now();
-
- char *buf = apr_palloc(p, CACHE_BUF_SIZE);
- if (!buf) {
- return APR_ENOMEM;
- }
-
- if(srcoff != 0) {
- rc = apr_file_seek(srcfd, APR_SET, &srcoff);
- if(rc != APR_SUCCESS) {
- return rc;
- }
- }
-
- if(destoff != 0) {
- rc = apr_file_seek(destfd, APR_SET, &destoff);
- if(rc != APR_SUCCESS) {
- return rc;
- }
- }
-
- /* Tried doing this with mmap, but sendfile on Linux got confused when
- sending a file while it was being written to from an mmapped area.
- The traditional way seems to be good enough, and less complex.
- */
- while(len > 0) {
- size=MIN(len, CACHE_BUF_SIZE);
-
- rc = apr_file_read_full (srcfd, buf, size, NULL);
- if(rc != APR_SUCCESS) {
- return rc;
- }
-
- rc = apr_file_write_full(destfd, buf, size, NULL);
- if(rc != APR_SUCCESS) {
- return rc;
- }
- len -= size;
- }
-
- /* Check if file has changed during copying. This is not 100% foolproof
- due to NFS attribute caching when on NFS etc. */
- /* FIXME: Can we assume that we're always copying an entire file? In that
- case we can check if the current filesize matches the length
- we think it is */
- rc = apr_file_info_get(&finfo, APR_FINFO_MTIME, srcfd);
- if(rc != APR_SUCCESS) {
- return rc;
- }
- if(starttime < finfo.mtime) {
- return APR_EGENERAL;
- }
-
- return APR_SUCCESS;
-}
-
-
-static apr_status_t replace_brigade_with_cache(cache_handle_t *h,
- request_rec *r,
- apr_bucket_brigade *bb)
-{
- apr_status_t rv;
- apr_bucket *e;
- disk_cache_object_t *dobj = (disk_cache_object_t *) h->cache_obj->vobj;
-
- if(dobj->fd) {
- apr_file_close(dobj->fd);
- dobj->fd = NULL;
- }
- rv = open_body_timeout(r, dobj->name, dobj);
- if (rv != APR_SUCCESS) {
- if(rv != CACHE_EDECLINED) {
- ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
- "disk_cache: Error opening datafile %s for URL %s",
- dobj->datafile, dobj->name);
- }
- return rv;
- }
-
- /* First, empty the brigade */
- e = APR_BRIGADE_FIRST(bb);
- while (e != APR_BRIGADE_SENTINEL(bb)) {
- apr_bucket *d;
- d = e;
- e = APR_BUCKET_NEXT(e);
- apr_bucket_delete(d);
- }
-
- /* Then, populate it with our cached instance */
- rv = recall_body(h, r->pool, bb);
- if (rv != APR_SUCCESS) {
- ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
- "disk_cache: Error serving URL %s from cache", dobj->name);
- return rv;
- }
- ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
- "disk_cache: Serving cached body for URL %s", dobj->name);
-
- return APR_SUCCESS;
-}
-
-
-static apr_status_t store_body(cache_handle_t *h, request_rec *r,
- apr_bucket_brigade *bb)
+/**
+ * Store the body of the response in the disk cache.
+ *
+ * As the data is written to the cache, it is also written to
+ * the filter provided. On network write failure, the full body
+ * will still be cached.
+ */
+static apr_status_t store_body(cache_handle_t *h, ap_filter_t *f, apr_bucket_brigade *bb)
{
- apr_bucket *e;
+ apr_bucket *e, *b;
+ request_rec *r = f->r;
apr_status_t rv;
- int copy_file = FALSE;
disk_cache_object_t *dobj = (disk_cache_object_t *) h->cache_obj->vobj;
disk_cache_conf *conf = ap_get_module_config(r->server->module_config,
&disk_cache_module);
dobj->store_body_called++;
-
+
if(r->no_cache) {
ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
"disk_cache: store_body called for URL %s even though"
"no_cache is set", dobj->name);
file_cache_errorcleanup(dobj, r);
- return APR_EGENERAL;
+ ap_remove_output_filter(f);
+ return ap_pass_brigade(f->next, bb);
}
if(dobj->initial_size == 0) {
/* Don't waste a body cachefile on a 0 length body */
- return APR_SUCCESS;
+ return ap_pass_brigade(f->next, bb);
}
if(!dobj->skipstore && dobj->fd == NULL) {
rv = open_new_file(r, dobj->datafile, &(dobj->fd), conf);
- if(rv == CACHE_EEXIST) {
+ if (rv == CACHE_EEXIST) {
/* Someone else beat us to storing this */
dobj->skipstore = TRUE;
}
- else if(rv != APR_SUCCESS) {
- return rv;
+ else if (rv != APR_SUCCESS) {
+ ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
+ "disk_cache: store_body tried to open cached file "
+ "for URL %s and this failed", dobj->name);
+ ap_remove_output_filter(f);
+ return ap_pass_brigade(f->next, bb);
}
else {
dobj->file_size = 0;
@@ -1762,194 +1674,227 @@
}
if(dobj->skipstore) {
- /* Someone else beat us to storing this object */
- if(dobj->store_body_called == 1 &&
- dobj->initial_size > 0 &&
- APR_BUCKET_IS_EOS(APR_BRIGADE_LAST(bb)) )
- {
- /* Yay, we can replace the body with the cached instance */
- return replace_brigade_with_cache(h, r, bb);
- }
-
- return APR_SUCCESS;
+ /* Someone else beat us to storing this object.
+ * We are too late to take advantage of this storage :( */
+ ap_remove_output_filter(f);
+ return ap_pass_brigade(f->next, bb);
}
- /* Check if this is a complete single sequential file, eligable for
- * file copy.
- */
- if(dobj->file_size == 0 && APR_BUCKET_IS_EOS(APR_BRIGADE_LAST(bb)))
- {
- apr_off_t begin = -1;
- apr_off_t pos = -1;
- apr_file_t *fd = NULL;
- apr_bucket_file *a;
-
- copy_file = TRUE;
-
- for (e = APR_BRIGADE_FIRST(bb);
- e != APR_BRIGADE_SENTINEL(bb);
- e = APR_BUCKET_NEXT(e))
- {
- if(APR_BUCKET_IS_EOS(e)) {
- break;
- }
- if(!APR_BUCKET_IS_FILE(e)) {
- copy_file = FALSE;
- break;
- }
-
- a = e->data;
-
- if(begin < 0) {
- begin = pos = e->start;
- fd = a->fd;
- }
-
- if(fd != a->fd || pos != e->start) {
- copy_file = FALSE;
- break;
- }
-
- pos += e->length;
- }
-
- if(copy_file) {
- dobj->file_size = pos;
- }
+ /* set up our temporary brigade */
+ if (!dobj->tmpbb) {
+ dobj->tmpbb = apr_brigade_create(r->pool, r->connection->bucket_alloc);
+ }
+ else {
+ apr_brigade_cleanup(dobj->tmpbb);
}
- if(copy_file) {
- apr_bucket_file *a;
-
- ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
- "disk_cache: Copying body for URL %s, len %"
- APR_OFF_T_FMT, dobj->name, dobj->file_size);
-
- e = APR_BRIGADE_FIRST(bb);
- a = e->data;
-
- rv = copy_body(r->pool, a->fd, e->start, dobj->fd, 0,
- dobj->file_size);
- if(rv != APR_SUCCESS) {
- ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
- "disk_cache: Copying body failed, "
- "URL %s", dobj->name);
- file_cache_errorcleanup(dobj, r);
- return rv;
- }
+ /* start caching the brigade */
+ ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
+ "disk_cache: Caching body for URL %s", dobj->name);
- }
- else {
- ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
- "disk_cache: Caching body for URL %s", dobj->name);
+ e = APR_BRIGADE_FIRST(bb);
+ while (e != APR_BRIGADE_SENTINEL(bb)) {
- for (e = APR_BRIGADE_FIRST(bb);
- e != APR_BRIGADE_SENTINEL(bb);
- e = APR_BUCKET_NEXT(e))
- {
- const char *str;
- apr_size_t length, written;
+ const char *str;
+ apr_size_t length, written;
+ apr_off_t offset = 0;
- /* Ignore the non-data-buckets */
- if(APR_BUCKET_IS_METADATA(e)) {
- continue;
- }
+ /* try write all data buckets to the cache, except for metadata buckets */
+ if(!APR_BUCKET_IS_METADATA(e)) {
+ /* read in a bucket fragment */
rv = apr_bucket_read(e, &str, &length, APR_BLOCK_READ);
if (rv != APR_SUCCESS) {
ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
- "disk_cache: Error when reading bucket for URL %s",
+ "disk_cache: Error when reading bucket for URL %s, aborting request",
dobj->name);
file_cache_errorcleanup(dobj, r);
+ /* not being able to read the bucket is fatal,
+ * return this up the filter stack
+ */
return rv;
}
+
+ /* try write the bucket fragment to the cache */
+ apr_file_seek(dobj->fd, APR_END, &offset);
rv = apr_file_write_full(dobj->fd, str, length, &written);
- if (rv != APR_SUCCESS) {
+ offset = - (apr_off_t)written;
+ apr_file_seek(dobj->fd, APR_END, &offset);
+
+ /* if the cache write was successful, swap the original bucket
+ * with a file bucket pointing to the same data in the cache.
+ *
+ * This is done because:
+ *
+ * - The ap_core_output_filter can take advantage of its ability
+ * to do non blocking writes on file buckets.
+ *
+ * - We are prevented from the need to read the original bucket
+ * a second time inside ap_core_output_filter, which could be
+ * expensive or memory consuming.
+ *
+ * - The cache, in theory, should be faster than the backend,
+ * otherwise there would be little point in caching in the first
+ * place.
+ */
+ if (APR_SUCCESS == rv) {
+
+ /* remove and destroy the original bucket from the brigade */
+ b = e;
+ e = APR_BUCKET_NEXT(e);
+ APR_BUCKET_REMOVE(b);
+ apr_bucket_destroy(b);
+
+ /* Is our network connection still alive?
+ * If not, we must continue caching the file, so keep looping.
+ * We will return the error at the end when caching is done.
+ */
+ if (APR_SUCCESS == dobj->frv) {
+
+ /* insert a file bucket pointing to the cache into out temporary brigade */
+ if (diskcache_brigade_insert(dobj->tmpbb, dobj->fd, dobj->file_size,
+ written,
+ dobj->updtimeout, r->pool) == NULL) {
+ return APR_ENOMEM;
+ }
+
+ /* TODO: If we are not able to guarantee that
+ * apr_core_output_filter() will not block on our
+ * file buckets, then the check for whether the
+ * socket will block must go here.
+ */
+
+ /* send our new brigade to the network */
+ dobj->frv = ap_pass_brigade(f->next, dobj->tmpbb);
+
+ }
+
+ /* update the write counter, and sanity check the size */
+ dobj->file_size += written;
+ if (dobj->file_size > conf->maxfs) {
+ ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+ "disk_cache: URL %s failed the size check "
+ "(%" APR_OFF_T_FMT " > %" APR_OFF_T_FMT ")",
+ dobj->name, dobj->file_size, conf->maxfs);
+ file_cache_errorcleanup(dobj, r);
+ ap_remove_output_filter(f);
+ return ap_pass_brigade(f->next, bb);
+ }
+
+ }
+
+ /*
+ * If the cache write failed, continue to loop and pass data to
+ * the network. Remove the cache filter from the output filters
+ * so we don't inadvertently try to cache write again, leaving
+ * a hole in the cached data.
+ */
+ else {
+
+ /* mark the write as having failed */
ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
"disk_cache: Error when writing cache file for "
"URL %s", dobj->name);
+
+ /* step away gracefully */
file_cache_errorcleanup(dobj, r);
- return rv;
+ ap_remove_output_filter(f);
+
+ /* write the rest of the brigade to the network, and leave */
+ return ap_pass_brigade(f->next, bb);
+
}
- dobj->file_size += written;
- if (dobj->file_size > conf->maxfs) {
- ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
- "disk_cache: URL %s failed the size check "
- "(%" APR_OFF_T_FMT " > %" APR_OFF_T_FMT ")",
- dobj->name, dobj->file_size, conf->maxfs);
- file_cache_errorcleanup(dobj, r);
- return APR_EGENERAL;
+
+
+ }
+
+ /* write metadata buckets direct to the output filter */
+ else {
+
+ /* move the metadata bucket to our temporary brigade */
+ b = e;
+ e = APR_BUCKET_NEXT(e);
+ APR_BUCKET_REMOVE(b);
+ APR_BRIGADE_INSERT_HEAD(dobj->tmpbb, b);
+
+ /* Is our network connection still alive?
+ * If not, we must continue looping, but stop writing to the network.
+ */
+ if (APR_SUCCESS == dobj->frv) {
+
+ /* TODO: If we are not able to guarantee that
+ * apr_core_output_filter() will not block on our
+ * file buckets, then the check for whether the
+ * socket will block must go here.
+ */
+
+ /* send our new brigade to the network */
+ dobj->frv = ap_pass_brigade(f->next, dobj->tmpbb);
+
}
+
}
- }
+ apr_brigade_cleanup(dobj->tmpbb);
+ }
+
+
/* Drop out here if this wasn't the end */
if (!APR_BUCKET_IS_EOS(APR_BRIGADE_LAST(bb))) {
return APR_SUCCESS;
}
- if(!copy_file) {
- ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
- "disk_cache: Done caching URL %s, len %" APR_OFF_T_FMT,
- dobj->name, dobj->file_size);
+ ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+ "disk_cache: Done caching URL %s, len %" APR_OFF_T_FMT,
+ dobj->name, dobj->file_size);
- /* FIXME: Do we really need to check r->no_cache here since we checked
- it in the beginning? */
- if (r->no_cache || r->connection->aborted) {
- ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
- "disk_cache: Discarding body for URL %s "
- "because connection has been aborted.",
- h->cache_obj->key);
- /* Remove the intermediate cache file and return non-APR_SUCCESS */
- file_cache_errorcleanup(dobj, r);
- return APR_EGENERAL;
- }
- if (dobj->file_size < conf->minfs) {
- ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
- "disk_cache: URL %s failed the size check "
- "(%" APR_OFF_T_FMT "<%" APR_OFF_T_FMT ")",
- h->cache_obj->key, dobj->file_size, conf->minfs);
- /* Remove the intermediate cache file and return non-APR_SUCCESS */
- file_cache_errorcleanup(dobj, r);
- return APR_EGENERAL;
- }
- if(dobj->initial_size < 0) {
- /* Update header information now that we know the size */
- dobj->initial_size = dobj->file_size;
- rv = store_headers(h, r, &(h->cache_obj->info));
- if(rv != APR_SUCCESS) {
- file_cache_errorcleanup(dobj, r);
- return rv;
- }
- }
- else if(dobj->initial_size != dobj->file_size) {
- ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
- "disk_cache: URL %s - body size mismatch: suggested %"
- APR_OFF_T_FMT " bodysize %" APR_OFF_T_FMT ")",
- dobj->name, dobj->initial_size, dobj->file_size);
+ if (APR_SUCCESS != dobj->frv) {
+ ap_log_error(APLOG_MARK, APLOG_ERR, dobj->frv, r->server,
+ "disk_cache: An error occurred while writing to the "
+ "network for URL %s.",
+ h->cache_obj->key);
+ }
+
+ if (dobj->file_size < conf->minfs) {
+ ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+ "disk_cache: URL %s failed the size check "
+ "(%" APR_OFF_T_FMT "<%" APR_OFF_T_FMT ")",
+ h->cache_obj->key, dobj->file_size, conf->minfs);
+ /* Remove the intermediate cache file and return filter status */
+ file_cache_errorcleanup(dobj, r);
+ return dobj->frv;
+ }
+ if (dobj->initial_size < 0) {
+ /* Update header information now that we know the size */
+ dobj->initial_size = dobj->file_size;
+ rv = store_headers(h, r, &(h->cache_obj->info));
+ if (rv != APR_SUCCESS) {
file_cache_errorcleanup(dobj, r);
- return APR_EGENERAL;
+ return dobj->frv;
}
}
+ else if (dobj->initial_size != dobj->file_size) {
+ ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+ "disk_cache: URL %s - body size mismatch: suggested %"
+ APR_OFF_T_FMT " bodysize %" APR_OFF_T_FMT ")",
+ dobj->name, dobj->initial_size, dobj->file_size);
+ file_cache_errorcleanup(dobj, r);
+ return dobj->frv;
+ }
/* All checks were fine, close output file */
rv = apr_file_close(dobj->fd);
dobj->fd = NULL;
- if(rv != APR_SUCCESS) {
+ if (rv != APR_SUCCESS) {
+ ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
+ "disk_cache: While trying to close the cache file for "
+ "URL %s, the close failed", dobj->name);
file_cache_errorcleanup(dobj, r);
- return rv;
+ return dobj->frv;
}
- /* Redirect to cachefile if we copied a plain file */
- if(copy_file) {
- rv = replace_brigade_with_cache(h, r, bb);
- if(rv != APR_SUCCESS) {
- return rv;
- }
- }
-
- return APR_SUCCESS;
+ return dobj->frv;
}
Modified: httpd/httpd/trunk/modules/cache/mod_disk_cache.h
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/cache/mod_disk_cache.h?view=diff&rev=468373&r1=468372&r2=468373
==============================================================================
--- httpd/httpd/trunk/modules/cache/mod_disk_cache.h (original)
+++ httpd/httpd/trunk/modules/cache/mod_disk_cache.h Fri Oct 27 06:28:56 2006
@@ -86,6 +86,8 @@
int skipstore; /* Set if we should skip storing stuff */
int store_body_called; /* Number of times store_body() has executed */
+ apr_bucket_brigade *tmpbb; /* Temporary bucket brigade. */
+ apr_status_t frv; /* Last known status of network write */
} disk_cache_object_t;
Modified: httpd/httpd/trunk/modules/cache/mod_mem_cache.c
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/cache/mod_mem_cache.c?view=diff&rev=468373&r1=468372&r2=468373
==============================================================================
--- httpd/httpd/trunk/modules/cache/mod_mem_cache.c (original)
+++ httpd/httpd/trunk/modules/cache/mod_mem_cache.c Fri Oct 27 06:28:56 2006
@@ -71,6 +71,7 @@
long total_refs; /**< total number of references this entry has had */
apr_uint32_t pos; /**< the position of this entry in the cache */
+ apr_status_t frv; /* last known status of writing to the output filter */
} mem_cache_object_t;
@@ -101,7 +102,7 @@
/* Forward declarations */
static int remove_entity(cache_handle_t *h);
static apr_status_t store_headers(cache_handle_t *h, request_rec *r, cache_info *i);
-static apr_status_t store_body(cache_handle_t *h, request_rec *r, apr_bucket_brigade *b);
+static apr_status_t store_body(cache_handle_t *h, ap_filter_t *f, apr_bucket_brigade *b);
static apr_status_t recall_headers(cache_handle_t *h, request_rec *r);
static apr_status_t recall_body(cache_handle_t *h, apr_pool_t *p, apr_bucket_brigade *bb);
@@ -620,9 +621,10 @@
return APR_SUCCESS;
}
-static apr_status_t store_body(cache_handle_t *h, request_rec *r, apr_bucket_brigade *b)
+static apr_status_t store_body(cache_handle_t *h, ap_filter_t *f, apr_bucket_brigade *b)
{
apr_status_t rv;
+ request_rec *r = f->r;
cache_object_t *obj = h->cache_obj;
cache_object_t *tobj = NULL;
mem_cache_object_t *mobj = (mem_cache_object_t*) obj->vobj;
@@ -667,7 +669,9 @@
rv = apr_file_open(&tmpfile, name, mobj->flags,
APR_OS_DEFAULT, r->pool);
if (rv != APR_SUCCESS) {
- return rv;
+ ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
+ "mem_cache: Failed to open file '%s' while attempting to cache the file descriptor.", name);
+ return ap_pass_brigade(f->next, b);
}
apr_file_inherit_unset(tmpfile);
apr_os_file_get(&(mobj->fd), tmpfile);
@@ -676,7 +680,7 @@
ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
"mem_cache: Cached file: %s with key: %s", name, obj->key);
obj->complete = 1;
- return APR_SUCCESS;
+ return ap_pass_brigade(f->next, b);
}
/* Content not suitable for fd caching. Cache in-memory instead. */
@@ -690,7 +694,12 @@
if (mobj->m == NULL) {
mobj->m = malloc(mobj->m_len);
if (mobj->m == NULL) {
- return APR_ENOMEM;
+ /* we didn't have space to cache it, fall back gracefully */
+ cleanup_cache_object(obj);
+ ap_remove_output_filter(f);
+ ap_log_error(APLOG_MARK, APLOG_ERR, APR_ENOMEM, r->server,
+ "mem_cache: Could not store body - not enough memory.");
+ return ap_pass_brigade(f->next, b);
}
obj->count = 0;
}
@@ -711,7 +720,12 @@
* buffer */
mobj->m = realloc(mobj->m, obj->count);
if (!mobj->m) {
- return APR_ENOMEM;
+ /* we didn't have space to cache it, fall back gracefully */
+ cleanup_cache_object(obj);
+ ap_remove_output_filter(f);
+ ap_log_error(APLOG_MARK, APLOG_ERR, APR_ENOMEM, r->server,
+ "mem_cache: Could not store next bit of body - not enough memory.");
+ return ap_pass_brigade(f->next, b);
}
/* Now comes the crufty part... there is no way to tell the
@@ -767,26 +781,36 @@
}
rv = apr_bucket_read(e, &s, &len, eblock);
if (rv != APR_SUCCESS) {
+ cleanup_cache_object(obj);
+ /* not being able to read the bucket is fatal,
+ * return this up the filter stack
+ */
return rv;
}
if (len) {
/* Check for buffer overflow */
- if ((obj->count + len) > mobj->m_len) {
- return APR_ENOMEM;
- }
- else {
+ if ((obj->count + len) > mobj->m_len) {
+ /* we didn't have space to cache it, fall back gracefully */
+ cleanup_cache_object(obj);
+ ap_remove_output_filter(f);
+ ap_log_error(APLOG_MARK, APLOG_ERR, APR_ENOMEM, r->server,
+ "mem_cache: Could not store body - buffer overflow.");
+ return ap_pass_brigade(f->next, b);
+ }
+ else {
memcpy(cur, s, len);
cur+=len;
obj->count+=len;
- }
+ }
}
/* This should not fail, but if it does, we are in BIG trouble
* cause we just stomped all over the heap.
*/
AP_DEBUG_ASSERT(obj->count <= mobj->m_len);
}
- return APR_SUCCESS;
+ return ap_pass_brigade(f->next, b);
}
+
/**
* Configuration and start-up
*/
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Ruediger Pluem <rp...@apache.org>.
On 11/01/2006 06:02 PM, Graham Leggett wrote:
> On Wed, November 1, 2006 5:48 pm, Ruediger Pluem wrote:
>
>
>>I have svn up to the latest revision and it is not fixed.
>>If you still think that it is fixed please point me to the revision
>>that should have fixed it and I will have a look if something is wrong
>>on my side.
>
>
> If you do an svn log on that file it should give you the revision.
>
> The problem came about when the line that initialises pconf was deleted to
> remove a compiler warning - it needed to be there in the APR_HAS_SENDFILE
> case but wasn't surrounded in an #if. It was put back, and surrounded
> with an #if.
The problem is that pconf is not declared.
>
> Can you post the compile error?
Sure:
make[3]: Verlassen des Verzeichnisses »/usr/src/apache/httpd-trunk/modules/aaa«
make[2]: Verlassen des Verzeichnisses »/usr/src/apache/httpd-trunk/modules/aaa«
Making install in cache
make[2]: Wechsel in das Verzeichnis »/usr/src/apache/httpd-trunk/modules/cache«
make[3]: Wechsel in das Verzeichnis »/usr/src/apache/httpd-trunk/modules/cache«
/usr/src/apache/httpd-trunk/srclib/apr/libtool --silent --mode=compile gcc -pthread -O2 -g -Wall -DLINUX=2
-D_REENTRANT -D_GNU_SOURCE -D_LARGEFILE64_SOURCE -I/usr/src/apache/httpd-trunk/srclib/pcre -I.
-I/usr/src/apache/httpd-trunk/os/unix -I/usr/src/apache/httpd-trunk/server/mpm/worker
-I/usr/src/apache/httpd-trunk/modules/http -I/usr/src/apache/httpd-trunk/modules/filters
-I/usr/src/apache/httpd-trunk/modules/proxy -I/usr/src/apache/httpd-trunk/include
-I/usr/src/apache/httpd-trunk/modules/generators -I/usr/src/apache/httpd-trunk/modules/mappers
-I/usr/src/apache/httpd-trunk/modules/database -I/usr/src/apache/httpd-trunk/srclib/apr/include
-I/usr/src/apache/httpd-trunk/srclib/apr-util/include -I/usr/src/apache/httpd-trunk/server
-I/usr/src/apache/httpd-trunk/modules/proxy/../generators -I/usr/src/apache/httpd-trunk/modules/ssl
-I/usr/src/apache/httpd-trunk/modules/dav/main -prefer-pic -c mod_disk_cache.c && touch mod_disk_cache.slo
mod_disk_cache.c: In function `open_new_file':
mod_disk_cache.c:1304: `pdconf' undeclared (first use in this function)
mod_disk_cache.c:1304: (Each undeclared identifier is reported only once
mod_disk_cache.c:1304: for each function it appears in.)
make[3]: *** [mod_disk_cache.slo] Fehler 1
make[3]: Verlassen des Verzeichnisses »/usr/src/apache/httpd-trunk/modules/cache«
make[2]: *** [install-recursive] Fehler 1
make[2]: Verlassen des Verzeichnisses »/usr/src/apache/httpd-trunk/modules/cache«
make[1]: *** [install-recursive] Fehler 1
make[1]: Verlassen des Verzeichnisses »/usr/src/apache/httpd-trunk/modules«
make: *** [install-recursive] Fehler 1
svn info modules/cache/mod_disk_cache.c
Path: modules/cache/mod_disk_cache.c
Name: mod_disk_cache.c
URL: https://svn.apache.org/repos/asf/httpd/httpd/trunk/modules/cache/mod_disk_cache.c
Repository UUID: 13f79535-47bb-0310-9956-ffa450edef68
Revision: 469941
Node Kind: file
Schedule: normal
Last Changed Author: minfrin
Last Changed Rev: 468409
Last Changed Date: 2006-10-27 17:15:24 +0200 (Fri, 27 Oct 2006)
Text Last Updated: 2006-11-01 20:32:46 +0100 (Wed, 01 Nov 2006)
Properties Last Updated: 2006-11-01 13:29:15 +0100 (Wed, 01 Nov 2006)
Checksum: f64b1dba6692fbd4978ff7709a0bd67c
Regards
Rüdiger
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Wed, November 1, 2006 5:48 pm, Ruediger Pluem wrote:
> I have svn up to the latest revision and it is not fixed.
> If you still think that it is fixed please point me to the revision
> that should have fixed it and I will have a look if something is wrong
> on my side.
If you do an svn log on that file it should give you the revision.
The problem came about when the line that initialises pconf was deleted to
remove a compiler warning - it needed to be there in the APR_HAS_SENDFILE
case but wasn't surrounded in an #if. It was put back, and surrounded
with an #if.
Can you post the compile error?
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Ruediger Pluem <rp...@apache.org>.
On 11/01/2006 02:30 PM, Graham Leggett wrote:
> On Wed, November 1, 2006 2:51 pm, Ruediger Pluem wrote:
>
>
>>This breaks compilation of mod_disk_cache on systems that have
>>APR_HAS_SENDFILE set.
>
>
> This was fixed in a commit shortly after.
I have svn up to the latest revision and it is not fixed.
If you still think that it is fixed please point me to the revision
that should have fixed it and I will have a look if something is wrong
on my side.
Regards
Rüdiger
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Wed, November 1, 2006 2:51 pm, Ruediger Pluem wrote:
> This breaks compilation of mod_disk_cache on systems that have
> APR_HAS_SENDFILE set.
This was fixed in a commit shortly after.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Ruediger Pluem <rp...@apache.org>.
On 10/27/2006 03:28 PM, minfrin@apache.org wrote:
> Author: minfrin
> Date: Fri Oct 27 06:28:56 2006
> New Revision: 468373
>
> URL: http://svn.apache.org/viewvc?view=rev&rev=468373
> Log:
> mod_cache: Pass the output filter stack through the store_body()
> hook, giving each cache backend the ability to make a better
> decision as to how it will allocate the tasks of writing to the
> cache and writing to the network. Previously the write to the
> cache task needed to be complete before the same brigade was
> written to the network, and this caused timing and memory issues
> on large cached files. This fix replaces the previous fix for
> PR39380.
>
>
> Modified: httpd/httpd/trunk/modules/cache/mod_disk_cache.c
> URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/cache/mod_disk_cache.c?view=diff&rev=468373&r1=468372&r2=468373
> ==============================================================================
> --- httpd/httpd/trunk/modules/cache/mod_disk_cache.c (original)
> +++ httpd/httpd/trunk/modules/cache/mod_disk_cache.c Fri Oct 27 06:28:56 2006
> @@ -1286,9 +1292,15 @@
> static apr_status_t open_new_file(request_rec *r, const char *filename,
> apr_file_t **fd, disk_cache_conf *conf)
> {
> - int flags = APR_CREATE | APR_WRITE | APR_BINARY | APR_BUFFERED | APR_EXCL;
> + int flags;
> apr_status_t rv;
>
> + flags = APR_CREATE | APR_WRITE | APR_READ | APR_BINARY | APR_BUFFERED | APR_EXCL | APR_TRUNCATE;
> +#if APR_HAS_SENDFILE
> + flags |= ((pdconf->enable_sendfile == ENABLE_SENDFILE_OFF)
> + ? 0 : APR_SENDFILE_ENABLED);
> +#endif
> +
This breaks compilation of mod_disk_cache on systems that have APR_HAS_SENDFILE set.
Regards
Rüdiger
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Mon, October 30, 2006 2:44 pm, Nick Kew wrote:
> Hang on! Where's the file coming from? If it's local and static,
> what is mod_cache supposed to gain you? And if not, what put it
> in a (single) file bucket before it reached mod_cache?
In the case of the person who reported this issue, the file is coming from
a very slow, but very large NFS mounted multi TB filesystem, and is served
as a normal file. 10% of the files are responsible for 90% of the traffic,
so it makes sense to have a small fast disk cache in front of the slow NFS
backend.
The normal serve-a-static-file content generator puts the entire file into
a single bucket.
>> - apr_bucket_read() assumes that a bucket will only ever be read once.
>>
>> In so doing, it may morph buckets into heap buckets while reading,
>> when buckets are too large to be read in one go. This behaviour is
>> undocumented (I plan to fix that).
>
> Yes. But what is reading them?
First, the cache_body() hook, for the purposes of caching the file.
Second, the next filter in the filter stack.
> If mod_disk_cache gets a single file bucket as input, does it
> actually need to read the file? It can send the file bucket
> down the chain as-is, having given it a filesystem entry in
> cache space.
>
> OK, that falls down if the cache's filespace is not on the same
> disc as the file bucket. But that in itself is a major overhead,
> and begs my first question: what is mod_cache supposed to gain?
>
> Mod_cache fronting a jukebox? Right, then you do want to copy
> the file: can't the cache filter itself pass buckets as it reads
> them? Of course it can. But just because this case exists
> doesn't mean the cache filter should insist on reading every
> file bucket it gets!
How will the cache filter cache the bucket without reading the bucket?
> OK, how about this for an alternative: introduce an apr_bucket_clone
> method, that works by reference-counting and lazy copying, and
> in the case of a file bucket, asynchronous copying. The filter
> can clone the bucket, pass one copy on immediately, and save the
> other: then the save will actually read the file if and only
> if it's copying between filesystems, and the filter chain can
> use sendfile.
>
> I haven't thought this through: I put it forward as the kind of
> proposal that might fix the problem without breaking Justin's
> expectations.
You are basically describing Niklas' solution as originally proposed.
His code isolated file buckets, and when found triggered special code that
knew how to copy a large file bucket to the cache using a normal file copy
from one fd buried in the file bucket to the fd of the cached file.
This was objected to by various people, who believed the cache should not
isolate buckets and treat them specially in this way.
An alternative (and in testing, better performing) method was to write the
original bucket regardless of type to the cache, and then replace the (now
heap) bucket with an equivalent bucket pointing at the cached file. As
this cached file has just been written, it is present in kernel caching
buffers. Reading back from this file is therefore very fast.
This removed the need to read the same bucket twice - the second read is
of a brand new file bucket.
It also allowed us to read the response to completion very quickly,
serving the many-orders-of-magnitude-slower network client at leisure from
the cache file.
This has enormous advantages for mod_cgi and other heavy processes - the
heavy process can complete and release resources, without waiting for the
slow client.
This is how the disk cache works right now this minute.
> Really? So if a DVD image comes in 8K chunks from mod_proxy,
> mod_cache is going to buffer everything? Erm .... why?
It won't - so far only the file content generator puts large chunks of
content in one bucket.
But that's no guarantee that any other content generator might not do the
same.
> Are you saying mod_cache enforces that? Or mod_disk_cache?
> In the latter case, there's always the option of introducing
> a new provider for large files.
Joe proposed that, it's not a bad idea.
> OK, I plead guilty to not reviewing them. Did you motivate review
> by accompanying them with an explanation (as above) of what
> brokenness they fixed?
The patches fixed a number of issues, including but not limited to:
- The issue above (caching of large files causes out of memory, timeouts)
- The thundering herd issue for request bodies. This issue is
significantly more complex than the issue above, I think it would be best
to discuss that in a separate thread.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Nick Kew <ni...@webthing.com>.
On Mon, 30 Oct 2006 14:03:03 +0200 (SAST)
"Graham Leggett" <mi...@sharp.fm> wrote:
> On Mon, October 30, 2006 12:57 pm, Nick Kew wrote:
>
> >> The current expectation that it be possible to separate completely
> >> the storing of the cached response and the delivery of the content
> >> is broken.
> >
> > Why is that?
>
> Because:
>
> - the cache_body() hook is expected to swallow an entire brigade
> completely and write it to cache completely before this brigade is
> written to the network.
>
> In the case of files, that means one brigade, containing one bucket,
> containing one entire file. For a 4.7GB DVD ISO file, that means many
> minutes before the response starts arriving at the client, which has
> timed out at this point.
Hang on! Where's the file coming from? If it's local and static,
what is mod_cache supposed to gain you? And if not, what put it
in a (single) file bucket before it reached mod_cache?
> - apr_bucket_read() assumes that a bucket will only ever be read once.
>
> In so doing, it may morph buckets into heap buckets while reading,
> when buckets are too large to be read in one go. This behaviour is
> undocumented (I plan to fix that).
Yes. But what is reading them?
> If these heap buckets are not immediately deleted, they will last the
> lifetime of a request. They are not deleted in mod_disk_cache because
> later, we need to write these same buckets to the network. Out of
> memory ensues.
If mod_disk_cache gets a single file bucket as input, does it
actually need to read the file? It can send the file bucket
down the chain as-is, having given it a filesystem entry in
cache space.
OK, that falls down if the cache's filespace is not on the same
disc as the file bucket. But that in itself is a major overhead,
and begs my first question: what is mod_cache supposed to gain?
Mod_cache fronting a jukebox? Right, then you do want to copy
the file: can't the cache filter itself pass buckets as it reads
them? Of course it can. But just because this case exists
doesn't mean the cache filter should insist on reading every
file bucket it gets!
OK, how about this for an alternative: introduce an apr_bucket_clone
method, that works by reference-counting and lazy copying, and
in the case of a file bucket, asynchronous copying. The filter
can clone the bucket, pass one copy on immediately, and save the
other: then the save will actually read the file if and only
if it's copying between filesystems, and the filter chain can
use sendfile.
I haven't thought this through: I put it forward as the kind of
proposal that might fix the problem without breaking Justin's
expectations.
> Previous discussion is just noise, it would be better if I explain
> again.
:-)
> > That suggests broken or implementation and/or inappropriate usage.
> > It says nothing about expectation.
>
> Sorry, but when Google buys YouTube for a Googol dollars, the argument
> that nobody wants to serve large files makes no sense.
Nobody said that.
> The existing mod_cache, regardless of configuration, and regardless of
> cache disk size, can under no circumstances cache a file bigger than
> available RAM.
>
> This is well and truly broken.
Really? So if a DVD image comes in 8K chunks from mod_proxy,
mod_cache is going to buffer everything? Erm .... why?
Are you saying mod_cache enforces that? Or mod_disk_cache?
In the latter case, there's always the option of introducing
a new provider for large files.
> The patches were posted to this dev list a long time ago, and nobody
> took any time to review them. I see no reason why anybody is going to
> review patches going in on some parallel dev branch either.
OK, I plead guilty to not reviewing them. Did you motivate review
by accompanying them with an explanation (as above) of what
brokenness they fixed?
--
Nick Kew
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Mon, October 30, 2006 12:57 pm, Nick Kew wrote:
>> The current expectation that it be possible to separate completely
>> the storing of the cached response and the delivery of the content is
>> broken.
>
> Why is that?
Because:
- the cache_body() hook is expected to swallow an entire brigade
completely and write it to cache completely before this brigade is written
to the network.
In the case of files, that means one brigade, containing one bucket,
containing one entire file. For a 4.7GB DVD ISO file, that means many
minutes before the response starts arriving at the client, which has timed
out at this point.
- apr_bucket_read() assumes that a bucket will only ever be read once.
In so doing, it may morph buckets into heap buckets while reading, when
buckets are too large to be read in one go. This behaviour is undocumented
(I plan to fix that).
If these heap buckets are not immediately deleted, they will last the
lifetime of a request. They are not deleted in mod_disk_cache because
later, we need to write these same buckets to the network. Out of memory
ensues.
> (references to previous discussion will do, if applicable)
Previous discussion is just noise, it would be better if I explain again.
> That suggests broken or implementation and/or inappropriate usage.
> It says nothing about expectation.
Sorry, but when Google buys YouTube for a Googol dollars, the argument
that nobody wants to serve large files makes no sense.
The existing mod_cache, regardless of configuration, and regardless of
cache disk size, can under no circumstances cache a file bigger than
available RAM.
This is well and truly broken.
> There are quite a lot of contributors, and there may be competing
> designs. That sounds like the kind of situation where a single
> /trunk/ and a single cache framework instance is hopelessly
> inadequate. If necessary, let it fork, demo your ideas on a
> branch, then present that for discussion.
>
> Isn't that approximately what Paul suggested some time back?
No - as Will Rowe pointed out, developing these ideas under a rock with no
oversight, and then suddenly producing a "finished work" does not benefit
anybody.
The patches were posted to this dev list a long time ago, and nobody took
any time to review them. I see no reason why anybody is going to review
patches going in on some parallel dev branch either.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Nick Kew <ni...@webthing.com>.
On Mon, 30 Oct 2006 01:53:18 +0200
Graham Leggett <mi...@sharp.fm> wrote:
> The current expectation that it be possible to separate completely
> the storing of the cached response and the delivery of the content is
> broken.
Why is that?
(references to previous discussion will do, if applicable)
> We have a real world case where the cache is expected to process a
> many MB or many GB file completely, before sending that same response
> to the network. This is too slow, and takes up too much RAM,
> resulting in a broken response to the client.
That suggests broken or implementation and/or inappropriate usage.
It says nothing about expectation.
> On wednesday night I wrote a patch that solved the large file
> problem, while maintaining the current separation between
> write-to-cache and write-to-network as you assert. This mod_cache
> code broke up the brigade into bite sized chunks inside mod_cache
> before passing it to write-to-cache, then write-to-network, and so on.
>
> Joe vetoed the patch, saying that it duplicated the natural behaviour
> of apr_bucket_read().
>
> The wednesday night patch was reverted, and thursday night was spent
> instead changing the cache_body() signature to make its own better
> judgement on how to handle cached files.
>
> Now you veto this next patch, saying it breaks the abstraction.
All of which says this work is in a state of flux.
> I agree with you that a design needs to be found on list first, as I
> have wasted enough time going round in circles coming up with
> solution after solution nobody is happy with.
>
> Do we put this to a vote?
There are quite a lot of contributors, and there may be competing
designs. That sounds like the kind of situation where a single
/trunk/ and a single cache framework instance is hopelessly
inadequate. If necessary, let it fork, demo your ideas on a
branch, then present that for discussion.
Isn't that approximately what Paul suggested some time back?
--
Nick Kew
Re: cache API visibility (was Re: svn commit: r468373...)
Posted by Justin Erenkrantz <ju...@erenkrantz.com>.
On 10/30/06, Joe Orton <jo...@redhat.com> wrote:
> So here's another assumption to consider with this work...
>
> I keep seeing people talk about the mod_cache interface as an
> externally-visible API for which we must maintain API/ABI compatibility.
> But mod_cache.h is not installed, so this API is not actually exposed
> externally, and never has been.
There are some folks who have custom cache modules flying around that
use the provider interface from 2.2. (They just drop into the cache
directory...)
> Given that fact I don't see the point in jumping through hoops to
> maintain backwards API compat, only two modules use the API.
I view it as that we can create a new provider interface without
breaking the old implementations. This reduces the amount of churn
when we want to rev the interface.
So, we could create a new "mod_large_disk_cache" and not alter
mod_mem_cache or mod_disk_cache in any way. Currently, we have to
keep them all in sync and I view that as a waste of time. I view it
as simpler to consider the current interface as fixed and come up with
a new interface. -- justin
Re: cache API visibility (was Re: svn commit: r468373...)
Posted by "William A. Rowe, Jr." <wr...@rowe-clan.net>.
Joe Orton wrote:
>
> I keep seeing people talk about the mod_cache interface as an
> externally-visible API for which we must maintain API/ABI compatibility.
> But mod_cache.h is not installed, so this API is not actually exposed
> externally, and never has been.
Uhm - it's our front for an extensible schema - we CANNOT go breaking
those sorts of things.
Even if it's mod_propritary (and no, I don't have one, but...) we can't
go on assuming nobody is extending our frameworks outside the project.
> Given that fact I don't see the point in jumping through hoops to
> maintain backwards API compat, only two modules use the API.
I don't either. Let's get this fixed and roll 2.4. EOS
(end of showstopper)
Bill
cache API visibility (was Re: svn commit: r468373...)
Posted by Joe Orton <jo...@redhat.com>.
On Mon, Oct 30, 2006 at 09:51:30AM -0700, Justin Erenkrantz wrote:
> >It would make
> >custom 3rd party modules easier to write too; we could define, say 10000
> >as PROVIDER_ID_CUSTOM, making it easier to write add-ons to modules
> >which use the provider interface to just write new providers with that
> >ID (or that ID +1 or -1) rather than having to both write the new
> >provider and also change the code which calls ap_lookup_provider to
> >request the new provider version. Or did I miss the point somewhere?
>
> Nah, that's not the point - you use the name value of the provider for
> third-party modules. The version number is fixed across all providers
> (generally).
>
> If I implement version "0" of the cache interface say in
> mod_disk_cache, then I call ap_register_provider() with version "0"
> with the name "disk". In a hypothetical mod_large_disk_cache that
> implements version "1" of the cache interface, I then register version
> "1" with the name "thumper" (say).
>
> mod_cache is then taught how to handle version "0" and "1" providers.
So here's another assumption to consider with this work...
I keep seeing people talk about the mod_cache interface as an
externally-visible API for which we must maintain API/ABI compatibility.
But mod_cache.h is not installed, so this API is not actually exposed
externally, and never has been.
Given that fact I don't see the point in jumping through hoops to
maintain backwards API compat, only two modules use the API.
joe
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Issac Goldstand <ma...@beamartyr.net>.
Justin Erenkrantz wrote:
> On 10/30/06, Issac Goldstand <ma...@beamartyr.net> wrote:
>> Looking at provider.c, a couple of questions spring to mind:
>> 1) Why isn't this part of apr-util? (it seems similar to apr_optional.h
>> - just intended for vtables rather than functions, and with this version
>> info)
>
> apr_optional is an ugly hack without a good API. =)
Point still stands, though, that this might still fit there.
>
> The generic provider code grew out of mod_dav's provider interface.
> The aaa modules also uses this same interface as well.
Right, I saw those this morning when I looked at provider.c
>
>> 2) It seems that if trunk would want to use version "1" and fallback to
>> "0", the control logic for that would need to be in
>> ap_cache_get_providers and would need to check every version string
>> mod_cache is interested in (for an exact string match, unless the hash
>> function's doing something funky). Wouldn't it be smarter to use a
>> numeric version and simply have ap_lookup_provider simply return the
>> provider of the requested group/type with the highest id?
>
> Yes, ap_cache_get_providers() would have the logic for checking "0"
> and "1" versions. Note that the APR hash abstraction that the
> provider is built on top of only works with strings - not integers.
> The general provider API is intended to only return specific versions
> that it's asked for.
>
> Let me explain...
>
>> It would make
>> custom 3rd party modules easier to write too; we could define, say 10000
>> as PROVIDER_ID_CUSTOM, making it easier to write add-ons to modules
>> which use the provider interface to just write new providers with that
>> ID (or that ID +1 or -1) rather than having to both write the new
>> provider and also change the code which calls ap_lookup_provider to
>> request the new provider version. Or did I miss the point somewhere?
>
> Nah, that's not the point - you use the name value of the provider for
> third-party modules. The version number is fixed across all providers
> (generally).
>
> If I implement version "0" of the cache interface say in
> mod_disk_cache, then I call ap_register_provider() with version "0"
> with the name "disk". In a hypothetical mod_large_disk_cache that
> implements version "1" of the cache interface, I then register version
> "1" with the name "thumper" (say).
>
> mod_cache is then taught how to handle version "0" and "1" providers.
>
> The user comes along with the following directive:
>
> CacheEnable thumper /
>
> mod_cache will then look up "thumper" version "1" and then "0". It'll
> find version "1" of the vtable and use that for its cache.
>
> However, if we wanted to do so, "thumper" could also register as
> "disk" version "1". So, a mod_cache that understands version "1"
> would use mod_large_disk_cache instead of mod_disk_cache (which
> doesn't implement version "0") if it understands version "1"
> providers.
>
> I hope this makes it a bit clearer. -- justin
It did. I think I need to digest it a bit more. I completely understand
the logic of using provider names for third-party stuff, but some little
voice in the back of my mind is still nagging me about the versioning
logic; it's just going to be a pain to repeat the selection code for
every module (eg, dav, aaa) that wants to bump a version and would be
easier if the API just did it for you. Can't have everything, I guess...
Regardless, many thanks for the insight!
Issac
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Justin Erenkrantz <ju...@erenkrantz.com>.
On 10/30/06, Issac Goldstand <ma...@beamartyr.net> wrote:
> Looking at provider.c, a couple of questions spring to mind:
> 1) Why isn't this part of apr-util? (it seems similar to apr_optional.h
> - just intended for vtables rather than functions, and with this version
> info)
apr_optional is an ugly hack without a good API. =)
The generic provider code grew out of mod_dav's provider interface.
The aaa modules also uses this same interface as well.
> 2) It seems that if trunk would want to use version "1" and fallback to
> "0", the control logic for that would need to be in
> ap_cache_get_providers and would need to check every version string
> mod_cache is interested in (for an exact string match, unless the hash
> function's doing something funky). Wouldn't it be smarter to use a
> numeric version and simply have ap_lookup_provider simply return the
> provider of the requested group/type with the highest id?
Yes, ap_cache_get_providers() would have the logic for checking "0"
and "1" versions. Note that the APR hash abstraction that the
provider is built on top of only works with strings - not integers.
The general provider API is intended to only return specific versions
that it's asked for.
Let me explain...
> It would make
> custom 3rd party modules easier to write too; we could define, say 10000
> as PROVIDER_ID_CUSTOM, making it easier to write add-ons to modules
> which use the provider interface to just write new providers with that
> ID (or that ID +1 or -1) rather than having to both write the new
> provider and also change the code which calls ap_lookup_provider to
> request the new provider version. Or did I miss the point somewhere?
Nah, that's not the point - you use the name value of the provider for
third-party modules. The version number is fixed across all providers
(generally).
If I implement version "0" of the cache interface say in
mod_disk_cache, then I call ap_register_provider() with version "0"
with the name "disk". In a hypothetical mod_large_disk_cache that
implements version "1" of the cache interface, I then register version
"1" with the name "thumper" (say).
mod_cache is then taught how to handle version "0" and "1" providers.
The user comes along with the following directive:
CacheEnable thumper /
mod_cache will then look up "thumper" version "1" and then "0". It'll
find version "1" of the vtable and use that for its cache.
However, if we wanted to do so, "thumper" could also register as
"disk" version "1". So, a mod_cache that understands version "1"
would use mod_large_disk_cache instead of mod_disk_cache (which
doesn't implement version "0") if it understands version "1"
providers.
I hope this makes it a bit clearer. -- justin
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Issac Goldstand <ma...@beamartyr.net>.
Justin Erenkrantz wrote:
> On 10/30/06, Issac Goldstand <ma...@beamartyr.net> wrote:
>> Can you clarify the above a bit? I don't understand what you're
>> referring to. Looking at the 2.2.3 tag, what versioning is currently in
>> place?
>
> Look at disk_cache_register_hook.
>
> /* cache initializer */
> ap_register_provider(p, CACHE_PROVIDER_GROUP, "disk", "0",
> &cache_disk_provider);
>
> The "0" is the version string with the cache_disk_provider vtable
> acting as the 0 version..
>
> When we have a new cache provider version (i.e. the vtable changes),
> that gets bumped up to "1". Then, mod_cache can select a "0" or
> "1"-class backend. Preferably, it should be able to use both; but
> trunk would prefer "1" and fallback to "0" if it's not available.
Gotcha. Thanks for the insight.
Looking at provider.c, a couple of questions spring to mind:
1) Why isn't this part of apr-util? (it seems similar to apr_optional.h
- just intended for vtables rather than functions, and with this version
info)
2) It seems that if trunk would want to use version "1" and fallback to
"0", the control logic for that would need to be in
ap_cache_get_providers and would need to check every version string
mod_cache is interested in (for an exact string match, unless the hash
function's doing something funky). Wouldn't it be smarter to use a
numeric version and simply have ap_lookup_provider simply return the
provider of the requested group/type with the highest id? It would make
custom 3rd party modules easier to write too; we could define, say 10000
as PROVIDER_ID_CUSTOM, making it easier to write add-ons to modules
which use the provider interface to just write new providers with that
ID (or that ID +1 or -1) rather than having to both write the new
provider and also change the code which calls ap_lookup_provider to
request the new provider version. Or did I miss the point somewhere?
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Justin Erenkrantz <ju...@erenkrantz.com>.
On 10/30/06, Issac Goldstand <ma...@beamartyr.net> wrote:
> Can you clarify the above a bit? I don't understand what you're
> referring to. Looking at the 2.2.3 tag, what versioning is currently in
> place?
Look at disk_cache_register_hook.
/* cache initializer */
ap_register_provider(p, CACHE_PROVIDER_GROUP, "disk", "0",
&cache_disk_provider);
The "0" is the version string with the cache_disk_provider vtable
acting as the 0 version..
When we have a new cache provider version (i.e. the vtable changes),
that gets bumped up to "1". Then, mod_cache can select a "0" or
"1"-class backend. Preferably, it should be able to use both; but
trunk would prefer "1" and fallback to "0" if it's not available.
HTH. -- justin
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Issac Goldstand <ma...@beamartyr.net>.
Justin Erenkrantz wrote:
>
> I will also note that the mod_cache provider system has explicit
> versioning, so any modifications to the providers should be
> represented with a new version number. (i.e. providers for version
> "0" should work while offering new features in version "1"-class
> providers.) We do not arbitrarily tweak the old provider structures
> any more - instead, we introduce new versions. -- justin
Justin,
Can you clarify the above a bit? I don't understand what you're
referring to. Looking at the 2.2.3 tag, what versioning is currently in
place?
Thanks,
Issac
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Mon, October 30, 2006 7:58 am, Justin Erenkrantz wrote:
>> We have a real world case where the cache is expected to process a many
>> MB or many GB file completely, before sending that same response to the
>> network. This is too slow, and takes up too much RAM, resulting in a
>> broken response to the client.
>
> In short, I haven't seen any evidence presented by you or others that
> this is due to a design flaw in the cache/provider abstraction.
+1.
Unfortunately, the patch to mod_cache that solved this problem and kept
the cache/provider abstraction intact was -1'd by Joe Orton.
What I should have done was challenge this point with Joe, but the signal
to noise ratio was too high at that point for there to have been any
chance of that happening.
Hopefully this can be remedied now.
> At
> best, mod_disk_cache could be smarter about storing the file when it's
> large - but that's just a small lines of code to fix - it doesn't
> require any massive changes or new bucket types. Just copy the file
> bucket and consume that within store_body()'s implementation.
The apr_bucket_read() function, when given a large file, splits the bucket
on each read into a heap bucket and a file bucket.
This works throughout the server, because the most common use case for
this is "read bucket, do something with bucket, delete (newly created
heap) bucket, repeat".
In the cache however, the "delete bucket" step is missing - because the
same brigade needs to later be given to the output filter stack.
So we have a leak. An attempt to cache any file larger than RAM (an ISO
file, any typical video file) results in one file bucket being replaced
with a brigade of heap buckets the size of the file.
Out of memory, boom.
The second problem, even if we had enough memory, is that disk drives are
just too slow. The current implementation will try and write the entire
file from backend to cache disk, before a single byte is written to the
network.
On some drives, this can be minutes later - assuming the browser hasn't
timed out at this point, the human definitely has, boom.
> If that isn't enough, please identify here on list and backed by
> references to the implementation and timings that it is 'too slow',
> 'takes up too much RAM', and results in a 'broken response to the
> client' and why we must break all of our cache structures and
> abstractions.
Hopefully the explanation above makes sense.
I agree with you - changing the abstraction is wrong, I far prefer the
original solution from wednesday last week.
> I'm tired of reviewing large code changes with only vague generalities
> about why the code must change. These decisions need to be explained
> and reviewed on list before any more code is committed. Your recent
> commits have been chock full of mistakes and style violations - which
> make it almost impossible to review what's going on. If you are going
> to commit, please take care to follow all of our style guidelines and
> please ensure the commit works before using our trunk as your personal
> dumping ground. If your changes are incomplete, feel free to post
> them to the list instead of committing.
The bulk of these patches are a contribution from Niklas Edmundsson who
has put a lot of work into making the cache functional on a high traffic
ftp mirror. These were posted to the list, and added to bugzilla, but this
was not enough to generate any interest or review.
I made the request of Niklas to break the monolithic patch into smaller
reviewable patches, and this was done. I then started reviewing and
committing the patches one by one, starting a few weeks ago.
Repeated requests were made for help reviewing, installing and testing the
patches, but very few reponses were forthcoming, possibly because people
didn't want to get involved in the noisy parts of the thread, or possibly
because they had no time.
Now, a few weeks later, people are suddenly revewing patches committed
weeks ago. I realise that you personally were tied up in infrastructure
issues and so have come late to this discussion, but I have to ask:
Where were the comments when the original patches went in? Of the few
comments that were made, where was the ongoing discussion on these points?
> We're not even close to knowing what we'd be voting on. So, please
> draft up a proposed design that explains in detail why you think there
> is a problem and what specific changes you propose and why that is the
> best solution. If you want to submit a patch to go with your
> rationale, cool. Yet, I certainly don't see any fundamental reason
> why Joe's concerns and mine can't both be addressed in the same
> design.
The current set of patches attempts to solve a list of problems, not a
single problem. The fact that multiple problems were being solved had
muddied the water and made it difficult to review.
I am now trying to address issues one single issue at a time, and the
issue I am trying to resolve here is "solve the out of memory problem when
trying to cache large files".
> I will also note that the mod_cache provider system has explicit
> versioning, so any modifications to the providers should be
> represented with a new version number. (i.e. providers for version
> "0" should work while offering new features in version "1"-class
> providers.) We do not arbitrarily tweak the old provider structures
> any more - instead, we introduce new versions. -- justin
Ok.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Justin Erenkrantz <ju...@erenkrantz.com>.
On 10/29/06, Graham Leggett <mi...@sharp.fm> wrote:
> The current expectation that it be possible to separate completely the
> storing of the cached response and the delivery of the content is broken.
>
> We have a real world case where the cache is expected to process a many
> MB or many GB file completely, before sending that same response to the
> network. This is too slow, and takes up too much RAM, resulting in a
> broken response to the client.
In short, I haven't seen any evidence presented by you or others that
this is due to a design flaw in the cache/provider abstraction. At
best, mod_disk_cache could be smarter about storing the file when it's
large - but that's just a small lines of code to fix - it doesn't
require any massive changes or new bucket types. Just copy the file
bucket and consume that within store_body()'s implementation.
If that isn't enough, please identify here on list and backed by
references to the implementation and timings that it is 'too slow',
'takes up too much RAM', and results in a 'broken response to the
client' and why we must break all of our cache structures and
abstractions.
I'm tired of reviewing large code changes with only vague generalities
about why the code must change. These decisions need to be explained
and reviewed on list before any more code is committed. Your recent
commits have been chock full of mistakes and style violations - which
make it almost impossible to review what's going on. If you are going
to commit, please take care to follow all of our style guidelines and
please ensure the commit works before using our trunk as your personal
dumping ground. If your changes are incomplete, feel free to post
them to the list instead of committing.
Looking at the current implementation of mod_disk_cache, someone has
turned it into unreadable and unmanagable code.
Take a look at:
http://svn.apache.org/repos/asf/httpd/httpd/tags/2.2.3/modules/cache/mod_disk_cache.c
versus
http://svn.apache.org/repos/asf/httpd/httpd/trunk/modules/cache/mod_disk_cache.c
The code somehow went from 9KB total to 15KB for no good reason.
Somewhere, we went really wrong here.
> So, we have disgreement over the right way to solve the problem of the
> cache being expected to swallow mouthfuls too big for it to handle.
>
> I agree with you that a design needs to be found on list first, as I
> have wasted enough time going round in circles coming up with solution
> after solution nobody is happy with.
>
> Do we put this to a vote?
We're not even close to knowing what we'd be voting on. So, please
draft up a proposed design that explains in detail why you think there
is a problem and what specific changes you propose and why that is the
best solution. If you want to submit a patch to go with your
rationale, cool. Yet, I certainly don't see any fundamental reason
why Joe's concerns and mine can't both be addressed in the same
design.
I will also note that the mod_cache provider system has explicit
versioning, so any modifications to the providers should be
represented with a new version number. (i.e. providers for version
"0" should work while offering new features in version "1"-class
providers.) We do not arbitrarily tweak the old provider structures
any more - instead, we introduce new versions. -- justin
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
Justin Erenkrantz wrote:
> -1.
>
> This breaks the abstraction between the cache providers and the filter streams.
> The cache providers should not be in the business of delivering content down to
> the next filter - that is the job of mod_cache. Following this route is
> completely anti-thetical to the separation between storing the cache response
> and delivery of the content.
The current expectation that it be possible to separate completely the
storing of the cached response and the delivery of the content is broken.
We have a real world case where the cache is expected to process a many
MB or many GB file completely, before sending that same response to the
network. This is too slow, and takes up too much RAM, resulting in a
broken response to the client.
On wednesday night I wrote a patch that solved the large file problem,
while maintaining the current separation between write-to-cache and
write-to-network as you assert. This mod_cache code broke up the brigade
into bite sized chunks inside mod_cache before passing it to
write-to-cache, then write-to-network, and so on.
Joe vetoed the patch, saying that it duplicated the natural behaviour of
apr_bucket_read().
The wednesday night patch was reverted, and thursday night was spent
instead changing the cache_body() signature to make its own better
judgement on how to handle cached files.
Now you veto this next patch, saying it breaks the abstraction.
So, we have disgreement over the right way to solve the problem of the
cache being expected to swallow mouthfuls too big for it to handle.
I agree with you that a design needs to be found on list first, as I
have wasted enough time going round in circles coming up with solution
after solution nobody is happy with.
Do we put this to a vote?
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
Justin Erenkrantz wrote:
> -1.
>
> This breaks the abstraction between the cache providers and the filter streams.
> The cache providers should not be in the business of delivering content down to
> the next filter - that is the job of mod_cache. Following this route is
> completely anti-thetical to the separation between storing the cache response
> and delivery of the content.
The current expectation that it be possible to separate completely the
storing of the cached response and the delivery of the content is broken.
We have a real world case where the cache is expected to process a many
MB or many GB file completely, before sending that same response to the
network. This is too slow, and takes up too much RAM, resulting in a
broken response to the client.
On wednesday night I wrote a patch that solved the large file problem,
while maintaining the current separation between write-to-cache and
write-to-network as you assert. This mod_cache code broke up the brigade
into bite sized chunks inside mod_cache before passing it to
write-to-cache, then write-to-network, and so on.
Joe vetoed the patch, saying that it duplicated the natural behaviour of
apr_bucket_read().
The wednesday night patch was reverted, and thursday night was spent
instead changing the cache_body() signature to make its own better
judgement on how to handle cached files.
Now you veto this next patch, saying it breaks the abstraction.
So, we have disgreement over the right way to solve the problem of the
cache being expected to swallow mouthfuls too big for it to handle.
I agree with you that a design needs to be found on list first, as I
have wasted enough time going round in circles coming up with solution
after solution nobody is happy with.
Do we put this to a vote?
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Justin Erenkrantz <ju...@erenkrantz.com>.
On Fri, Oct 27, 2006 at 01:28:57PM -0000, minfrin@apache.org wrote:
> Author: minfrin
> Date: Fri Oct 27 06:28:56 2006
> New Revision: 468373
>
> URL: http://svn.apache.org/viewvc?view=rev&rev=468373
> Log:
> mod_cache: Pass the output filter stack through the store_body()
> hook, giving each cache backend the ability to make a better
> decision as to how it will allocate the tasks of writing to the
> cache and writing to the network. Previously the write to the
> cache task needed to be complete before the same brigade was
> written to the network, and this caused timing and memory issues
> on large cached files. This fix replaces the previous fix for
> PR39380.
-1.
This breaks the abstraction between the cache providers and the filter streams.
The cache providers should not be in the business of delivering content down to
the next filter - that is the job of mod_cache. Following this route is
completely anti-thetical to the separation between storing the cache response
and delivery of the content.
As others have mentioned, I would highly recommend that you take a step back
and come up with a design on-list first, run it through the gauntlet on dev@,
and test it before breaking carefully designed abstractions. -- justin
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Justin Erenkrantz <ju...@erenkrantz.com>.
On Fri, Oct 27, 2006 at 01:28:57PM -0000, minfrin@apache.org wrote:
> Author: minfrin
> Date: Fri Oct 27 06:28:56 2006
> New Revision: 468373
>
> URL: http://svn.apache.org/viewvc?view=rev&rev=468373
> Log:
> mod_cache: Pass the output filter stack through the store_body()
> hook, giving each cache backend the ability to make a better
> decision as to how it will allocate the tasks of writing to the
> cache and writing to the network. Previously the write to the
> cache task needed to be complete before the same brigade was
> written to the network, and this caused timing and memory issues
> on large cached files. This fix replaces the previous fix for
> PR39380.
-1.
This breaks the abstraction between the cache providers and the filter streams.
The cache providers should not be in the business of delivering content down to
the next filter - that is the job of mod_cache. Following this route is
completely anti-thetical to the separation between storing the cache response
and delivery of the content.
As others have mentioned, I would highly recommend that you take a step back
and come up with a design on-list first, run it through the gauntlet on dev@,
and test it before breaking carefully designed abstractions. -- justin
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
Davi Arnaut wrote:
> But that's a corner case. There is no reason in doing this for small
> files (common case). For example, in a enterprise grade server memory is
> cheap and permanent storage is slow and expensive.
If permanent storage is slow and expensive, what on earth is the admin
doing trying to turn on a disk cache?
However luckily though the typical modern OS helps by RAM caching common
disk access, the argument has been made here before that a disk cache
gives same or better performance than the memory cache.
Please don't forget that mod_cache, by design, is a generic RFC2616
caching proxy bolted onto the frontend of the webserver, and that means
you can cache virtually any well behaved application - from mod_perl, to
CGI, to mod_python, mod_proxy, whatever.
And also don't forget that compared to the speed at which a webserver
can generate data, the typical client is many orders of maginitude
slower at swallowing that data. So your big expensive mod_perl, CGI,
mod_python, etc sits around waiting for the client to get on with it and
download the response, using resources better spent elsewhere.
A very common use case is for a caching proxy is to take the load off an
expensive application server, and usually this is deployed as a separate
piece of kit in front of the webserver. With mod_cache, you can offload
this inside the server itself, which simplifies your setup.
In short, this is not a corner case.
>> Also, httpd seems to be faster delivering data by sendfile than
>> delivering data from memory buckets. That's more of a performance bug
>> in httpd though.
>
> It's not all httpd fault. Sendfile avoids memory copying/context
> switches/etc behind the scenes (in the kernel) because it "talks"
> directly to the underlying file system. When you have the memory already
> in user space there isn't much that you can do, the kernel has to copy
> it..etc.
Exactly - another strong argument to make sure that when you turn on the
disk cache, you serve the network off the disk cache.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
Davi Arnaut wrote:
> Because the data is already in memory. Why should I write something to
> disk, erase it from memory, and read it again shortly ? Why should I
> take care of something that is the job of the OS ? Why should I trash
> the VM constantly ?
Because reading slowly from a cache disk is less resource intensive than
reading from CGI, proxy, slow NFS drives, etc. And reading from cache
disk gives you SENDFILE. Try it out, you will see.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Davi Arnaut <da...@haxent.com.br>.
Davi Arnaut wrote:
> Greg Marr wrote:
>> At 01:41 PM 10/27/2006, Davi Arnaut wrote:
>>> Niklas Edmundsson wrote:
>>>> And when you have a file backend, you want to hit your disk cache
>>> and
>>>> not the backend when delivering data to a client. People might
>>> think
>>>> that this doesn't matter, but for large files, especially larger
>>> than
>>>> RAM in your machine, you usually go disk-bound without much help
>>> from
>>>> the OS disk cache.
>>> But that's a corner case. There is no reason in doing this for small
>>> files (common case). For example, in a enterprise grade server
>>> memory is
>>> cheap and permanent storage is slow and expensive.
>> So why would this server be using mod_disk_cache in that
>> case? Shouldn't this server be using mod_mem_cache? Selecting
>> mod_disk_cache over mod_mem_cache means it's better to serve the
>> cache from disk rather than from memory. If serving from the disk on
>> the original request is too slow, then wouldn't serving from the disk
>> on the subsequent requests be too slow as well?
>>
>
> Because the data is already in memory. Why should I write something to
> disk, erase it from memory, and read it again shortly ? Why should I
> take care of something that is the job of the OS ? Why should I trash
> the VM constantly ?
I should note that if there is enough memory those reads will be served
by the page cache. Later we could think about reading directly to
mmapable files, changing it's protection (from rw to ro) as soon as we
are done writing to.
--
Davi Arnaut
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Davi Arnaut <da...@haxent.com.br>.
Greg Marr wrote:
> At 01:41 PM 10/27/2006, Davi Arnaut wrote:
>> Niklas Edmundsson wrote:
>>> And when you have a file backend, you want to hit your disk cache
>> and
>>> not the backend when delivering data to a client. People might
>> think
>>> that this doesn't matter, but for large files, especially larger
>> than
>>> RAM in your machine, you usually go disk-bound without much help
>> from
>>> the OS disk cache.
>> But that's a corner case. There is no reason in doing this for small
>> files (common case). For example, in a enterprise grade server
>> memory is
>> cheap and permanent storage is slow and expensive.
>
> So why would this server be using mod_disk_cache in that
> case? Shouldn't this server be using mod_mem_cache? Selecting
> mod_disk_cache over mod_mem_cache means it's better to serve the
> cache from disk rather than from memory. If serving from the disk on
> the original request is too slow, then wouldn't serving from the disk
> on the subsequent requests be too slow as well?
>
Because the data is already in memory. Why should I write something to
disk, erase it from memory, and read it again shortly ? Why should I
take care of something that is the job of the OS ? Why should I trash
the VM constantly ?
A proxy/cache common purpose is to cache web content, this means we have
a large number of small files (think html pages, images, etc) that we
must keep in a permanent storage, but if we have memory, let's use it!
--
Davi Arnaut
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Greg Marr <gr...@alum.wpi.edu>.
At 01:41 PM 10/27/2006, Davi Arnaut wrote:
>Niklas Edmundsson wrote:
> > And when you have a file backend, you want to hit your disk cache
> and
> > not the backend when delivering data to a client. People might
> think
> > that this doesn't matter, but for large files, especially larger
> than
> > RAM in your machine, you usually go disk-bound without much help
> from
> > the OS disk cache.
>
>But that's a corner case. There is no reason in doing this for small
>files (common case). For example, in a enterprise grade server
>memory is
>cheap and permanent storage is slow and expensive.
So why would this server be using mod_disk_cache in that
case? Shouldn't this server be using mod_mem_cache? Selecting
mod_disk_cache over mod_mem_cache means it's better to serve the
cache from disk rather than from memory. If serving from the disk on
the original request is too slow, then wouldn't serving from the disk
on the subsequent requests be too slow as well?
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Davi Arnaut <da...@haxent.com.br>.
Niklas Edmundsson wrote:
> On Fri, 27 Oct 2006, Graham Leggett wrote:
>
>>>> Err. We had the data in memory, we are going to read it back from disk
>>>> again just in order to not block ? That's nonsense.
>>> Agreed.
>> Please explain.
>>
>> This is a disk cache. Why would you write expensive bucket data to cache,
>> and then expensive bucket data to the network?
>>
>> That's plain stupid.
>
> And when you have a file backend, you want to hit your disk cache and
> not the backend when delivering data to a client. People might think
> that this doesn't matter, but for large files, especially larger than
> RAM in your machine, you usually go disk-bound without much help from
> the OS disk cache.
But that's a corner case. There is no reason in doing this for small
files (common case). For example, in a enterprise grade server memory is
cheap and permanent storage is slow and expensive.
> Also, httpd seems to be faster delivering data by sendfile than
> delivering data from memory buckets. That's more of a performance bug
> in httpd though.
It's not all httpd fault. Sendfile avoids memory copying/context
switches/etc behind the scenes (in the kernel) because it "talks"
directly to the underlying file system. When you have the memory already
in user space there isn't much that you can do, the kernel has to copy
it..etc.
--
Davi Arnaut
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Niklas Edmundsson <ni...@acc.umu.se>.
On Fri, 27 Oct 2006, Graham Leggett wrote:
>>> Err. We had the data in memory, we are going to read it back from disk
>>> again just in order to not block ? That's nonsense.
>>
>> Agreed.
>
> Please explain.
>
> This is a disk cache. Why would you write expensive bucket data to cache,
> and then expensive bucket data to the network?
>
> That's plain stupid.
And when you have a file backend, you want to hit your disk cache and
not the backend when delivering data to a client. People might think
that this doesn't matter, but for large files, especially larger than
RAM in your machine, you usually go disk-bound without much help from
the OS disk cache.
Also, httpd seems to be faster delivering data by sendfile than
delivering data from memory buckets. That's more of a performance bug
in httpd though.
/Nikke
--
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
Niklas Edmundsson, Admin @ {acc,hpc2n}.umu.se | nikke@acc.umu.se
---------------------------------------------------------------------------
Monolith Auto Sales Center: "My God! It's full of cars!"
=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Fri, October 27, 2006 6:00 pm, Joe Orton wrote:
>> Err. We had the data in memory, we are going to read it back from disk
>> again just in order to not block ? That's nonsense.
>
> Agreed.
Please explain.
This is a disk cache. Why would you write expensive bucket data to cache,
and then expensive bucket data to the network?
That's plain stupid.
>> We don't need a special bucket type!
>
> Agreed. So -1 on this, on r462696, and on r450105.
Sigh. So not only have you not reviewed the code, you have not reviewed
the comment in CHANGES either. r450105 was reversed by this patch.
I'm afraid we do need a special bucket type.
The current file bucket type does not currently have the ability to
support the same fd across multiple buckets. Sure you can split one bucket
to make two, but you cannot make two file buckets out of one fd without
some low level hocus pocus.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Joe Orton <jo...@redhat.com>.
On Fri, Oct 27, 2006 at 11:38:02AM -0300, Davi Arnaut wrote:
> > + /* Is our network connection still alive?
> > + * If not, we must continue caching the file, so keep looping.
> > + * We will return the error at the end when caching is done.
> > + */
> > + if (APR_SUCCESS == dobj->frv) {
> > +
> > + /* insert a file bucket pointing to the cache into out temporary brigade */
> > + if (diskcache_brigade_insert(dobj->tmpbb, dobj->fd, dobj->file_size,
> > + written,
> > + dobj->updtimeout, r->pool) == NULL) {
> > + return APR_ENOMEM;
> > + }
>
> Err. We had the data in memory, we are going to read it back from disk
> again just in order to not block ? That's nonsense.
Agreed.
> We don't need a special bucket type!
Agreed. So -1 on this, on r462696, and on r450105.
joe
Re: [PATCH] mod_disk_cache fails to compile [Was: svn commit:
r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c]
Posted by Graham Leggett <mi...@sharp.fm>.
On Thu, November 2, 2006 12:34 pm, Martin Kraemer wrote:
> Still, since 2 days, again mod_disk_cache fails to compile on
> platforms with sendfile (e.g., FreeBSD).
I missed the patch you sent - looks the same as r468409 though?
Regards,
Graham
--
Re: [PATCH] mod_disk_cache fails to compile [Was: svn commit:
r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c]
Posted by Graham Leggett <mi...@sharp.fm>.
On Thu, November 2, 2006 2:20 pm, Martin Kraemer wrote:
> Oops - I forgot to mention the branch I'm working on - it's a bug in
> the trunk. I am at Revision 470348, but the pdconf variable occurs at
> *two* places in the file, however the declaration appears only once.
Aaah... the penny drops.
I deleted the pdconf definition in one place (it was giving a compiler
warning for the APR_HAS_SENDFILE == 0 case), and when it was pointed out I
put it back.
Looks like this code hasn't built in the SENDFILE case for a while.
Regards,
Graham
--
Re: [PATCH] mod_disk_cache fails to compile [Was: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c]
Posted by Martin Kraemer <ma...@apache.org>.
On Thu, Nov 02, 2006 at 01:00:29PM +0200, Graham Leggett wrote:
> On Thu, November 2, 2006 12:34 pm, Martin Kraemer wrote:
>
> > Still, since 2 days, again mod_disk_cache fails to compile on
> > platforms with sendfile (e.g., FreeBSD).
>
Oops - I forgot to mention the branch I'm working on - it's a bug in
the trunk. I am at Revision 470348, but the pdconf variable occurs at
*two* places in the file, however the declaration appears only once.
Martin
--
<Ma...@Fujitsu-Siemens.com> | Fujitsu Siemens
Fon: +49-89-636-46021, FAX: +49-89-636-48332 | 81730 Munich, Germany
Re: [PATCH] mod_disk_cache fails to compile [Was: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c]
Posted by Martin Kraemer <ma...@apache.org>.
On Thu, Nov 02, 2006 at 01:00:29PM +0200, Graham Leggett wrote:
> If you can send more details, I can get to the bottom of it -
> "fails to compile" doesn't tell me anything useful.
mod_disk_cache.c: In function `open_new_file':
mod_disk_cache.c:1304: `pdconf' undeclared (first use in this function)
mod_disk_cache.c:1304: (Each undeclared identifier is reported only once
mod_disk_cache.c:1304: for each function it appears in.)
--
<Ma...@Fujitsu-Siemens.com> | Fujitsu Siemens
Fon: +49-89-636-46021, FAX: +49-89-636-48332 | 81730 Munich, Germany
Re: [PATCH] mod_disk_cache fails to compile [Was: svn commit:
r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c]
Posted by Graham Leggett <mi...@sharp.fm>.
On Thu, November 2, 2006 12:34 pm, Martin Kraemer wrote:
> Still, since 2 days, again mod_disk_cache fails to compile on
> platforms with sendfile (e.g., FreeBSD).
Is r468409 present in your tree?
Unfortunately I don't have a FreeBSD box, and so cannot reproduce the
problem. If you can send more details, I can get to the bottom of it -
"fails to compile" doesn't tell me anything useful.
Regards,
Graham
--
Index: modules/cache/mod_disk_cache.c
===================================================================
--- modules/cache/mod_disk_cache.c (revision 468408)
+++ modules/cache/mod_disk_cache.c (revision 468409)
@@ -720,6 +720,10 @@
apr_time_t starttime = apr_time_now();
int flags;
apr_status_t rc;
+#if APR_HAS_SENDFILE
+ core_dir_config *pdconf = ap_get_module_config(r->per_dir_config,
+ &core_module);
+#endif
flags = APR_READ|APR_BINARY|APR_BUFFERED;
#if APR_HAS_SENDFILE
[PATCH] mod_disk_cache fails to compile [Was: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c]
Posted by Martin Kraemer <ma...@apache.org>.
On Fri, Oct 27, 2006 at 05:05:29PM +0200, Graham Leggett wrote:
> On Fri, October 27, 2006 4:38 pm, Davi Arnaut wrote:
>
> > Where is pdconf ? Check out all those APR_HAS_SENDFILE.
>
> Aaargh... will fix.
Still, since 2 days, again mod_disk_cache fails to compile on
platforms with sendfile (e.g., FreeBSD).
Martin
--
<Ma...@Fujitsu-Siemens.com> | Fujitsu Siemens
Fon: +49-89-636-46021, FAX: +49-89-636-48332 | 81730 Munich, Germany
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Fri, October 27, 2006 6:14 pm, Paul Querna wrote:
> Honestly, I am -1 on any module needing knowledge of the core module
> setting. We should be making the abstraction better, not pulling things
> from the core config.
This is true, but right now we don't have this level of abstraction, and
I'm trying to get the disk cache put to bed first. Baby steps :)
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Paul Querna <ch...@force-elite.com>.
Graham Leggett wrote:
> On Fri, October 27, 2006 6:05 pm, Niklas Edmundsson wrote:
>
>> The purpose of that code was originally to make "EnableSendfile Off"
>> in the config file work. APR_HAS_SENDFILE only tells you that APR has
>> sendfile.
>
> It still does that as far as I can see:
>
> #if APR_HAS_SENDFILE
> core_dir_config *pdconf = ap_get_module_config(r->per_dir_config,
> &core_module);
> #endif
>
> flags = APR_READ|APR_BINARY|APR_BUFFERED;
> #if APR_HAS_SENDFILE
> flags |= ((pdconf->enable_sendfile == ENABLE_SENDFILE_OFF)
> ? 0 : APR_SENDFILE_ENABLED);
> #endif
>
> It was removed because in the absence of APR_HAS_SENDFILE, it gave a
> compiler warning. The real fix was to surround it in an ifdef, which has
> been done.
Honestly, I am -1 on any module needing knowledge of the core module
setting. We should be making the abstraction better, not pulling things
from the core config.
-Paul
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Fri, October 27, 2006 6:05 pm, Niklas Edmundsson wrote:
> The purpose of that code was originally to make "EnableSendfile Off"
> in the config file work. APR_HAS_SENDFILE only tells you that APR has
> sendfile.
It still does that as far as I can see:
#if APR_HAS_SENDFILE
core_dir_config *pdconf = ap_get_module_config(r->per_dir_config,
&core_module);
#endif
flags = APR_READ|APR_BINARY|APR_BUFFERED;
#if APR_HAS_SENDFILE
flags |= ((pdconf->enable_sendfile == ENABLE_SENDFILE_OFF)
? 0 : APR_SENDFILE_ENABLED);
#endif
It was removed because in the absence of APR_HAS_SENDFILE, it gave a
compiler warning. The real fix was to surround it in an ifdef, which has
been done.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h modules/cache/mod_disk_cache.c
modules/cache/mod_disk_cache.h modules/cache/mod_mem_cache.c
Posted by Niklas Edmundsson <ni...@acc.umu.se>.
On Fri, 27 Oct 2006, Graham Leggett wrote:
> On Fri, October 27, 2006 4:38 pm, Davi Arnaut wrote:
>
>> Where is pdconf ? Check out all those APR_HAS_SENDFILE.
>
> Aaargh... will fix.
The purpose of that code was originally to make "EnableSendfile Off"
in the config file work. APR_HAS_SENDFILE only tells you that APR has
sendfile.
/Nikke
--
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
Niklas Edmundsson, Admin @ {acc,hpc2n}.umu.se | nikke@acc.umu.se
---------------------------------------------------------------------------
My favorite color? Red. No, BluAAAAAHHH!
=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES
modules/cache/mod_cache.c modules/cache/mod_cache.h
modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Graham Leggett <mi...@sharp.fm>.
On Fri, October 27, 2006 4:38 pm, Davi Arnaut wrote:
> Where is pdconf ? Check out all those APR_HAS_SENDFILE.
Aaargh... will fix.
> Those "if(" and "if (" are going on my nerves! :)
Damn, missed some.
> Swapping the metadata check would make the code much more readable:
>
> if (is_metadata)
> deal with it
> continue
>
> handle normal bucket
Cool. I also need to handle flush buckets as well.
> Err. We had the data in memory, we are going to read it back from disk
> again just in order to not block ? That's nonsense.
Yep, we have the data in memory. Now we have it on disk, and the memory
can be used for something else while the extremely slow client consumes
the download.
Don't forget, the cache is supposed to improve performance, and the
entities being cached could be large and expensive, like CGIs or mod_perl,
etc. The sooner their work is done and their memory cleared the better.
> We don't need a special bucket type!
>
> Also, are you really sure you are tracking correctly all those buckets
> with the correct offset ? The file pointer offset is unique for a single
> fd.
So far in testing with 1.5GB files, yes. Downloaded files, cached files,
original files all cmp the same.
I tried a normal file bucket type and it didn't work, as it kept closing
the underlying fd, while using the special type as contributed by Niklas
it did work. Niklas has obviously found and fixed this issue, so I didn't
pursue it further.
Regards,
Graham
--
Re: svn commit: r468373 - in /httpd/httpd/trunk: CHANGES modules/cache/mod_cache.c
modules/cache/mod_cache.h modules/cache/mod_disk_cache.c modules/cache/mod_disk_cache.h
modules/cache/mod_mem_cache.c
Posted by Davi Arnaut <da...@haxent.com.br>.
minfrin@apache.org wrote:
> Author: minfrin
> Date: Fri Oct 27 06:28:56 2006
> New Revision: 468373
>
> URL: http://svn.apache.org/viewvc?view=rev&rev=468373
> Log:
> mod_cache: Pass the output filter stack through the store_body()
> hook, giving each cache backend the ability to make a better
> decision as to how it will allocate the tasks of writing to the
> cache and writing to the network. Previously the write to the
> cache task needed to be complete before the same brigade was
> written to the network, and this caused timing and memory issues
> on large cached files. This fix replaces the previous fix for
> PR39380.
>
> Modified:
> httpd/httpd/trunk/CHANGES
> httpd/httpd/trunk/modules/cache/mod_cache.c
> httpd/httpd/trunk/modules/cache/mod_cache.h
> httpd/httpd/trunk/modules/cache/mod_disk_cache.c
> httpd/httpd/trunk/modules/cache/mod_disk_cache.h
> httpd/httpd/trunk/modules/cache/mod_mem_cache.c
>
..
> @@ -1286,9 +1292,15 @@
> static apr_status_t open_new_file(request_rec *r, const char *filename,
> apr_file_t **fd, disk_cache_conf *conf)
> {
> - int flags = APR_CREATE | APR_WRITE | APR_BINARY | APR_BUFFERED | APR_EXCL;
> + int flags;
> apr_status_t rv;
>
> + flags = APR_CREATE | APR_WRITE | APR_READ | APR_BINARY | APR_BUFFERED | APR_EXCL | APR_TRUNCATE;
Nitpick: APR_TRUNCATE here is useless.
> +#if APR_HAS_SENDFILE
> + flags |= ((pdconf->enable_sendfile == ENABLE_SENDFILE_OFF)
> + ? 0 : APR_SENDFILE_ENABLED);
> +#endif
> +
Where is pdconf ? Check out all those APR_HAS_SENDFILE.
> while(1) {
> rv = apr_file_open(fd, filename, flags,
> APR_FPROT_UREAD | APR_FPROT_UWRITE, r->pool);
> @@ -1611,150 +1623,50 @@
> return APR_SUCCESS;
> }
>
Looking at open_new_file, it is somewhat unreliable:
if(finfo.mtime < (apr_time_now() - conf->updtimeout) ) {
We use APR_BUFFERED and under various circumstances files may get
modified without having the mtime updated.
..
> +/**
> + * Store the body of the response in the disk cache.
> + *
> + * As the data is written to the cache, it is also written to
> + * the filter provided. On network write failure, the full body
> + * will still be cached.
> + */
> +static apr_status_t store_body(cache_handle_t *h, ap_filter_t *f, apr_bucket_brigade *bb)
> {
> - apr_bucket *e;
> + apr_bucket *e, *b;
> + request_rec *r = f->r;
> apr_status_t rv;
> - int copy_file = FALSE;
> disk_cache_object_t *dobj = (disk_cache_object_t *) h->cache_obj->vobj;
> disk_cache_conf *conf = ap_get_module_config(r->server->module_config,
> &disk_cache_module);
>
> dobj->store_body_called++;
> -
> +
White space trash.
> if(r->no_cache) {
> ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
> "disk_cache: store_body called for URL %s even though"
> "no_cache is set", dobj->name);
> file_cache_errorcleanup(dobj, r);
> - return APR_EGENERAL;
> + ap_remove_output_filter(f);
> + return ap_pass_brigade(f->next, bb);
> }
>
> if(dobj->initial_size == 0) {
> /* Don't waste a body cachefile on a 0 length body */
> - return APR_SUCCESS;
> + return ap_pass_brigade(f->next, bb);
> }
>
> if(!dobj->skipstore && dobj->fd == NULL) {
> rv = open_new_file(r, dobj->datafile, &(dobj->fd), conf);
> - if(rv == CACHE_EEXIST) {
> + if (rv == CACHE_EEXIST) {
> /* Someone else beat us to storing this */
> dobj->skipstore = TRUE;
> }
> - else if(rv != APR_SUCCESS) {
> - return rv;
> + else if (rv != APR_SUCCESS) {
> + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
> + "disk_cache: store_body tried to open cached file "
> + "for URL %s and this failed", dobj->name);
> + ap_remove_output_filter(f);
> + return ap_pass_brigade(f->next, bb);
> }
> else {
> dobj->file_size = 0;
> @@ -1762,194 +1674,227 @@
> }
>
> if(dobj->skipstore) {
> - /* Someone else beat us to storing this object */
> - if(dobj->store_body_called == 1 &&
> - dobj->initial_size > 0 &&
> - APR_BUCKET_IS_EOS(APR_BRIGADE_LAST(bb)) )
> - {
> - /* Yay, we can replace the body with the cached instance */
> - return replace_brigade_with_cache(h, r, bb);
> - }
> -
> - return APR_SUCCESS;
> + /* Someone else beat us to storing this object.
> + * We are too late to take advantage of this storage :( */
> + ap_remove_output_filter(f);
> + return ap_pass_brigade(f->next, bb);
> }
Those "if(" and "if (" are going on my nerves! :)
..
> + /* start caching the brigade */
> + ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
> + "disk_cache: Caching body for URL %s", dobj->name);
>
> - }
> - else {
> - ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
> - "disk_cache: Caching body for URL %s", dobj->name);
> + e = APR_BRIGADE_FIRST(bb);
> + while (e != APR_BRIGADE_SENTINEL(bb)) {
>
> - for (e = APR_BRIGADE_FIRST(bb);
> - e != APR_BRIGADE_SENTINEL(bb);
> - e = APR_BUCKET_NEXT(e))
> - {
> - const char *str;
> - apr_size_t length, written;
> + const char *str;
> + apr_size_t length, written;
> + apr_off_t offset = 0;
>
> - /* Ignore the non-data-buckets */
> - if(APR_BUCKET_IS_METADATA(e)) {
> - continue;
> - }
> + /* try write all data buckets to the cache, except for metadata buckets */
> + if(!APR_BUCKET_IS_METADATA(e)) {
Swapping the metadata check would make the code much more readable:
if (is_metadata)
deal with it
continue
handle normal bucket
> + /* read in a bucket fragment */
> rv = apr_bucket_read(e, &str, &length, APR_BLOCK_READ);
> if (rv != APR_SUCCESS) {
> ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
> - "disk_cache: Error when reading bucket for URL %s",
> + "disk_cache: Error when reading bucket for URL %s, aborting request",
> dobj->name);
> file_cache_errorcleanup(dobj, r);
> + /* not being able to read the bucket is fatal,
> + * return this up the filter stack
> + */
> return rv;
> }
> +
> + /* try write the bucket fragment to the cache */
> + apr_file_seek(dobj->fd, APR_END, &offset);
> rv = apr_file_write_full(dobj->fd, str, length, &written);
> - if (rv != APR_SUCCESS) {
> + offset = - (apr_off_t)written;
> + apr_file_seek(dobj->fd, APR_END, &offset);
> + /* if the cache write was successful, swap the original bucket
> + * with a file bucket pointing to the same data in the cache.
> + *
> + * This is done because:
> + *
> + * - The ap_core_output_filter can take advantage of its ability
> + * to do non blocking writes on file buckets.
> + *
> + * - We are prevented from the need to read the original bucket
> + * a second time inside ap_core_output_filter, which could be
> + * expensive or memory consuming.
> + *
> + * - The cache, in theory, should be faster than the backend,
> + * otherwise there would be little point in caching in the first
> + * place.
> + */
> + if (APR_SUCCESS == rv) {
> +
> + /* remove and destroy the original bucket from the brigade */
> + b = e;
> + e = APR_BUCKET_NEXT(e);
> + APR_BUCKET_REMOVE(b);
> + apr_bucket_destroy(b);
aka as apr_bucket_delete
> + /* Is our network connection still alive?
> + * If not, we must continue caching the file, so keep looping.
> + * We will return the error at the end when caching is done.
> + */
> + if (APR_SUCCESS == dobj->frv) {
> +
> + /* insert a file bucket pointing to the cache into out temporary brigade */
> + if (diskcache_brigade_insert(dobj->tmpbb, dobj->fd, dobj->file_size,
> + written,
> + dobj->updtimeout, r->pool) == NULL) {
> + return APR_ENOMEM;
> + }
Err. We had the data in memory, we are going to read it back from disk
again just in order to not block ? That's nonsense.
We don't need a special bucket type!
Also, are you really sure you are tracking correctly all those buckets
with the correct offset ? The file pointer offset is unique for a single fd.
I will stop the review here, I have to calm down to really digest what's
happening.
--
Davi Arnaut