You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2010/08/12 23:05:00 UTC
svn commit: r984973 - in /subversion/branches/performance/subversion:
include/private/svn_cache.h libsvn_fs_fs/fs_fs.c
libsvn_subr/cache-inprocess.c libsvn_subr/cache-membuffer.c
libsvn_subr/cache-memcache.c libsvn_subr/cache.c libsvn_subr/cache.h
Author: stefan2
Date: Thu Aug 12 21:04:59 2010
New Revision: 984973
URL: http://svn.apache.org/viewvc?rev=984973&view=rev
Log:
Cache (almost) all representations as full text. To handle contents longer
than one MB, let the caches decide upon their limits. Also, reps with
rb->len == 0 may have actual content but we can only cache them once
the stream gets closed because the content end won't be detected before
that.
* subversion/include/private/svn_cache.h
(svn_cache__is_cachable): declare new API function
* subversion/libsvn_subr/cache.h
(svn_cache__vtable_t): add method pointer for is_cachable
* subversion/libsvn_subr/cache.c
(svn_cache__is_cachable): implement by forwarding to the actual cache object
* subversion/libsvn_subr/cache-memcache.c
(memcache_is_cachable): implement is_cachable method
(memcache_vtable): adapt vtable
* subversion/libsvn_subr/cache-membuffer.c
(svn_membuffer_cache_is_cachable): implement is_cachable method
(membuffer_cache_vtable): adapt vtable
* subversion/libsvn_subr/cache-inprocess.c
(inprocess_cache_is_cachable): implement is_cachable method
(inprocess_cache_vtable): adapt vtable
* subversion/libsvn_fs_fs/fs_fs.c
(fulltext_size_is_cachable): reimplement, add fs_fs_data_t parameter for cache access
(cache_rep): extract this utility function from rep_read_contents
(rep_read_contents_close): make sure the fulltext gets cached at latest when the stream gets closed
(rep_read_contents): call cache_rep
(read_representation): adapt to called function signature change
Modified:
subversion/branches/performance/subversion/include/private/svn_cache.h
subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c
subversion/branches/performance/subversion/libsvn_subr/cache-inprocess.c
subversion/branches/performance/subversion/libsvn_subr/cache-membuffer.c
subversion/branches/performance/subversion/libsvn_subr/cache-memcache.c
subversion/branches/performance/subversion/libsvn_subr/cache.c
subversion/branches/performance/subversion/libsvn_subr/cache.h
Modified: subversion/branches/performance/subversion/include/private/svn_cache.h
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/include/private/svn_cache.h?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/include/private/svn_cache.h (original)
+++ subversion/branches/performance/subversion/include/private/svn_cache.h Thu Aug 12 21:04:59 2010
@@ -245,6 +245,16 @@ svn_cache__set_error_handler(svn_cache__
void *baton,
apr_pool_t *pool);
+/**
+ * Returns @c TRUE if the @a cache supports objects of the given @a size.
+ * There is no guarantee, that @ref svn_cache__set will actually store the
+ * respective object in that case. However, a @c FALSE return value indicates
+ * that an attempt to cache the item will either fail or impair the overall
+ * cache performance. @c FALSE will also be returned if @a cache is @c NULL.
+ */
+svn_boolean_t
+svn_cache__is_cachable(svn_cache__t *cache,
+ apr_size_t size);
#define SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS "memcached-servers"
Modified: subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c Thu Aug 12 21:04:59 2010
@@ -3398,11 +3398,46 @@ get_combined_window(svn_txdelta_window_t
return SVN_NO_ERROR;
}
+/* Returns whether or not the expanded fulltext of the file is cachable
+ * based on its size SIZE. The decision depends on the cache used by RB.
+ */
+static svn_boolean_t
+fulltext_size_is_cachable(fs_fs_data_t *ffd, svn_filesize_t size)
+{
+ return svn_cache__is_cachable(ffd->fulltext_cache, size);
+}
+
+/* Store fulltext in RB in the fulltext cache used by said RB. Items that
+ * are too large to be cached won't. Also, this will be a no-op if no
+ * fulltext cache has been enabled in RB.
+ */
+static svn_error_t *
+cache_rep(struct rep_read_baton *rb)
+{
+ fs_fs_data_t *ffd = rb->fs->fsap_data;
+ if (rb->current_fulltext &&
+ fulltext_size_is_cachable(ffd, rb->current_fulltext->len))
+ {
+ SVN_ERR(svn_cache__set(ffd->fulltext_cache, rb->fulltext_cache_key,
+ rb->current_fulltext, rb->pool));
+ rb->current_fulltext = NULL;
+ }
+
+ return SVN_NO_ERROR;
+}
+
static svn_error_t *
rep_read_contents_close(void *baton)
{
struct rep_read_baton *rb = baton;
+ if (rb->len == 0)
+ {
+ /* This has not yet been attempted to be added to the cache.
+ * Now, the data should be in. */
+ cache_rep(rb);
+ }
+
svn_pool_destroy(rb->pool);
svn_pool_destroy(rb->filehandle_pool);
@@ -3596,30 +3631,15 @@ rep_read_contents(void *baton,
}
}
- if (rb->off == rb->len && rb->current_fulltext)
- {
- fs_fs_data_t *ffd = rb->fs->fsap_data;
- SVN_ERR(svn_cache__set(ffd->fulltext_cache, rb->fulltext_cache_key,
- rb->current_fulltext, rb->pool));
- rb->current_fulltext = NULL;
- }
+ /* Ff we read the whole content, cache it.
+ * Otherwise, the closing the read stream will take care of that. */
+ if (rb->len && rb->len == rb->off)
+ cache_rep(rb);
return SVN_NO_ERROR;
}
-/* Returns whether or not the expanded fulltext of the file is
- * cachable based on its size SIZE. Specifically, if it will fit
- * into a memcached value. The memcached cutoff seems to be a bit
- * (header length?) under a megabyte; we round down a little to be
- * safe.
- */
-static svn_boolean_t
-fulltext_size_is_cachable(svn_filesize_t size)
-{
- return size < 1000000;
-}
-
/* Return a stream in *CONTENTS_P that will read the contents of a
representation stored at the location given by REP. Appropriate
for any kind of immutable representation, but only for file
@@ -3646,7 +3666,7 @@ read_representation(svn_stream_t **conte
struct rep_read_baton *rb;
if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision)
- && fulltext_size_is_cachable(rep->expanded_size))
+ && fulltext_size_is_cachable(ffd, rep->expanded_size))
{
svn_string_t *fulltext;
svn_boolean_t is_cached;
Modified: subversion/branches/performance/subversion/libsvn_subr/cache-inprocess.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_subr/cache-inprocess.c?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_subr/cache-inprocess.c (original)
+++ subversion/branches/performance/subversion/libsvn_subr/cache-inprocess.c Thu Aug 12 21:04:59 2010
@@ -428,10 +428,23 @@ inprocess_cache_iter(svn_boolean_t *comp
}
+static svn_boolean_t
+inprocess_cache_is_cachable(void *cache_void, apr_size_t size)
+{
+ /* Be relatively strict: per page we should not allocate more than
+ * we could spare as "unused" memory.
+ * But in most cases, nobody will ask anyway. And in no case, this
+ * will be used for checks _inside_ the cache code.
+ */
+ inprocess_cache_t *cache = cache_void;
+ return size < SVN_ALLOCATOR_RECOMMENDED_MAX_FREE / cache->items_per_page;
+}
+
static svn_cache__vtable_t inprocess_cache_vtable = {
inprocess_cache_get,
inprocess_cache_set,
- inprocess_cache_iter
+ inprocess_cache_iter,
+ inprocess_cache_is_cachable
};
svn_error_t *
Modified: subversion/branches/performance/subversion/libsvn_subr/cache-membuffer.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_subr/cache-membuffer.c?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_subr/cache-membuffer.c (original)
+++ subversion/branches/performance/subversion/libsvn_subr/cache-membuffer.c Thu Aug 12 21:04:59 2010
@@ -1051,12 +1051,23 @@ svn_membuffer_cache_iter(svn_boolean_t *
_("Can't iterate a membuffer-based cache"));
}
+static svn_boolean_t
+svn_membuffer_cache_is_cachable(void *cache_void, apr_size_t size)
+{
+ /* Don't allow extremely large element sizes. Otherwise, the cache
+ * might by thrashed by a few extremely large entries.
+ */
+ svn_membuffer_cache_t *cache = cache_void;
+ return size < cache->membuffer->data_size / 16;
+}
+
/* the v-table for membuffer-based caches
*/
static svn_cache__vtable_t membuffer_cache_vtable = {
svn_membuffer_cache_get,
svn_membuffer_cache_set,
- svn_membuffer_cache_iter
+ svn_membuffer_cache_iter,
+ svn_membuffer_cache_is_cachable
};
/* standard serialization function for svn_stringbuf_t items
Modified: subversion/branches/performance/subversion/libsvn_subr/cache-memcache.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_subr/cache-memcache.c?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_subr/cache-memcache.c (original)
+++ subversion/branches/performance/subversion/libsvn_subr/cache-memcache.c Thu Aug 12 21:04:59 2010
@@ -222,10 +222,20 @@ memcache_iter(svn_boolean_t *completed,
_("Can't iterate a memcached cache"));
}
+static svn_boolean_t
+memcache_is_cachable(void *, apr_size_t size)
+{
+ /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
+ * We round down a little to be safe.
+ */
+ return size < 1000000;
+}
+
static svn_cache__vtable_t memcache_vtable = {
memcache_get,
memcache_set,
- memcache_iter
+ memcache_iter,
+ memcache_is_cachable
};
svn_error_t *
Modified: subversion/branches/performance/subversion/libsvn_subr/cache.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_subr/cache.c?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_subr/cache.c (original)
+++ subversion/branches/performance/subversion/libsvn_subr/cache.c Thu Aug 12 21:04:59 2010
@@ -34,6 +34,16 @@ svn_cache__set_error_handler(svn_cache__
return SVN_NO_ERROR;
}
+svn_boolean_t
+svn_cache__is_cachable(svn_cache__t *cache,
+ apr_size_t size)
+{
+ /* having no cache means we can't cache anything */
+ if (cache == NULL)
+ return FALSE;
+
+ return cache->vtable->is_cachable(cache->cache_internal, size);
+}
/* Give the error handler callback a chance to replace or ignore the
error. */
Modified: subversion/branches/performance/subversion/libsvn_subr/cache.h
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_subr/cache.h?rev=984973&r1=984972&r2=984973&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_subr/cache.h (original)
+++ subversion/branches/performance/subversion/libsvn_subr/cache.h Thu Aug 12 21:04:59 2010
@@ -47,6 +47,9 @@ typedef struct {
svn_iter_apr_hash_cb_t func,
void *baton,
apr_pool_t *pool);
+
+ svn_boolean_t (*is_cachable)(void *cache_implementation,
+ apr_size_t size);
} svn_cache__vtable_t;
struct svn_cache__t {