You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2010/08/08 17:34:48 UTC
svn commit: r983430 [1/3] - in
/subversion/branches/performance/subversion/libsvn_fs_fs: caching.c fs.h
fs_fs.c temp_serializer.c temp_serializer.h
Author: stefan2
Date: Sun Aug 8 15:34:48 2010
New Revision: 983430
URL: http://svn.apache.org/viewvc?rev=983430&view=rev
Log:
Implement txdelta window caching. This will be used by svnadmin only
since fulltext caches are mainly ineffective during dump etc. The opposite
is true for server functions (get / set fulltext data).
The cache processing for txdelta windows is relatively straightforward
but requires some auxiliary information. Thus, we wrap them in the
svn_fs_fs__txdelta_cached_window_t structure.
* subversion/libsvn_fs_fs/temp_serializer.h
(svn_fs_fs__combine_number_and_string, svn_fs_fs__combine_two_numbers):
declare key construction utility functions
(svn_fs_fs__txdelta_cached_window_t): auxilliary struct used to represent cached txdelta windows
(svn_fs_fs__serialize_txdelta_window, svn_fs_fs__deserialize_txdelta_window):
declare API to (de)serialize txdelta windows for caching
* subversion/libsvn_fs_fs/temp_serializer.c
* subversion/libsvn_fs_fs/fs_fs.c
(rep_state): add a reference to the txdelta window cache to use
(create_rep_state_body): init that reference
(get_window_key, get_cached_window, set_cached_window):
utility functions to get / set txdelta window objects from / to the cache
(read_window): speed up using the window cache
* subversion/libsvn_fs_fs/fs.h
(fs_fs_data_t): add txdelta_window_cache window
* subversion/libsvn_fs_fs/caching.c
(svn_fs_fs__initialize_caches): add initialization of the txdelta window cache
Added:
subversion/branches/performance/subversion/libsvn_fs_fs/temp_serializer.c (contents, props changed)
- copied, changed from r982417, subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c
subversion/branches/performance/subversion/libsvn_fs_fs/temp_serializer.h (contents, props changed)
- copied, changed from r982376, subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.h
Modified:
subversion/branches/performance/subversion/libsvn_fs_fs/caching.c
subversion/branches/performance/subversion/libsvn_fs_fs/fs.h
subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c
Modified: subversion/branches/performance/subversion/libsvn_fs_fs/caching.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_fs_fs/caching.c?rev=983430&r1=983429&r2=983430&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_fs_fs/caching.c (original)
+++ subversion/branches/performance/subversion/libsvn_fs_fs/caching.c Sun Aug 8 15:34:48 2010
@@ -195,7 +195,7 @@ warn_on_cache_errors(svn_error_t *err,
static svn_fs_fs__cache_config_t cache_settings =
{
/* default configuration:
- */
+ */
0x8000000, /* 128 MB for caches */
16, /* up to 16 files kept open */
FALSE, /* don't cache fulltexts */
@@ -416,5 +416,27 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
/* initialize file handle cache as configured */
ffd->file_handle_cache = get_global_file_handle_cache();
+ /* if enabled, enable the txdelta window cache */
+ if (get_global_membuffer_cache() &&
+ svn_fs_fs__get_cache_config()->cache_txdeltas)
+ {
+ SVN_ERR(svn_cache__create_membuffer_cache
+ (&(ffd->txdelta_window_cache),
+ get_global_membuffer_cache(),
+ svn_fs_fs__serialize_txdelta_window,
+ svn_fs_fs__deserialize_txdelta_window,
+ APR_HASH_KEY_STRING,
+ apr_pstrcat(pool, prefix, "TXDELTA_WINDOW", NULL),
+ fs->pool));
+ }
+ else
+ {
+ ffd->txdelta_window_cache = NULL;
+ }
+
+ if (ffd->txdelta_window_cache && ! no_handler)
+ SVN_ERR(svn_cache__set_error_handler(ffd->txdelta_window_cache,
+ warn_on_cache_errors, fs, pool));
+
return SVN_NO_ERROR;
}
Modified: subversion/branches/performance/subversion/libsvn_fs_fs/fs.h
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_fs_fs/fs.h?rev=983430&r1=983429&r2=983430&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_fs_fs/fs.h (original)
+++ subversion/branches/performance/subversion/libsvn_fs_fs/fs.h Sun Aug 8 15:34:48 2010
@@ -244,6 +244,9 @@ typedef struct
/* Reference to the process-global open file handle cache */
svn_file_handle_cache_t *file_handle_cache;
+ /* Cache for txdelta_window_t objects; the key is (revFilePath, offset) */
+ svn_cache__t *txdelta_window_cache;
+
/* Data shared between all svn_fs_t objects for a given filesystem. */
fs_fs_shared_data_t *shared;
Modified: subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c?rev=983430&r1=983429&r2=983430&view=diff
==============================================================================
--- subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/performance/subversion/libsvn_fs_fs/fs_fs.c Sun Aug 8 15:34:48 2010
@@ -47,6 +47,7 @@
#include "svn_time.h"
#include "svn_mergeinfo.h"
#include "svn_config.h"
+#include "svn_ctype.h"
#include "fs.h"
#include "err.h"
@@ -56,6 +57,7 @@
#include "fs_fs.h"
#include "id.h"
#include "rep-cache.h"
+#include "temp_serializer.h"
#include "revprops-db.h"
@@ -2913,6 +2915,8 @@ struct rep_state
/* For convenience, store the APR file handle
along with the surrounding cached file handle. */
apr_file_t *apr_file;
+ /* The txdelta window cache to use or NULL. */
+ svn_cache__t *window_cache;
apr_off_t start; /* The starting offset for the raw
svndiff/plaintext data minus header. */
apr_off_t off; /* The current offset into the file. */
@@ -2929,12 +2933,14 @@ create_rep_state_body(struct rep_state *
svn_fs_t *fs,
apr_pool_t *pool)
{
+ fs_fs_data_t *ffd = fs->fsap_data;
struct rep_state *rs = apr_pcalloc(pool, sizeof(*rs));
struct rep_args *ra;
unsigned char buf[4];
SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool));
rs->apr_file = svn_file_handle_cache__get_apr_handle(rs->file);
+ rs->window_cache = ffd->txdelta_window_cache;
SVN_ERR(read_rep_line(&ra, rs->apr_file, pool));
SVN_ERR(get_file_offset(&rs->start, rs->apr_file, pool));
@@ -3126,6 +3132,113 @@ rep_read_get_baton(struct rep_read_baton
return SVN_NO_ERROR;
}
+/* Combine the name of the rev file in RS with the given OFFSET to form
+ * a cache lookup key. Allocations will be made from POOL. */
+static const char*
+get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool)
+{
+ const char *name;
+ const char *last_part;
+ const char *name_last;
+
+ /* the rev file name containing the txdelta window.
+ * If this fails we are in serious trouble anyways.
+ * And if nobody else detects the problems, the file content checksum
+ * comparison _will_ find them.
+ */
+ if (apr_file_name_get(&name, rs->apr_file))
+ return "";
+
+ /* We care about the file name only as it represents the start revision
+ * of this rev file. Handle packed files as well. */
+ name_last = name + strlen(name) - 1;
+ while (! svn_ctype_isdigit(*name_last))
+ --name_last;
+
+ last_part = name_last;
+ while (svn_ctype_isdigit(*last_part))
+ --last_part;
+
+ /* copy one char MORE than the actual number to mark packed files,
+ * i.e. packed revision file content uses different key space then
+ * non-packed ones: keys for packed rev file content ends with a dot
+ * for non-packed rev files they end with a digit. */
+ name = apr_pstrndup(pool, last_part + 1, name_last - last_part);
+ return svn_fs_fs__combine_number_and_string(offset, name, pool);
+}
+
+/* Read the WINDOW_P for the rep state RS from the current FSFS session's
+ * cache. This will be a no-op and IS_CACHED will be set to FALSE if no
+ * cache has been given. If a cache is available IS_CACHED will inform
+ * the caller about the success of the lookup. Allocations (of the window
+ * in particualar) will be made from POOL.
+ *
+ * If the information could be found, put RS and the position within the
+ * rev file into the same state as if the data had just been read from it.
+ */
+static svn_error_t *
+get_cached_window(svn_txdelta_window_t **window_p,
+ struct rep_state *rs,
+ svn_boolean_t *is_cached,
+ apr_pool_t *pool)
+{
+ if (! rs->window_cache)
+ {
+ /* txdelta window has not been enabled */
+ *is_cached = FALSE;
+ }
+ else
+ {
+ /* ask the cache for the desired txdelta window */
+ svn_fs_fs__txdelta_cached_window_t *cached_window;
+ SVN_ERR(svn_cache__get((void **) &cached_window,
+ is_cached,
+ rs->window_cache,
+ get_window_key(rs, rs->off, pool),
+ pool));
+
+ if (*is_cached)
+ {
+ /* found it. Pass it back to the caller. */
+ *window_p = cached_window->window;
+
+ /* manipulate the RS as if we just read the data */
+ rs->chunk_index++;
+ rs->off = cached_window->end_offset;
+
+ /* manipulate the rev file as if we just read from it */
+ SVN_ERR(svn_io_file_seek(rs->apr_file, APR_SET, &rs->off, pool));
+ }
+ }
+
+ return SVN_NO_ERROR;
+}
+
+/* Store the WINDOW read at OFFSET for the rep state RS in the current
+ * FSFS session's cache. This will be a no-op if no cache has been given.
+ * Temporary allocations will be made from SCRATCH_POOL. */
+static svn_error_t *
+set_cached_window(svn_txdelta_window_t *window,
+ struct rep_state *rs,
+ apr_off_t offset,
+ apr_pool_t *scratch_pool)
+{
+ if (rs->window_cache)
+ {
+ /* store the window and the first offset _past_ it */
+ svn_fs_fs__txdelta_cached_window_t cached_window = { window, rs->off };
+
+ /* but key it with the start offset because that is the known state
+ * when we will look it up */
+ return svn_cache__set(rs->window_cache,
+ get_window_key(rs, offset, scratch_pool),
+ &cached_window,
+ scratch_pool);
+ }
+
+ return SVN_NO_ERROR;
+}
+
/* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta
window into *NWIN. */
static svn_error_t *
@@ -3133,6 +3246,8 @@ read_window(svn_txdelta_window_t **nwin,
apr_pool_t *pool)
{
svn_stream_t *stream;
+ svn_boolean_t is_cached;
+ apr_off_t old_offset;
SVN_ERR_ASSERT(rs->chunk_index <= this_chunk);
@@ -3149,7 +3264,13 @@ read_window(svn_txdelta_window_t **nwin,
"representation"));
}
- /* Read the next window. */
+ /* Read the next window. But first, try to find it in the cache. */
+ SVN_ERR(get_cached_window(nwin, rs, &is_cached, pool));
+ if (is_cached)
+ return SVN_NO_ERROR;
+
+ /* Actually read the next window. */
+ old_offset = rs->off;
stream = svn_stream__from_cached_file_handle(rs->file, TRUE, pool);
SVN_ERR(svn_txdelta_read_svndiff_window(nwin, stream, rs->ver, pool));
rs->chunk_index++;
@@ -3160,7 +3281,9 @@ read_window(svn_txdelta_window_t **nwin,
_("Reading one svndiff window read beyond "
"the end of the representation"));
- return SVN_NO_ERROR;
+ /* the window has not been cached before, thus cache it now
+ * (if caching is used for them at all) */
+ return set_cached_window(*nwin, rs, old_offset, pool);
}
/* Get one delta window that is a result of combining all but the last deltas