You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2012/08/28 23:30:27 UTC
svn commit: r1378358 - in /subversion/trunk/subversion/libsvn_fs_fs:
caching.c fs.h fs_fs.c temp_serializer.c temp_serializer.h
Author: stefan2
Date: Tue Aug 28 21:30:27 2012
New Revision: 1378358
URL: http://svn.apache.org/viewvc?rev=1378358&view=rev
Log:
As it turned out, nobody had the revisions' change list on his
radar of performance-critical data structures. This patch
introduces an unconditional cache (always on just like noderev
cache and others) for those change lists.
The code is pretty straight-forward as the data structure to
cache is simple and change list-related functions are quite
isolated from the rest.
Now all FSFS data structures should be cacheable and repeated
svn log runs are sped up by one order of magnitude.
* subversion/libsvn_fs_fs/fs.h
(fs_fs_data_t): add a new cache
* subversion/libsvn_fs_fs/caching.c
(svn_fs_fs__initialize_caches): create new cache as well
* subversion/libsvn_fs_fs/fs_fs.c
(fetch_all_changes): split up into ...
(process_changes, read_all_changes): ... these two
(get_changes): new function, adds cache lookup & write
(svn_fs_fs__txn_changes_fetch): adapt
(svn_fs_fs__paths_changed): use the new, cache-aware functions
* subversion/libsvn_fs_fs/temp_serializer.h
(svn_fs_fs__serialize_changes, svn_fs_fs__deserialize_changes):
declare new private API
* subversion/libsvn_fs_fs/temp_serializer.c
(serialize_change, deserialize_change): new utility functions
(changes_data_t): new temporary data structure
(svn_fs_fs__serialize_changes, svn_fs_fs__deserialize_changes):
implement the new private API
Modified:
subversion/trunk/subversion/libsvn_fs_fs/caching.c
subversion/trunk/subversion/libsvn_fs_fs/fs.h
subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c
subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.c
subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.h
Modified: subversion/trunk/subversion/libsvn_fs_fs/caching.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/caching.c?rev=1378358&r1=1378357&r2=1378358&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/caching.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/caching.c Tue Aug 28 21:30:27 2012
@@ -429,6 +429,19 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
SVN_ERR(init_callbacks(ffd->node_revision_cache, fs, no_handler, pool));
+ /* initialize node change list cache, if caching has been enabled */
+ SVN_ERR(create_cache(&(ffd->changes_cache),
+ NULL,
+ membuffer,
+ 0, 0, /* Do not use inprocess cache */
+ svn_fs_fs__serialize_changes,
+ svn_fs_fs__deserialize_changes,
+ sizeof(svn_revnum_t),
+ apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL),
+ fs->pool));
+
+ SVN_ERR(init_callbacks(ffd->changes_cache, fs, no_handler, pool));
+
return SVN_NO_ERROR;
}
Modified: subversion/trunk/subversion/libsvn_fs_fs/fs.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs.h?rev=1378358&r1=1378357&r2=1378358&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/fs.h (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/fs.h Tue Aug 28 21:30:27 2012
@@ -285,6 +285,10 @@ typedef struct fs_fs_data_t
/* Cache for node_revision_t objects; the key is (revision, id offset) */
svn_cache__t *node_revision_cache;
+ /* Cache for change lists as APR arrays of change_t * objects; the key
+ is the revision */
+ svn_cache__t *changes_cache;
+
/* If set, there are or have been more than one concurrent transaction */
svn_boolean_t concurrent_transactions;
Modified: subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c?rev=1378358&r1=1378357&r2=1378358&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c Tue Aug 28 21:30:27 2012
@@ -5748,7 +5748,7 @@ read_change(change_t **change_p,
return SVN_NO_ERROR;
}
-/* Fetch all the changed path entries from FILE and store then in
+/* Examine all the changed path entries in CHANGES and store them in
*CHANGED_PATHS. Folding is done to remove redundant or unnecessary
*data. Store a hash of paths to copyfrom "REV PATH" strings in
COPYFROM_HASH if it is non-NULL. If PREFOLDED is true, assume that
@@ -5757,22 +5757,22 @@ read_change(change_t **change_p,
remove children of replaced or deleted directories. Do all
allocations in POOL. */
static svn_error_t *
-fetch_all_changes(apr_hash_t *changed_paths,
- apr_hash_t *copyfrom_cache,
- apr_file_t *file,
- svn_boolean_t prefolded,
- apr_pool_t *pool)
+process_changes(apr_hash_t *changed_paths,
+ apr_hash_t *copyfrom_cache,
+ apr_array_header_t *changes,
+ svn_boolean_t prefolded,
+ apr_pool_t *pool)
{
- change_t *change;
apr_pool_t *iterpool = svn_pool_create(pool);
+ int i;
/* Read in the changes one by one, folding them into our local hash
as necessary. */
- SVN_ERR(read_change(&change, file, iterpool));
-
- while (change)
+ for (i = 0; i < changes->nelts; ++i)
{
+ change_t *change = APR_ARRAY_IDX(changes, i, change_t *);
+
SVN_ERR(fold_change(changed_paths, change, copyfrom_cache));
/* Now, if our change was a deletion or replacement, we have to
@@ -5826,8 +5826,6 @@ fetch_all_changes(apr_hash_t *changed_pa
/* Clear the per-iteration subpool. */
svn_pool_clear(iterpool);
-
- SVN_ERR(read_change(&change, file, iterpool));
}
/* Destroy the per-iteration subpool. */
@@ -5836,6 +5834,29 @@ fetch_all_changes(apr_hash_t *changed_pa
return SVN_NO_ERROR;
}
+/* Fetch all the changes from FILE and store them in *CHANGES. Do all
+ allocations in POOL. */
+static svn_error_t *
+read_all_changes(apr_array_header_t **changes,
+ apr_file_t *file,
+ apr_pool_t *pool)
+{
+ change_t *change;
+
+ /* pre-allocate enough room for most change lists
+ (will be auto-expanded as necessary) */
+ *changes = apr_array_make(pool, 30, sizeof(change_t *));
+
+ SVN_ERR(read_change(&change, file, pool));
+ while (change)
+ {
+ APR_ARRAY_PUSH(*changes, change_t*) = change;
+ SVN_ERR(read_change(&change, file, pool));
+ }
+
+ return SVN_NO_ERROR;
+}
+
svn_error_t *
svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p,
svn_fs_t *fs,
@@ -5844,11 +5865,15 @@ svn_fs_fs__txn_changes_fetch(apr_hash_t
{
apr_file_t *file;
apr_hash_t *changed_paths = apr_hash_make(pool);
+ apr_array_header_t *changes;
+ apr_pool_t *scratch_pool = svn_pool_create(pool);
SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool),
APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
- SVN_ERR(fetch_all_changes(changed_paths, NULL, file, FALSE, pool));
+ SVN_ERR(read_all_changes(&changes, file, scratch_pool));
+ SVN_ERR(process_changes(changed_paths, NULL, changes, FALSE, pool));
+ svn_pool_destroy(scratch_pool);
SVN_ERR(svn_io_file_close(file, pool));
@@ -5857,17 +5882,29 @@ svn_fs_fs__txn_changes_fetch(apr_hash_t
return SVN_NO_ERROR;
}
-svn_error_t *
-svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p,
- svn_fs_t *fs,
- svn_revnum_t rev,
- apr_hash_t *copyfrom_cache,
- apr_pool_t *pool)
+/* Fetch the list of change in revision REV in FS and return it in *CHANGES.
+ * Allocate the result in POOL.
+ */
+static svn_error_t *
+get_changes(apr_array_header_t **changes,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_pool_t *pool)
{
apr_off_t changes_offset;
- apr_hash_t *changed_paths;
apr_file_t *revision_file;
+ svn_boolean_t found;
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ /* try cache lookup first */
+
+ SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache,
+ &rev, pool));
+ if (found)
+ return SVN_NO_ERROR;
+ /* read changes from revision file */
+
SVN_ERR(ensure_revision_exists(fs, rev, pool));
SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
@@ -5876,14 +5913,36 @@ svn_fs_fs__paths_changed(apr_hash_t **ch
rev, pool));
SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool));
+ SVN_ERR(read_all_changes(changes, revision_file, pool));
+
+ SVN_ERR(svn_io_file_close(revision_file, pool));
- changed_paths = apr_hash_make(pool);
+ /* cache for future reference */
+
+ SVN_ERR(svn_cache__set(ffd->changes_cache, &rev, *changes, pool));
+
+ return SVN_NO_ERROR;
+}
- SVN_ERR(fetch_all_changes(changed_paths, copyfrom_cache, revision_file,
- TRUE, pool));
- /* Close the revision file. */
- SVN_ERR(svn_io_file_close(revision_file, pool));
+svn_error_t *
+svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_hash_t *copyfrom_cache,
+ apr_pool_t *pool)
+{
+ apr_hash_t *changed_paths;
+ apr_array_header_t *changes;
+ apr_pool_t *scratch_pool = svn_pool_create(pool);
+
+ SVN_ERR(get_changes(&changes, fs, rev, scratch_pool));
+
+ changed_paths = apr_hash_make(pool);
+
+ SVN_ERR(process_changes(changed_paths, copyfrom_cache, changes,
+ TRUE, pool));
+ svn_pool_destroy(scratch_pool);
*changed_paths_p = changed_paths;
Modified: subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.c?rev=1378358&r1=1378357&r2=1378358&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.c Tue Aug 28 21:30:27 2012
@@ -1079,3 +1079,135 @@ svn_fs_fs__replace_dir_entry(void **data
return SVN_NO_ERROR;
}
+
+/* Utility function to serialize change CHANGE_P in the given serialization
+ * CONTEXT.
+ */
+static void
+serialize_change(svn_temp_serializer__context_t *context,
+ change_t * const *change_p)
+{
+ const change_t * change = *change_p;
+ if (change == NULL)
+ return;
+
+ /* serialize the change struct itself */
+ svn_temp_serializer__push(context,
+ (const void * const *)change_p,
+ sizeof(*change));
+
+ /* serialize sub-structures */
+ svn_fs_fs__id_serialize(context, &change->noderev_id);
+
+ svn_temp_serializer__add_string(context, &change->path);
+ svn_temp_serializer__add_string(context, &change->copyfrom_path);
+
+ /* return to the caller's nesting level */
+ svn_temp_serializer__pop(context);
+}
+
+/* Utility function to serialize the CHANGE_P within the given
+ * serialization CONTEXT.
+ */
+static void
+deserialize_change(void *buffer, change_t **change_p)
+{
+ change_t * change;
+
+ /* fix-up of the pointer to the struct in question */
+ svn_temp_deserializer__resolve(buffer, (void **)change_p);
+
+ change = *change_p;
+ if (change == NULL)
+ return;
+
+ /* fix-up of sub-structures */
+ svn_fs_fs__id_deserialize(change, (svn_fs_id_t **)&change->noderev_id);
+
+ svn_temp_deserializer__resolve(change, (void **)&change->path);
+ svn_temp_deserializer__resolve(change, (void **)&change->copyfrom_path);
+}
+
+/* Auxiliary structure representing the content of a change_t array.
+ This structure is much easier to (de-)serialize than an APR array.
+ */
+typedef struct changes_data_t
+{
+ /* number of entries in the array */
+ int count;
+
+ /* reference to the changes */
+ change_t **changes;
+} changes_data_t;
+
+svn_error_t *
+svn_fs_fs__serialize_changes(void **data,
+ apr_size_t *data_len,
+ void *in,
+ apr_pool_t *pool)
+{
+ apr_array_header_t *array = in;
+ changes_data_t changes;
+ svn_temp_serializer__context_t *context;
+ svn_stringbuf_t *serialized;
+ int i;
+
+ /* initialize our auxiliary data structure */
+ changes.count = array->nelts;
+ changes.changes = apr_palloc(pool, sizeof(change_t*) * changes.count);
+
+ /* populate it with the array elements */
+ for (i = 0; i < changes.count; ++i)
+ changes.changes[i] = APR_ARRAY_IDX(array, i, change_t*);
+
+ /* serialize it and all its elements */
+ context = svn_temp_serializer__init(&changes,
+ sizeof(changes),
+ changes.count * 100,
+ pool);
+
+ svn_temp_serializer__push(context,
+ (const void * const *)&changes.changes,
+ changes.count * sizeof(change_t*));
+
+ for (i = 0; i < changes.count; ++i)
+ serialize_change(context, &changes.changes[i]);
+
+ svn_temp_serializer__pop(context);
+
+ /* return the serialized result */
+ serialized = svn_temp_serializer__get(context);
+
+ *data = serialized->data;
+ *data_len = serialized->len;
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_fs__deserialize_changes(void **out,
+ void *data,
+ apr_size_t data_len,
+ apr_pool_t *pool)
+{
+ int i;
+ changes_data_t *changes = (changes_data_t *)data;
+ apr_array_header_t *array = apr_array_make(pool, changes->count,
+ sizeof(change_t *));
+
+ /* de-serialize our auxiliary data structure */
+ svn_temp_deserializer__resolve(changes, (void**)&changes->changes);
+
+ /* de-serialize each entry and add it to the array */
+ for (i = 0; i < changes->count; ++i)
+ {
+ deserialize_change((void*)changes->changes,
+ (change_t **)&changes->changes[i]);
+ APR_ARRAY_PUSH(array, change_t *) = changes->changes[i];
+ }
+
+ /* done */
+ *out = array;
+
+ return SVN_NO_ERROR;
+}
Modified: subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.h?rev=1378358&r1=1378357&r2=1378358&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.h (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/temp_serializer.h Tue Aug 28 21:30:27 2012
@@ -235,4 +235,24 @@ svn_fs_fs__replace_dir_entry(void **data
void *baton,
apr_pool_t *pool);
+/**
+ * Implements #svn_cache__serialize_func_t for an #apr_array_header_t of
+ * #change_t *.
+ */
+svn_error_t *
+svn_fs_fs__serialize_changes(void **data,
+ apr_size_t *data_len,
+ void *in,
+ apr_pool_t *pool);
+
+/**
+ * Implements #svn_cache__deserialize_func_t for an #apr_array_header_t of
+ * #change_t *.
+ */
+svn_error_t *
+svn_fs_fs__deserialize_changes(void **out,
+ void *data,
+ apr_size_t data_len,
+ apr_pool_t *pool);
+
#endif