You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2011/03/07 23:28:25 UTC
svn commit: r1078990 - in /subversion/trunk: ./
subversion/libsvn_fs_fs/caching.c subversion/libsvn_fs_fs/fs.h
subversion/libsvn_fs_fs/fs_fs.c
Author: stefan2
Date: Mon Mar 7 22:28:25 2011
New Revision: 1078990
URL: http://svn.apache.org/viewvc?rev=1078990&view=rev
Log:
Merge r983760 and r1035882 from the performance branch.
These patches introduce node_revision_t caching.
Without that, the txdelta caching is basically ineffective
because reading the noderevs is very I/O intensive.
Modified:
subversion/trunk/ (props changed)
subversion/trunk/subversion/libsvn_fs_fs/caching.c
subversion/trunk/subversion/libsvn_fs_fs/fs.h
subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c
Propchange: subversion/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 7 22:28:25 2011
@@ -35,7 +35,7 @@
/subversion/branches/log-g-performance:870941-871032
/subversion/branches/merge-skips-obstructions:874525-874615
/subversion/branches/nfc-nfd-aware-client:870276,870376
-/subversion/branches/performance:979193,980118,981087,981090,981189,981194,981287,981684,981827,982043,982355,983398,983406,983430,983474,983488,983490,983764,983766,983770,984927,984973,984984,985014,985037,985046,985472,985477,985482,985487-985488,985493,985497,985500,985514,985601,985603,985606,985669,985695,986453,986465,986485,986491-986492,986517,986521,986605,986608,986817,986832,987865,987868-987869,987872,987886-987888,987893,988319,988898,990330,990533,990535-990537,990541,990568,990572,990574-990575,990600,990759,992899,992911,993127,993141,994956,995478,995507,995603,998012,998858,999098,1001413,1001417,1004291,1022668,1022670,1022676,1022715,1022719,1025660,1025672,1027193,1027203,1027206,1027214,1027227,1028077,1028092,1028094,1028104,1028107,1028111,1028354,1029038,1029042-1029043,1029054-1029055,1029062-1029063,1029078,1029080,1029090,1029092-1029093,1029111,1029151,1029158,1029229,1029232,1029335-1029336,1029339-1029340,1029342,1029344,1030763,1030827,103120
3,1031235,1032285,1032333,1033040,1033057,1033294,1035869,1039511,1043705,1053735,1056015,1066452,1067683
+/subversion/branches/performance:979193,980118,981087,981090,981189,981194,981287,981684,981827,982043,982355,983398,983406,983430,983474,983488,983490,983760,983764,983766,983770,984927,984973,984984,985014,985037,985046,985472,985477,985482,985487-985488,985493,985497,985500,985514,985601,985603,985606,985669,985695,986453,986465,986485,986491-986492,986517,986521,986605,986608,986817,986832,987865,987868-987869,987872,987886-987888,987893,988319,988898,990330,990533,990535-990537,990541,990568,990572,990574-990575,990600,990759,992899,992911,993127,993141,994956,995478,995507,995603,998012,998858,999098,1001413,1001417,1004291,1022668,1022670,1022676,1022715,1022719,1025660,1025672,1027193,1027203,1027206,1027214,1027227,1028077,1028092,1028094,1028104,1028107,1028111,1028354,1029038,1029042-1029043,1029054-1029055,1029062-1029063,1029078,1029080,1029090,1029092-1029093,1029111,1029151,1029158,1029229,1029232,1029335-1029336,1029339-1029340,1029342,1029344,1030763,1030827
,1031203,1031235,1032285,1032333,1033040,1033057,1033294,1035869,1035882,1039511,1043705,1053735,1056015,1066452,1067683
/subversion/branches/py-tests-as-modules:956579-1033052
/subversion/branches/ra_serf-digest-authn:875693-876404
/subversion/branches/reintegrate-improvements:873853-874164
Modified: subversion/trunk/subversion/libsvn_fs_fs/caching.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/caching.c?rev=1078990&r1=1078989&r2=1078990&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/caching.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/caching.c Mon Mar 7 22:28:25 2011
@@ -205,7 +205,7 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
SVN_ERR(svn_cache__set_error_handler(ffd->fulltext_cache,
warn_on_cache_errors, fs, pool));
- /* if enabled, enable the txdelta window cache */
+ /* initialize txdelta window cache, if that has been enabled */
if (svn_fs__get_global_membuffer_cache() &&
svn_fs_get_cache_config()->cache_txdeltas)
{
@@ -227,5 +227,28 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
SVN_ERR(svn_cache__set_error_handler(ffd->txdelta_window_cache,
warn_on_cache_errors, fs, pool));
+ /* initialize node revision cache, if caching has been enabled */
+ if (svn_fs__get_global_membuffer_cache())
+ {
+ SVN_ERR(svn_cache__create_membuffer_cache(&(ffd->node_revision_cache),
+ svn_fs__get_global_membuffer_cache(),
+ svn_fs_fs__serialize_node_revision,
+ svn_fs_fs__deserialize_node_revision,
+ APR_HASH_KEY_STRING,
+ apr_pstrcat(pool,
+ prefix,
+ "NODEREVS",
+ NULL),
+ fs->pool));
+ }
+ else
+ {
+ ffd->node_revision_cache = NULL;
+ }
+
+ if (ffd->node_revision_cache && ! no_handler)
+ SVN_ERR(svn_cache__set_error_handler(ffd->node_revision_cache,
+ warn_on_cache_errors, fs, pool));
+
return SVN_NO_ERROR;
}
Modified: subversion/trunk/subversion/libsvn_fs_fs/fs.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs.h?rev=1078990&r1=1078989&r2=1078990&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/fs.h (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/fs.h Mon Mar 7 22:28:25 2011
@@ -245,6 +245,9 @@ typedef struct fs_fs_data_t
/* Cache for txdelta_window_t objects; the key is (revFilePath, offset) */
svn_cache__t *txdelta_window_cache;
+ /* Cache for node_revision_t objects; the key is (revision, id offset) */
+ svn_cache__t *node_revision_cache;
+
/* Data shared between all svn_fs_t objects for a given filesystem. */
fs_fs_shared_data_t *shared;
Modified: subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c?rev=1078990&r1=1078989&r2=1078990&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/fs_fs.c Mon Mar 7 22:28:25 2011
@@ -2164,6 +2164,64 @@ err_dangling_id(svn_fs_t *fs, const svn_
id_str->data, fs->path);
}
+/* Return a string that uniquely identifies the noderev with the
+ * given ID, for use as a cache key.
+ */
+static const char *
+get_noderev_cache_key(const svn_fs_id_t *id, apr_pool_t *pool)
+{
+ const svn_string_t *id_unparsed = svn_fs_fs__id_unparse(id, pool);
+ return id_unparsed->data;
+}
+
+/* Look up the NODEREV_P for ID in FS' node revsion cache. If noderev
+ * caching has been enabled and the data can be found, IS_CACHED will
+ * be set to TRUE. The noderev will be allocated from POOL.
+ *
+ * Non-permanent ids (e.g. ids within a TXN) will not be cached.
+ */
+static svn_error_t *
+get_cached_node_revision_body(node_revision_t **noderev_p,
+ svn_fs_t *fs,
+ const svn_fs_id_t *id,
+ svn_boolean_t *is_cached,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ if (! ffd->node_revision_cache || svn_fs_fs__id_txn_id(id))
+ *is_cached = FALSE;
+ else
+ SVN_ERR(svn_cache__get((void **) noderev_p,
+ is_cached,
+ ffd->node_revision_cache,
+ get_noderev_cache_key(id, pool),
+ pool));
+
+ return SVN_NO_ERROR;
+}
+
+/* If noderev caching has been enabled, store the NODEREV_P for the given ID
+ * in FS' node revsion cache. SCRATCH_POOL is used for temporary allcations.
+ *
+ * Non-permanent ids (e.g. ids within a TXN) will not be cached.
+ */
+static svn_error_t *
+set_cached_node_revision_body(node_revision_t *noderev_p,
+ svn_fs_t *fs,
+ const svn_fs_id_t *id,
+ apr_pool_t *scratch_pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ if (ffd->node_revision_cache && !svn_fs_fs__id_txn_id(id))
+ return svn_cache__set(ffd->node_revision_cache,
+ get_noderev_cache_key(id, scratch_pool),
+ noderev_p,
+ scratch_pool);
+
+ return SVN_NO_ERROR;
+}
+
/* Get the node-revision for the node ID in FS.
Set *NODEREV_P to the new node-revision structure, allocated in POOL.
See svn_fs_fs__get_node_revision, which wraps this and adds another
@@ -2176,6 +2234,12 @@ get_node_revision_body(node_revision_t *
{
apr_file_t *revision_file;
svn_error_t *err;
+ svn_boolean_t is_cached = FALSE;
+
+ /* First, try a cache lookup. If that succeeds, we are done here. */
+ SVN_ERR(get_cached_node_revision_body(noderev_p, fs, id, &is_cached, pool));
+ if (is_cached)
+ return SVN_NO_ERROR;
if (svn_fs_fs__id_txn_id(id))
{
@@ -2203,10 +2267,13 @@ get_node_revision_body(node_revision_t *
return svn_error_return(err);
}
- return svn_fs_fs__read_noderev(noderev_p,
- svn_stream_from_aprfile2(revision_file, FALSE,
- pool),
- pool);
+ SVN_ERR(svn_fs_fs__read_noderev(noderev_p,
+ svn_stream_from_aprfile2(revision_file, FALSE,
+ pool),
+ pool));
+
+ /* The noderev is not in cache, yet. Add it, if caching has been enabled. */
+ return set_cached_node_revision_body(*noderev_p, fs, id, pool);
}
svn_error_t *
Re: svn commit: r1078990 - in /subversion/trunk: ./ subversion/libsvn_fs_fs/caching.c
subversion/libsvn_fs_fs/fs.h subversion/libsvn_fs_fs/fs_fs.c
Posted by Stefan Fuhrmann <eq...@web.de>.
On 07.03.2011 23:40, Daniel Shahaf wrote:
> stefan2@apache.org wrote on Mon, Mar 07, 2011 at 22:28:25 -0000:
>> Modified: subversion/trunk/subversion/libsvn_fs_fs/caching.c
>> URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/caching.c?rev=1078990&r1=1078989&r2=1078990&view=diff
>> ==============================================================================
>> --- subversion/trunk/subversion/libsvn_fs_fs/caching.c (original)
>> +++ subversion/trunk/subversion/libsvn_fs_fs/caching.c Mon Mar 7 22:28:25 2011
>> @@ -227,5 +227,28 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
>> SVN_ERR(svn_cache__set_error_handler(ffd->txdelta_window_cache,
>> warn_on_cache_errors, fs, pool));
>>
>> + /* initialize node revision cache, if caching has been enabled */
>> + if (svn_fs__get_global_membuffer_cache())
>> + {
>> + SVN_ERR(svn_cache__create_membuffer_cache(&(ffd->node_revision_cache),
>> + svn_fs__get_global_membuffer_cache(),
>> + svn_fs_fs__serialize_node_revision,
>> + svn_fs_fs__deserialize_node_revision,
>> + APR_HASH_KEY_STRING,
>> + apr_pstrcat(pool,
>> + prefix,
>> + "NODEREVS",
>> + NULL),
>> + fs->pool));
>> + }
> /me wonders if we should maintain somewhere a list of all the "cache key" namespaces...
>
> (i.e., should someone else use the "fsfs:$uuid:NODEREVS" prefix, the end
> results won't be nice :-[ )
It wouldn't, indeed. However, all these prefixes are being constructed,
i.e. implicitly listed, in the svn_fs_fs__initialize_caches function.
So, unless somebody else calls svn_cache__create_membuffer_cache
the risk of key space collisions is minimal.
-- Stefan^2.
Re: svn commit: r1078990 - in /subversion/trunk: ./
subversion/libsvn_fs_fs/caching.c subversion/libsvn_fs_fs/fs.h
subversion/libsvn_fs_fs/fs_fs.c
Posted by Daniel Shahaf <d....@daniel.shahaf.name>.
stefan2@apache.org wrote on Mon, Mar 07, 2011 at 22:28:25 -0000:
> Modified: subversion/trunk/subversion/libsvn_fs_fs/caching.c
> URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/caching.c?rev=1078990&r1=1078989&r2=1078990&view=diff
> ==============================================================================
> --- subversion/trunk/subversion/libsvn_fs_fs/caching.c (original)
> +++ subversion/trunk/subversion/libsvn_fs_fs/caching.c Mon Mar 7 22:28:25 2011
> @@ -227,5 +227,28 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
> SVN_ERR(svn_cache__set_error_handler(ffd->txdelta_window_cache,
> warn_on_cache_errors, fs, pool));
>
> + /* initialize node revision cache, if caching has been enabled */
> + if (svn_fs__get_global_membuffer_cache())
> + {
> + SVN_ERR(svn_cache__create_membuffer_cache(&(ffd->node_revision_cache),
> + svn_fs__get_global_membuffer_cache(),
> + svn_fs_fs__serialize_node_revision,
> + svn_fs_fs__deserialize_node_revision,
> + APR_HASH_KEY_STRING,
> + apr_pstrcat(pool,
> + prefix,
> + "NODEREVS",
> + NULL),
> + fs->pool));
> + }
/me wonders if we should maintain somewhere a list of all the "cache key" namespaces...
(i.e., should someone else use the "fsfs:$uuid:NODEREVS" prefix, the end
results won't be nice :-[ )
Re: svn commit: r1078990 - in /subversion/trunk: ./
subversion/libsvn_fs_fs/caching.c subversion/libsvn_fs_fs/fs.h
subversion/libsvn_fs_fs/fs_fs.c
Posted by Daniel Shahaf <d....@daniel.shahaf.name>.
stefan2@apache.org wrote on Mon, Mar 07, 2011 at 22:28:25 -0000:
> Modified: subversion/trunk/subversion/libsvn_fs_fs/caching.c
> URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/caching.c?rev=1078990&r1=1078989&r2=1078990&view=diff
> ==============================================================================
> --- subversion/trunk/subversion/libsvn_fs_fs/caching.c (original)
> +++ subversion/trunk/subversion/libsvn_fs_fs/caching.c Mon Mar 7 22:28:25 2011
> @@ -227,5 +227,28 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
> SVN_ERR(svn_cache__set_error_handler(ffd->txdelta_window_cache,
> warn_on_cache_errors, fs, pool));
>
> + /* initialize node revision cache, if caching has been enabled */
> + if (svn_fs__get_global_membuffer_cache())
> + {
> + SVN_ERR(svn_cache__create_membuffer_cache(&(ffd->node_revision_cache),
> + svn_fs__get_global_membuffer_cache(),
> + svn_fs_fs__serialize_node_revision,
> + svn_fs_fs__deserialize_node_revision,
> + APR_HASH_KEY_STRING,
> + apr_pstrcat(pool,
> + prefix,
> + "NODEREVS",
> + NULL),
> + fs->pool));
> + }
/me wonders if we should maintain somewhere a list of all the "cache key" namespaces...
(i.e., should someone else use the "fsfs:$uuid:NODEREVS" prefix, the end
results won't be nice :-[ )