You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2014/04/11 00:07:53 UTC
svn commit: r1586481 - in /subversion/branches/thunder/subversion:
libsvn_fs_util/thunder.c tests/libsvn_fs/fs-test.c
Author: stefan2
Date: Thu Apr 10 22:07:53 2014
New Revision: 1586481
URL: http://svn.apache.org/r1586481
Log:
On the thunder branch:
Implement the new svn_fs__thunder_t API and provide a test for it.
* subversion/libsvn_fs_util/thunder.c
(): New file.
* subversion/tests/libsvn_fs/fs-test.c
(basic_thunder_interface): New test.
(test_func): Register the new test.
Added:
subversion/branches/thunder/subversion/libsvn_fs_util/thunder.c
Modified:
subversion/branches/thunder/subversion/tests/libsvn_fs/fs-test.c
Added: subversion/branches/thunder/subversion/libsvn_fs_util/thunder.c
URL: http://svn.apache.org/viewvc/subversion/branches/thunder/subversion/libsvn_fs_util/thunder.c?rev=1586481&view=auto
==============================================================================
--- subversion/branches/thunder/subversion/libsvn_fs_util/thunder.c (added)
+++ subversion/branches/thunder/subversion/libsvn_fs_util/thunder.c Thu Apr 10 22:07:53 2014
@@ -0,0 +1,481 @@
+/* thunder.c : logic to mitigate the "thundering herd" effects.
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+
+#include <string.h>
+
+#include <apr_pools.h>
+#include <apr_strings.h>
+#include <apr_thread_cond.h>
+#include <apr_portable.h>
+#include <sys/stat.h>
+
+#include "svn_private_config.h"
+#include "svn_hash.h"
+#include "svn_fs.h"
+#include "svn_dirent_uri.h"
+#include "svn_path.h"
+#include "svn_pools.h"
+#include "svn_version.h"
+
+#include "private/svn_fs_util.h"
+#include "private/svn_fspath.h"
+#include "private/svn_subr_private.h"
+#include "private/svn_string_private.h"
+#include "../libsvn_fs/fs-loader.h"
+
+#if APR_HAS_THREADS
+
+typedef struct access_t
+{
+ svn_stringbuf_t *key;
+
+ apr_time_t started;
+ apr_thread_cond_t *condition;
+ svn_mutex__t *mutex;
+
+ apr_os_thread_t owning_thread;
+} access_t;
+
+struct svn_fs__thunder_t
+{
+ svn_mutex__t *mutex;
+ apr_pool_t *pool;
+ apr_pool_t *owning_pool;
+
+ apr_time_t timeout;
+
+ apr_hash_t *in_access;
+ apr_array_header_t *recyler;
+};
+
+struct svn_fs__thunder_access_t
+{
+ svn_fs__thunder_t *thunder;
+ access_t *access;
+ svn_stringbuf_t *key;
+};
+
+/* Forward declaration. */
+static apr_status_t
+thunder_root_cleanup(void *baton);
+
+/* Pool cleanup function to be called when the registry's owning pool gets
+ * cleared up (first). The registry is being provided as BATON.
+
+ * Exactly one of thunder_cleanup, thunder_root_cleanup and
+ * svn_fs__thunder_destroywill be called. */
+static apr_status_t
+thunder_cleanup(void *baton)
+{
+ svn_fs__thunder_t *thunder = baton;
+
+ /* No double cleanup. */
+ apr_pool_cleanup_kill(thunder->pool, thunder, thunder_root_cleanup);
+
+ /* We don't want our independent root pool to linger until the end of
+ * this process. */
+ svn_pool_destroy(thunder->pool);
+
+ return APR_SUCCESS;
+}
+
+/* Pool cleanup function to be called when the registry's private root pool
+ * gets cleared up (first). The registry is being provided as BATON.
+
+ * Exactly one of thunder_cleanup, thunder_root_cleanup and
+ * svn_fs__thunder_destroywill be called. */
+static apr_status_t
+thunder_root_cleanup(void *baton)
+{
+ svn_fs__thunder_t *thunder = baton;
+
+ /* No double cleanup. */
+ apr_pool_cleanup_kill(thunder->owning_pool, thunder, thunder_cleanup);
+
+ return APR_SUCCESS;
+}
+
+svn_error_t *
+svn_fs__thunder_create(svn_fs__thunder_t **thunder,
+ apr_time_t timeout,
+ apr_pool_t *pool)
+{
+ svn_fs__thunder_t *result = apr_pcalloc(pool, sizeof(*result));
+ result->pool = apr_allocator_owner_get(svn_pool_create_allocator(FALSE));
+ result->owning_pool = pool;
+
+ /* From now on, make sure we clean up everything internal (i.e coming
+ * from our root pool / allocator) gets cleaned up nicely as soon as
+ * the struct gets cleaned up. */
+ apr_pool_cleanup_register(result->owning_pool, result, thunder_cleanup,
+ apr_pool_cleanup_null);
+ apr_pool_cleanup_register(result->pool, result, thunder_root_cleanup,
+ apr_pool_cleanup_null);
+
+ /* Simply initialize the remaining struct members. We use our own pool
+ * here since we know it is single-threaded, incurring the least overhead.
+ */
+ result->timeout = timeout;
+ result->in_access = svn_hash__make(result->pool);
+ result->recyler = apr_array_make(result->pool, 256, sizeof(access_t *));
+ SVN_ERR(svn_mutex__init(&result->mutex, TRUE, result->pool));
+
+ *thunder = result;
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs__thunder_destroy(svn_fs__thunder_t *thunder)
+{
+ apr_pool_cleanup_kill(thunder->owning_pool, thunder, thunder_cleanup);
+ apr_pool_cleanup_kill(thunder->pool, thunder, thunder_root_cleanup);
+ svn_pool_destroy(thunder->pool);
+
+ return SVN_NO_ERROR;
+}
+
+/* Return the combination of PATH and LOCATION as a single key allocated in
+ * POOL.
+ */
+static svn_stringbuf_t *
+construct_key(const char *path,
+ apr_uint64_t location,
+ apr_pool_t *pool)
+{
+ /* There are certainly more efficient ways to do it, but this good enough
+ * because the amount of data that the caller wants to process as part of
+ * the data access is several kB. So, we can afford to trade a few cycles
+ * for simplicity. */
+ return svn_stringbuf_createf(pool, "%s:%" APR_UINT64_T_FMT, path, location);
+}
+
+/* Mark ACCESS as begin used for KEY.
+ *
+ * Callers must serialize for ACCESS.
+ */
+static svn_error_t *
+set_access(access_t *access,
+ svn_stringbuf_t *key)
+{
+ svn_stringbuf_set(access->key, key->data);
+
+ return SVN_NO_ERROR;
+}
+
+/* Retrieve the internal access description for KEY in THUNDER and return
+ * it in *ACCESS. If there is no such entry, create a new one / recycle an
+ * unused one, start the access and set *FIRST to TRUE. Set it to FALSE
+ * otherwise.
+ *
+ * Callers must serialize for THUNDER.
+ */
+static svn_error_t *
+get_access(access_t **access,
+ svn_boolean_t *first,
+ svn_fs__thunder_t *thunder,
+ svn_stringbuf_t *key,
+ void *owner)
+{
+ access_t *result = apr_hash_get(thunder->in_access, key->data, key->len);
+ if (result)
+ {
+ /* There is already an access object for KEY
+ * (might have timed out already but we let the caller handle that). */
+ *first = FALSE;
+ }
+ else
+ {
+ /* A new entry is needed. */
+ *first = TRUE;
+
+ /* Recycle old, unused access description objects whenever we can. */
+ if (thunder->recyler->nelts)
+ {
+ result = *(access_t **)apr_array_pop(thunder->recyler);
+
+ /* Make sure that access to the key (also acting as a usage marker)
+ * gets serialized. */
+ SVN_MUTEX__WITH_LOCK(result->mutex, set_access(result, key));
+ }
+ else
+ {
+ apr_status_t status;
+ result = apr_pcalloc(thunder->pool, sizeof(*result));
+ result->key = svn_stringbuf_dup(key, thunder->pool);
+ SVN_ERR(svn_mutex__init(&result->mutex, TRUE, thunder->pool));
+ status = apr_thread_cond_create(&result->condition, thunder->pool);
+ if (status != APR_SUCCESS)
+ return svn_error_trace(svn_error_wrap_apr(status,
+ _("Can't create condition")));
+ }
+
+ /* Start the access and remember which thread we will be waiting for. */
+ result->started = apr_time_now();
+ result->owning_thread = apr_os_thread_current();
+
+ /* Add it to the list of accesses currently under way. */
+ apr_hash_set(thunder->in_access, result->key->data, key->len, result);
+ }
+
+ *access = result;
+ return SVN_NO_ERROR;
+}
+
+/* Remove *ACCESS from THUNDER's list of accesses currently in progress.
+ * This is a no-op when *ACCESS is not the current entry for KEY. In that
+ * case, we set *ACCESS to NULL.
+ *
+ * Callers must serialize for THUNDER.
+ */
+static svn_error_t *
+remove_access(access_t **access,
+ svn_fs__thunder_t *thunder,
+ svn_stringbuf_t *key)
+{
+ void *value = apr_hash_get(thunder->in_access, key->data, key->len);
+ if (value == *access)
+ {
+ /* remove entry from hash */
+ apr_hash_set(thunder->in_access, key->data, key->len, NULL);
+ }
+ else
+ {
+ /* Access has already been removed (and possibly re-used for another
+ * key later). Leave it alone. */
+ *access = NULL;
+ }
+
+ return SVN_NO_ERROR;
+}
+
+/* Mark ACCESS as unused.
+ *
+ * Callers must serialize for ACCESS.
+ */
+static svn_error_t *
+reset_access(access_t *access)
+{
+ svn_stringbuf_setempty(access->key);
+
+ return SVN_NO_ERROR;
+}
+
+/* Move the unused ACCESS to the list of recyclable objects in THUNDER.
+ *
+ * Callers must serialize for THUNDER.
+ */
+static svn_error_t *
+recycle_access(svn_fs__thunder_t *thunder,
+ access_t *access)
+{
+ APR_ARRAY_PUSH(thunder->recyler, access_t *) = access;
+ return SVN_NO_ERROR;
+}
+
+/* Safely remove ACCESS from THUNDER's list of ongoing accesses for KEY and
+ * unblock any threads waiting on it.
+ */
+static svn_error_t *
+release_access(svn_fs__thunder_t *thunder,
+ access_t *access,
+ svn_stringbuf_t *key)
+{
+ /* No longer report KEY as "in access", i.e. don't block any additional
+ * threads t. */
+ SVN_MUTEX__WITH_LOCK(thunder->mutex, remove_access(&access, thunder, key));
+
+ /* This was racy up to here but now we know whether we are the ones
+ * releasing ACCESS. */
+ if (access)
+ {
+ apr_status_t status;
+
+ /* Sync with the time-out test in svn_fs__thunder_begin_access. */
+ SVN_MUTEX__WITH_LOCK(access->mutex, reset_access(access));
+
+ /* At this point, no thread will attempt to wait for this access,
+ * so we only have to wake up those who already wait. */
+
+ /* Tell / wake everybody that the access has been completed now. */
+ status = apr_thread_cond_broadcast(access->condition);
+ if (status != APR_SUCCESS)
+ return svn_error_trace(svn_error_wrap_apr(status,
+ _("Can't signal condition")));
+
+ /* Some threads may still be in the process of waking up or at least
+ * hold ACCESS->MUTEX. That's fine since the object remains valid.
+ *
+ * It might happen that some threads are still waiting for ACCESS->MUTEX
+ * on their early time-out check. If ACCESS should get re-used quickly,
+ * those threads would end up waiting for the new access to finish.
+ * This is inefficient but rare and safe. */
+
+ /* Object is now ready to be recycled */
+ SVN_MUTEX__WITH_LOCK(thunder->mutex, recycle_access(thunder, access));
+ }
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs__thunder_begin_access(svn_fs__thunder_access_t **access,
+ svn_fs__thunder_t *thunder,
+ const char *path,
+ apr_uint64_t location,
+ apr_pool_t *pool)
+{
+ svn_stringbuf_t *key = construct_key(path, location, pool);
+ access_t *internal_access;
+ svn_boolean_t first;
+
+ /* Get the current hash entry or create a new one (FIRST will then be TRUE).
+ */
+ SVN_MUTEX__WITH_LOCK(thunder->mutex,
+ get_access(&internal_access, &first, thunder, key,
+ pool));
+
+ if (first)
+ {
+ /* No concurrent access. Hand out an access token. */
+ svn_fs__thunder_access_t *result = apr_pcalloc(pool, sizeof(*result));
+ result->thunder = thunder;
+ result->access = internal_access;
+ result->key = key;
+
+ *access = result;
+ }
+ else if (apr_os_thread_equal(apr_os_thread_current(),
+ internal_access->owning_thread))
+ {
+ /* The current thread already holds a token for this KEY.
+ * There is no point in making it block on itself since it would
+ * simply time out. */
+ *access = NULL;
+ return SVN_NO_ERROR;
+ }
+ else
+ {
+ apr_time_t timeout;
+ *access = NULL;
+
+ timeout = internal_access->started + thunder->timeout - apr_time_now();
+ if (timeout <= 0)
+ {
+ /* Something went wrong (probably just some hold-up but still ...).
+ * No longer let anyone wait on this access. This is racy but we
+ * allow multiple attempts to release the same access. */
+ SVN_ERR(release_access(thunder, internal_access, key));
+ }
+ else
+ {
+ /* Sync. with reset and signaling code.
+ * Don't use the SVN_MUTEX__WITH_LOCK macro here since we need
+ * to hold the lock when waiting for the condition variable. */
+ SVN_ERR(svn_mutex__lock(internal_access->mutex));
+
+ /* Has there been a reset in the meantime?
+ * There is a chance that the access got recycled and used for
+ * the same key. But in that case, we would simply wait for that
+ * access to complete. It's the same data block so we simply
+ * don't care _who_ is reading it. */
+ if (svn_stringbuf_compare(internal_access->key, key))
+ {
+ /* We will receive the signal. The only way to time out here
+ * is a holdup in the thread holding the access. */
+ apr_thread_cond_timedwait(internal_access->condition,
+ internal_access->mutex,
+ timeout);
+ }
+
+ /* Done with the access struct.
+ * Others may now do with it as they please. */
+ SVN_ERR(svn_mutex__unlock(internal_access->mutex, SVN_NO_ERROR));
+ }
+ }
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs__thunder_end_access(svn_fs__thunder_access_t *access)
+{
+ /* NULL is valid for ACCESS. */
+ return access
+ ? release_access(access->thunder, access->access, access->key)
+ : SVN_NO_ERROR;
+}
+
+#else
+
+/* There are no other threads, thus there is no need to synchronize anything.
+ * Satisfy the API and always hand out access tokens.
+ */
+
+/* We don't have to manage anything, hence define structs as basically empty.
+ */
+struct svn_fs__thunder_t
+{
+ /* Handling empty structs is compiler-dependent in C.
+ * So, make this non-empty. */
+ int dummy;
+};
+
+struct svn_fs__thunder_access_t
+{
+ /* Handling empty structs is compiler-dependent in C.
+ * So, make this non-empty. */
+ int dummy;
+};
+
+svn_error_t *
+svn_fs__thunder_create(svn_fs__thunder_t **thunder,
+ apr_time_t timeout,
+ apr_pool_t *pool)
+{
+ *thunder = apr_pcalloc(pool, sizeof(**thunder));
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs__thunder_destroy(svn_fs__thunder_t *thunder)
+{
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs__thunder_begin_access(svn_fs__thunder_access_t **access,
+ svn_fs__thunder_t *thunder,
+ const char *path,
+ apr_uint64_t location,
+ apr_pool_t *pool)
+{
+ *access = apr_pcalloc(pool, sizeof(**access));
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs__thunder_end_access(svn_fs__thunder_access_t *access)
+{
+ return SVN_NO_ERROR;
+}
+
+#endif
\ No newline at end of file
Modified: subversion/branches/thunder/subversion/tests/libsvn_fs/fs-test.c
URL: http://svn.apache.org/viewvc/subversion/branches/thunder/subversion/tests/libsvn_fs/fs-test.c?rev=1586481&r1=1586480&r2=1586481&view=diff
==============================================================================
--- subversion/branches/thunder/subversion/tests/libsvn_fs/fs-test.c (original)
+++ subversion/branches/thunder/subversion/tests/libsvn_fs/fs-test.c Thu Apr 10 22:07:53 2014
@@ -5307,6 +5307,55 @@ dir_prop_merge(const svn_test_opts_t *op
return SVN_NO_ERROR;
}
+static svn_error_t *
+basic_thunder_interface(const svn_test_opts_t *opts,
+ apr_pool_t *pool)
+{
+ svn_fs__thunder_t *thunder;
+ svn_fs__thunder_access_t *access, *access2, *access3;
+
+ /* Create thunder object with some small timeout
+ * (this is a single-threaded test anyways which would not block) */
+ SVN_ERR(svn_fs__thunder_create(&thunder, 10000, pool));
+
+ /* Gain access and release it. */
+ SVN_ERR(svn_fs__thunder_begin_access(&access, thunder, "/some/path",
+ 123456, pool));
+ SVN_TEST_ASSERT(access);
+ SVN_ERR(svn_fs__thunder_end_access(access));
+
+ /* Double release is fine */
+ SVN_ERR(svn_fs__thunder_end_access(access));
+
+ /* Re-acquiring an access is fine, too. This time, we won't release it
+ * to verify that unreleased access objects don't mess up the destruction
+ * of the thunder_t instance. */
+ SVN_ERR(svn_fs__thunder_begin_access(&access, thunder, "/some/path",
+ 123456, pool));
+ SVN_TEST_ASSERT(access);
+
+ /* Acquiring the same path twice is legal but the second attempt causes
+ * a time out and no access object gets returned. */
+ SVN_ERR(svn_fs__thunder_begin_access(&access, thunder, "path2", 9, pool));
+ SVN_ERR(svn_fs__thunder_begin_access(&access2, thunder, "path2", 9, pool));
+
+ SVN_TEST_ASSERT(access);
+ SVN_TEST_ASSERT(access2 == NULL);
+
+ /* Gaining access to another path should be passible. */
+ SVN_ERR(svn_fs__thunder_begin_access(&access3, thunder, "path2", 91, pool));
+ SVN_TEST_ASSERT(access3);
+
+ /* Now, release all three. */
+ SVN_ERR(svn_fs__thunder_end_access(access));
+ SVN_ERR(svn_fs__thunder_end_access(access2));
+ SVN_ERR(svn_fs__thunder_end_access(access3));
+
+ /* Clean up the container (while still holding an access object). */
+ svn_fs__thunder_destroy(thunder);
+
+ return SVN_NO_ERROR;
+}
/* ------------------------------------------------------------------------ */
@@ -5402,6 +5451,8 @@ static struct svn_test_descriptor_t test
"test svn_fs__compatible_version"),
SVN_TEST_OPTS_PASS(dir_prop_merge,
"test merge directory properties"),
+ SVN_TEST_OPTS_PASS(basic_thunder_interface,
+ "test basic thunder_t interface"),
SVN_TEST_NULL
};