You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2012/09/24 14:40:53 UTC
svn commit: r1389338 - /subversion/trunk/tools/server-side/fsfs-reorg.c
Author: stefan2
Date: Mon Sep 24 12:40:53 2012
New Revision: 1389338
URL: http://svn.apache.org/viewvc?rev=1389338&view=rev
Log:
Make FSFS reorg tool more usable: add a proper warning to the usage
output and make the cache size configurable.
* tools/server-side/fsfs-reorg.c
(window_cache_t,
create_window_cache,
set_cached_window): use byte-granularity capacity and usage instead
estimating memory usage based on entry count
(read_revisions): replace hard-coded cache sizes with calculated values
(print_usage): update; add warning
(main): replace revision parameter with cache size
Modified:
subversion/trunk/tools/server-side/fsfs-reorg.c
Modified: subversion/trunk/tools/server-side/fsfs-reorg.c
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/server-side/fsfs-reorg.c?rev=1389338&r1=1389337&r2=1389338&view=diff
==============================================================================
--- subversion/trunk/tools/server-side/fsfs-reorg.c (original)
+++ subversion/trunk/tools/server-side/fsfs-reorg.c Mon Sep 24 12:40:53 2012
@@ -182,7 +182,8 @@ typedef struct window_cache_t
apr_pool_t *pool;
apr_size_t entry_count;
- apr_size_t insert_count;
+ apr_size_t capacity;
+ apr_size_t used;
} window_cache_t;
typedef struct fs_fs_t
@@ -386,8 +387,8 @@ get_cached_dir(fs_fs_t *fs,
svn_revnum_t revision = representation->revision->revision;
apr_off_t offset = representation->original.offset;
- apr_size_t index = get_dir_cache_index(fs, revision, offset);
- dir_cache_entry_t *entry = &fs->dir_cache->entries[index];
+ apr_size_t i = get_dir_cache_index(fs, revision, offset);
+ dir_cache_entry_t *entry = &fs->dir_cache->entries[i];
return entry->offset == offset && entry->revision == revision
? entry->hash
@@ -402,8 +403,8 @@ set_cached_dir(fs_fs_t *fs,
svn_revnum_t revision = representation->revision->revision;
apr_off_t offset = representation->original.offset;
- apr_size_t index = get_dir_cache_index(fs, revision, offset);
- dir_cache_entry_t *entry = &fs->dir_cache->entries[index];
+ apr_size_t i = get_dir_cache_index(fs, revision, offset);
+ dir_cache_entry_t *entry = &fs->dir_cache->entries[i];
fs->dir_cache->insert_count += apr_hash_count(hash);
if (fs->dir_cache->insert_count >= fs->dir_cache->entry_count * 100)
@@ -428,13 +429,15 @@ set_cached_dir(fs_fs_t *fs,
static window_cache_t *
create_window_cache(apr_pool_t *pool,
- apr_size_t entry_count)
+ apr_size_t entry_count,
+ apr_size_t capacity)
{
window_cache_t *result = apr_pcalloc(pool, sizeof(*result));
result->pool = svn_pool_create(pool);
result->entry_count = entry_count;
- result->insert_count = 0;
+ result->capacity = capacity;
+ result->used = 0;
result->entries = apr_pcalloc(pool, sizeof(*result->entries) * entry_count);
return result;
@@ -456,8 +459,8 @@ get_cached_window(fs_fs_t *fs,
svn_revnum_t revision = representation->revision->revision;
apr_off_t offset = representation->original.offset;
- apr_size_t index = get_window_cache_index(fs, revision, offset);
- window_cache_entry_t *entry = &fs->window_cache->entries[index];
+ apr_size_t i = get_window_cache_index(fs, revision, offset);
+ window_cache_entry_t *entry = &fs->window_cache->entries[i];
return entry->offset == offset && entry->revision == revision
? svn_stringbuf_dup(entry->window, pool)
@@ -472,17 +475,17 @@ set_cached_window(fs_fs_t *fs,
svn_revnum_t revision = representation->revision->revision;
apr_off_t offset = representation->original.offset;
- apr_size_t index = get_window_cache_index(fs, revision, offset);
- window_cache_entry_t *entry = &fs->window_cache->entries[index];
+ apr_size_t i = get_window_cache_index(fs, revision, offset);
+ window_cache_entry_t *entry = &fs->window_cache->entries[i];
- fs->window_cache->insert_count += window->len;
- if (fs->window_cache->insert_count >= fs->window_cache->entry_count * 10000)
+ fs->window_cache->used += window->len;
+ if (fs->window_cache->used >= fs->window_cache->capacity)
{
svn_pool_clear(fs->window_cache->pool);
memset(fs->window_cache->entries,
0,
sizeof(*fs->window_cache->entries) * fs->window_cache->entry_count);
- fs->window_cache->insert_count = 0;
+ fs->window_cache->used = window->len;
}
entry->window = svn_stringbuf_dup(window, fs->window_cache->pool);
@@ -1265,7 +1268,7 @@ read_pack_file(fs_fs_t *fs,
svn_revnum_t base,
apr_pool_t *pool)
{
- apr_array_header_t *manifest;
+ apr_array_header_t *manifest = NULL;
apr_pool_t *local_pool = svn_pool_create(pool);
apr_pool_t *iter_pool = svn_pool_create(local_pool);
int i;
@@ -1283,7 +1286,7 @@ read_pack_file(fs_fs_t *fs,
revisions->filesize = file_content->len;
APR_ARRAY_PUSH(fs->packs, revision_pack_t*) = revisions;
- read_manifest(&manifest, pack_folder, local_pool);
+ SVN_ERR(read_manifest(&manifest, pack_folder, local_pool));
if (manifest->nelts != fs->max_files_per_dir)
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, NULL);
@@ -1386,10 +1389,22 @@ static svn_error_t *
read_revisions(fs_fs_t **fs,
const char *path,
svn_revnum_t start_revision,
- apr_size_t limit,
+ apr_size_t memsize,
apr_pool_t *pool)
{
svn_revnum_t revision;
+ apr_size_t content_cache_size;
+ apr_size_t window_cache_size;
+ apr_size_t dir_cache_size;
+
+ /* determine cache sizes */
+
+ if (memsize < 100)
+ memsize = 100;
+
+ content_cache_size = memsize * 7 / 10 > 4000 ? 4000 : memsize * 7 / 10;
+ window_cache_size = memsize * 2 / 10 * 1024 * 1024;
+ dir_cache_size = (memsize / 10) * 16000;
SVN_ERR(fs_open(fs, path, pool));
@@ -1406,15 +1421,15 @@ read_revisions(fs_fs_t **fs,
(*fs)->cache = create_content_cache
(apr_allocator_owner_get
(svn_pool_create_allocator(FALSE)),
- limit * 1024 * 1024);
+ content_cache_size * 1024 * 1024);
(*fs)->dir_cache = create_dir_cache
(apr_allocator_owner_get
(svn_pool_create_allocator(FALSE)),
- 100000);
+ dir_cache_size);
(*fs)->window_cache = create_window_cache
(apr_allocator_owner_get
(svn_pool_create_allocator(FALSE)),
- 10000);
+ 10000, window_cache_size);
for ( revision = start_revision
; revision < (*fs)->min_unpacked_rev
@@ -2118,7 +2133,7 @@ update_id(svn_stringbuf_t *node_rev,
const char *key,
noderev_t *node)
{
- char *newline_pos;
+ char *newline_pos = 0;
char *pos;
pos = strstr(node_rev->data, key);
@@ -2378,10 +2393,17 @@ print_usage(svn_stream_t *ostream, const
apr_pool_t *pool)
{
svn_error_clear(svn_stream_printf(ostream, pool,
- "Usage: %s <repo> <rev>\n"
"\n"
- "Optimize the repository at local path <repo> staring from "
- "revision <rev>.\n",
+ "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+ "!!! This is an experimental tool. Don't use it on production data !!!\n"
+ "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+ "\n"
+ "Usage: %s <repo> <cachesize>\n"
+ "\n"
+ "Optimize the repository at local path <repo> staring from revision 0.\n"
+ "Use up to <cachesize> MB of memory for caching. This does not include\n"
+ "temporary representation of the repository structure, i.e. the actual\n"
+ "memory will be higher and <cachesize> be the lower limit.\n",
progname));
}
@@ -2392,7 +2414,7 @@ int main(int argc, const char *argv[])
svn_error_t *svn_err;
const char *repo_path = NULL;
svn_revnum_t start_revision = 0;
- apr_int64_t rev = 0;
+ apr_int64_t memsize = 0;
fs_fs_t *fs;
apr_initialize();
@@ -2413,7 +2435,7 @@ int main(int argc, const char *argv[])
return 2;
}
- svn_err = svn_cstring_atoi64(&rev, argv[2]);
+ svn_err = svn_cstring_atoi64(&memsize, argv[2]);
if (svn_err)
{
print_usage(ostream, argv[0], pool);
@@ -2422,15 +2444,15 @@ int main(int argc, const char *argv[])
}
repo_path = argv[1];
- start_revision = (long)rev;
+ start_revision = 0;
printf("\nPreparing repository\n");
svn_err = prepare_repo(repo_path, pool);
if (!svn_err)
{
- printf("\nReading revisions\n");
- svn_err = read_revisions(&fs, repo_path, start_revision, 4000, pool);
+ printf("Reading revisions\n");
+ svn_err = read_revisions(&fs, repo_path, start_revision, memsize, pool);
}
if (!svn_err)