You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2014/05/18 11:57:38 UTC
svn commit: r1595582 - in /subversion/trunk/subversion/libsvn_fs_x: ./
cached_data.c fs_x.c index.c index.h pack.c verify.c
Author: stefan2
Date: Sun May 18 09:57:37 2014
New Revision: 1595582
URL: http://svn.apache.org/r1595582
Log:
Sync'ing FSX with FSFS:
Merge r1584235,1590460,1590464 from /subversion/libsvn_fs_fs into
subversion/libsvn_fs_x. Conflicts were due structural and naming
differences between FSX and FSFS.
This ports the "independent p2l index page size" patches.
Modified:
subversion/trunk/subversion/libsvn_fs_x/ (props changed)
subversion/trunk/subversion/libsvn_fs_x/cached_data.c
subversion/trunk/subversion/libsvn_fs_x/fs_x.c
subversion/trunk/subversion/libsvn_fs_x/index.c
subversion/trunk/subversion/libsvn_fs_x/index.h
subversion/trunk/subversion/libsvn_fs_x/pack.c
subversion/trunk/subversion/libsvn_fs_x/verify.c
Propchange: subversion/trunk/subversion/libsvn_fs_x/
------------------------------------------------------------------------------
Merged /subversion/trunk/subversion/libsvn_fs_fs:r1584235,1590460,1590464
Modified: subversion/trunk/subversion/libsvn_fs_x/cached_data.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/cached_data.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/cached_data.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/cached_data.c Sun May 18 09:57:37 2014
@@ -2701,9 +2701,10 @@ block_read(void **result,
offset = wanted_offset;
do
{
- SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, revision, offset,
- scratch_pool));
+ /* fetch list of items in the block surrounding OFFSET */
SVN_ERR(aligned_seek(fs, revision_file, &block_start, offset, iterpool));
+ SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, revision, block_start,
+ ffd->block_size, scratch_pool));
/* read all items from the block */
for (i = 0; i < entries->nelts; ++i)
Modified: subversion/trunk/subversion/libsvn_fs_x/fs_x.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/fs_x.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/fs_x.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/fs_x.c Sun May 18 09:57:37 2014
@@ -252,7 +252,7 @@ read_config(fs_x_data_t *ffd,
SVN_ERR(svn_config_get_int64(config, &ffd->p2l_page_size,
CONFIG_SECTION_IO,
CONFIG_OPTION_P2L_PAGE_SIZE,
- 64));
+ 0x400));
ffd->block_size *= 0x400;
ffd->p2l_page_size *= 0x400;
@@ -428,11 +428,10 @@ write_config(svn_fs_t *fs,
"### of tens of MB each, increasing the page size will shorten the index" NL
"### file at the expense of a slightly increased latency in sections with" NL
"### smaller changes." NL
-"### For practical reasons, this should match block-size. Differing" NL
-"### values are perfectly legal but may result in some processing overhead." NL
+"### For source code repositories, this should be about 16x the block-size." NL
"### Must be a power of 2." NL
-"### p2l-page-size is 64 kBytes by default." NL
-"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 64" NL
+"### p2l-page-size is 1024 kBytes by default." NL
+"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 1024" NL
;
#undef NL
return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool),
Modified: subversion/trunk/subversion/libsvn_fs_x/index.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/index.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/index.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/index.c Sun May 18 09:57:37 2014
@@ -1436,6 +1436,8 @@ prefetch_l2p_pages(svn_boolean_t *end,
for (i = 0; i < pages->nelts && !*end; ++i)
{
+ svn_boolean_t is_cached;
+
l2p_page_table_entry_t *entry
= &APR_ARRAY_IDX(pages, i, l2p_page_table_entry_t);
if (i == exlcuded_page_no)
@@ -1451,9 +1453,9 @@ prefetch_l2p_pages(svn_boolean_t *end,
/* page already in cache? */
key.page = i;
- SVN_ERR(svn_cache__has_key(end, ffd->l2p_page_cache,
+ SVN_ERR(svn_cache__has_key(&is_cached, ffd->l2p_page_cache,
&key, iterpool));
- if (!*end)
+ if (!is_cached)
{
/* no in cache -> read from stream (data already buffered in APR)
* and cache the result */
@@ -1527,8 +1529,8 @@ l2p_index_lookup(apr_off_t *offset,
svn_boolean_t end;
apr_off_t max_offset
= APR_ALIGN(info_baton.entry.offset + info_baton.entry.size,
- 0x10000);
- apr_off_t min_offset = max_offset - 0x10000;
+ ffd->block_size);
+ apr_off_t min_offset = max_offset - ffd->block_size;
/* read the relevant page */
SVN_ERR(get_l2p_page(&page, &stream, fs, info_baton.first_revision,
@@ -2420,27 +2422,140 @@ get_p2l_keys(p2l_page_info_baton_t *page
return SVN_NO_ERROR;
}
-/* Body of svn_fs_x__p2l_index_lookup. Use / autoconstruct *STREAM as
- * your input based on REVISION.
+/* qsort-compatible compare function that compares the OFFSET of the
+ * svn_fs_x__p2l_entry_t in *LHS with the apr_off_t in *RHS. */
+static int
+compare_start_p2l_entry(const void *lhs,
+ const void *rhs)
+{
+ const svn_fs_x__p2l_entry_t *entry = lhs;
+ apr_off_t start = *(const apr_off_t*)rhs;
+ apr_off_t diff = entry->offset - start;
+
+ /* restrict result to int */
+ return diff < 0 ? -1 : (diff == 0 ? 0 : 1);
+}
+
+/* From the PAGE_ENTRIES array of svn_fs_x__p2l_entry_t, ordered
+ * by their OFFSET member, copy all elements overlapping the range
+ * [BLOCK_START, BLOCK_END) to ENTRIES. If RESOLVE_PTR is set, the ITEMS
+ * sub-array in each entry needs to be de-serialized. */
+static void
+append_p2l_entries(apr_array_header_t *entries,
+ apr_array_header_t *page_entries,
+ apr_off_t block_start,
+ apr_off_t block_end,
+ svn_boolean_t resolve_ptr)
+{
+ const svn_fs_x__p2l_entry_t *entry;
+ int idx = svn_sort__bsearch_lower_bound(page_entries, &block_start,
+ compare_start_p2l_entry);
+
+ /* start at the first entry that overlaps with BLOCK_START */
+ if (idx > 0)
+ {
+ entry = &APR_ARRAY_IDX(page_entries, idx - 1, svn_fs_x__p2l_entry_t);
+ if (entry->offset + entry->size > block_start)
+ --idx;
+ }
+
+ /* copy all entries covering the requested range */
+ for ( ; idx < page_entries->nelts; ++idx)
+ {
+ svn_fs_x__p2l_entry_t *copy;
+ entry = &APR_ARRAY_IDX(page_entries, idx, svn_fs_x__p2l_entry_t);
+ if (entry->offset >= block_end)
+ break;
+
+ /* Copy the entry record. */
+ copy = apr_array_push(entries);
+ *copy = *entry;
+
+ /* Copy the items of that entries. */
+ if (entry->item_count)
+ {
+ const svn_fs_x__id_part_t *items
+ = resolve_ptr
+ ? svn_temp_deserializer__ptr(page_entries->elts,
+ (const void * const *)&entry->items)
+ : entry->items;
+
+ copy->items = apr_pmemdup(entries->pool, items,
+ entry->item_count * sizeof(*items));
+ }
+ }
+}
+
+/* Auxilliary struct passed to p2l_entries_func selecting the relevant
+ * data range. */
+typedef struct p2l_entries_baton_t
+{
+ apr_off_t start;
+ apr_off_t end;
+} p2l_entries_baton_t;
+
+/* Implement svn_cache__partial_getter_func_t: extract p2l entries from
+ * the page in DATA which overlap the p2l_entries_baton_t in BATON.
+ * The target array is already provided in *OUT.
*/
static svn_error_t *
-p2l_index_lookup(apr_array_header_t **entries,
+p2l_entries_func(void **out,
+ const void *data,
+ apr_size_t data_len,
+ void *baton,
+ apr_pool_t *result_pool)
+{
+ apr_array_header_t *entries = *(apr_array_header_t **)out;
+ const apr_array_header_t *raw_page = data;
+ p2l_entries_baton_t *block = baton;
+ int i;
+
+ /* Make PAGE a readable APR array. */
+ apr_array_header_t page = *raw_page;
+ page.elts = (void *)svn_temp_deserializer__ptr(raw_page,
+ (const void * const *)&raw_page->elts);
+
+ /* append relevant information to result */
+ append_p2l_entries(entries, &page, block->start, block->end, TRUE);
+
+ return SVN_NO_ERROR;
+}
+
+
+/* Body of svn_fs_x__p2l_index_lookup. However, do a single index page
+ * lookup and append the result to the ENTRIES array provided by the caller.
+ * Use successive calls to cover larger ranges.
+ */
+static svn_error_t *
+p2l_index_lookup(apr_array_header_t *entries,
packed_number_stream_t **stream,
svn_fs_t *fs,
svn_revnum_t revision,
- apr_off_t offset,
+ apr_off_t block_start,
+ apr_off_t block_end,
apr_pool_t *pool)
{
fs_x_data_t *ffd = fs->fsap_data;
svn_fs_x__page_cache_key_t key;
svn_boolean_t is_cached = FALSE;
p2l_page_info_baton_t page_info;
+ apr_array_header_t *local_result = entries;
+
+ /* baton selecting the relevant entries from the one page we access */
+ p2l_entries_baton_t block;
+ block.start = block_start;
+ block.end = block_end;
- /* look for this page in our cache */
- SVN_ERR(get_p2l_keys(&page_info, &key, stream, fs, revision, offset,
+ /* if we requested an empty range, the result would be empty */
+ SVN_ERR_ASSERT(block_start < block_end);
+
+ /* look for the fist page of the range in our cache */
+ SVN_ERR(get_p2l_keys(&page_info, &key, stream, fs, revision, block_start,
pool));
- SVN_ERR(svn_cache__get((void**)entries, &is_cached, ffd->p2l_page_cache,
- &key, pool));
+ SVN_ERR(svn_cache__get_partial((void**)&local_result, &is_cached,
+ ffd->p2l_page_cache, &key, p2l_entries_func,
+ &block, pool));
+
if (!is_cached)
{
svn_boolean_t end;
@@ -2448,6 +2563,7 @@ p2l_index_lookup(apr_array_header_t **en
apr_off_t original_page_start = page_info.page_start;
int leaking_bucket = 4;
p2l_page_info_baton_t prefetch_info = page_info;
+ apr_array_header_t *page_entries;
apr_off_t max_offset
= APR_ALIGN(page_info.next_offset, ffd->block_size);
@@ -2472,14 +2588,18 @@ p2l_index_lookup(apr_array_header_t **en
}
/* fetch page from disk and put it into the cache */
- SVN_ERR(get_p2l_page(entries, stream, fs,
+ SVN_ERR(get_p2l_page(&page_entries, stream, fs,
page_info.first_revision,
page_info.start_offset,
page_info.next_offset,
page_info.page_start,
- page_info.page_size, pool, pool));
+ page_info.page_size, pool, iterpool));
- SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, *entries, pool));
+ SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, page_entries,
+ iterpool));
+
+ /* append relevant information to result */
+ append_p2l_entries(entries, page_entries, block_start, block_end, FALSE);
/* pre-fetch following pages */
end = FALSE;
@@ -2500,6 +2620,40 @@ p2l_index_lookup(apr_array_header_t **en
svn_pool_destroy(iterpool);
}
+ /* We access a valid page (otherwise, we had seen an error in the
+ * get_p2l_keys request). Hence, at least one entry must be found. */
+ SVN_ERR_ASSERT(entries->nelts > 0);
+
+ /* Add an "unused" entry if it extends beyond the end of the data file.
+ * Since the index page size might be smaller than the current data
+ * read block size, the trailing "unused" entry in this index may not
+ * fully cover the end of the last block. */
+ if (page_info.page_no + 1 >= page_info.page_count)
+ {
+ svn_fs_x__p2l_entry_t *entry
+ = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t);
+
+ apr_off_t entry_end = entry->offset + entry->size;
+ if (entry_end < block_end)
+ {
+ if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED)
+ {
+ /* extend the terminal filler */
+ entry->size = block_end - entry->offset;
+ }
+ else
+ {
+ /* No terminal filler. Add one. */
+ entry = apr_array_push(entries);
+ entry->offset = entry_end;
+ entry->size = block_end - entry_end;
+ entry->type = SVN_FS_X__ITEM_TYPE_UNUSED;
+ entry->item_count = 0;
+ entry->items = NULL;
+ }
+ }
+ }
+
return SVN_NO_ERROR;
}
@@ -2507,17 +2661,51 @@ svn_error_t *
svn_fs_x__p2l_index_lookup(apr_array_header_t **entries,
svn_fs_t *fs,
svn_revnum_t revision,
- apr_off_t offset,
+ apr_off_t block_start,
+ apr_off_t block_size,
apr_pool_t *pool)
{
packed_number_stream_t *stream = NULL;
- /* look for this page in our cache */
- SVN_ERR(p2l_index_lookup(entries, &stream, fs, revision, offset, pool));
+ apr_off_t block_end = block_start + block_size;
+
+ /* the receiving container */
+ int last_count = 0;
+ apr_array_header_t *result = apr_array_make(pool, 16,
+ sizeof(svn_fs_x__p2l_entry_t));
+
+ /* Fetch entries page-by-page. Since the p2l index is supposed to cover
+ * every single byte in the rev / pack file - even unused sections -
+ * every iteration must result in some progress. */
+ while (block_start < block_end)
+ {
+ svn_fs_x__p2l_entry_t *entry;
+ SVN_ERR(p2l_index_lookup(result, &stream, fs, revision, block_start,
+ block_end, pool));
+ SVN_ERR_ASSERT(result->nelts > 0);
+
+ /* continue directly behind last item */
+ entry = &APR_ARRAY_IDX(result, result->nelts-1, svn_fs_x__p2l_entry_t);
+ block_start = entry->offset + entry->size;
+
+ /* Some paranoia check. Successive iterations should never return
+ * duplicates but if it did, we might get into trouble later on. */
+ if (last_count > 0 && last_count < result->nelts)
+ {
+ entry = &APR_ARRAY_IDX(result, last_count - 1,
+ svn_fs_x__p2l_entry_t);
+ SVN_ERR_ASSERT(APR_ARRAY_IDX(result, last_count,
+ svn_fs_x__p2l_entry_t).offset
+ >= entry->offset + entry->size);
+ }
+
+ last_count = result->nelts;
+ }
/* make sure we close files after usage */
SVN_ERR(packed_stream_close(stream));
+ *entries = result;
return SVN_NO_ERROR;
}
@@ -2611,8 +2799,9 @@ p2l_entry_lookup(svn_fs_x__p2l_entry_t *
{
/* do a standard index lookup. This is will automatically prefetch
* data to speed up future lookups. */
- apr_array_header_t *entries;
- SVN_ERR(p2l_index_lookup(&entries, stream, fs, revision, offset, pool));
+ apr_array_header_t *entries = apr_array_make(pool, 1, sizeof(**entry_p));
+ SVN_ERR(p2l_index_lookup(entries, stream, fs, revision, offset,
+ offset + 1, pool));
/* Find the entry that we want. */
*entry_p = svn_sort__array_lookup(entries, &offset, NULL,
Modified: subversion/trunk/subversion/libsvn_fs_x/index.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/index.h?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/index.h (original)
+++ subversion/trunk/subversion/libsvn_fs_x/index.h Sun May 18 09:57:37 2014
@@ -168,9 +168,9 @@ svn_fs_x__p2l_index_create(svn_fs_t *fs,
apr_pool_t *pool);
/* Use the phys-to-log mapping files in FS to build a list of entries
- * that (partly) share in the same cluster as the item at global OFFSET
- * in the rep file containing REVISION. Return the array in *ENTRIES,
- * elements being of type svn_fs_x__p2l_entry_t.
+ * that (at least partly) overlap with the range given by BLOCK_START
+ * offset and BLOCK_SIZE in the rep / pack file containing REVISION.
+ * Return the array in *ENTRIES with svn_fs_fs__p2l_entry_t as elements.
* Use POOL for allocations.
*
* Note that (only) the first and the last mapping may cross a cluster
@@ -180,7 +180,8 @@ svn_error_t *
svn_fs_x__p2l_index_lookup(apr_array_header_t **entries,
svn_fs_t *fs,
svn_revnum_t revision,
- apr_off_t offset,
+ apr_off_t block_start,
+ apr_off_t block_size,
apr_pool_t *pool);
/* Use the phys-to-log mapping files in FS to return the entry for the
Modified: subversion/trunk/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/pack.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/pack.c Sun May 18 09:57:37 2014
@@ -1756,6 +1756,7 @@ static svn_error_t *
pack_range(pack_context_t *context,
apr_pool_t *pool)
{
+ fs_x_data_t *ffd = context->fs->fsap_data;
apr_pool_t *revpool = svn_pool_create(pool);
apr_pool_t *iterpool = svn_pool_create(pool);
@@ -1788,9 +1789,11 @@ pack_range(pack_context_t *context,
/* read one cluster */
int i;
apr_array_header_t *entries;
+ svn_pool_clear(iterpool);
+
SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs,
revision, offset,
- iterpool));
+ ffd->p2l_page_size, iterpool));
for (i = 0; i < entries->nelts; ++i)
{
@@ -1840,8 +1843,6 @@ pack_range(pack_context_t *context,
if (context->cancel_func)
SVN_ERR(context->cancel_func(context->cancel_baton));
-
- svn_pool_clear(iterpool);
}
svn_pool_clear(revpool);
@@ -1887,6 +1888,7 @@ static svn_error_t *
append_revision(pack_context_t *context,
apr_pool_t *pool)
{
+ fs_x_data_t *ffd = context->fs->fsap_data;
apr_off_t offset = 0;
apr_pool_t *iterpool = svn_pool_create(pool);
apr_file_t *rev_file;
@@ -1919,7 +1921,7 @@ append_revision(pack_context_t *context,
apr_array_header_t *entries;
SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs,
context->start_rev, offset,
- iterpool));
+ ffd->p2l_page_size, iterpool));
for (i = 0; i < entries->nelts; ++i)
{
Modified: subversion/trunk/subversion/libsvn_fs_x/verify.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/verify.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/verify.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/verify.c Sun May 18 09:57:37 2014
@@ -250,6 +250,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
void *cancel_baton,
apr_pool_t *pool)
{
+ fs_x_data_t *ffd = fs->fsap_data;
apr_pool_t *iterpool = svn_pool_create(pool);
apr_off_t max_offset;
apr_off_t offset = 0;
@@ -267,7 +268,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
/* get all entries for the current block */
SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, start, offset,
- iterpool));
+ ffd->p2l_page_size, iterpool));
if (entries->nelts == 0)
return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION,
NULL,