You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by br...@apache.org on 2015/01/03 15:00:44 UTC
svn commit: r1649205 [15/30] - in /subversion/branches/authzperf: ./ build/
build/ac-macros/ notes/ subversion/bindings/ctypes-python/
subversion/bindings/cxxhl/
subversion/bindings/javahl/tests/org/apache/subversion/javahl/
subversion/bindings/swig/ s...
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h Sat Jan 3 14:00:41 2015
@@ -33,7 +33,7 @@
*
* In its serialized form, the svn_fs_x__noderevs_t container extracts
* most of that redundancy and the run-time representation is also much
- * smaller than sum of the respective node_revision_t objects.
+ * smaller than sum of the respective svn_fs_x__noderev_t objects.
*
* As with other containers, this one has two modes: 'construction', in
* which you may add data to it, and 'getter' in which there is only r/o
@@ -47,7 +47,7 @@ typedef struct svn_fs_x__noderevs_t svn_
/* Create and populate noderev containers. */
/* Create and return a new noderevs container with an initial capacity of
- * INITIAL_COUNT node_revision_t objects. Allocate the result in POOL.
+ * INITIAL_COUNT svn_fs_x__noderev_t objects. Allocate the result in POOL.
*/
svn_fs_x__noderevs_t *
svn_fs_x__noderevs_create(int initial_count,
@@ -58,7 +58,7 @@ svn_fs_x__noderevs_create(int initial_co
*/
apr_size_t
svn_fs_x__noderevs_add(svn_fs_x__noderevs_t *container,
- node_revision_t *noderev);
+ svn_fs_x__noderev_t *noderev);
/* Return a rough estimate in bytes for the serialized representation
* of CONTAINER.
@@ -72,7 +72,7 @@ svn_fs_x__noderevs_estimate_size(const s
* the result in POOL and return it in *NODEREV_P.
*/
svn_error_t *
-svn_fs_x__noderevs_get(node_revision_t **noderev_p,
+svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p,
const svn_fs_x__noderevs_t *container,
apr_size_t idx,
apr_pool_t *pool);
@@ -116,7 +116,7 @@ svn_fs_x__deserialize_noderevs_container
apr_pool_t *pool);
/* Implements svn_cache__partial_getter_func_t for svn_fs_x__noderevs_t,
- * setting *OUT to the node_revision_t selected by the apr_uint32_t index
+ * setting *OUT to the svn_fs_x__noderev_t selected by the apr_uint32_t index
* passed in as *BATON. This function is similar to svn_fs_x__noderevs_get
* but operates on the cache serialized representation of the container.
*/
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c Sat Jan 3 14:00:41 2015
@@ -101,7 +101,7 @@ typedef struct path_order_t
svn_prefix_string__t *path;
/* node ID for this PATH in REVISION */
- svn_fs_x__id_part_t node_id;
+ svn_fs_x__id_t node_id;
/* when this change happened */
svn_revnum_t revision;
@@ -113,10 +113,10 @@ typedef struct path_order_t
apr_int64_t expanded_size;
/* item ID of the noderev linked to the change. May be (0, 0). */
- svn_fs_x__id_part_t noderev_id;
+ svn_fs_x__id_t noderev_id;
/* item ID of the representation containing the new data. May be (0, 0). */
- svn_fs_x__id_part_t rep_id;
+ svn_fs_x__id_t rep_id;
} path_order_t;
/* Represents a reference from item FROM to item TO. FROM may be a noderev
@@ -125,8 +125,8 @@ typedef struct path_order_t
*/
typedef struct reference_t
{
- svn_fs_x__id_part_t to;
- svn_fs_x__id_part_t from;
+ svn_fs_x__id_t to;
+ svn_fs_x__id_t from;
} reference_t;
/* This structure keeps track of all the temporary data and status that
@@ -251,7 +251,7 @@ initialize_pack_context(pack_context_t *
void *cancel_baton,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
const char *temp_dir;
int max_revs = MIN(ffd->max_files_per_dir, max_items);
@@ -357,12 +357,6 @@ static svn_error_t *
close_pack_context(pack_context_t *context,
apr_pool_t *pool)
{
- const char *l2p_index_path
- = apr_pstrcat(pool, context->pack_file_path, PATH_EXT_L2P_INDEX,
- SVN_VA_NULL);
- const char *p2l_index_path
- = apr_pstrcat(pool, context->pack_file_path, PATH_EXT_P2L_INDEX,
- SVN_VA_NULL);
const char *proto_l2p_index_path;
const char *proto_p2l_index_path;
@@ -371,18 +365,17 @@ close_pack_context(pack_context_t *conte
context->proto_l2p_index, pool));
SVN_ERR(svn_io_file_name_get(&proto_p2l_index_path,
context->proto_p2l_index, pool));
-
+
/* finalize proto index files */
SVN_ERR(svn_io_file_close(context->proto_l2p_index, pool));
SVN_ERR(svn_io_file_close(context->proto_p2l_index, pool));
- /* Create the actual index files*/
- SVN_ERR(svn_fs_x__l2p_index_create(context->fs, l2p_index_path,
- proto_l2p_index_path,
- context->shard_rev, pool));
- SVN_ERR(svn_fs_x__p2l_index_create(context->fs, p2l_index_path,
- proto_p2l_index_path,
- context->shard_rev, pool));
+ /* Append the actual index data to the pack file. */
+ SVN_ERR(svn_fs_x__add_index_data(context->fs, context->pack_file,
+ proto_l2p_index_path,
+ proto_p2l_index_path,
+ context->shard_rev,
+ pool));
/* remove proto index files */
SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, pool));
@@ -421,7 +414,7 @@ copy_file_data(pack_context_t *context,
/* use streaming copies for larger data blocks. That may require
* the allocation of larger buffers and we should make sure that
* this extra memory is released asap. */
- fs_x_data_t *ffd = context->fs->fsap_data;
+ svn_fs_x__data_t *ffd = context->fs->fsap_data;
apr_pool_t *copypool = svn_pool_create(pool);
char *buffer = apr_palloc(copypool, ffd->block_size);
@@ -476,17 +469,18 @@ static svn_error_t *
copy_item_to_temp(pack_context_t *context,
apr_array_header_t *entries,
apr_file_t *temp_file,
- apr_file_t *rev_file,
+ svn_fs_x__revision_file_t *rev_file,
svn_fs_x__p2l_entry_t *entry,
apr_pool_t *pool)
{
svn_fs_x__p2l_entry_t *new_entry
= svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- new_entry->offset = 0;
- SVN_ERR(svn_io_file_seek(temp_file, SEEK_CUR, &new_entry->offset, pool));
+
+ SVN_ERR(svn_fs_x__get_file_offset(&new_entry->offset, temp_file, pool));
APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry;
- SVN_ERR(copy_file_data(context, temp_file, rev_file, entry->size, pool));
+ SVN_ERR(copy_file_data(context, temp_file, rev_file->file, entry->size,
+ pool));
return SVN_NO_ERROR;
}
@@ -536,7 +530,7 @@ add_item_rep_mapping(pack_context_t *con
*/
static svn_fs_x__p2l_entry_t *
get_item(pack_context_t *context,
- const svn_fs_x__id_part_t *id,
+ const svn_fs_x__id_t *id,
svn_boolean_t reset)
{
svn_fs_x__p2l_entry_t *result = NULL;
@@ -561,26 +555,22 @@ get_item(pack_context_t *context,
*/
static svn_error_t *
copy_rep_to_temp(pack_context_t *context,
- apr_file_t *rev_file,
+ svn_fs_x__revision_file_t *rev_file,
svn_fs_x__p2l_entry_t *entry,
apr_pool_t *pool)
{
svn_fs_x__rep_header_t *rep_header;
- svn_stream_t *stream;
apr_off_t source_offset = entry->offset;
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- entry->offset = 0;
- SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, &entry->offset,
- pool));
+ SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file, pool));
add_item_rep_mapping(context, entry);
/* read & parse the representation header */
- stream = svn_stream_from_aprfile2(rev_file, TRUE, pool);
- SVN_ERR(svn_fs_x__read_rep_header(&rep_header, stream, pool));
- svn_stream_close(stream);
+ SVN_ERR(svn_fs_x__read_rep_header(&rep_header, rev_file->stream, pool,
+ pool));
/* if the representation is a delta against some other rep, link the two */
if ( rep_header->type == svn_fs_x__rep_delta
@@ -596,9 +586,9 @@ copy_rep_to_temp(pack_context_t *context
}
/* copy the whole rep (including header!) to our temp file */
- SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool));
- SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size,
- pool));
+ SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset, pool));
+ SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file,
+ entry->size, pool));
return SVN_NO_ERROR;
}
@@ -679,34 +669,30 @@ tweak_path_for_ordering(const char *orig
*/
static svn_error_t *
copy_node_to_temp(pack_context_t *context,
- apr_file_t *rev_file,
+ svn_fs_x__revision_file_t *rev_file,
svn_fs_x__p2l_entry_t *entry,
apr_pool_t *pool)
{
path_order_t *path_order = apr_pcalloc(context->info_pool,
sizeof(*path_order));
- node_revision_t *noderev;
+ svn_fs_x__noderev_t *noderev;
const char *sort_path;
- svn_stream_t *stream;
apr_off_t source_offset = entry->offset;
/* read & parse noderev */
- stream = svn_stream_from_aprfile2(rev_file, TRUE, pool);
- SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, pool));
- svn_stream_close(stream);
+ SVN_ERR(svn_fs_x__read_noderev(&noderev, rev_file->stream, pool, pool));
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- entry->offset = 0;
- SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR,
- &entry->offset, pool));
+ SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
+ pool));
add_item_rep_mapping(context, entry);
/* copy the noderev to our temp file */
- SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool));
- SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size,
- pool));
+ SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset, pool));
+ SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file,
+ entry->size, pool));
/* if the node has a data representation, make that the node's "base".
* This will (often) cause the noderev to be placed right in front of
@@ -731,10 +717,10 @@ copy_node_to_temp(pack_context_t *contex
* It will not be stored in the final pack file. */
sort_path = tweak_path_for_ordering(noderev->created_path, pool);
path_order->path = svn_prefix_string__create(context->paths, sort_path);
- path_order->node_id = *svn_fs_x__id_node_id(noderev->id);
- path_order->revision = svn_fs_x__id_rev(noderev->id);
+ path_order->node_id = noderev->node_id;
+ path_order->revision = svn_fs_x__get_revnum(noderev->noderev_id.change_set);
path_order->is_dir = noderev->kind == svn_node_dir;
- path_order->noderev_id = *svn_fs_x__id_noderev_id(noderev->id);
+ path_order->noderev_id = noderev->noderev_id;
APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order;
return SVN_NO_ERROR;
@@ -768,58 +754,6 @@ sort_items(apr_array_header_t *entries)
(int (*)(const void *, const void *))compare_p2l_info);
}
-/* Decorator for svn_fs_x__p2l_entry_t that associates it with a sorted
- * variant of its ITEMS array.
- */
-typedef struct sub_item_ordered_t
-{
- /* ENTRY that got wrapped */
- svn_fs_x__p2l_entry_t *entry;
-
- /* Array of pointers into ENTRY->ITEMS, sorted by their revision member
- * _descending_ order. May be NULL if ENTRY->ITEM_COUNT < 2. */
- svn_fs_x__id_part_t **order;
-} sub_item_ordered_t;
-
-/* implements compare_fn_t. Place LHS before RHS, if the latter is younger.
- * Used to sort sub_item_ordered_t::order
- */
-static int
-compare_sub_items(const svn_fs_x__id_part_t * const * lhs,
- const svn_fs_x__id_part_t * const * rhs)
-{
- return (*lhs)->change_set < (*rhs)->change_set
- ? 1
- : ((*lhs)->change_set > (*rhs)->change_set ? -1 : 0);
-}
-
-/* implements compare_fn_t. Place LHS before RHS, if the latter belongs to
- * a newer revision.
- */
-static int
-compare_p2l_info_rev(const sub_item_ordered_t * lhs,
- const sub_item_ordered_t * rhs)
-{
- svn_fs_x__id_part_t *lhs_part;
- svn_fs_x__id_part_t *rhs_part;
-
- assert(lhs != rhs);
- if (lhs->entry->item_count == 0)
- return rhs->entry->item_count == 0 ? 0 : -1;
- if (rhs->entry->item_count == 0)
- return 1;
-
- lhs_part = lhs->order ? lhs->order[lhs->entry->item_count - 1]
- : &lhs->entry->items[0];
- rhs_part = rhs->order ? rhs->order[rhs->entry->item_count - 1]
- : &rhs->entry->items[0];
-
- if (lhs_part->change_set == rhs_part->change_set)
- return 0;
-
- return lhs_part->change_set < rhs_part->change_set ? -1 : 1;
-}
-
/* implements compare_fn_t. Sort descending by PATH, NODE_ID and REVISION.
*/
static int
@@ -840,7 +774,7 @@ compare_path_order(const path_order_t *
return diff;
/* reverse order on node (i.e. latest first) */
- diff = svn_fs_x__id_part_compare(&rhs->node_id, &lhs->node_id);
+ diff = svn_fs_x__id_compare(&rhs->node_id, &lhs->node_id);
if (diff)
return diff;
@@ -860,8 +794,8 @@ compare_references(const reference_t * c
const reference_t * lhs = *lhs_p;
const reference_t * rhs = *rhs_p;
- int diff = svn_fs_x__id_part_compare(&lhs->to, &rhs->to);
- return diff ? diff : svn_fs_x__id_part_compare(&lhs->from, &rhs->from);
+ int diff = svn_fs_x__id_compare(&lhs->to, &rhs->to);
+ return diff ? diff : svn_fs_x__id_compare(&lhs->from, &rhs->from);
}
/* Order the data collected in CONTEXT such that we can place them in the
@@ -882,7 +816,7 @@ sort_reps(pack_context_t *context)
static apr_ssize_t
get_block_left(pack_context_t *context)
{
- fs_x_data_t *ffd = context->fs->fsap_data;
+ svn_fs_x__data_t *ffd = context->fs->fsap_data;
return ffd->block_size - (context->pack_offset % ffd->block_size);
}
@@ -895,7 +829,7 @@ static svn_error_t *
auto_pad_block(pack_context_t *context,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = context->fs->fsap_data;
+ svn_fs_x__data_t *ffd = context->fs->fsap_data;
/* This is the maximum number of bytes "wasted" that way per block.
* Larger items will cross the block boundaries. */
@@ -944,7 +878,7 @@ find_first_reference(pack_context_t *con
reference_t *reference
= APR_ARRAY_IDX(context->references, current, reference_t *);
- if (svn_fs_x__id_part_compare(&reference->to, item->items) < 0)
+ if (svn_fs_x__id_compare(&reference->to, item->items) < 0)
lower = current + 1;
else
upper = current - 1;
@@ -965,7 +899,7 @@ is_reference_match(pack_context_t *conte
return FALSE;
reference = APR_ARRAY_IDX(context->references, idx, reference_t *);
- return svn_fs_x__id_part_eq(&reference->to, item->items);
+ return svn_fs_x__id_eq(&reference->to, item->items);
}
/* Starting at IDX in CONTEXT->PATH_ORDER, select all representations and
@@ -1002,8 +936,7 @@ select_reps(pack_context_t *context,
path_order_t *current_path
= APR_ARRAY_IDX(path_order, idx, path_order_t *);
- if (!svn_fs_x__id_part_eq(&start_path->node_id,
- ¤t_path->node_id))
+ if (!svn_fs_x__id_eq(&start_path->node_id, ¤t_path->node_id))
break;
APR_ARRAY_IDX(path_order, idx, path_order_t *) = NULL;
@@ -1102,7 +1035,7 @@ write_nodes_container(pack_context_t *co
container_entry->type = SVN_FS_X__ITEM_TYPE_NODEREVS_CONT;
container_entry->item_count = items->nelts;
container_entry->items = apr_palloc(context->info_pool,
- sizeof(svn_fs_x__id_part_t) * container_entry->item_count);
+ sizeof(svn_fs_x__id_t) * container_entry->item_count);
for (i = 0; i < items->nelts; ++i)
container_entry->items[i]
@@ -1157,7 +1090,7 @@ store_nodes(pack_context_t *context,
apr_size_t pack_savings = 0;
for (i = 0; i < node_parts->nelts; ++i)
{
- node_revision_t *noderev;
+ svn_fs_x__noderev_t *noderev;
svn_fs_x__p2l_entry_t *entry
= APR_ARRAY_IDX(node_parts, i, svn_fs_x__p2l_entry_t *);
@@ -1207,7 +1140,7 @@ store_nodes(pack_context_t *context,
/* item will fit into the block. */
SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool));
- SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, iterpool));
+ SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, iterpool, iterpool));
svn_fs_x__noderevs_add(*container, noderev);
container_size += entry->size;
@@ -1245,13 +1178,13 @@ write_reps_container(pack_context_t *con
SVN_ERR(svn_fs_x__write_reps_container(pack_stream, container, pool));
SVN_ERR(svn_stream_close(pack_stream));
- SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool));
+ SVN_ERR(svn_io_file_seek(context->pack_file, APR_CUR, &offset, pool));
container_entry.offset = context->pack_offset;
container_entry.size = offset - container_entry.offset;
container_entry.type = SVN_FS_X__ITEM_TYPE_REPS_CONT;
container_entry.item_count = sub_items->nelts;
- container_entry.items = (svn_fs_x__id_part_t *)sub_items->elts;
+ container_entry.items = (svn_fs_x__id_t *)sub_items->elts;
context->pack_offset = offset;
APR_ARRAY_PUSH(new_entries, svn_fs_x__p2l_entry_t *)
@@ -1283,14 +1216,15 @@ write_reps_containers(pack_context_t *co
svn_fs_x__reps_builder_t *container
= svn_fs_x__reps_builder_create(context->fs, container_pool);
apr_array_header_t *sub_items
- = apr_array_make(pool, 64, sizeof(svn_fs_x__id_part_t));
- svn_stream_t *temp_stream
- = svn_stream_from_aprfile2(temp_file, TRUE, pool);
+ = apr_array_make(pool, 64, sizeof(svn_fs_x__id_t));
+ svn_fs_x__revision_file_t *file;
+
+ SVN_ERR(svn_fs_x__wrap_temp_rev_file(&file, context->fs, temp_file, pool));
/* copy all items in strict order */
for (i = entries->nelts-1; i >= 0; --i)
{
- representation_t representation = { 0 };
+ svn_fs_x__representation_t representation = { 0 };
svn_stringbuf_t *contents;
svn_stream_t *stream;
apr_size_t list_index;
@@ -1327,12 +1261,12 @@ write_reps_containers(pack_context_t *co
/* select the change list in the source file, parse it and add it to
* the container */
- SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset,
+ SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset,
iterpool));
SVN_ERR(svn_fs_x__get_representation_length(&representation.size,
&representation.expanded_size,
- context->fs, temp_file,
- temp_stream, entry, iterpool));
+ context->fs, file,
+ entry, iterpool));
SVN_ERR(svn_fs_x__get_contents(&stream, context->fs, &representation,
FALSE, iterpool));
contents = svn_stringbuf_create_ensure(representation.expanded_size,
@@ -1348,7 +1282,7 @@ write_reps_containers(pack_context_t *co
SVN_ERR_ASSERT(list_index == sub_items->nelts);
block_left -= entry->size;
- APR_ARRAY_PUSH(sub_items, svn_fs_x__id_part_t) = entry->items[0];
+ APR_ARRAY_PUSH(sub_items, svn_fs_x__id_t) = entry->items[0];
svn_pool_clear(iterpool);
}
@@ -1414,7 +1348,7 @@ store_items(pack_context_t *context,
/* select the item in the source file and copy it into the target
* pack file */
- SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset,
+ SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset,
iterpool));
SVN_ERR(copy_file_data(context, context->pack_file, temp_file,
entry->size, iterpool));
@@ -1444,7 +1378,7 @@ copy_reps_from_temp(pack_context_t *cont
apr_file_t *temp_file,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = context->fs->fsap_data;
+ svn_fs_x__data_t *ffd = context->fs->fsap_data;
apr_pool_t *iterpool = svn_pool_create(pool);
apr_pool_t *container_pool = svn_pool_create(pool);
@@ -1558,13 +1492,13 @@ write_changes_container(pack_context_t *
container,
pool));
SVN_ERR(svn_stream_close(pack_stream));
- SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool));
+ SVN_ERR(svn_io_file_seek(context->pack_file, APR_CUR, &offset, pool));
container_entry.offset = context->pack_offset;
container_entry.size = offset - container_entry.offset;
container_entry.type = SVN_FS_X__ITEM_TYPE_CHANGES_CONT;
container_entry.item_count = sub_items->nelts;
- container_entry.items = (svn_fs_x__id_part_t *)sub_items->elts;
+ container_entry.items = (svn_fs_x__id_t *)sub_items->elts;
context->pack_offset = offset;
APR_ARRAY_PUSH(new_entries, svn_fs_x__p2l_entry_t *)
@@ -1596,7 +1530,7 @@ write_changes_containers(pack_context_t
svn_fs_x__changes_t *container
= svn_fs_x__changes_create(1000, container_pool);
apr_array_header_t *sub_items
- = apr_array_make(pool, 64, sizeof(svn_fs_x__id_part_t));
+ = apr_array_make(pool, 64, sizeof(svn_fs_x__id_t));
apr_array_header_t *new_entries
= apr_array_make(context->info_pool, 16, entries->elt_size);
svn_stream_t *temp_stream
@@ -1654,15 +1588,15 @@ write_changes_containers(pack_context_t
/* select the change list in the source file, parse it and add it to
* the container */
- SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset,
+ SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset,
iterpool));
- SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, iterpool));
+ SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, pool, iterpool));
SVN_ERR(svn_fs_x__changes_append_list(&list_index, container, changes));
SVN_ERR_ASSERT(list_index == sub_items->nelts);
block_left -= estimated_size;
estimated_addition += estimated_size;
- APR_ARRAY_PUSH(sub_items, svn_fs_x__id_part_t) = entry->items[0];
+ APR_ARRAY_PUSH(sub_items, svn_fs_x__id_t) = entry->items[0];
svn_pool_clear(iterpool);
}
@@ -1720,13 +1654,10 @@ static svn_error_t *
write_l2p_index(pack_context_t *context,
apr_pool_t *pool)
{
- apr_pool_t *iterpool = svn_pool_create(pool);
- svn_revnum_t prev_rev = SVN_INVALID_REVNUM;
- int i;
- apr_uint32_t k;
- svn_priority_queue__t *queue;
- apr_size_t count = 0;
- apr_array_header_t *sub_item_orders;
+ apr_pool_t *scratch_pool = svn_pool_create(pool);
+ const char *temp_name;
+ const char *proto_index;
+ apr_off_t offset = 0;
/* lump all items into one bucket. As target, use the bucket that
* probably has the most entries already. */
@@ -1734,88 +1665,24 @@ write_l2p_index(pack_context_t *context,
append_entries(context->reps, context->file_props);
append_entries(context->reps, context->dir_props);
- /* wrap P2L entries such that we have access to the sub-items in revision
- order. The ENTRY_COUNT member will point to the next item to read+1. */
- sub_item_orders
- = apr_array_make(pool, context->reps->nelts, sizeof(sub_item_ordered_t));
- sub_item_orders->nelts = context->reps->nelts;
-
- for (i = 0; i < context->reps->nelts; ++i)
- {
- svn_fs_x__p2l_entry_t *entry
- = APR_ARRAY_IDX(context->reps, i, svn_fs_x__p2l_entry_t *);
- sub_item_ordered_t *ordered
- = &APR_ARRAY_IDX(sub_item_orders, i, sub_item_ordered_t);
-
- /* skip unused regions (e.g. padding) */
- if (entry->item_count == 0)
- continue;
-
- assert(entry);
- ordered->entry = entry;
- count += entry->item_count;
-
- if (entry->item_count > 1)
- {
- ordered->order
- = apr_palloc(pool, sizeof(*ordered->order) * entry->item_count);
- for (k = 0; k < entry->item_count; ++k)
- ordered->order[k] = &entry->items[k];
-
- qsort(ordered->order, entry->item_count, sizeof(*ordered->order),
- (int (*)(const void *, const void *))compare_sub_items);
- }
- }
-
- /* we need to write the index in ascending revision order */
- queue = svn_priority_queue__create
- (sub_item_orders,
- (int (*)(const void *, const void *))compare_p2l_info_rev);
-
- /* write index entries */
- for (i = 0; i < count; ++i)
- {
- svn_fs_x__id_part_t *sub_item;
- sub_item_ordered_t *ordered = svn_priority_queue__peek(queue);
-
- if (ordered->entry->item_count > 0)
- {
- /* if there is only one item, we skip the overhead of having an
- extra array for the item order */
- sub_item = ordered->order
- ? ordered->order[ordered->entry->item_count - 1]
- : &ordered->entry->items[0];
-
- /* next revision? */
- if (prev_rev != svn_fs_x__get_revnum(sub_item->change_set))
- {
- prev_rev = svn_fs_x__get_revnum(sub_item->change_set);
- SVN_ERR(svn_fs_x__l2p_proto_index_add_revision
- (context->proto_l2p_index, iterpool));
- }
-
- /* add entry */
- SVN_ERR(svn_fs_x__l2p_proto_index_add_entry
- (context->proto_l2p_index, ordered->entry->offset,
- (apr_uint32_t)(sub_item - ordered->entry->items),
- sub_item->number, iterpool));
-
- /* make ITEM_COUNT point the next sub-item to use+1 */
- --ordered->entry->item_count;
- }
-
- /* process remaining sub-items (if any) of that container later */
- if (ordered->entry->item_count)
- svn_priority_queue__update(queue);
- else
- svn_priority_queue__pop(queue);
-
- /* keep memory usage in check */
- if (i % 256 == 0)
- svn_pool_clear(iterpool);
- }
+ /* Let the index code do the expensive L2P -> P2L transformation. */
+ SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&temp_name,
+ context->fs,
+ context->reps,
+ pool, scratch_pool));
+
+ /* Append newly written segment to exisiting proto index file. */
+ SVN_ERR(svn_io_file_name_get(&proto_index, context->proto_l2p_index,
+ scratch_pool));
+
+ SVN_ERR(svn_io_file_flush(context->proto_l2p_index, scratch_pool));
+ SVN_ERR(svn_io_append_file(temp_name, proto_index, scratch_pool));
+ SVN_ERR(svn_io_remove_file2(temp_name, FALSE, scratch_pool));
+ SVN_ERR(svn_io_file_seek(context->proto_l2p_index, APR_END, &offset,
+ scratch_pool));
- svn_pool_destroy(iterpool);
+ /* Done. */
+ svn_pool_destroy(scratch_pool);
return SVN_NO_ERROR;
}
@@ -1827,7 +1694,7 @@ static svn_error_t *
pack_range(pack_context_t *context,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = context->fs->fsap_data;
+ svn_fs_x__data_t *ffd = context->fs->fsap_data;
apr_pool_t *revpool = svn_pool_create(pool);
apr_pool_t *iterpool = svn_pool_create(pool);
@@ -1836,26 +1703,19 @@ pack_range(pack_context_t *context,
for (revision = context->start_rev; revision < context->end_rev; ++revision)
{
apr_off_t offset = 0;
- apr_finfo_t finfo;
- apr_file_t *rev_file;
+ svn_fs_x__revision_file_t *rev_file;
- /* Get the size of the file. */
- const char *path = svn_dirent_join(context->shard_dir,
- apr_psprintf(revpool, "%ld",
- revision),
- revpool);
- SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, revpool));
-
- SVN_ERR(svn_io_file_open(&rev_file, path,
- APR_READ | APR_BUFFERED | APR_BINARY,
- APR_OS_DEFAULT, revpool));
+ /* Get the rev file dimensions (mainly index locations). */
+ SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs,
+ revision, revpool, iterpool));
+ SVN_ERR(svn_fs_x__auto_read_footer(rev_file));
/* store the indirect array index */
APR_ARRAY_PUSH(context->rev_offsets, int) = context->reps->nelts;
/* read the phys-to-log index file until we covered the whole rev file.
* That index contains enough info to build both target indexes from it. */
- while (offset < finfo.size)
+ while (offset < rev_file->l2p_offset)
{
/* read one cluster */
int i;
@@ -1863,8 +1723,9 @@ pack_range(pack_context_t *context,
svn_pool_clear(iterpool);
SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs,
- revision, offset,
- ffd->p2l_page_size, iterpool));
+ rev_file, revision, offset,
+ ffd->p2l_page_size, iterpool,
+ iterpool));
for (i = 0; i < entries->nelts; ++i)
{
@@ -1878,9 +1739,9 @@ pack_range(pack_context_t *context,
/* process entry while inside the rev file */
offset = entry->offset;
- if (offset < finfo.size)
+ if (offset < rev_file->l2p_offset)
{
- SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &offset,
+ SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &offset,
iterpool));
if (entry->type == SVN_FS_X__ITEM_TYPE_CHANGES)
@@ -1959,10 +1820,10 @@ static svn_error_t *
append_revision(pack_context_t *context,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = context->fs->fsap_data;
+ svn_fs_x__data_t *ffd = context->fs->fsap_data;
apr_off_t offset = 0;
apr_pool_t *iterpool = svn_pool_create(pool);
- apr_file_t *rev_file;
+ svn_fs_x__revision_file_t *rev_file;
apr_finfo_t finfo;
/* Get the size of the file. */
@@ -1973,11 +1834,11 @@ append_revision(pack_context_t *context,
SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, pool));
/* Copy all the bits from the rev file to the end of the pack file. */
- SVN_ERR(svn_io_file_open(&rev_file, path,
- APR_READ | APR_BUFFERED | APR_BINARY,
- APR_OS_DEFAULT, pool));
- SVN_ERR(copy_file_data(context, context->pack_file, rev_file, finfo.size,
- iterpool));
+ SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs,
+ context->start_rev, pool,
+ iterpool));
+ SVN_ERR(copy_file_data(context, context->pack_file, rev_file->file,
+ finfo.size, iterpool));
/* mark the start of a new revision */
SVN_ERR(svn_fs_x__l2p_proto_index_add_revision(context->proto_l2p_index,
@@ -1990,9 +1851,10 @@ append_revision(pack_context_t *context,
/* read one cluster */
int i;
apr_array_header_t *entries;
- SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs,
- context->start_rev, offset,
- ffd->p2l_page_size, iterpool));
+ SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs, rev_file,
+ context->start_rev, offset,
+ ffd->p2l_page_size, iterpool,
+ iterpool));
for (i = 0; i < entries->nelts; ++i)
{
@@ -2075,7 +1937,7 @@ pack_log_addressed(svn_fs_t *fs,
/* phase 1: determine the size of the revisions to pack */
SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, shard_rev,
context.shard_end_rev - shard_rev,
- pool));
+ pool, pool));
/* pack revisions in ranges that don't exceed MAX_MEM */
for (i = 0; i < max_ids->nelts; ++i)
@@ -2130,7 +1992,7 @@ svn_fs_x__get_packed_offset(apr_off_t *r
svn_revnum_t rev,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_stream_t *manifest_stream;
svn_boolean_t is_cached;
svn_revnum_t shard;
@@ -2256,7 +2118,7 @@ pack_shard(const char *revs_dir,
void *cancel_baton,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
const char *rev_shard_path, *rev_pack_file_dir;
const char *revprops_shard_path, *revprops_pack_file_dir;
@@ -2372,7 +2234,7 @@ pack_body(void *baton,
apr_pool_t *pool)
{
struct pack_baton *pb = baton;
- fs_x_data_t *ffd = pb->fs->fsap_data;
+ svn_fs_x__data_t *ffd = pb->fs->fsap_data;
apr_int64_t completed_shards;
apr_int64_t i;
svn_revnum_t youngest;
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c Sat Jan 3 14:00:41 2015
@@ -55,10 +55,11 @@ recover_get_largest_revision(svn_fs_t *f
while (1)
{
svn_error_t *err;
- apr_file_t *file;
+ svn_fs_x__revision_file_t *file;
svn_pool_clear(iterpool);
- err = svn_fs_x__open_pack_or_rev_file(&file, fs, right, iterpool);
+ err = svn_fs_x__open_pack_or_rev_file(&file, fs, right, iterpool,
+ iterpool);
if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION)
{
svn_error_clear(err);
@@ -78,10 +79,11 @@ recover_get_largest_revision(svn_fs_t *f
{
svn_revnum_t probe = left + ((right - left) / 2);
svn_error_t *err;
- apr_file_t *file;
+ svn_fs_x__revision_file_t *file;
svn_pool_clear(iterpool);
- err = svn_fs_x__open_pack_or_rev_file(&file, fs, probe, iterpool);
+ err = svn_fs_x__open_pack_or_rev_file(&file, fs, probe, iterpool,
+ iterpool);
if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION)
{
svn_error_clear(err);
@@ -116,13 +118,18 @@ recover_body(void *baton, apr_pool_t *po
{
struct recover_baton *b = baton;
svn_fs_t *fs = b->fs;
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_revnum_t max_rev;
svn_revnum_t youngest_rev;
svn_node_kind_t youngest_revprops_kind;
/* Lose potentially corrupted data in temp files */
- SVN_ERR(svn_fs_x__cleanup_revprop_namespace(fs));
+ SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, pool));
+
+ /* The admin may have created a plain copy of this repo before attempting
+ to recover it (hotcopy may or may not work with corrupted repos).
+ Bump the instance ID. */
+ SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, pool));
/* We need to know the largest revision in the filesystem. */
SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool));
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c Sat Jan 3 14:00:41 2015
@@ -61,7 +61,7 @@ open_rep_cache(void *baton,
apr_pool_t *pool)
{
svn_fs_t *fs = baton;
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_sqlite__db_t *sdb;
const char *db_path;
int version;
@@ -118,7 +118,7 @@ svn_error_t *
svn_fs_x__open_rep_cache(svn_fs_t *fs,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_error_t *err = svn_atomic__init_once(&ffd->rep_cache_db_opened,
open_rep_cache, fs, pool);
return svn_error_quick_wrap(err, _("Couldn't open rep-cache database"));
@@ -141,7 +141,7 @@ svn_error_t *
svn_fs_x__walk_rep_reference(svn_fs_t *fs,
svn_revnum_t start,
svn_revnum_t end,
- svn_error_t *(*walker)(representation_t *,
+ svn_error_t *(*walker)(svn_fs_x__representation_t *,
void *,
svn_fs_t *,
apr_pool_t *),
@@ -150,7 +150,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f
void *cancel_baton,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_sqlite__stmt_t *stmt;
svn_boolean_t have_row;
int iterations = 0;
@@ -183,7 +183,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f
SVN_ERR(svn_sqlite__step(&have_row, stmt));
while (have_row)
{
- representation_t *rep;
+ svn_fs_x__representation_t *rep;
const char *sha1_digest;
svn_error_t *err;
svn_checksum_t *checksum;
@@ -200,7 +200,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f
return svn_error_compose_create(err, svn_sqlite__reset(stmt));
}
- /* Construct a representation_t. */
+ /* Construct a svn_fs_x__representation_t. */
rep = apr_pcalloc(iterpool, sizeof(*rep));
sha1_digest = svn_sqlite__column_text(stmt, 0, iterpool);
err = svn_checksum_parse_hex(&checksum, svn_checksum_sha1,
@@ -234,12 +234,12 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f
If you extend this function, check the callsite to see if you have
to make it not-ignore additional error codes. */
svn_error_t *
-svn_fs_x__get_rep_reference(representation_t **rep,
+svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep,
svn_fs_t *fs,
svn_checksum_t *checksum,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_sqlite__stmt_t *stmt;
svn_boolean_t have_row;
@@ -291,10 +291,10 @@ svn_fs_x__get_rep_reference(representati
svn_error_t *
svn_fs_x__set_rep_reference(svn_fs_t *fs,
- representation_t *rep,
+ svn_fs_x__representation_t *rep,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_sqlite__stmt_t *stmt;
svn_error_t *err;
svn_checksum_t checksum;
@@ -322,7 +322,7 @@ svn_fs_x__set_rep_reference(svn_fs_t *fs
err = svn_sqlite__insert(NULL, stmt);
if (err)
{
- representation_t *old_rep;
+ svn_fs_x__representation_t *old_rep;
if (err->apr_err != SVN_ERR_SQLITE_CONSTRAINT)
return svn_error_trace(err);
@@ -351,7 +351,7 @@ svn_fs_x__del_rep_reference(svn_fs_t *fs
svn_revnum_t youngest,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_sqlite__stmt_t *stmt;
if (! ffd->rep_cache_db)
@@ -369,7 +369,7 @@ svn_error_t *
svn_fs_x__lock_rep_cache(svn_fs_t *fs,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
if (! ffd->rep_cache_db)
SVN_ERR(svn_fs_x__open_rep_cache(fs, pool));
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h Sat Jan 3 14:00:41 2015
@@ -51,7 +51,7 @@ svn_error_t *
svn_fs_x__walk_rep_reference(svn_fs_t *fs,
svn_revnum_t start,
svn_revnum_t end,
- svn_error_t *(*walker)(representation_t *rep,
+ svn_error_t *(*walker)(svn_fs_x__representation_t *rep,
void *walker_baton,
svn_fs_t *fs,
apr_pool_t *scratch_pool),
@@ -65,7 +65,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f
opened, just set *REP to NULL. Returns SVN_ERR_FS_CORRUPT if
a reference beyond HEAD is detected. */
svn_error_t *
-svn_fs_x__get_rep_reference(representation_t **rep,
+svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep,
svn_fs_t *fs,
svn_checksum_t *checksum,
apr_pool_t *pool);
@@ -77,7 +77,7 @@ svn_fs_x__get_rep_reference(representati
If the rep cache database has not been opened, this may be a no op. */
svn_error_t *
svn_fs_x__set_rep_reference(svn_fs_t *fs,
- representation_t *rep,
+ svn_fs_x__representation_t *rep,
apr_pool_t *pool);
/* Delete from the cache all reps corresponding to revisions younger
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c Sat Jan 3 14:00:41 2015
@@ -401,7 +401,7 @@ svn_fs_x__reps_builder_create(svn_fs_t *
svn_error_t *
svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder,
- representation_t *rep,
+ svn_fs_x__representation_t *rep,
int priority,
apr_pool_t *scratch_pool)
{
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h Sat Jan 3 14:00:41 2015
@@ -91,7 +91,7 @@ svn_fs_x__reps_builder_create(svn_fs_t *
*/
svn_error_t *
svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder,
- representation_t *rep,
+ svn_fs_x__representation_t *rep,
int priority,
apr_pool_t *scratch_pool);
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c Sat Jan 3 14:00:41 2015
@@ -21,6 +21,7 @@
*/
#include <assert.h>
+#include <apr_md5.h>
#include "svn_pools.h"
#include "svn_hash.h"
@@ -42,11 +43,15 @@
process got aborted and that we have re-read revprops. */
#define REVPROP_CHANGE_TIMEOUT (10 * 1000000)
-/* The following are names of atomics that will be used to communicate
- * revprop updates across all processes on this machine. */
-#define ATOMIC_REVPROP_GENERATION "rev-prop-generation"
-#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout"
-#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics"
+/* In case of an inconsistent read, close the generation file, yield,
+ re-open and re-read. This is the number of times we try this before
+ giving up. */
+#define GENERATION_READ_RETRY_COUNT 100
+
+/* Maximum size of the generation number file contents (including NUL). */
+#define CHECKSUMMED_NUMBER_BUFFER_LEN \
+ (SVN_INT64_BUFFER_SIZE + 3 + APR_MD5_DIGESTSIZE * 2)
+
svn_error_t *
svn_fs_x__upgrade_pack_revprops(svn_fs_t *fs,
@@ -56,7 +61,7 @@ svn_fs_x__upgrade_pack_revprops(svn_fs_t
void *cancel_baton,
apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
const char *revprops_shard_path;
const char *revprops_pack_file_dir;
apr_int64_t shard;
@@ -108,7 +113,7 @@ svn_fs_x__upgrade_cleanup_pack_revprops(
void *cancel_baton,
apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
const char *revprops_shard_path;
apr_int64_t shard;
apr_int64_t first_unpacked_shard
@@ -147,179 +152,251 @@ svn_fs_x__upgrade_cleanup_pack_revprops(
*
* Revprop caching needs to be activated and will be deactivated for the
* respective FS instance if the necessary infrastructure could not be
- * initialized. In deactivated mode, there is almost no runtime overhead
- * associated with revprop caching. As long as no revprops are being read
- * or changed, revprop caching imposes no overhead.
+ * initialized. As long as no revprops are being read or changed, revprop
+ * caching imposes no overhead.
*
* When activated, we cache revprops using (revision, generation) pairs
* as keys with the generation being incremented upon every revprop change.
* Since the cache is process-local, the generation needs to be tracked
* for at least as long as the process lives but may be reset afterwards.
*
- * To track the revprop generation, we use two-layer approach. On the lower
- * level, we use named atomics to have a system-wide consistent value for
- * the current revprop generation. However, those named atomics will only
- * remain valid for as long as at least one process / thread in the system
- * accesses revprops in the respective repository. The underlying shared
- * memory gets cleaned up afterwards.
- *
- * On the second level, we will use a persistent file to track the latest
- * revprop generation. It will be written upon each revprop change but
- * only be read if we are the first process to initialize the named atomics
- * with that value.
- *
- * The overhead for the second and following accesses to revprops is
- * almost zero on most systems.
- *
- *
- * Tech aspects:
- * -------------
- *
- * A problem is that we need to provide a globally available file name to
- * back the SHM implementation on OSes that need it. We can only assume
- * write access to some file within the respective repositories. Because
- * a given server process may access thousands of repositories during its
- * lifetime, keeping the SHM data alive for all of them is also not an
- * option.
- *
- * So, we store the new revprop generation on disk as part of each
- * setrevprop call, i.e. this write will be serialized and the write order
- * be guaranteed by the repository write lock.
- *
- * The only racy situation occurs when the data is being read again by two
- * processes concurrently but in that situation, the first process to
- * finish that procedure is guaranteed to be the only one that initializes
- * the SHM data. Since even writers will first go through that
- * initialization phase, they will never operate on stale data.
+ * We track the revprop generation in a persistent, unbuffered file that
+ * we may keep open for the lifetime of the svn_fs_t. It is the OS'
+ * responsibility to provide us with the latest contents upon read. To
+ * detect incomplete updates due to non-atomic reads, we put a MD5 checksum
+ * next to the actual generation number and verify that it matches.
+ *
+ * Since we cannot guarantee that the OS will provide us with up-to-date
+ * data buffers for open files, we re-open and re-read the file before
+ * modifying it. This will prevent lost updates.
+ *
+ * A race condition exists between switching to the modified revprop data
+ * and bumping the generation number. In particular, the process may crash
+ * just after switching to the new revprop data and before bumping the
+ * generation. To be able to detect this scenario, we bump the generation
+ * twice per revprop change: once immediately before (creating an odd number)
+ * and once after the atomic switch (even generation).
+ *
+ * A writer holding the write lock can immediately assume a crashed writer
+ * in case of an odd generation or they would not have been able to acquire
+ * the lock. A reader detecting an odd generation will use that number and
+ * be forced to re-read any revprop data - usually getting the new revprops
+ * already. If the generation file modification timestamp is too old, the
+ * reader will assume a crashed writer, acquire the write lock and bump
+ * the generation if it is still odd. So, for about REVPROP_CHANGE_TIMEOUT
+ * after the crash, reader caches may be stale.
*/
-/* Read revprop generation as stored on disk for repository FS. The result
- * is returned in *CURRENT. Default to 2 if no such file is available.
+/* If the revprop generation file in FS is open, close it. This is a no-op
+ * if the file is not open.
*/
static svn_error_t *
-read_revprop_generation_file(apr_int64_t *current,
- svn_fs_t *fs,
- apr_pool_t *pool)
+close_revprop_generation_file(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
{
- svn_error_t *err;
- apr_file_t *file;
- char buf[80];
- apr_size_t len;
- const char *path = svn_fs_x__path_revprop_generation(fs, pool);
-
- err = svn_io_file_open(&file, path,
- APR_READ | APR_BUFFERED,
- APR_OS_DEFAULT, pool);
- if (err && APR_STATUS_IS_ENOENT(err->apr_err))
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ if (ffd->revprop_generation_file)
{
- svn_error_clear(err);
- *current = 2;
-
- return SVN_NO_ERROR;
+ SVN_ERR(svn_io_file_close(ffd->revprop_generation_file, scratch_pool));
+ ffd->revprop_generation_file = NULL;
}
- SVN_ERR(err);
- len = sizeof(buf);
- SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
+ return SVN_NO_ERROR;
+}
- /* Check that the first line contains only digits. */
- SVN_ERR(svn_fs_x__check_file_buffer_numeric(buf, 0, path,
- "Revprop Generation", pool));
- SVN_ERR(svn_cstring_atoi64(current, buf));
+/* Make sure the revprop_generation member in FS is set. If READ_ONLY is
+ * set, open the file w/o write permission if the file is not open yet.
+ * The file is kept open if it has sufficient rights (or more) but will be
+ * closed and re-opened if it provided insufficient access rights.
+ *
+ * Call only for repos that support revprop caching.
+ */
+static svn_error_t *
+open_revprop_generation_file(svn_fs_t *fs,
+ svn_boolean_t read_only,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ apr_int32_t flags = read_only ? APR_READ : (APR_READ | APR_WRITE);
+
+ /* Close the current file handle if it has insufficient rights. */
+ if ( ffd->revprop_generation_file
+ && (apr_file_flags_get(ffd->revprop_generation_file) & flags) != flags)
+ SVN_ERR(close_revprop_generation_file(fs, scratch_pool));
+
+ /* If not open already, open with sufficient rights. */
+ if (ffd->revprop_generation_file == NULL)
+ {
+ const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool);
+ SVN_ERR(svn_io_file_open(&ffd->revprop_generation_file, path,
+ flags, APR_OS_DEFAULT, fs->pool));
+ }
- return svn_io_file_close(file, pool);
+ return SVN_NO_ERROR;
}
-/* Write the CURRENT revprop generation to disk for repository FS.
+/* Return the textual representation of NUMBER and its checksum in *BUFFER.
*/
-svn_error_t *
-svn_fs_x__write_revprop_generation_file(svn_fs_t *fs,
- apr_int64_t current,
- apr_pool_t *pool)
+static svn_error_t *
+checkedsummed_number(svn_stringbuf_t **buffer,
+ apr_int64_t number,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool)
{
- char buf[SVN_INT64_BUFFER_SIZE];
- apr_size_t len = svn__i64toa(buf, current);
- buf[len] = '\n';
+ svn_checksum_t *checksum;
+ const char *digest;
+
+ char str[SVN_INT64_BUFFER_SIZE];
+ apr_size_t len = svn__i64toa(str, number);
+ str[len] = 0;
- SVN_ERR(svn_io_write_atomic(svn_fs_x__path_revprop_generation(fs, pool),
- buf, len + 1,
- NULL /* copy_perms */, pool));
+ SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, str, len, scratch_pool));
+ digest = svn_checksum_to_cstring_display(checksum, scratch_pool);
+
+ *buffer = svn_stringbuf_createf(result_pool, "%s %s\n", digest, str);
return SVN_NO_ERROR;
}
-/* Make sure the revprop_namespace member in FS is set. */
+/* Extract the generation number from the text BUFFER of LEN bytes and
+ * verify it against the checksum in the same BUFFER. If they match, return
+ * the generation in *NUMBER. Otherwise, return an error.
+ * BUFFER does not need to be NUL-terminated.
+ */
static svn_error_t *
-ensure_revprop_namespace(svn_fs_t *fs)
+verify_extract_number(apr_int64_t *number,
+ const char *buffer,
+ apr_size_t len,
+ apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ const char *digest_end = strchr(buffer, ' ');
- return ffd->revprop_namespace == NULL
- ? svn_atomic_namespace__create(&ffd->revprop_namespace,
- svn_dirent_join(fs->path,
- ATOMIC_REVPROP_NAMESPACE,
- fs->pool),
- fs->pool)
- : SVN_NO_ERROR;
-}
+ /* Does the buffer even contain checksum _and_ number? */
+ if (digest_end != NULL)
+ {
+ svn_checksum_t *expected;
+ svn_checksum_t *actual;
-svn_error_t *
-svn_fs_x__cleanup_revprop_namespace(svn_fs_t *fs)
-{
- const char *name = svn_dirent_join(fs->path,
- ATOMIC_REVPROP_NAMESPACE,
- fs->pool);
- return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool));
+ SVN_ERR(svn_checksum_parse_hex(&expected, svn_checksum_md5, buffer,
+ scratch_pool));
+ SVN_ERR(svn_checksum(&actual, svn_checksum_md5, digest_end + 1,
+ (buffer + len) - (digest_end + 1), scratch_pool));
+
+ if (svn_checksum_match(expected, actual))
+ return svn_error_trace(svn_cstring_atoi64(number, digest_end + 1));
+ }
+
+ /* Incomplete buffer or not a match. */
+ return svn_error_create(SVN_ERR_FS_INVALID_GENERATION, NULL,
+ _("Invalid generation number data."));
}
-/* Make sure the revprop_generation member in FS is set and, if necessary,
- * initialized with the latest value stored on disk.
+/* Read revprop generation as stored on disk for repository FS. The result is
+ * returned in *CURRENT. Call only for repos that support revprop caching.
*/
static svn_error_t *
-ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool)
+read_revprop_generation_file(apr_int64_t *current,
+ svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+ char buf[CHECKSUMMED_NUMBER_BUFFER_LEN];
+ apr_size_t len;
+ apr_off_t offset = 0;
+ int i;
+ svn_error_t *err = SVN_NO_ERROR;
- SVN_ERR(ensure_revprop_namespace(fs));
- if (ffd->revprop_generation == NULL)
+ /* Retry in case of incomplete file buffer updates. */
+ for (i = 0; i < GENERATION_READ_RETRY_COUNT; ++i)
{
- apr_int64_t current;
-
- SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation,
- ffd->revprop_namespace,
- ATOMIC_REVPROP_GENERATION,
- TRUE));
+ svn_error_clear(err);
+ svn_pool_clear(iterpool);
- /* If the generation is at 0, we just created a new namespace
- * (it would be at least 2 otherwise). Read the latest generation
- * from disk and if we are the first one to initialize the atomic
- * (i.e. is still 0), set it to the value just gotten.
+ /* If we can't even access the data, things are very wrong.
+ * Don't retry in that case.
*/
- SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation));
- if (current == 0)
- {
- SVN_ERR(read_revprop_generation_file(¤t, fs, pool));
- SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0,
- ffd->revprop_generation));
- }
+ SVN_ERR(open_revprop_generation_file(fs, TRUE, iterpool));
+ SVN_ERR(svn_io_file_seek(ffd->revprop_generation_file, APR_SET, &offset,
+ iterpool));
+
+ len = sizeof(buf);
+ SVN_ERR(svn_io_read_length_line(ffd->revprop_generation_file, buf, &len,
+ iterpool));
+
+ /* Some data has been read. It will most likely be complete and
+ * consistent. Extract and verify anyway. */
+ err = verify_extract_number(current, buf, len, iterpool);
+ if (!err)
+ break;
+
+ /* Got unlucky and data was invalid. Retry. */
+ SVN_ERR(close_revprop_generation_file(fs, iterpool));
+
+#if APR_HAS_THREADS
+ apr_thread_yield();
+#else
+ apr_sleep(0);
+#endif
}
- return SVN_NO_ERROR;
+ svn_pool_destroy(iterpool);
+
+ /* If we had to give up, propagate the error. */
+ return svn_error_trace(err);
}
-/* Make sure the revprop_timeout member in FS is set. */
+/* Write the CURRENT revprop generation to disk for repository FS.
+ * Call only for repos that support revprop caching.
+ */
static svn_error_t *
-ensure_revprop_timeout(svn_fs_t *fs)
+write_revprop_generation_file(svn_fs_t *fs,
+ apr_int64_t current,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ svn_stringbuf_t *buffer;
+ apr_off_t offset = 0;
+
+ SVN_ERR(checkedsummed_number(&buffer, current, scratch_pool, scratch_pool));
+
+ SVN_ERR(open_revprop_generation_file(fs, FALSE, scratch_pool));
+ SVN_ERR(svn_io_file_seek(ffd->revprop_generation_file, APR_SET, &offset,
+ scratch_pool));
+ SVN_ERR(svn_io_file_write_full(ffd->revprop_generation_file, buffer->data,
+ buffer->len, NULL, scratch_pool));
+ SVN_ERR(svn_io_file_flush_to_disk(ffd->revprop_generation_file,
+ scratch_pool));
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_x__reset_revprop_generation_file(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool);
+ svn_stringbuf_t *buffer;
+
+ /* Unconditionally close the revprop generation file.
+ * Don't care about FS formats. This ensures consistent internal state. */
+ SVN_ERR(close_revprop_generation_file(fs, scratch_pool));
+
+ /* Unconditionally remove any old revprop generation file.
+ * Don't care about FS formats. This ensures consistent on-disk state
+ * for old format repositories. */
+ SVN_ERR(svn_io_remove_file2(path, TRUE, scratch_pool));
+
+ /* Write the initial revprop generation file contents, if supported by
+ * the current format. This ensures consistent on-disk state for new
+ * format repositories. */
+ SVN_ERR(checkedsummed_number(&buffer, 0, scratch_pool, scratch_pool));
+ SVN_ERR(svn_io_write_atomic(path, buffer->data, buffer->len, NULL,
+ scratch_pool));
- SVN_ERR(ensure_revprop_namespace(fs));
- return ffd->revprop_timeout == NULL
- ? svn_named_atomic__get(&ffd->revprop_timeout,
- ffd->revprop_namespace,
- ATOMIC_REVPROP_TIMEOUT,
- TRUE)
- : SVN_NO_ERROR;
+ /* ffd->revprop_generation_file will be re-opened on demand. */
+
+ return SVN_NO_ERROR;
}
/* Create an error object with the given MESSAGE and pass it to the
@@ -344,32 +421,18 @@ log_revprop_cache_init_warning(svn_fs_t
/* Test whether revprop cache and necessary infrastructure are
available in FS. */
static svn_boolean_t
-has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool)
+has_revprop_cache(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_error_t *error;
/* is the cache (still) enabled? */
if (ffd->revprop_cache == NULL)
return FALSE;
- /* is it efficient? */
- if (!svn_named_atomic__is_efficient())
- {
- /* access to it would be quite slow
- * -> disable the revprop cache for good
- */
- ffd->revprop_cache = NULL;
- log_revprop_cache_init_warning(fs, NULL,
- "Revprop caching for '%s' disabled"
- " because it would be inefficient.",
- pool);
-
- return FALSE;
- }
-
- /* try to access our SHM-backed infrastructure */
- error = ensure_revprop_generation(fs, pool);
+ /* try initialize our file-backed infrastructure */
+ error = open_revprop_generation_file(fs, TRUE, scratch_pool);
if (error)
{
/* failure -> disable revprop cache for good */
@@ -377,9 +440,9 @@ has_revprop_cache(svn_fs_t *fs, apr_pool
ffd->revprop_cache = NULL;
log_revprop_cache_init_warning(fs, error,
"Revprop caching for '%s' disabled "
- "because SHM infrastructure for revprop "
+ "because infrastructure for revprop "
"caching failed to initialize.",
- pool);
+ scratch_pool);
return FALSE;
}
@@ -393,8 +456,8 @@ typedef struct revprop_generation_fixup_
/* revprop generation to read */
apr_int64_t *generation;
- /* containing the revprop_generation member to query */
- fs_x_data_t *ffd;
+ /* file system context */
+ svn_fs_t *fs;
} revprop_generation_upgrade_t;
/* If the revprop generation has an odd value, it means the original writer
@@ -406,23 +469,31 @@ typedef struct revprop_generation_fixup_
*/
static svn_error_t *
revprop_generation_fixup(void *void_baton,
- apr_pool_t *pool)
+ apr_pool_t *scratch_pool)
{
revprop_generation_upgrade_t *baton = void_baton;
- assert(baton->ffd->has_write_lock);
+ svn_fs_x__data_t *ffd = baton->fs->fsap_data;
+ assert(ffd->has_write_lock);
+
+ /* Make sure we don't operate on stale OS buffers. */
+ SVN_ERR(close_revprop_generation_file(baton->fs, scratch_pool));
/* Maybe, either the original revprop writer or some other reader has
already corrected / bumped the revprop generation. Thus, we need
- to read it again. */
- SVN_ERR(svn_named_atomic__read(baton->generation,
- baton->ffd->revprop_generation));
+ to read it again. However, we will now be the only ones changing
+ the file contents due to us holding the write lock. */
+ SVN_ERR(read_revprop_generation_file(baton->generation, baton->fs,
+ scratch_pool));
/* Cause everyone to re-read revprops upon their next access, if the
last revprop write did not complete properly. */
- while (*baton->generation % 2)
- SVN_ERR(svn_named_atomic__add(baton->generation,
- 1,
- baton->ffd->revprop_generation));
+ if (*baton->generation % 2)
+ {
+ ++*baton->generation;
+ SVN_ERR(write_revprop_generation_file(baton->fs,
+ *baton->generation,
+ scratch_pool));
+ }
return SVN_NO_ERROR;
}
@@ -433,42 +504,46 @@ revprop_generation_fixup(void *void_bato
static svn_error_t *
read_revprop_generation(apr_int64_t *generation,
svn_fs_t *fs,
- apr_pool_t *pool)
+ apr_pool_t *scratch_pool)
{
apr_int64_t current = 0;
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
/* read the current revprop generation number */
- SVN_ERR(ensure_revprop_generation(fs, pool));
- SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation));
+ SVN_ERR(read_revprop_generation_file(¤t, fs, scratch_pool));
/* is an unfinished revprop write under the way? */
if (current % 2)
{
- apr_int64_t timeout = 0;
+ svn_boolean_t timeout = FALSE;
- /* read timeout for the write operation */
- SVN_ERR(ensure_revprop_timeout(fs));
- SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout));
-
- /* has the writer process been aborted,
- * i.e. has the timeout been reached?
+ /* Has the writer process been aborted?
+ * Either by timeout or by us being the writer now.
*/
- if (apr_time_now() > timeout)
+ if (!ffd->has_write_lock)
+ {
+ apr_time_t mtime;
+ SVN_ERR(svn_io_file_affected_time(&mtime,
+ svn_fs_x__path_revprop_generation(fs, scratch_pool),
+ scratch_pool));
+ timeout = apr_time_now() > mtime + REVPROP_CHANGE_TIMEOUT;
+ }
+
+ if (ffd->has_write_lock || timeout)
{
revprop_generation_upgrade_t baton;
baton.generation = ¤t;
- baton.ffd = ffd;
+ baton.fs = fs;
/* Ensure that the original writer process no longer exists by
* acquiring the write lock to this repository. Then, fix up
* the revprop generation.
*/
if (ffd->has_write_lock)
- SVN_ERR(revprop_generation_fixup(&baton, pool));
+ SVN_ERR(revprop_generation_fixup(&baton, scratch_pool));
else
SVN_ERR(svn_fs_x__with_write_lock(fs, revprop_generation_fixup,
- &baton, pool));
+ &baton, scratch_pool));
}
}
@@ -477,64 +552,54 @@ read_revprop_generation(apr_int64_t *gen
return SVN_NO_ERROR;
}
-/* Set the revprop generation to the next odd number to indicate that
- there is a revprop write process under way. If that times out,
- readers shall recover from that state & re-read revprops.
- Use the access object in FS to set the shared mem value. */
-static svn_error_t *
-begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool)
-{
- apr_int64_t current;
- fs_x_data_t *ffd = fs->fsap_data;
-
- /* set the timeout for the write operation */
- SVN_ERR(ensure_revprop_timeout(fs));
- SVN_ERR(svn_named_atomic__write(NULL,
- apr_time_now() + REVPROP_CHANGE_TIMEOUT,
- ffd->revprop_timeout));
+/* Set the revprop generation in FS to the next odd number to indicate
+ that there is a revprop write process under way. Return that value
+ in *GENERATION. If the change times out, readers shall recover from
+ that state & re-read revprops.
+ This is a no-op for repo formats that don't support revprop caching. */
+static svn_error_t *
+begin_revprop_change(apr_int64_t *generation,
+ svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ SVN_ERR_ASSERT(ffd->has_write_lock);
+
+ /* Close and re-open to make sure we read the latest data. */
+ SVN_ERR(close_revprop_generation_file(fs, scratch_pool));
+ SVN_ERR(open_revprop_generation_file(fs, FALSE, scratch_pool));
- /* set the revprop generation to an odd value to indicate
- * that a write is in progress
+ /* Set the revprop generation to an odd value to indicate
+ * that a write is in progress.
*/
- SVN_ERR(ensure_revprop_generation(fs, pool));
- do
- {
- SVN_ERR(svn_named_atomic__add(¤t,
- 1,
- ffd->revprop_generation));
- }
- while (current % 2 == 0);
+ SVN_ERR(read_revprop_generation(generation, fs, scratch_pool));
+ ++*generation;
+ SVN_ERR(write_revprop_generation_file(fs, *generation, scratch_pool));
return SVN_NO_ERROR;
}
-/* Set the revprop generation to the next even number to indicate that
+/* Set the revprop generation in FS to the next even generation after
+ the odd value in GENERATION to indicate that
a) readers shall re-read revprops, and
- b) the write process has been completed (no recovery required)
- Use the access object in FS to set the shared mem value. */
+ b) the write process has been completed (no recovery required).
+ This is a no-op for repo formats that don't support revprop caching. */
static svn_error_t *
-end_revprop_change(svn_fs_t *fs, apr_pool_t *pool)
-{
- apr_int64_t current = 1;
- fs_x_data_t *ffd = fs->fsap_data;
-
- /* set the revprop generation to an even value to indicate
- * that a write has been completed
- */
- SVN_ERR(ensure_revprop_generation(fs, pool));
- do
- {
- SVN_ERR(svn_named_atomic__add(¤t,
- 1,
- ffd->revprop_generation));
- }
- while (current % 2);
-
- /* Save the latest generation to disk. FS is currently in a "locked"
- * state such that we can be sure the be the only ones to write that
- * file.
+end_revprop_change(svn_fs_t *fs,
+ apr_int64_t generation,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ SVN_ERR_ASSERT(ffd->has_write_lock);
+ SVN_ERR_ASSERT(generation % 2);
+
+ /* Set the revprop generation to an even value to indicate
+ * that a write has been completed. Since we held the write
+ * lock, nobody else could have updated the file contents.
*/
- return svn_fs_x__write_revprop_generation_file(fs, current, pool);
+ SVN_ERR(write_revprop_generation_file(fs, generation + 1, scratch_pool));
+
+ return SVN_NO_ERROR;
}
/* Container for all data required to access the packed revprop file
@@ -613,8 +678,8 @@ parse_revprop(apr_hash_t **properties,
SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool));
if (has_revprop_cache(fs, pool))
{
- fs_x_data_t *ffd = fs->fsap_data;
- pair_cache_key_t key = { 0 };
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ svn_fs_x__pair_cache_key_t key = { 0 };
key.revision = revision;
key.second = generation;
@@ -668,6 +733,19 @@ read_non_packed_revprop(apr_hash_t **pro
return SVN_NO_ERROR;
}
+/* Return the minimum length of any packed revprop file name in REVPROPS. */
+static apr_size_t
+get_min_filename_len(packed_revprops_t *revprops)
+{
+ char number_buffer[SVN_INT64_BUFFER_SIZE];
+
+ /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being
+ * at least the first rev in the shard and <COUNT> having at least one
+ * digit. Thus, the minimum is 2 + #decimal places in the start rev.
+ */
+ return svn__i64toa(number_buffer, revprops->manifest_start) + 2;
+}
+
/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
* members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
*/
@@ -677,49 +755,98 @@ get_revprop_packname(svn_fs_t *fs,
apr_pool_t *pool,
apr_pool_t *scratch_pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_stringbuf_t *content = NULL;
const char *manifest_file_path;
- int idx;
+ int idx, rev_count;
+ char *buffer, *buffer_end;
+ const char **filenames, **filenames_end;
+ apr_size_t min_filename_len;
+
+ /* Determine the dimensions. Rev 0 is excluded from the first shard. */
+ rev_count = ffd->max_files_per_dir;
+ revprops->manifest_start
+ = revprops->revision - (revprops->revision % rev_count);
+ if (revprops->manifest_start == 0)
+ {
+ ++revprops->manifest_start;
+ --rev_count;
+ }
+
+ revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*));
- /* read content of the manifest file */
+ /* No line in the file can be less than this number of chars long. */
+ min_filename_len = get_min_filename_len(revprops);
+
+ /* Read the content of the manifest file */
revprops->folder
= svn_fs_x__path_revprops_pack_shard(fs, revprops->revision, pool);
manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
SVN_ERR(svn_fs_x__read_content(&content, manifest_file_path, pool));
- /* parse the manifest. Every line is a file name */
- revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir,
- sizeof(const char*));
-
- /* Read all lines. Since the last line ends with a newline, we will
- end up with a valid but empty string after the last entry. */
- while (content->data && *content->data)
- {
- APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data;
- content->data = strchr(content->data, '\n');
- if (content->data)
- {
- *content->data = 0;
- content->data++;
- }
+ /* There CONTENT must have a certain minimal size and there no
+ * unterminated lines at the end of the file. Both guarantees also
+ * simplify the parser loop below.
+ */
+ if ( content->len < rev_count * (min_filename_len + 1)
+ || content->data[content->len - 1] != '\n')
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ _("Packed revprop manifest for r%ld not "
+ "properly terminated"), revprops->revision);
+
+ /* Chop (parse) the manifest CONTENT into filenames, one per line.
+ * We only have to replace all newlines with NUL and add all line
+ * starts to REVPROPS->MANIFEST.
+ *
+ * There must be exactly REV_COUNT lines and that is the number of
+ * lines we parse from BUFFER to FILENAMES. Set the end pointer for
+ * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid
+ * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN.
+ *
+ * Please note that this loop is performance critical for e.g. 'svn log'.
+ * It is run 1000x per revprop access, i.e. per revision and about
+ * 50 million times per sec (and CPU core).
+ */
+ for (filenames = (const char **)revprops->manifest->elts,
+ filenames_end = filenames + rev_count,
+ buffer = content->data,
+ buffer_end = buffer + content->len - min_filename_len;
+ (filenames < filenames_end) && (buffer < buffer_end);
+ ++filenames)
+ {
+ /* BUFFER always points to the start of the next line / filename. */
+ *filenames = buffer;
+
+ /* Find the next EOL. This is guaranteed to stay within the CONTENT
+ * buffer because we left enough room after BUFFER_END and we know
+ * we will always see a newline as the last non-NUL char. */
+ buffer += min_filename_len;
+ while (*buffer != '\n')
+ ++buffer;
+
+ /* Found EOL. Turn it into the filename terminator and move BUFFER
+ * to the start of the next line or CONTENT buffer end. */
+ *buffer = '\0';
+ ++buffer;
}
- content = NULL; /* No longer a valid stringbuf. */
- /* Index for our revision. Rev 0 is excluded from the first shard. */
- revprops->manifest_start = revprops->revision
- - (revprops->revision % ffd->max_files_per_dir);
- if (revprops->manifest_start == 0)
- ++revprops->manifest_start;
- idx = (int)(revprops->revision - revprops->manifest_start);
+ /* We must have reached the end of both buffers. */
+ if (buffer < content->data + content->len)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ _("Packed revprop manifest for r%ld "
+ "has too many entries"), revprops->revision);
- if (revprops->manifest->nelts <= idx)
+ if (filenames < filenames_end)
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
- _("Packed revprop manifest for r%ld too "
- "small"), revprops->revision);
+ _("Packed revprop manifest for r%ld "
+ "has too few entries"), revprops->revision);
+
+ /* The target array has now exactly one entry per revision. */
+ revprops->manifest->nelts = rev_count;
/* Now get the file name */
+ idx = (int)(revprops->revision - revprops->manifest_start);
revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
return SVN_NO_ERROR;
@@ -732,13 +859,14 @@ same_shard(svn_fs_t *fs,
svn_revnum_t r1,
svn_revnum_t r2)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir);
}
/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
- * fill the START_REVISION, SIZES, OFFSETS members. Also, make
- * PACKED_REVPROPS point to the first serialized revprop.
+ * fill the START_REVISION member, and make PACKED_REVPROPS point to the
+ * first serialized revprop. If READ_ALL is set, initialize the SIZES
+ * and OFFSETS members as well.
*
* Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
* well as the SERIALIZED_SIZE member. If revprop caching has been
@@ -747,6 +875,7 @@ same_shard(svn_fs_t *fs,
static svn_error_t *
parse_packed_revprops(svn_fs_t *fs,
packed_revprops_t *revprops,
+ svn_boolean_t read_all,
apr_pool_t *pool,
apr_pool_t *scratch_pool)
{
@@ -802,11 +931,14 @@ parse_packed_revprops(svn_fs_t *fs,
revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset);
revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset);
- /* STREAM still points to the first entry in the sizes list.
- * Init / construct REVPROPS members. */
+ /* STREAM still points to the first entry in the sizes list. */
revprops->start_revision = (svn_revnum_t)first_rev;
- revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
- revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
+ if (read_all)
+ {
+ /* Init / construct REVPROPS members. */
+ revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
+ revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
+ }
/* Now parse, revision by revision, the size and content of each
* revisions' revprops. */
@@ -814,7 +946,6 @@ parse_packed_revprops(svn_fs_t *fs,
{
apr_int64_t size;
svn_string_t serialized;
- apr_hash_t *properties;
svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
svn_pool_clear(iterpool);
@@ -835,20 +966,18 @@ parse_packed_revprops(svn_fs_t *fs,
revprops->generation, &serialized,
pool, iterpool));
revprops->serialized_size = serialized.len;
+
+ /* If we only wanted the revprops for REVISION then we are done. */
+ if (!read_all)
+ break;
}
- else
+
+ if (read_all)
{
- /* If revprop caching is enabled, parse any revprops.
- * They will get cached as a side-effect of this. */
- if (has_revprop_cache(fs, pool))
- SVN_ERR(parse_revprop(&properties, fs, revision,
- revprops->generation, &serialized,
- iterpool, iterpool));
+ /* fill REVPROPS data structures */
+ APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
+ APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
}
-
- /* fill REVPROPS data structures */
- APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
- APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
revprops->total_size += serialized.len;
offset += serialized.len;
@@ -859,6 +988,8 @@ parse_packed_revprops(svn_fs_t *fs,
/* In filesystem FS, read the packed revprops for revision REV into
* *REVPROPS. Use GENERATION to populate the revprop cache, if enabled.
+ * If you want to modify revprop contents / update REVPROPS, READ_ALL
+ * must be set. Otherwise, only the properties of REV are being provided.
* Allocate data in POOL.
*/
static svn_error_t *
@@ -866,6 +997,7 @@ read_pack_revprop(packed_revprops_t **re
svn_fs_t *fs,
svn_revnum_t rev,
apr_int64_t generation,
+ svn_boolean_t read_all,
apr_pool_t *pool)
{
apr_pool_t *iterpool = svn_pool_create(pool);
@@ -924,7 +1056,7 @@ read_pack_revprop(packed_revprops_t **re
_("Failed to read revprop pack file for r%ld"), rev);
/* parse it. RESULT will be complete afterwards. */
- err = parse_packed_revprops(fs, result, pool, iterpool);
+ err = parse_packed_revprops(fs, result, read_all, pool, iterpool);
svn_pool_destroy(iterpool);
if (err)
return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
@@ -943,9 +1075,10 @@ svn_error_t *
svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p,
svn_fs_t *fs,
svn_revnum_t rev,
+ svn_boolean_t bypass_cache,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
apr_int64_t generation = 0;
/* not found, yet */
@@ -955,10 +1088,10 @@ svn_fs_x__get_revision_proplist(apr_hash
SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool));
/* Try cache lookup first. */
- if (has_revprop_cache(fs, pool))
+ if (!bypass_cache && has_revprop_cache(fs, pool))
{
svn_boolean_t is_cached;
- pair_cache_key_t key = { 0 };
+ svn_fs_x__pair_cache_key_t key = { 0 };
SVN_ERR(read_revprop_generation(&generation, fs, pool));
@@ -993,7 +1126,7 @@ svn_fs_x__get_revision_proplist(apr_hash
if (!*proplist_p)
{
packed_revprops_t *revprops;
- SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool));
+ SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool));
*proplist_p = revprops->properties;
}
@@ -1054,17 +1187,19 @@ switch_to_new_revprop(svn_fs_t *fs,
svn_boolean_t bump_generation,
apr_pool_t *pool)
{
+ apr_int64_t generation;
+
/* Now, we may actually be replacing revprops. Make sure that all other
threads and processes will know about this. */
if (bump_generation)
- SVN_ERR(begin_revprop_change(fs, pool));
+ SVN_ERR(begin_revprop_change(&generation, fs, pool));
SVN_ERR(svn_fs_x__move_into_place(tmp_path, final_path, perms_reference,
pool));
/* Indicate that the update (if relevant) has been completed. */
if (bump_generation)
- SVN_ERR(end_revprop_change(fs, pool));
+ SVN_ERR(end_revprop_change(fs, generation, pool));
/* Clean up temporary files, if necessary. */
if (files_to_delete)
@@ -1146,7 +1281,7 @@ repack_revprops(svn_fs_t *fs,
svn_stream_t *file_stream,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_stream_t *stream;
int i;
@@ -1274,7 +1409,7 @@ write_packed_revprop(const char **final_
apr_hash_t *proplist,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
packed_revprops_t *revprops;
apr_int64_t generation = 0;
svn_stream_t *stream;
@@ -1288,7 +1423,7 @@ write_packed_revprop(const char **final_
SVN_ERR(read_revprop_generation(&generation, fs, pool));
/* read contents of the current pack file */
- SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool));
+ SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool));
/* serialize the new revprops */
serialized = svn_stringbuf_create_empty(pool);
@@ -1433,20 +1568,18 @@ svn_fs_x__set_revision_proplist(svn_fs_t
is_packed = svn_fs_x__is_packed_revprop(fs, rev);
/* Test whether revprops already exist for this revision.
- * Only then will we need to bump the revprop generation. */
- if (has_revprop_cache(fs, pool))
+ * Only then will we need to bump the revprop generation.
+ * The fact that they did not yet exist is never cached. */
+ if (is_packed)
{
- if (is_packed)
- {
- bump_generation = TRUE;
- }
- else
- {
- svn_node_kind_t kind;
- SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, rev, pool),
- &kind, pool));
- bump_generation = kind != svn_node_none;
- }
+ bump_generation = TRUE;
+ }
+ else
+ {
+ svn_node_kind_t kind;
+ SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, rev, pool),
+ &kind, pool));
+ bump_generation = kind != svn_node_none;
}
/* Serialize the new revprop data */
@@ -1481,7 +1614,7 @@ svn_fs_x__packed_revprop_available(svn_b
svn_revnum_t revision,
apr_pool_t *pool)
{
- fs_x_data_t *ffd = fs->fsap_data;
+ svn_fs_x__data_t *ffd = fs->fsap_data;
svn_stringbuf_t *content = NULL;
/* try to read the manifest file */