You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2016/04/29 20:38:56 UTC
svn commit: r1741682 [7/26] - in /subversion/branches/authzperf: ./ build/
build/ac-macros/ build/generator/ contrib/server-side/svncutter/ notes/
notes/api-errata/1.9/ notes/move-tracking/ subversion/
subversion/bindings/ctypes-python/ subversion/bind...
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/pack.c Fri Apr 29 18:38:53 2016
@@ -108,8 +108,8 @@ typedef struct path_order_t
/* noderev predecessor count */
int predecessor_count;
- /* this is a directory node */
- svn_boolean_t is_dir;
+ /* this is a node is the latest for this PATH in this rev / pack file */
+ svn_boolean_t is_head;
/* length of the expanded representation content */
apr_int64_t expanded_size;
@@ -233,6 +233,9 @@ typedef struct pack_context_t
/* pool used for temporary data structures that will be cleaned up when
* the next range of revisions is being processed */
apr_pool_t *info_pool;
+
+ /* ensure that all filesystem changes are written to disk. */
+ svn_boolean_t flush_to_disk;
} pack_context_t;
/* Create and initialize a new pack context for packing shard SHARD_REV in
@@ -240,7 +243,7 @@ typedef struct pack_context_t
* and return the structure in *CONTEXT.
*
* Limit the number of items being copied per iteration to MAX_ITEMS.
- * Set CANCEL_FUNC and CANCEL_BATON as well.
+ * Set FLUSH_TO_DISK, CANCEL_FUNC and CANCEL_BATON as well.
*/
static svn_error_t *
initialize_pack_context(pack_context_t *context,
@@ -249,6 +252,7 @@ initialize_pack_context(pack_context_t *
const char *shard_dir,
svn_revnum_t shard_rev,
int max_items,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *pool)
@@ -325,6 +329,8 @@ initialize_pack_context(pack_context_t *
context->info_pool = svn_pool_create(pool);
context->paths = svn_prefix_tree__create(context->info_pool);
+ context->flush_to_disk = flush_to_disk;
+
return SVN_NO_ERROR;
}
@@ -385,7 +391,8 @@ close_pack_context(pack_context_t *conte
SVN_ERR(svn_io_remove_file2(proto_p2l_index_path, FALSE, pool));
/* Ensure that packed file is written to disk.*/
- SVN_ERR(svn_io_file_flush_to_disk(context->pack_file, pool));
+ if (context->flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(context->pack_file, pool));
SVN_ERR(svn_io_file_close(context->pack_file, pool));
return SVN_NO_ERROR;
@@ -481,7 +488,7 @@ copy_item_to_temp(pack_context_t *contex
svn_fs_fs__p2l_entry_t *new_entry
= apr_pmemdup(context->info_pool, entry, sizeof(*entry));
- SVN_ERR(svn_fs_fs__get_file_offset(&new_entry->offset, temp_file, pool));
+ SVN_ERR(svn_io_file_get_offset(&new_entry->offset, temp_file, pool));
APR_ARRAY_PUSH(entries, svn_fs_fs__p2l_entry_t *) = new_entry;
SVN_ERR(copy_file_data(context, temp_file, rev_file, entry->size, pool));
@@ -568,7 +575,7 @@ copy_rep_to_temp(pack_context_t *context
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = apr_pmemdup(context->info_pool, entry, sizeof(*entry));
- SVN_ERR(svn_fs_fs__get_file_offset(&entry->offset, context->reps_file, pool));
+ SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file, pool));
add_item_rep_mapping(context, entry);
/* read & parse the representation header */
@@ -719,8 +726,8 @@ copy_node_to_temp(pack_context_t *contex
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = apr_pmemdup(context->info_pool, entry, sizeof(*entry));
- SVN_ERR(svn_fs_fs__get_file_offset(&entry->offset, context->reps_file,
- pool));
+ SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+ pool));
add_item_rep_mapping(context, entry);
/* copy the noderev to our temp file */
@@ -746,7 +753,6 @@ copy_node_to_temp(pack_context_t *contex
path_order->node_id = *svn_fs_fs__id_node_id(noderev->id);
path_order->revision = svn_fs_fs__id_rev(noderev->id);
path_order->predecessor_count = noderev->predecessor_count;
- path_order->is_dir = noderev->kind == svn_node_dir;
path_order->noderev_id = *svn_fs_fs__id_rev_item(noderev->id);
APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order;
@@ -763,13 +769,8 @@ compare_path_order(const path_order_t *
const path_order_t * lhs = *lhs_p;
const path_order_t * rhs = *rhs_p;
- /* cluster all directories */
- int diff = rhs->is_dir - lhs->is_dir;
- if (diff)
- return diff;
-
/* lexicographic order on path and node (i.e. latest first) */
- diff = svn_prefix_string__compare(lhs->path, rhs->path);
+ int diff = svn_prefix_string__compare(lhs->path, rhs->path);
if (diff)
return diff;
@@ -809,11 +810,46 @@ compare_ref_to_item(const reference_t *
/* Look for the least significant bit set in VALUE and return the smallest
* number with the same property, i.e. the largest power of 2 that is a
- * factor in VALUE. */
+ * factor in VALUE. Edge case: roundness(0) := 0 . */
static int
roundness(int value)
{
- return value ? value - (value & (value - 1)) : INT_MAX;
+ return value - (value & (value - 1));
+}
+
+/* For all paths in first COUNT entries in PATH_ORDER, mark their latest
+ * node as "HEAD". PATH_ORDER must be ordered by path, revision.
+ */
+static void
+classify_nodes(path_order_t **path_order,
+ int count)
+{
+ const svn_prefix_string__t *path;
+ int i;
+
+ /* The logic below would fail for empty ranges. */
+ if (count == 0)
+ return;
+
+ /* All entries are sorted by path, followed by revision.
+ * So, the first index is also HEAD for the first path.
+ */
+ path = path_order[0]->path;
+ path_order[0]->is_head = TRUE;
+
+ /* Since the sorting implicitly groups all entries by path and then sorts
+ * by descending revision within the group, whenever we encounter a new
+ * path, the first entry is "HEAD" for that path.
+ */
+ for (i = 1; i < count; ++i)
+ {
+ /* New path? */
+ if (svn_prefix_string__compare(path, path_order[i]->path))
+ {
+ path = path_order[i]->path;
+ path_order[i]->is_head = TRUE;
+ }
+ }
}
/* Order a range of data collected in CONTEXT such that we can place them
@@ -822,13 +858,13 @@ roundness(int value)
*/
static void
sort_reps_range(pack_context_t *context,
- const path_order_t **path_order,
- const path_order_t **temp,
+ path_order_t **path_order,
+ path_order_t **temp,
int first,
int last)
{
const svn_prefix_string__t *path;
- int i, dest, best;
+ int i, dest;
svn_fs_fs__id_part_t rep_id;
fs_fs_data_t *ffd = context->fs->fsap_data;
@@ -845,49 +881,52 @@ sort_reps_range(pack_context_t *context,
* We simply pick & chose from the existing path, rev order.
*/
dest = first;
- path = path_order[first]->path;
- best = first;
- /* (1) For each path, pick the "roundest" representation and put it in
- * front of all other nodes in the pack file. The "roundest" rep is
- * the one most likely to be referenced from future pack files, i.e. we
- * concentrate those potential "foreign link targets" in one section of
- * the pack file.
+ /* (1) There are two classes of representations that are likely to be
+ * referenced from future shards. These form a "hot zone" of mostly
+ * relevant data, i.e. we try to include as many reps as possible that
+ * are needed for future checkouts while trying to exclude as many as
+ * possible that are likely not needed in future checkouts.
+ *
+ * First, "very round" representations from frequently changing nodes.
+ * That excludes many in-between representations not accessed from HEAD.
*
- * And we only apply this to reps outside the linear deltification
- * sections because references *into* linear deltification ranges are
- * much less likely.
+ * The second class are infrequently changing nodes. Because they are
+ * unlikely to change often in the future, they will remain relevant for
+ * HEAD even over long spans of revisions. They are most likely the only
+ * thing we need from very old pack files.
*/
for (i = first; i < last; ++i)
{
- /* Investigated all nodes for the current path? */
- if (svn_prefix_string__compare(path, path_order[i]->path))
- {
- /* next path */
- path = path_order[i]->path;
-
- /* Pick roundest non-linear deltified node. */
- if (roundness(path_order[best]->predecessor_count)
- >= ffd->max_linear_deltification)
- {
- temp[dest++] = path_order[best];
- path_order[best] = NULL;
- best = i;
- }
- }
+ int round = roundness(path_order[i]->predecessor_count);
- /* next entry */
- if ( roundness(path_order[best]->predecessor_count)
- < roundness(path_order[i]->predecessor_count))
- best = i;
- }
+ /* Class 1:
+ * Pretty round _and_ a significant stop in the node's delta chain.
+ * This may pick up more than one representation from the same chain
+ * but that's rare and not a problem. Prefer simple checks here.
+ *
+ * The divider of 4 is arbitrary but seems to work well in practice.
+ * Larger values increase the number of items in the "hot zone".
+ * Smaller values make delta chains at HEAD more likely to contain
+ * "cold zone" representations. */
+ svn_boolean_t likely_target
+ = (round >= ffd->max_linear_deltification)
+ && (round >= path_order[i]->predecessor_count / 4);
+
+ /* Class 2:
+ * Anything from short node chains. The default of 16 is generous
+ * but we'd rather include too many than too few nodes here to keep
+ * seeks between different regions of this pack file at a minimum. */
+ svn_boolean_t likely_head
+ = path_order[i]->predecessor_count
+ < ffd->max_linear_deltification;
- /* Treat the last path the same as all others. */
- if (roundness(path_order[best]->predecessor_count)
- >= ffd->max_linear_deltification)
- {
- temp[dest++] = path_order[best];
- path_order[best] = NULL;
+ /* Pick any node that from either class. */
+ if (likely_target || likely_head)
+ {
+ temp[dest++] = path_order[i];
+ path_order[i] = NULL;
+ }
}
/* (2) For each (remaining) path, pick the nodes along the delta chain
@@ -953,7 +992,7 @@ static void
sort_reps(pack_context_t *context)
{
apr_pool_t *temp_pool;
- const path_order_t **temp, **path_order;
+ path_order_t **temp, **path_order;
int i, count;
/* We will later assume that there is at least one node / path.
@@ -979,7 +1018,10 @@ sort_reps(pack_context_t *context)
temp = apr_pcalloc(temp_pool, count * sizeof(*temp));
path_order = (void *)context->path_order->elts;
- /* Sort those sub-sections separately. */
+ /* Mark nodes depending on what other nodes exist for the same path etc. */
+ classify_nodes(path_order, count);
+
+ /* Rearrange those sub-sections separately. */
sort_reps_range(context, path_order, temp, 0, count);
/* We now know the final ordering. */
@@ -1016,7 +1058,7 @@ sort_items(apr_array_header_t *entries)
/* Return the remaining unused bytes in the current block in CONTEXT's
* pack file.
*/
-static apr_ssize_t
+static apr_off_t
get_block_left(pack_context_t *context)
{
fs_fs_data_t *ffd = context->fs->fsap_data;
@@ -1147,7 +1189,7 @@ copy_reps_from_temp(pack_context_t *cont
apr_array_header_t *path_order = context->path_order;
int i;
- /* copy items in path order. */
+ /* copy items in path order. Exclude the non-HEAD noderevs. */
for (i = 0; i < path_order->nelts; ++i)
{
path_order_t *current_path;
@@ -1157,13 +1199,30 @@ copy_reps_from_temp(pack_context_t *cont
svn_pool_clear(iterpool);
current_path = APR_ARRAY_IDX(path_order, i, path_order_t *);
- node_part = get_item(context, ¤t_path->noderev_id, TRUE);
+ if (current_path->is_head)
+ {
+ node_part = get_item(context, ¤t_path->noderev_id, TRUE);
+ if (node_part)
+ SVN_ERR(store_item(context, temp_file, node_part, iterpool));
+ }
+
rep_part = get_item(context, ¤t_path->rep_id, TRUE);
+ if (rep_part)
+ SVN_ERR(store_item(context, temp_file, rep_part, iterpool));
+ }
+
+ /* copy the remaining non-head noderevs. */
+ for (i = 0; i < path_order->nelts; ++i)
+ {
+ path_order_t *current_path;
+ svn_fs_fs__p2l_entry_t *node_part;
+ svn_pool_clear(iterpool);
+
+ current_path = APR_ARRAY_IDX(path_order, i, path_order_t *);
+ node_part = get_item(context, ¤t_path->noderev_id, TRUE);
if (node_part)
SVN_ERR(store_item(context, temp_file, node_part, iterpool));
- if (rep_part)
- SVN_ERR(store_item(context, temp_file, rep_part, iterpool));
}
svn_pool_destroy(iterpool);
@@ -1391,21 +1450,17 @@ append_revision(pack_context_t *context,
apr_off_t offset = 0;
apr_pool_t *iterpool = svn_pool_create(pool);
svn_fs_fs__revision_file_t *rev_file;
- apr_finfo_t finfo;
-
- /* Get the size of the file. */
- const char *path = svn_dirent_join(context->shard_dir,
- apr_psprintf(iterpool, "%ld",
- context->start_rev),
- pool);
- SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, pool));
+ svn_filesize_t revfile_size;
/* Copy all the bits from the rev file to the end of the pack file. */
SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, context->fs,
context->start_rev, pool,
iterpool));
+ /* Get the size of the file. */
+ SVN_ERR(svn_io_file_size_get(&revfile_size, rev_file->file, pool));
+
SVN_ERR(copy_file_data(context, context->pack_file, rev_file->file,
- finfo.size, iterpool));
+ revfile_size, iterpool));
/* mark the start of a new revision */
SVN_ERR(svn_fs_fs__l2p_proto_index_add_revision(context->proto_l2p_index,
@@ -1413,7 +1468,7 @@ append_revision(pack_context_t *context,
/* read the phys-to-log index file until we covered the whole rev file.
* That index contains enough info to build both target indexes from it. */
- while (offset < finfo.size)
+ while (offset < revfile_size)
{
/* read one cluster */
int i;
@@ -1437,7 +1492,7 @@ append_revision(pack_context_t *context,
/* process entry while inside the rev file */
offset = entry->offset;
- if (offset < finfo.size)
+ if (offset < revfile_size)
{
entry->offset += context->pack_offset;
offset += entry->size;
@@ -1451,7 +1506,7 @@ append_revision(pack_context_t *context,
}
svn_pool_destroy(iterpool);
- context->pack_offset += finfo.size;
+ context->pack_offset += revfile_size;
SVN_ERR(svn_fs_fs__close_revision_file(rev_file));
@@ -1462,8 +1517,9 @@ append_revision(pack_context_t *context,
*
* Pack the revision shard starting at SHARD_REV in filesystem FS from
* SHARD_DIR into the PACK_FILE_DIR, using POOL for allocations. Limit
- * the extra memory consumption to MAX_MEM bytes. CANCEL_FUNC and
- * CANCEL_BATON are what you think they are.
+ * the extra memory consumption to MAX_MEM bytes. If FLUSH_TO_DISK is
+ * non-zero, do not return until the data has actually been written on
+ * the disk. CANCEL_FUNC and CANCEL_BATON are what you think they are.
*/
static svn_error_t *
pack_log_addressed(svn_fs_t *fs,
@@ -1471,6 +1527,7 @@ pack_log_addressed(svn_fs_t *fs,
const char *shard_dir,
svn_revnum_t shard_rev,
apr_size_t max_mem,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *pool)
@@ -1503,8 +1560,8 @@ pack_log_addressed(svn_fs_t *fs,
/* set up a pack context */
SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir,
- shard_rev, max_items, cancel_func,
- cancel_baton, pool));
+ shard_rev, max_items, flush_to_disk,
+ cancel_func, cancel_baton, pool));
/* phase 1: determine the size of the revisions to pack */
SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, shard_rev,
@@ -1513,7 +1570,8 @@ pack_log_addressed(svn_fs_t *fs,
/* pack revisions in ranges that don't exceed MAX_MEM */
for (i = 0; i < max_ids->nelts; ++i)
- if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + item_count <= max_items)
+ if ( APR_ARRAY_IDX(max_ids, i, apr_uint64_t)
+ <= (apr_uint64_t)max_items - item_count)
{
context.end_rev++;
}
@@ -1626,14 +1684,16 @@ svn_fs_fs__get_packed_offset(apr_off_t *
*
* Pack the revision shard starting at SHARD_REV containing exactly
* MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
- * using POOL for allocations. CANCEL_FUNC and CANCEL_BATON are what you
- * think they are.
+ * using POOL for allocations. If FLUSH_TO_DISK is non-zero, do not
+ * return until the data has actually been written on the disk.
+ * CANCEL_FUNC and CANCEL_BATON are what you think they are.
*/
static svn_error_t *
pack_phys_addressed(const char *pack_file_dir,
const char *shard_path,
svn_revnum_t start_rev,
int max_files_per_dir,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *pool)
@@ -1643,7 +1703,6 @@ pack_phys_addressed(const char *pack_fil
apr_file_t *manifest_file;
svn_stream_t *manifest_stream;
svn_revnum_t end_rev, rev;
- apr_off_t next_offset;
apr_pool_t *iterpool;
/* Some useful paths. */
@@ -1664,27 +1723,26 @@ pack_phys_addressed(const char *pack_fil
manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE, pool);
end_rev = start_rev + max_files_per_dir - 1;
- next_offset = 0;
iterpool = svn_pool_create(pool);
/* Iterate over the revisions in this shard, squashing them together. */
for (rev = start_rev; rev <= end_rev; rev++)
{
svn_stream_t *rev_stream;
- apr_finfo_t finfo;
const char *path;
+ apr_off_t offset;
svn_pool_clear(iterpool);
- /* Get the size of the file. */
path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
iterpool);
- SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
+
+ /* Obtain current offset in pack file. */
+ SVN_ERR(svn_io_file_get_offset(&offset, pack_file, iterpool));
/* build manifest */
SVN_ERR(svn_stream_printf(manifest_stream, iterpool,
- "%" APR_OFF_T_FMT "\n", next_offset));
- next_offset += finfo.size;
+ "%" APR_OFF_T_FMT "\n", offset));
/* Copy all the bits from the rev file to the end of the pack file. */
SVN_ERR(svn_stream_open_readonly(&rev_stream, path, iterpool, iterpool));
@@ -1697,14 +1755,16 @@ pack_phys_addressed(const char *pack_fil
SVN_ERR(svn_stream_close(manifest_stream));
/* Ensure that pack file is written to disk. */
- SVN_ERR(svn_io_file_flush_to_disk(manifest_file, pool));
+ if (flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(manifest_file, pool));
SVN_ERR(svn_io_file_close(manifest_file, pool));
/* disallow write access to the manifest file */
SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool));
/* Ensure that pack file is written to disk. */
- SVN_ERR(svn_io_file_flush_to_disk(pack_file, pool));
+ if (flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(pack_file, pool));
SVN_ERR(svn_io_file_close(pack_file, pool));
svn_pool_destroy(iterpool);
@@ -1715,8 +1775,9 @@ pack_phys_addressed(const char *pack_fil
/* In filesystem FS, pack the revision SHARD containing exactly
* MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
* using POOL for allocations. Try to limit the amount of temporary
- * memory needed to MAX_MEM bytes. CANCEL_FUNC and CANCEL_BATON are what
- * you think they are.
+ * memory needed to MAX_MEM bytes. If FLUSH_TO_DISK is non-zero, do
+ * not return until the data has actually been written on the disk.
+ * CANCEL_FUNC and CANCEL_BATON are what you think they are.
*
* If for some reason we detect a partial packing already performed, we
* remove the pack file and start again.
@@ -1730,6 +1791,7 @@ pack_rev_shard(svn_fs_t *fs,
apr_int64_t shard,
int max_files_per_dir,
apr_size_t max_mem,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *pool)
@@ -1749,12 +1811,13 @@ pack_rev_shard(svn_fs_t *fs,
/* Index information files */
if (svn_fs_fs__use_log_addressing(fs))
- SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev,
- max_mem, cancel_func, cancel_baton, pool));
+ SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path,
+ shard_rev, max_mem, flush_to_disk,
+ cancel_func, cancel_baton, pool));
else
SVN_ERR(pack_phys_addressed(pack_file_dir, shard_path, shard_rev,
- max_files_per_dir, cancel_func,
- cancel_baton, pool));
+ max_files_per_dir, flush_to_disk,
+ cancel_func, cancel_baton, pool));
SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, pool));
SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, pool));
@@ -1801,6 +1864,8 @@ synced_pack_shard(void *baton,
/* if enabled, pack the revprops in an equivalent way */
if (pb->revsprops_dir)
{
+ apr_int64_t pack_size_limit = 0.9 * ffd->revprop_pack_size;
+
revprops_pack_file_dir = svn_dirent_join(pb->revsprops_dir,
apr_psprintf(pool,
"%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
@@ -1814,10 +1879,11 @@ synced_pack_shard(void *baton,
revprops_shard_path,
pb->shard,
ffd->max_files_per_dir,
- (int)(0.9*ffd->revprop_pack_size),
+ pack_size_limit,
ffd->compress_packed_revprops
? SVN__COMPRESSION_ZLIB_DEFAULT
: SVN__COMPRESSION_NONE,
+ ffd->flush_to_disk,
pb->cancel_func,
pb->cancel_baton,
pool));
@@ -1894,8 +1960,8 @@ pack_shard(struct pack_baton *baton,
/* pack the revision content */
SVN_ERR(pack_rev_shard(baton->fs, rev_pack_file_dir, baton->rev_shard_path,
baton->shard, ffd->max_files_per_dir,
- DEFAULT_MAX_MEM, baton->cancel_func,
- baton->cancel_baton, pool));
+ DEFAULT_MAX_MEM, ffd->flush_to_disk,
+ baton->cancel_func, baton->cancel_baton, pool));
/* For newer repo formats, we only acquired the pack lock so far.
Before modifying the repo state by switching over to the packed
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.c Fri Apr 29 18:38:53 2016
@@ -51,6 +51,13 @@ path_rep_cache_db(const char *fs_path,
return svn_dirent_join(fs_path, REP_CACHE_DB_NAME, result_pool);
}
+#define SVN_ERR_CLOSE(x, db) do \
+{ \
+ svn_error_t *svn__err = (x); \
+ if (svn__err) \
+ return svn_error_compose_create(svn__err, svn_sqlite__close(db)); \
+} while (0)
+
/** Library-private API's. **/
@@ -100,12 +107,12 @@ open_rep_cache(void *baton,
0, NULL, 0,
fs->pool, pool));
- SVN_ERR(svn_sqlite__read_schema_version(&version, sdb, pool));
+ SVN_ERR_CLOSE(svn_sqlite__read_schema_version(&version, sdb, pool), sdb);
if (version < REP_CACHE_SCHEMA_FORMAT)
{
/* Must be 0 -- an uninitialized (no schema) database. Create
the schema. Results in schema version of 1. */
- SVN_ERR(svn_sqlite__exec_statements(sdb, STMT_CREATE_SCHEMA));
+ SVN_ERR_CLOSE(svn_sqlite__exec_statements(sdb, STMT_CREATE_SCHEMA), sdb);
}
/* This is used as a flag that the database is available so don't
@@ -126,6 +133,21 @@ svn_fs_fs__open_rep_cache(svn_fs_t *fs,
}
svn_error_t *
+svn_fs_fs__close_rep_cache(svn_fs_t *fs)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ if (ffd->rep_cache_db)
+ {
+ SVN_ERR(svn_sqlite__close(ffd->rep_cache_db));
+ ffd->rep_cache_db = NULL;
+ ffd->rep_cache_db_opened = 0;
+ }
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
svn_fs_fs__exists_rep_cache(svn_boolean_t *exists,
svn_fs_t *fs, apr_pool_t *pool)
{
@@ -239,7 +261,7 @@ svn_fs_fs__walk_rep_reference(svn_fs_t *
If you extend this function, check the callsite to see if you have
to make it not-ignore additional error codes. */
svn_error_t *
-svn_fs_fs__get_rep_reference(representation_t **rep,
+svn_fs_fs__get_rep_reference(representation_t **rep_p,
svn_fs_t *fs,
svn_checksum_t *checksum,
apr_pool_t *pool)
@@ -247,6 +269,7 @@ svn_fs_fs__get_rep_reference(representat
fs_fs_data_t *ffd = fs->fsap_data;
svn_sqlite__stmt_t *stmt;
svn_boolean_t have_row;
+ representation_t *rep;
SVN_ERR_ASSERT(ffd->rep_sharing_allowed);
if (! ffd->rep_cache_db)
@@ -265,28 +288,28 @@ svn_fs_fs__get_rep_reference(representat
SVN_ERR(svn_sqlite__step(&have_row, stmt));
if (have_row)
{
- *rep = apr_pcalloc(pool, sizeof(**rep));
- svn_fs_fs__id_txn_reset(&(*rep)->txn_id);
- memcpy((*rep)->sha1_digest, checksum->digest,
- sizeof((*rep)->sha1_digest));
- (*rep)->has_sha1 = TRUE;
- (*rep)->revision = svn_sqlite__column_revnum(stmt, 0);
- (*rep)->item_index = svn_sqlite__column_int64(stmt, 1);
- (*rep)->size = svn_sqlite__column_int64(stmt, 2);
- (*rep)->expanded_size = svn_sqlite__column_int64(stmt, 3);
-
- SVN_ERR(svn_fs_fs__fixup_expanded_size(fs, *rep, pool));
+ rep = apr_pcalloc(pool, sizeof(*rep));
+ svn_fs_fs__id_txn_reset(&(rep->txn_id));
+ memcpy(rep->sha1_digest, checksum->digest, sizeof(rep->sha1_digest));
+ rep->has_sha1 = TRUE;
+ rep->revision = svn_sqlite__column_revnum(stmt, 0);
+ rep->item_index = svn_sqlite__column_int64(stmt, 1);
+ rep->size = svn_sqlite__column_int64(stmt, 2);
+ rep->expanded_size = svn_sqlite__column_int64(stmt, 3);
}
else
- *rep = NULL;
+ rep = NULL;
SVN_ERR(svn_sqlite__reset(stmt));
- if (*rep)
+ if (rep)
{
+ svn_error_t *err;
+
+ SVN_ERR(svn_fs_fs__fixup_expanded_size(fs, rep, pool));
+
/* Check that REP refers to a revision that exists in FS. */
- svn_error_t *err = svn_fs_fs__ensure_revision_exists((*rep)->revision,
- fs, pool);
+ err = svn_fs_fs__ensure_revision_exists(rep->revision, fs, pool);
if (err)
return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
"Checksum '%s' in rep-cache is beyond HEAD",
@@ -294,6 +317,7 @@ svn_fs_fs__get_rep_reference(representat
pool));
}
+ *rep_p = rep;
return SVN_NO_ERROR;
}
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/rep-cache.h Fri Apr 29 18:38:53 2016
@@ -40,6 +40,10 @@ svn_error_t *
svn_fs_fs__open_rep_cache(svn_fs_t *fs,
apr_pool_t *pool);
+/* Close the rep cache database associated with FS. */
+svn_error_t *
+svn_fs_fs__close_rep_cache(svn_fs_t *fs);
+
/* Set *EXISTS to TRUE iff the rep-cache DB file exists. */
svn_error_t *
svn_fs_fs__exists_rep_cache(svn_boolean_t *exists,
@@ -60,11 +64,11 @@ svn_fs_fs__walk_rep_reference(svn_fs_t *
apr_pool_t *pool);
/* Return the representation REP in FS which has fulltext CHECKSUM.
- REP is allocated in POOL. If the rep cache database has not been
- opened, just set *REP to NULL. Returns SVN_ERR_FS_CORRUPT if
+ *REP_P is allocated in POOL. If the rep cache database has not been
+ opened, just set *REP_P to NULL. Returns SVN_ERR_FS_CORRUPT if
a reference beyond HEAD is detected. */
svn_error_t *
-svn_fs_fs__get_rep_reference(representation_t **rep,
+svn_fs_fs__get_rep_reference(representation_t **rep_p,
svn_fs_t *fs,
svn_checksum_t *checksum,
apr_pool_t *pool);
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.c Fri Apr 29 18:38:53 2016
@@ -25,9 +25,11 @@
#include "svn_pools.h"
#include "svn_hash.h"
#include "svn_dirent_uri.h"
+#include "svn_sorts.h"
#include "fs_fs.h"
#include "revprops.h"
+#include "temp_serializer.h"
#include "util.h"
#include "private/svn_subr_private.h"
@@ -77,6 +79,7 @@ svn_fs_fs__upgrade_pack_revprops(svn_fs_
shard, ffd->max_files_per_dir,
(int)(0.9 * ffd->revprop_pack_size),
compression_level,
+ ffd->flush_to_disk,
cancel_func, cancel_baton,
iterpool));
if (notify_func)
@@ -139,9 +142,6 @@ typedef struct packed_revprops_t
/* revision number to read (not necessarily the first in the pack) */
svn_revnum_t revision;
- /* current revprop generation. Used when populating the revprop cache */
- apr_int64_t generation;
-
/* the actual revision properties */
apr_hash_t *properties;
@@ -184,35 +184,73 @@ typedef struct packed_revprops_t
/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
* Also, put them into the revprop cache, if activated, for future use.
- * Three more parameters are being used to update the revprop cache: FS is
- * our file system, the revprops belong to REVISION and the global revprop
- * GENERATION is used as well.
*
- * The returned hash will be allocated in POOL, SCRATCH_POOL is being used
- * for temporary allocations.
+ * The returned hash will be allocated in RESULT_POOL, SCRATCH_POOL is being
+ * used for temporary allocations.
*/
static svn_error_t *
parse_revprop(apr_hash_t **properties,
svn_fs_t *fs,
svn_revnum_t revision,
- apr_int64_t generation,
svn_string_t *content,
- apr_pool_t *pool,
+ apr_pool_t *result_pool,
apr_pool_t *scratch_pool)
{
svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
- *properties = apr_hash_make(pool);
+ *properties = apr_hash_make(result_pool);
- SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool),
+ SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR,
+ result_pool),
apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.",
revision));
return SVN_NO_ERROR;
}
+void
+svn_fs_fs__reset_revprop_cache(svn_fs_t *fs)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ ffd->revprop_prefix = 0;
+}
+
+/* If FS has not a revprop cache prefix set, generate one.
+ * Always call this before accessing the revprop cache.
+ */
+static svn_error_t *
+prepare_revprop_cache(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ if (!ffd->revprop_prefix)
+ SVN_ERR(svn_atomic__unique_counter(&ffd->revprop_prefix));
+
+ return SVN_NO_ERROR;
+}
+
+/* Store the unparsed revprop hash CONTENT for REVISION in FS's revprop
+ * cache. Use SCRATCH_POOL for temporary allocations. */
+static svn_error_t *
+cache_revprops(svn_fs_t *fs,
+ svn_revnum_t revision,
+ svn_string_t *content,
+ apr_pool_t *scratch_pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ pair_cache_key_t key;
+
+ /* Make sure prepare_revprop_cache() has been called. */
+ SVN_ERR_ASSERT(ffd->revprop_prefix);
+ key.revision = revision;
+ key.second = ffd->revprop_prefix;
+
+ SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, content, scratch_pool));
+
+ return SVN_NO_ERROR;
+}
+
/* Read the non-packed revprops for revision REV in FS, put them into the
- * revprop cache if activated and return them in *PROPERTIES. GENERATION
- * is the current revprop generation.
+ * revprop cache if PROPULATE_CACHE is set and return them in *PROPERTIES.
*
* If the data could not be read due to an otherwise recoverable error,
* leave *PROPERTIES unchanged. No error will be returned in that case.
@@ -223,7 +261,7 @@ static svn_error_t *
read_non_packed_revprop(apr_hash_t **properties,
svn_fs_t *fs,
svn_revnum_t rev,
- apr_int64_t generation,
+ svn_boolean_t populate_cache,
apr_pool_t *pool)
{
svn_stringbuf_t *content = NULL;
@@ -244,9 +282,13 @@ read_non_packed_revprop(apr_hash_t **pro
}
if (content)
- SVN_ERR(parse_revprop(properties, fs, rev, generation,
- svn_stringbuf__morph_into_string(content),
- pool, iterpool));
+ {
+ svn_string_t *as_string = svn_stringbuf__morph_into_string(content);
+ SVN_ERR(parse_revprop(properties, fs, rev, as_string, pool, iterpool));
+
+ if (populate_cache)
+ SVN_ERR(cache_revprops(fs, rev, as_string, iterpool));
+ }
svn_pool_clear(iterpool);
@@ -267,12 +309,13 @@ get_min_filename_len(packed_revprops_t *
}
/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
- * members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
+ * members. Use RESULT_POOL for allocating results and SCRATCH_POOL for
+ * temporaries.
*/
static svn_error_t *
get_revprop_packname(svn_fs_t *fs,
packed_revprops_t *revprops,
- apr_pool_t *pool,
+ apr_pool_t *result_pool,
apr_pool_t *scratch_pool)
{
fs_fs_data_t *ffd = fs->fsap_data;
@@ -293,18 +336,20 @@ get_revprop_packname(svn_fs_t *fs,
--rev_count;
}
- revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*));
+ revprops->manifest = apr_array_make(result_pool, rev_count,
+ sizeof(const char*));
/* No line in the file can be less than this number of chars long. */
min_filename_len = get_min_filename_len(revprops);
/* Read the content of the manifest file */
revprops->folder
- = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool);
+ = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision,
+ result_pool);
manifest_file_path
- = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
+ = svn_dirent_join(revprops->folder, PATH_MANIFEST, result_pool);
- SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool));
+ SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, result_pool));
/* There CONTENT must have a certain minimal size and there no
* unterminated lines at the end of the file. Both guarantees also
@@ -387,7 +432,8 @@ same_shard(svn_fs_t *fs,
/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
* fill the START_REVISION member, and make PACKED_REVPROPS point to the
* first serialized revprop. If READ_ALL is set, initialize the SIZES
- * and OFFSETS members as well.
+ * and OFFSETS members as well. If POPULATE_CACHE is set, cache all
+ * revprops found in this pack.
*
* Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
* well as the SERIALIZED_SIZE member. If revprop caching has been
@@ -397,19 +443,20 @@ static svn_error_t *
parse_packed_revprops(svn_fs_t *fs,
packed_revprops_t *revprops,
svn_boolean_t read_all,
- apr_pool_t *pool,
+ svn_boolean_t populate_cache,
+ apr_pool_t *result_pool,
apr_pool_t *scratch_pool)
{
svn_stream_t *stream;
apr_int64_t first_rev, count, i;
- apr_off_t offset;
+ apr_size_t offset;
const char *header_end;
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
/* decompress (even if the data is only "stored", there is still a
* length header to remove) */
svn_stringbuf_t *compressed = revprops->packed_revprops;
- svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool);
+ svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(result_pool);
SVN_ERR(svn__decompress(compressed->data, compressed->len,
uncompressed, APR_SIZE_MAX));
@@ -449,18 +496,21 @@ parse_packed_revprops(svn_fs_t *fs,
offset = header_end - uncompressed->data + 2;
- revprops->packed_revprops = svn_stringbuf_create_empty(pool);
+ revprops->packed_revprops = svn_stringbuf_create_empty(result_pool);
revprops->packed_revprops->data = uncompressed->data + offset;
revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset);
- revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset);
+ revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize
+ - offset);
/* STREAM still points to the first entry in the sizes list. */
revprops->start_revision = (svn_revnum_t)first_rev;
if (read_all)
{
/* Init / construct REVPROPS members. */
- revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
- revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
+ revprops->sizes = apr_array_make(result_pool, (int)count,
+ sizeof(offset));
+ revprops->offsets = apr_array_make(result_pool, (int)count,
+ sizeof(offset));
}
/* Now parse, revision by revision, the size and content of each
@@ -475,7 +525,7 @@ parse_packed_revprops(svn_fs_t *fs,
/* read & check the serialized size */
SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream,
iterpool));
- if (size + offset > (apr_int64_t)revprops->packed_revprops->len)
+ if (size > (apr_int64_t)revprops->packed_revprops->len - offset)
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
_("Packed revprop size exceeds pack file size"));
@@ -485,21 +535,24 @@ parse_packed_revprops(svn_fs_t *fs,
if (revision == revprops->revision)
{
+ /* Parse (and possibly cache) the one revprop list we care about. */
SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
- revprops->generation, &serialized,
- pool, iterpool));
+ &serialized, result_pool, iterpool));
revprops->serialized_size = serialized.len;
/* If we only wanted the revprops for REVISION then we are done. */
- if (!read_all)
+ if (!read_all && !populate_cache)
break;
}
+ if (populate_cache)
+ SVN_ERR(cache_revprops(fs, revision, &serialized, iterpool));
+
if (read_all)
{
/* fill REVPROPS data structures */
- APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
- APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
+ APR_ARRAY_PUSH(revprops->sizes, apr_size_t) = serialized.len;
+ APR_ARRAY_PUSH(revprops->offsets, apr_size_t) = offset;
}
revprops->total_size += serialized.len;
@@ -510,7 +563,7 @@ parse_packed_revprops(svn_fs_t *fs,
}
/* In filesystem FS, read the packed revprops for revision REV into
- * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled.
+ * *REVPROPS. Populate the revprop cache, if POPULATE_CACHE is set.
* If you want to modify revprop contents / update REVPROPS, READ_ALL
* must be set. Otherwise, only the properties of REV are being provided.
* Allocate data in POOL.
@@ -519,8 +572,8 @@ static svn_error_t *
read_pack_revprop(packed_revprops_t **revprops,
svn_fs_t *fs,
svn_revnum_t rev,
- apr_int64_t generation,
svn_boolean_t read_all,
+ svn_boolean_t populate_cache,
apr_pool_t *pool)
{
apr_pool_t *iterpool = svn_pool_create(pool);
@@ -540,7 +593,6 @@ read_pack_revprop(packed_revprops_t **re
/* initialize the result data structure */
result = apr_pcalloc(pool, sizeof(*result));
result->revision = rev;
- result->generation = generation;
/* try to read the packed revprops. This may require retries if we have
* concurrent writers. */
@@ -571,7 +623,8 @@ read_pack_revprop(packed_revprops_t **re
_("Failed to read revprop pack file for r%ld"), rev);
/* parse it. RESULT will be complete afterwards. */
- err = parse_packed_revprops(fs, result, read_all, pool, iterpool);
+ err = parse_packed_revprops(fs, result, read_all, populate_cache, pool,
+ iterpool);
svn_pool_destroy(iterpool);
if (err)
return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
@@ -590,16 +643,48 @@ svn_error_t *
svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p,
svn_fs_t *fs,
svn_revnum_t rev,
- apr_pool_t *pool)
+ svn_boolean_t refresh,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool)
{
fs_fs_data_t *ffd = fs->fsap_data;
- apr_int64_t generation = 0;
+
+ /* Only populate the cache if we did not just cross a sync barrier.
+ * This is to eliminate overhead from code that always sets REFRESH.
+ * For callers that want caching, the caching kicks in on read "later". */
+ svn_boolean_t populate_cache = !refresh;
/* not found, yet */
*proplist_p = NULL;
/* should they be available at all? */
- SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool));
+ SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, scratch_pool));
+
+ if (refresh)
+ {
+ /* Previous cache contents is invalid now. */
+ svn_fs_fs__reset_revprop_cache(fs);
+ }
+ else
+ {
+ /* Try cache lookup first. */
+ svn_boolean_t is_cached;
+ pair_cache_key_t key;
+
+ /* Auto-alloc prefix and construct the key. */
+ SVN_ERR(prepare_revprop_cache(fs, scratch_pool));
+ key.revision = rev;
+ key.second = ffd->revprop_prefix;
+
+ /* The only way that this might error out is due to parser error. */
+ SVN_ERR_W(svn_cache__get((void **) proplist_p, &is_cached,
+ ffd->revprop_cache, &key, result_pool),
+ apr_psprintf(scratch_pool,
+ "Failed to parse revprops for r%ld.",
+ rev));
+ if (is_cached)
+ return SVN_NO_ERROR;
+ }
/* if REV had not been packed when we began, try reading it from the
* non-packed shard. If that fails, we will fall through to packed
@@ -607,7 +692,7 @@ svn_fs_fs__get_revision_proplist(apr_has
if (!svn_fs_fs__is_packed_revprop(fs, rev))
{
svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev,
- generation, pool);
+ populate_cache, result_pool);
if (err)
{
if (!APR_STATUS_IS_ENOENT(err->apr_err)
@@ -625,7 +710,8 @@ svn_fs_fs__get_revision_proplist(apr_has
if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
{
packed_revprops_t *revprops;
- SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool));
+ SVN_ERR(read_pack_revprop(&revprops, fs, rev, FALSE, populate_cache,
+ result_pool));
*proplist_p = revprops->properties;
}
@@ -653,6 +739,7 @@ write_non_packed_revprop(const char **fi
apr_hash_t *proplist,
apr_pool_t *pool)
{
+ fs_fs_data_t *ffd = fs->fsap_data;
apr_file_t *file;
svn_stream_t *stream;
*final_path = svn_fs_fs__path_revprops(fs, rev, pool);
@@ -667,7 +754,8 @@ write_non_packed_revprop(const char **fi
SVN_ERR(svn_stream_close(stream));
/* Flush temporary file to disk and close it. */
- SVN_ERR(svn_io_file_flush_to_disk(file, pool));
+ if (ffd->flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(file, pool));
SVN_ERR(svn_io_file_close(file, pool));
return SVN_NO_ERROR;
@@ -690,8 +778,10 @@ switch_to_new_revprop(svn_fs_t *fs,
apr_array_header_t *files_to_delete,
apr_pool_t *pool)
{
+ fs_fs_data_t *ffd = fs->fsap_data;
+
SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference,
- pool));
+ ffd->flush_to_disk, pool));
/* Clean up temporary files, if necessary. */
if (files_to_delete)
@@ -740,8 +830,8 @@ serialize_revprops_header(svn_stream_t *
* We only allocate a few bytes each iteration -- even with a
* million iterations we would still be in good shape memory-wise.
*/
- apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t);
- SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n",
+ apr_size_t size = APR_ARRAY_IDX(sizes, i, apr_size_t);
+ SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_SIZE_T_FMT "\n",
size));
}
@@ -769,7 +859,7 @@ repack_revprops(svn_fs_t *fs,
int end,
int changed_index,
svn_stringbuf_t *new_serialized,
- apr_off_t new_total_size,
+ apr_size_t new_total_size,
apr_file_t *file,
apr_pool_t *pool)
{
@@ -798,10 +888,8 @@ repack_revprops(svn_fs_t *fs,
}
else
{
- apr_size_t size
- = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t);
- apr_size_t offset
- = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t);
+ apr_size_t size = APR_ARRAY_IDX(revprops->sizes, i, apr_size_t);
+ apr_size_t offset = APR_ARRAY_IDX(revprops->offsets, i, apr_size_t);
SVN_ERR(svn_stream_write(stream,
revprops->packed_revprops->data + offset,
@@ -821,7 +909,8 @@ repack_revprops(svn_fs_t *fs,
/* finally, write the content to the target file, flush and close it */
SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len,
NULL, pool));
- SVN_ERR(svn_io_file_flush_to_disk(file, pool));
+ if (ffd->flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(file, pool));
SVN_ERR(svn_io_file_close(file, pool));
return SVN_NO_ERROR;
@@ -903,15 +992,14 @@ write_packed_revprop(const char **final_
{
fs_fs_data_t *ffd = fs->fsap_data;
packed_revprops_t *revprops;
- apr_int64_t generation = 0;
svn_stream_t *stream;
apr_file_t *file;
svn_stringbuf_t *serialized;
- apr_off_t new_total_size;
+ apr_size_t new_total_size;
int changed_index;
/* read contents of the current pack file */
- SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool));
+ SVN_ERR(read_pack_revprop(&revprops, fs, rev, TRUE, FALSE, pool));
/* serialize the new revprops */
serialized = svn_stringbuf_create_empty(pool);
@@ -925,7 +1013,7 @@ write_packed_revprop(const char **final_
+ serialized->len
+ (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
- APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len;
+ APR_ARRAY_IDX(revprops->sizes, changed_index, apr_size_t) = serialized->len;
/* can we put the new data into the same pack as the before? */
if ( new_total_size < ffd->revprop_pack_size
@@ -949,23 +1037,23 @@ write_packed_revprop(const char **final_
int left = 0;
int right = revprops->sizes->nelts - 1;
- apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
- apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
+ apr_size_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
+ apr_size_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
/* let left and right side grow such that their size difference
* is minimal after each step. */
while (left <= right)
- if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
- < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t))
+ if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_size_t)
+ < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_size_t))
{
- left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
+ left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_size_t)
+ SVN_INT64_BUFFER_SIZE;
++left;
}
else
{
- right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)
- + SVN_INT64_BUFFER_SIZE;
+ right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_size_t)
+ + SVN_INT64_BUFFER_SIZE;
--right;
}
@@ -1029,7 +1117,8 @@ write_packed_revprop(const char **final_
SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename));
}
SVN_ERR(svn_stream_close(stream));
- SVN_ERR(svn_io_file_flush_to_disk(file, pool));
+ if (ffd->flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(file, pool));
SVN_ERR(svn_io_file_close(file, pool));
}
@@ -1063,6 +1152,9 @@ svn_fs_fs__set_revision_proplist(svn_fs_
SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
fs, rev, proplist, pool));
+ /* Previous cache contents is invalid now. */
+ svn_fs_fs__reset_revprop_cache(fs);
+
/* We use the rev file of this revision as the perms reference,
* because when setting revprops for the first time, the revprop
* file won't exist and therefore can't serve as its own reference.
@@ -1161,6 +1253,7 @@ svn_fs_fs__copy_revprops(const char *pac
apr_array_header_t *sizes,
apr_size_t total_size,
int compression_level,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool)
@@ -1217,7 +1310,8 @@ svn_fs_fs__copy_revprops(const char *pac
/* write the pack file content to disk */
SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len,
NULL, scratch_pool));
- SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool));
+ if (flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool));
SVN_ERR(svn_io_file_close(pack_file, scratch_pool));
svn_pool_destroy(iterpool);
@@ -1230,8 +1324,9 @@ svn_fs_fs__pack_revprops_shard(const cha
const char *shard_path,
apr_int64_t shard,
int max_files_per_dir,
- apr_off_t max_pack_size,
+ apr_int64_t max_pack_size,
int compression_level,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool)
@@ -1240,10 +1335,14 @@ svn_fs_fs__pack_revprops_shard(const cha
apr_file_t *manifest_file;
svn_stream_t *manifest_stream;
svn_revnum_t start_rev, end_rev, rev;
- apr_off_t total_size;
+ apr_size_t total_size;
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
apr_array_header_t *sizes;
+ /* Sanitize config file values. */
+ apr_size_t max_size = (apr_size_t)MIN(MAX(max_pack_size, 1),
+ SVN_MAX_OBJECT_SIZE);
+
/* Some useful paths. */
manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
scratch_pool);
@@ -1271,7 +1370,7 @@ svn_fs_fs__pack_revprops_shard(const cha
works. */
/* initialize the revprop size info */
- sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t));
+ sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_size_t));
total_size = 2 * SVN_INT64_BUFFER_SIZE;
/* Iterate over the revisions in this shard, determine their size and
@@ -1288,16 +1387,20 @@ svn_fs_fs__pack_revprops_shard(const cha
iterpool);
SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
- /* if we already have started a pack file and this revprop cannot be
- * appended to it, write the previous pack file. */
- if (sizes->nelts != 0 &&
- total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size)
+ /* If we already have started a pack file and this revprop cannot be
+ * appended to it, write the previous pack file. Note this overflow
+ * check works because we enforced MAX_SIZE <= SVN_MAX_OBJECT_SIZE. */
+ if (sizes->nelts != 0
+ && ( finfo.size > max_size
+ || total_size > max_size
+ || SVN_INT64_BUFFER_SIZE + finfo.size > max_size - total_size))
{
SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
shard_path, start_rev, rev-1,
- sizes, (apr_size_t)total_size,
- compression_level, cancel_func,
- cancel_baton, iterpool));
+ sizes, total_size,
+ compression_level, flush_to_disk,
+ cancel_func, cancel_baton,
+ iterpool));
/* next pack file starts empty again */
apr_array_clear(sizes);
@@ -1314,7 +1417,7 @@ svn_fs_fs__pack_revprops_shard(const cha
pack_filename));
/* add to list of files to put into the current pack file */
- APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size;
+ APR_ARRAY_PUSH(sizes, apr_size_t) = finfo.size;
total_size += SVN_INT64_BUFFER_SIZE + finfo.size;
}
@@ -1323,12 +1426,13 @@ svn_fs_fs__pack_revprops_shard(const cha
SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
shard_path, start_rev, rev-1,
sizes, (apr_size_t)total_size,
- compression_level, cancel_func,
- cancel_baton, iterpool));
+ compression_level, flush_to_disk,
+ cancel_func, cancel_baton, iterpool));
/* flush the manifest file to disk and update permissions */
SVN_ERR(svn_stream_close(manifest_stream));
- SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool));
+ if (flush_to_disk)
+ SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool));
SVN_ERR(svn_io_file_close(manifest_file, iterpool));
SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/revprops.h Fri Apr 29 18:38:53 2016
@@ -58,15 +58,23 @@ svn_fs_fs__upgrade_cleanup_pack_revprops
void *cancel_baton,
apr_pool_t *scratch_pool);
+/* Invalidate the revprop cache in FS. */
+void
+svn_fs_fs__reset_revprop_cache(svn_fs_t *fs);
+
/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
+ * If REFRESH is set, clear the revprop cache before accessing the data.
*
- * Allocations will be done in POOL.
+ * The result will be allocated in RESULT_POOL; SCRATCH_POOL is used for
+ * temporaries.
*/
svn_error_t *
svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p,
svn_fs_t *fs,
svn_revnum_t rev,
- apr_pool_t *pool);
+ svn_boolean_t refresh,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool);
/* Set the revision property list of revision REV in filesystem FS to
PROPLIST. Use POOL for temporary allocations. */
@@ -103,8 +111,9 @@ svn_fs_fs__packed_revprop_available(svn_
* a hint on which initial buffer size we should use to hold the pack file
* content.
*
- * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations
- * are done in SCRATCH_POOL.
+ * If FLUSH_TO_DISK is non-zero, do not return until the data has actually
+ * been written on the disk. CANCEL_FUNC and CANCEL_BATON are used as usual.
+ * Temporary allocations are done in SCRATCH_POOL.
*/
svn_error_t *
svn_fs_fs__copy_revprops(const char *pack_file_dir,
@@ -115,6 +124,7 @@ svn_fs_fs__copy_revprops(const char *pac
apr_array_header_t *sizes,
apr_size_t total_size,
int compression_level,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool);
@@ -126,16 +136,18 @@ svn_fs_fs__copy_revprops(const char *pac
* have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after
* the bump.
*
- * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are
- * used in the usual way. Temporary allocations are done in SCRATCH_POOL.
+ * If FLUSH_TO_DISK is non-zero, do not return until the data has actually
+ * been written on the disk. CANCEL_FUNC and CANCEL_BATON areused in the
+ * usual way. Temporary allocations are done in SCRATCH_POOL.
*/
svn_error_t *
svn_fs_fs__pack_revprops_shard(const char *pack_file_dir,
const char *shard_path,
apr_int64_t shard,
int max_files_per_dir,
- apr_off_t max_pack_size,
+ apr_int64_t max_pack_size,
int compression_level,
+ svn_boolean_t flush_to_disk,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool);
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.c Fri Apr 29 18:38:53 2016
@@ -663,6 +663,44 @@ svn_fs_fs__deserialize_properties(void *
}
svn_error_t *
+svn_fs_fs__serialize_revprops(void **data,
+ apr_size_t *data_len,
+ void *in,
+ apr_pool_t *pool)
+{
+ svn_string_t *buffer = in;
+
+ *data = (void *)buffer->data;
+ *data_len = buffer->len;
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_fs__deserialize_revprops(void **out,
+ void *data,
+ apr_size_t data_len,
+ apr_pool_t *pool)
+{
+ apr_hash_t *properties;
+ svn_stream_t *stream;
+
+ svn_string_t buffer;
+ buffer.data = data;
+ buffer.len = data_len;
+
+ stream = svn_stream_from_string(&buffer, pool);
+ properties = svn_hash__make(pool);
+
+ SVN_ERR(svn_hash_read2(properties, stream, SVN_HASH_TERMINATOR, pool));
+
+ /* done */
+ *out = properties;
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
svn_fs_fs__serialize_id(void **data,
apr_size_t *data_len,
void *in,
@@ -750,16 +788,18 @@ svn_fs_fs__deserialize_node_revision(voi
}
/* Utility function that returns the directory serialized inside CONTEXT
- * to DATA and DATA_LEN. */
+ * to DATA and DATA_LEN. If OVERPROVISION is set, allocate some extra
+ * room for future in-place changes by svn_fs_fs__replace_dir_entry. */
static svn_error_t *
return_serialized_dir_context(svn_temp_serializer__context_t *context,
void **data,
- apr_size_t *data_len)
+ apr_size_t *data_len,
+ svn_boolean_t overprovision)
{
svn_stringbuf_t *serialized = svn_temp_serializer__get(context);
*data = serialized->data;
- *data_len = serialized->blocksize;
+ *data_len = overprovision ? serialized->blocksize : serialized->len;
((dir_data_t *)serialized->data)->len = serialized->len;
return SVN_NO_ERROR;
@@ -777,7 +817,24 @@ svn_fs_fs__serialize_dir_entries(void **
* and return the serialized data */
return return_serialized_dir_context(serialize_dir(dir, pool),
data,
- data_len);
+ data_len,
+ FALSE);
+}
+
+svn_error_t *
+svn_fs_fs__serialize_txndir_entries(void **data,
+ apr_size_t *data_len,
+ void *in,
+ apr_pool_t *pool)
+{
+ svn_fs_fs__dir_data_t *dir = in;
+
+ /* serialize the dir content into a new serialization context
+ * and return the serialized data */
+ return return_serialized_dir_context(serialize_dir(dir, pool),
+ data,
+ data_len,
+ TRUE);
}
svn_error_t *
@@ -878,7 +935,7 @@ svn_fs_fs__extract_dir_entry(void **out,
apr_pool_t *pool)
{
const dir_data_t *dir_data = data;
- const extract_dir_entry_baton_t *entry_baton = baton;
+ extract_dir_entry_baton_t *entry_baton = baton;
svn_boolean_t found;
/* resolve the reference to the entries array */
@@ -897,8 +954,11 @@ svn_fs_fs__extract_dir_entry(void **out,
/* de-serialize that entry or return NULL, if no match has been found.
* Be sure to check that the directory contents is still up-to-date. */
+ entry_baton->out_of_date
+ = dir_data->txn_filesize != entry_baton->txn_filesize;
+
*out = NULL;
- if (found && dir_data->txn_filesize == entry_baton->txn_filesize)
+ if (found && !entry_baton->out_of_date)
{
const svn_fs_dirent_t *source =
svn_temp_deserializer__ptr(entries, (const void *const *)&entries[pos]);
@@ -1059,9 +1119,7 @@ svn_fs_fs__replace_dir_entry(void **data
serialize_dir_entry(context, &entries[pos], &length);
/* return the updated serialized data */
- SVN_ERR (return_serialized_dir_context(context,
- data,
- data_len));
+ SVN_ERR(return_serialized_dir_context(context, data, data_len, TRUE));
/* since the previous call may have re-allocated the buffer, the lengths
* pointer may no longer point to the entry in that buffer. Therefore,
@@ -1075,6 +1133,18 @@ svn_fs_fs__replace_dir_entry(void **data
return SVN_NO_ERROR;
}
+
+svn_error_t *
+svn_fs_fs__reset_txn_filesize(void **data,
+ apr_size_t *data_len,
+ void *baton,
+ apr_pool_t *pool)
+{
+ dir_data_t *dir_data = (dir_data_t *)*data;
+ dir_data->txn_filesize = SVN_INVALID_FILESIZE;
+
+ return SVN_NO_ERROR;
+}
svn_error_t *
svn_fs_fs__serialize_rep_header(void **data,
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/temp_serializer.h Fri Apr 29 18:38:53 2016
@@ -156,6 +156,26 @@ svn_fs_fs__deserialize_properties(void *
apr_pool_t *pool);
/**
+ * Implements #svn_cache__serialize_func_t for a properties hash
+ * (@a in is an #apr_hash_t of svn_string_t elements, keyed by const char*).
+ */
+svn_error_t *
+svn_fs_fs__serialize_revprops(void **data,
+ apr_size_t *data_len,
+ void *in,
+ apr_pool_t *pool);
+
+/**
+ * Implements #svn_cache__deserialize_func_t for a properties hash
+ * (@a *out is an #apr_hash_t of svn_string_t elements, keyed by const char*).
+ */
+svn_error_t *
+svn_fs_fs__deserialize_revprops(void **out,
+ void *data,
+ apr_size_t data_len,
+ apr_pool_t *pool);
+
+/**
* Implements #svn_cache__serialize_func_t for #svn_fs_id_t
*/
svn_error_t *
@@ -201,6 +221,16 @@ svn_fs_fs__serialize_dir_entries(void **
apr_pool_t *pool);
/**
+ * Same as svn_fs_fs__serialize_dir_entries but allocates extra room for
+ * in-place modification.
+ */
+svn_error_t *
+svn_fs_fs__serialize_txndir_entries(void **data,
+ apr_size_t *data_len,
+ void *in,
+ apr_pool_t *pool);
+
+/**
* Implements #svn_cache__deserialize_func_t for a #svn_fs_fs__dir_data_t
*/
svn_error_t *
@@ -244,6 +274,12 @@ typedef struct extract_dir_entry_baton_t
/** Current length of the in-txn in-disk representation of the directory.
* SVN_INVALID_FILESIZE if unknown. */
svn_filesize_t txn_filesize;
+
+ /** Will be set by the callback. If FALSE, the cached data is out of date.
+ * We need this indicator because the svn_cache__t interface will always
+ * report the lookup as a success (FOUND==TRUE) if the generic lookup was
+ * successful -- regardless of what the entry extraction callback does. */
+ svn_boolean_t out_of_date;
} extract_dir_entry_baton_t;
@@ -295,6 +331,17 @@ svn_fs_fs__replace_dir_entry(void **data
apr_pool_t *pool);
/**
+ * Implements #svn_cache__partial_setter_func_t for a #svn_fs_fs__dir_data_t
+ * at @a *data, resetting its txn_filesize field to SVN_INVALID_FILESIZE.
+ * &a baton should be NULL.
+ */
+svn_error_t *
+svn_fs_fs__reset_txn_filesize(void **data,
+ apr_size_t *data_len,
+ void *baton,
+ apr_pool_t *pool);
+
+/**
* Implements #svn_cache__serialize_func_t for a #svn_fs_fs__rep_header_t.
*/
svn_error_t *