You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2016/04/29 20:38:56 UTC
svn commit: r1741682 [10/26] - in /subversion/branches/authzperf: ./ build/
build/ac-macros/ build/generator/ contrib/server-side/svncutter/ notes/
notes/api-errata/1.9/ notes/move-tracking/ subversion/
subversion/bindings/ctypes-python/ subversion/bin...
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/index.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/index.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/index.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/index.c Fri Apr 29 18:38:53 2016
@@ -231,7 +231,7 @@ stream_error_create(svn_fs_x__packed_num
apr_off_t offset;
SVN_ERR(svn_io_file_name_get(&file_name, stream->file,
stream->pool));
- SVN_ERR(svn_fs_x__get_file_offset(&offset, stream->file, stream->pool));
+ SVN_ERR(svn_io_file_get_offset(&offset, stream->file, stream->pool));
return svn_error_createf(err, NULL, message, file_name,
apr_psprintf(stream->pool,
@@ -963,8 +963,8 @@ svn_fs_x__l2p_index_append(svn_checksum_
/* 1 page with up to L2P_PAGE_SIZE entries.
* fsfs.conf settings validation guarantees this to fit into
* our address space. */
- apr_size_t last_buffer_size
- = (apr_size_t)svn_spillbuf__get_size(buffer);
+ apr_uint64_t last_buffer_size
+ = (apr_uint64_t)svn_spillbuf__get_size(buffer);
svn_pool_clear(iterpool);
@@ -1932,7 +1932,7 @@ svn_fs_x__l2p_get_max_ids(apr_array_head
apr_uint64_t item_count;
apr_size_t first_page_index, last_page_index;
- if (revision >= header->first_revision + header->revision_count)
+ if (revision - header->first_revision >= header->revision_count)
{
/* need to read the next index. Clear up memory used for the
* previous one. Note that intermittent pack runs do not change
@@ -2182,8 +2182,8 @@ svn_fs_x__p2l_index_append(svn_checksum_
apr_uint64_t last_entry_end = 0;
apr_uint64_t last_page_end = 0;
- apr_size_t last_buffer_size = 0; /* byte offset in the spill buffer at
- the begin of the current revision */
+ apr_uint64_t last_buffer_size = 0; /* byte offset in the spill buffer at
+ the begin of the current revision */
apr_uint64_t file_size = 0;
/* temporary data structures that collect the data which will be moved
@@ -2650,6 +2650,13 @@ read_entry(svn_fs_x__packed_number_strea
}
}
+ /* Corrupted SIZE values might cause arithmetic overflow.
+ * The same can happen if you copy a repository from a system with 63 bit
+ * file lengths to one with 31 bit file lengths. */
+ if ((apr_uint64_t)entry.offset + (apr_uint64_t)entry.size > off_t_max)
+ return svn_error_create(SVN_ERR_FS_INDEX_OVERFLOW , NULL,
+ _("P2L index entry size overflow."));
+
APR_ARRAY_PUSH(result, svn_fs_x__p2l_entry_t) = entry;
*item_offset += entry.size;
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/lock.c Fri Apr 29 18:38:53 2016
@@ -235,7 +235,7 @@ write_digest_file(apr_hash_t *children,
if ((err = svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR,
scratch_pool)))
{
- svn_error_clear(svn_stream_close(stream));
+ err = svn_error_compose_create(err, svn_stream_close(stream));
return svn_error_createf(err->apr_err,
err,
_("Cannot write lock/entries hashfile '%s'"),
@@ -287,7 +287,7 @@ read_digest_file(apr_hash_t **children_p
hash = apr_hash_make(pool);
if ((err = svn_hash_read2(hash, stream, SVN_HASH_TERMINATOR, pool)))
{
- svn_error_clear(svn_stream_close(stream));
+ err = svn_error_compose_create(err, svn_stream_close(stream));
return svn_error_createf(err->apr_err,
err,
_("Can't parse lock/entries hashfile '%s'"),
@@ -951,6 +951,7 @@ lock_body(void *baton,
}
}
+ svn_pool_destroy(iterpool);
return SVN_NO_ERROR;
}
@@ -1163,7 +1164,7 @@ svn_fs_x__lock(svn_fs_t *fs,
lb.fs = fs;
lb.targets = sorted_targets;
- lb.infos = apr_array_make(scratch_pool, sorted_targets->nelts,
+ lb.infos = apr_array_make(result_pool, sorted_targets->nelts,
sizeof(struct lock_info_t));
lb.comment = comment;
lb.is_dav_comment = is_dav_comment;
@@ -1259,7 +1260,7 @@ svn_fs_x__unlock(svn_fs_t *fs,
ub.fs = fs;
ub.targets = sorted_targets;
- ub.infos = apr_array_make(scratch_pool, sorted_targets->nelts,
+ ub.infos = apr_array_make(result_pool, sorted_targets->nelts,
sizeof(struct unlock_info_t));
ub.skip_check = FALSE;
ub.break_lock = break_lock;
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.c Fri Apr 29 18:38:53 2016
@@ -102,6 +102,19 @@ parse_revnum(svn_revnum_t *rev,
return SVN_NO_ERROR;
}
+/* If ERR is not NULL, wrap it MESSAGE. The latter must have an %ld
+ * format parameter that will be filled with REV. */
+static svn_error_t *
+wrap_footer_error(svn_error_t *err,
+ const char *message,
+ svn_revnum_t rev)
+{
+ if (err)
+ return svn_error_quick_wrapf(err, message, rev);
+
+ return SVN_NO_ERROR;
+}
+
svn_error_t *
svn_fs_x__parse_footer(apr_off_t *l2p_offset,
svn_checksum_t **l2p_checksum,
@@ -109,6 +122,7 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
svn_checksum_t **p2l_checksum,
svn_stringbuf_t *footer,
svn_revnum_t rev,
+ apr_off_t footer_offset,
apr_pool_t *result_pool)
{
apr_int64_t val;
@@ -117,17 +131,20 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
/* Get the L2P offset. */
const char *str = svn_cstring_tokenize(" ", &last_str);
if (str == NULL)
- return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
- _("Invalid revision footer"));
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ "Invalid r%ld footer", rev);
- SVN_ERR(svn_cstring_atoi64(&val, str));
+ SVN_ERR(wrap_footer_error(svn_cstring_strtoi64(&val, str, 0,
+ footer_offset - 1, 10),
+ "Invalid L2P offset in r%ld footer",
+ rev));
*l2p_offset = (apr_off_t)val;
/* Get the L2P checksum. */
str = svn_cstring_tokenize(" ", &last_str);
if (str == NULL)
- return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
- _("Invalid revision footer"));
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ "Invalid r%ld footer", rev);
SVN_ERR(svn_checksum_parse_hex(l2p_checksum, svn_checksum_md5, str,
result_pool));
@@ -135,17 +152,33 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
/* Get the P2L offset. */
str = svn_cstring_tokenize(" ", &last_str);
if (str == NULL)
- return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
- _("Invalid revision footer"));
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ "Invalid r%ld footer", rev);
- SVN_ERR(svn_cstring_atoi64(&val, str));
+ SVN_ERR(wrap_footer_error(svn_cstring_strtoi64(&val, str, 0,
+ footer_offset - 1, 10),
+ "Invalid P2L offset in r%ld footer",
+ rev));
*p2l_offset = (apr_off_t)val;
+ /* The P2L indes follows the L2P index */
+ if (*p2l_offset <= *l2p_offset)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ "P2L offset %s must be larger than L2P offset %s"
+ " in r%ld footer",
+ apr_psprintf(result_pool,
+ "%" APR_UINT64_T_HEX_FMT,
+ (apr_uint64_t)*p2l_offset),
+ apr_psprintf(result_pool,
+ "%" APR_UINT64_T_HEX_FMT,
+ (apr_uint64_t)*l2p_offset),
+ rev);
+
/* Get the P2L checksum. */
str = svn_cstring_tokenize(" ", &last_str);
if (str == NULL)
- return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
- _("Invalid revision footer"));
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ "Invalid r%ld footer", rev);
SVN_ERR(svn_checksum_parse_hex(p2l_checksum, svn_checksum_md5, str,
result_pool));
@@ -796,14 +829,6 @@ read_change(svn_fs_x__change_t **change_
change = apr_pcalloc(result_pool, sizeof(*change));
last_str = line->data;
- /* Get the node-id of the change. */
- str = svn_cstring_tokenize(" ", &last_str);
- if (str == NULL)
- return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
- _("Invalid changes line in rev-file"));
-
- SVN_ERR(svn_fs_x__id_parse(&change->noderev_id, str));
-
/* Get the change type. */
str = svn_cstring_tokenize(" ", &last_str);
if (str == NULL)
@@ -1011,7 +1036,6 @@ write_change_entry(svn_stream_t *stream,
svn_fs_x__change_t *change,
apr_pool_t *scratch_pool)
{
- const char *idstr;
const char *change_string = NULL;
const char *kind_string = "";
svn_stringbuf_t *buf;
@@ -1037,8 +1061,6 @@ write_change_entry(svn_stream_t *stream,
change->change_kind);
}
- idstr = svn_fs_x__id_unparse(&change->noderev_id, scratch_pool)->data;
-
SVN_ERR_ASSERT(change->node_kind == svn_node_dir
|| change->node_kind == svn_node_file);
kind_string = apr_psprintf(scratch_pool, "-%s",
@@ -1046,8 +1068,8 @@ write_change_entry(svn_stream_t *stream,
? SVN_FS_X__KIND_DIR
: SVN_FS_X__KIND_FILE);
- buf = svn_stringbuf_createf(scratch_pool, "%s %s%s %s %s %s %s\n",
- idstr, change_string, kind_string,
+ buf = svn_stringbuf_createf(scratch_pool, "%s%s %s %s %s %s\n",
+ change_string, kind_string,
change->text_mod ? FLAG_TRUE : FLAG_FALSE,
change->prop_mod ? FLAG_TRUE : FLAG_FALSE,
change->mergeinfo_mod == svn_tristate_true
@@ -1113,3 +1135,107 @@ svn_fs_x__write_changes(svn_stream_t *st
return SVN_NO_ERROR;
}
+svn_error_t *
+svn_fs_x__parse_properties(apr_hash_t **properties,
+ const svn_string_t *content,
+ apr_pool_t *result_pool)
+{
+ const apr_byte_t *p = (const apr_byte_t *)content->data;
+ const apr_byte_t *end = p + content->len;
+ apr_uint64_t count;
+
+ *properties = apr_hash_make(result_pool);
+
+ /* Extract the number of properties we are expected to read. */
+ p = svn__decode_uint(&count, p, end);
+
+ /* Read all the properties we find.
+ Because prop-name and prop-value are nicely NUL-terminated
+ sub-strings of CONTENT, we can simply reference them there.
+ I.e. there is no need to copy them around.
+ */
+ while (p < end)
+ {
+ apr_uint64_t value_len;
+ svn_string_t *value;
+
+ const char *key = (const char *)p;
+
+ /* Note that this may never overflow / segfault because
+ CONTENT itself is NUL-terminated. */
+ apr_size_t key_len = strlen(key);
+ p += key_len + 1;
+ if (key[key_len])
+ return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+ "Property name not NUL terminated");
+
+ if (p >= end)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+ "Property value missing");
+ p = svn__decode_uint(&value_len, p, end);
+ if (value_len >= (end - p))
+ return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+ "Property value too long");
+
+ value = apr_pcalloc(result_pool, sizeof(*value));
+ value->data = (const char *)p;
+ value->len = (apr_size_t)value_len;
+ if (p[value->len])
+ return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+ "Property value not NUL terminated");
+
+ p += value->len + 1;
+
+ apr_hash_set(*properties, key, key_len, value);
+ }
+
+ /* Check that we read the expected number of properties. */
+ if ((apr_uint64_t)apr_hash_count(*properties) != count)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT_PROPLIST, NULL,
+ "Property count mismatch");
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_x__write_properties(svn_stream_t *stream,
+ apr_hash_t *proplist,
+ apr_pool_t *scratch_pool)
+{
+ apr_byte_t buffer[SVN__MAX_ENCODED_UINT_LEN];
+ apr_size_t len;
+ apr_hash_index_t *hi;
+
+ /* Write the number of properties in this list. */
+ len = svn__encode_uint(buffer, apr_hash_count(proplist)) - buffer;
+ SVN_ERR(svn_stream_write(stream, (const char *)buffer, &len));
+
+ /* Serialize each property as follows:
+ <Prop-name> <NUL>
+ <Value-len> <Prop-value> <NUL>
+ */
+ for (hi = apr_hash_first(scratch_pool, proplist);
+ hi;
+ hi = apr_hash_next(hi))
+ {
+ const char *key;
+ apr_size_t key_len;
+ svn_string_t *value;
+ apr_hash_this(hi, (const void **)&key, (apr_ssize_t *)&key_len,
+ (void **)&value);
+
+ /* Include the terminating NUL. */
+ ++key_len;
+ SVN_ERR(svn_stream_write(stream, key, &key_len));
+
+ len = svn__encode_uint(buffer, value->len) - buffer;
+ SVN_ERR(svn_stream_write(stream, (const char *)buffer, &len));
+ SVN_ERR(svn_stream_write(stream, value->data, &value->len));
+
+ /* Terminate with NUL. */
+ len = 1;
+ SVN_ERR(svn_stream_write(stream, "", &len));
+ }
+
+ return SVN_NO_ERROR;
+}
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/low_level.h Fri Apr 29 18:38:53 2016
@@ -50,6 +50,8 @@ extern "C" {
* *P2L_OFFSET, respectively. Also, return the expected checksums in
* in *L2P_CHECKSUM and *P2L_CHECKSUM.
*
+ * FOOTER_OFFSET is used for validation.
+ *
* Note that REV is only used to construct nicer error objects that
* mention this revision. Allocate the checksums in RESULT_POOL.
*/
@@ -60,6 +62,7 @@ svn_fs_x__parse_footer(apr_off_t *l2p_of
svn_checksum_t **p2l_checksum,
svn_stringbuf_t *footer,
svn_revnum_t rev,
+ apr_off_t footer_offset,
apr_pool_t *result_pool);
/* Given the offset of the L2P index data in L2P_OFFSET, the content
@@ -207,6 +210,23 @@ svn_fs_x__write_changes(svn_stream_t *st
svn_boolean_t terminate_list,
apr_pool_t *scratch_pool);
+/* Parse the property list serialized in CONTENT and return it in
+ *PROPERTIES, allocated from RESULT_POOL. CONTENT must remain
+ valid at least until the next cleanup of RESULT_POOL.
+ */
+svn_error_t *
+svn_fs_x__parse_properties(apr_hash_t **properties,
+ const svn_string_t *content,
+ apr_pool_t *result_pool);
+
+/* Write the property list PROPLIST to STREAM in serialized format.
+ Use SCRATCH_POOL for temporary allocations.
+ */
+svn_error_t *
+svn_fs_x__write_properties(svn_stream_t *stream,
+ apr_hash_t *proplist,
+ apr_pool_t *scratch_pool);
+
#ifdef __cplusplus
}
#endif /* __cplusplus */
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c Fri Apr 29 18:38:53 2016
@@ -106,9 +106,6 @@ typedef struct path_order_t
/* when this change happened */
svn_revnum_t revision;
- /* this is a directory node */
- svn_boolean_t is_dir;
-
/* length of the expanded representation content */
apr_int64_t expanded_size;
@@ -247,6 +244,7 @@ initialize_pack_context(pack_context_t *
const char *shard_dir,
svn_revnum_t shard_rev,
int max_items,
+ svn_fs_x__batch_fsync_t *batch,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *pool)
@@ -275,9 +273,9 @@ initialize_pack_context(pack_context_t *
context->pack_file_dir = pack_file_dir;
context->pack_file_path
= svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
- SVN_ERR(svn_io_file_open(&context->pack_file, context->pack_file_path,
- APR_WRITE | APR_BUFFERED | APR_BINARY | APR_EXCL
- | APR_CREATE, APR_OS_DEFAULT, pool));
+
+ SVN_ERR(svn_fs_x__batch_fsync_open_file(&context->pack_file, batch,
+ context->pack_file_path, pool));
/* Proto index files */
SVN_ERR(svn_fs_x__l2p_proto_index_open(
@@ -382,8 +380,6 @@ close_pack_context(pack_context_t *conte
SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, scratch_pool));
SVN_ERR(svn_io_remove_file2(proto_p2l_index_path, FALSE, scratch_pool));
- SVN_ERR(svn_io_file_close(context->pack_file, scratch_pool));
-
return SVN_NO_ERROR;
}
@@ -395,7 +391,7 @@ static svn_error_t *
copy_file_data(pack_context_t *context,
apr_file_t *dest,
apr_file_t *source,
- apr_off_t size,
+ svn_filesize_t size,
apr_pool_t *scratch_pool)
{
/* most non-representation items will be small. Minimize the buffer
@@ -482,8 +478,8 @@ copy_item_to_temp(pack_context_t *contex
svn_fs_x__p2l_entry_t *new_entry
= svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- SVN_ERR(svn_fs_x__get_file_offset(&new_entry->offset, temp_file,
- scratch_pool));
+ SVN_ERR(svn_io_file_get_offset(&new_entry->offset, temp_file,
+ scratch_pool));
APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry;
SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
@@ -576,8 +572,8 @@ copy_rep_to_temp(pack_context_t *context
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
- scratch_pool));
+ SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+ scratch_pool));
add_item_rep_mapping(context, entry);
/* read & parse the representation header */
@@ -621,9 +617,6 @@ compare_dir_entries(const svn_sort__item
const svn_fs_dirent_t *lhs = (const svn_fs_dirent_t *) a->value;
const svn_fs_dirent_t *rhs = (const svn_fs_dirent_t *) b->value;
- if (lhs->kind != rhs->kind)
- return lhs->kind == svn_node_dir ? -1 : 1;
-
return strcmp(lhs->name, rhs->name);
}
@@ -705,8 +698,8 @@ copy_node_to_temp(pack_context_t *contex
/* create a copy of ENTRY, make it point to the copy destination and
* store it in CONTEXT */
entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool);
- SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file,
- scratch_pool));
+ SVN_ERR(svn_io_file_get_offset(&entry->offset, context->reps_file,
+ scratch_pool));
add_item_rep_mapping(context, entry);
/* copy the noderev to our temp file */
@@ -740,7 +733,6 @@ copy_node_to_temp(pack_context_t *contex
path_order->path = svn_prefix_string__create(context->paths, sort_path);
path_order->node_id = noderev->node_id;
path_order->revision = svn_fs_x__get_revnum(noderev->noderev_id.change_set);
- path_order->is_dir = noderev->kind == svn_node_dir;
path_order->noderev_id = noderev->noderev_id;
APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order;
@@ -784,13 +776,8 @@ compare_path_order(const path_order_t *
const path_order_t * lhs = *lhs_p;
const path_order_t * rhs = *rhs_p;
- /* cluster all directories */
- int diff = rhs->is_dir - lhs->is_dir;
- if (diff)
- return diff;
-
/* lexicographic order on path and node (i.e. latest first) */
- diff = svn_prefix_string__compare(lhs->path, rhs->path);
+ int diff = svn_prefix_string__compare(lhs->path, rhs->path);
if (diff)
return diff;
@@ -834,7 +821,7 @@ sort_reps(pack_context_t *context)
/* Return the remaining unused bytes in the current block in CONTEXT's
* pack file.
*/
-static apr_ssize_t
+static apr_off_t
get_block_left(pack_context_t *context)
{
svn_fs_x__data_t *ffd = context->fs->fsap_data;
@@ -1852,20 +1839,15 @@ append_revision(pack_context_t *context,
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
svn_fs_x__revision_file_t *rev_file;
apr_file_t *file;
- apr_finfo_t finfo;
-
- /* Get the size of the file. */
- const char *path = svn_dirent_join(context->shard_dir,
- apr_psprintf(iterpool, "%ld",
- context->start_rev),
- scratch_pool);
- SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, scratch_pool));
+ svn_filesize_t revfile_size;
/* Copy all the bits from the rev file to the end of the pack file. */
SVN_ERR(svn_fs_x__rev_file_init(&rev_file, context->fs, context->start_rev,
scratch_pool));
SVN_ERR(svn_fs_x__rev_file_get(&file, rev_file));
- SVN_ERR(copy_file_data(context, context->pack_file, file, finfo.size,
+
+ SVN_ERR(svn_io_file_size_get(&revfile_size, file, scratch_pool));
+ SVN_ERR(copy_file_data(context, context->pack_file, file, revfile_size,
iterpool));
/* mark the start of a new revision */
@@ -1874,7 +1856,7 @@ append_revision(pack_context_t *context,
/* read the phys-to-log index file until we covered the whole rev file.
* That index contains enough info to build both target indexes from it. */
- while (offset < finfo.size)
+ while (offset < revfile_size)
{
/* read one cluster */
int i;
@@ -1896,7 +1878,7 @@ append_revision(pack_context_t *context,
/* process entry while inside the rev file */
offset = entry->offset;
- if (offset < finfo.size)
+ if (offset < revfile_size)
{
/* there should be true containers */
SVN_ERR_ASSERT(entry->item_count == 1);
@@ -1915,7 +1897,7 @@ append_revision(pack_context_t *context,
}
svn_pool_destroy(iterpool);
- context->pack_offset += finfo.size;
+ context->pack_offset += revfile_size;
return SVN_NO_ERROR;
}
@@ -1926,6 +1908,7 @@ append_revision(pack_context_t *context,
* SHARD_DIR into the PACK_FILE_DIR, using SCRATCH_POOL for temporary
* allocations. Limit the extra memory consumption to MAX_MEM bytes.
* CANCEL_FUNC and CANCEL_BATON are what you think they are.
+ * Schedule necessary fsync calls in BATCH.
*/
static svn_error_t *
pack_log_addressed(svn_fs_t *fs,
@@ -1933,6 +1916,7 @@ pack_log_addressed(svn_fs_t *fs,
const char *shard_dir,
svn_revnum_t shard_rev,
apr_size_t max_mem,
+ svn_fs_x__batch_fsync_t *batch,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool)
@@ -1959,7 +1943,7 @@ pack_log_addressed(svn_fs_t *fs,
/* set up a pack context */
SVN_ERR(initialize_pack_context(&context, fs, pack_file_dir, shard_dir,
- shard_rev, max_items, cancel_func,
+ shard_rev, max_items, batch, cancel_func,
cancel_baton, scratch_pool));
/* phase 1: determine the size of the revisions to pack */
@@ -1969,7 +1953,8 @@ pack_log_addressed(svn_fs_t *fs,
/* pack revisions in ranges that don't exceed MAX_MEM */
for (i = 0; i < max_ids->nelts; ++i)
- if (APR_ARRAY_IDX(max_ids, i, apr_uint64_t) + item_count <= max_items)
+ if ( APR_ARRAY_IDX(max_ids, i, apr_uint64_t)
+ <= (apr_uint64_t)max_items - item_count)
{
context.end_rev++;
}
@@ -2016,7 +2001,7 @@ pack_log_addressed(svn_fs_t *fs,
* MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
* using SCRATCH_POOL for temporary allocations. Try to limit the amount of
* temporary memory needed to MAX_MEM bytes. CANCEL_FUNC and CANCEL_BATON
- * are what you think they are.
+ * are what you think they are. Schedule necessary fsync calls in BATCH.
*
* If for some reason we detect a partial packing already performed, we
* remove the pack file and start again.
@@ -2030,6 +2015,7 @@ pack_rev_shard(svn_fs_t *fs,
apr_int64_t shard,
int max_files_per_dir,
apr_size_t max_mem,
+ svn_fs_x__batch_fsync_t *batch,
svn_cancel_func_t cancel_func,
void *cancel_baton,
apr_pool_t *scratch_pool)
@@ -2046,10 +2032,11 @@ pack_rev_shard(svn_fs_t *fs,
/* Create the new directory and pack file. */
SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
+ SVN_ERR(svn_fs_x__batch_fsync_new_path(batch, pack_file_dir, scratch_pool));
/* Index information files */
SVN_ERR(pack_log_addressed(fs, pack_file_dir, shard_path, shard_rev,
- max_mem, cancel_func, cancel_baton,
+ max_mem, batch, cancel_func, cancel_baton,
scratch_pool));
SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, scratch_pool));
@@ -2082,45 +2069,39 @@ pack_shard(const char *dir,
apr_pool_t *scratch_pool)
{
svn_fs_x__data_t *ffd = fs->fsap_data;
- const char *rev_shard_path, *rev_pack_file_dir;
- const char *revprops_shard_path, *revprops_pack_file_dir;
+ const char *shard_path, *pack_file_dir;
+ svn_fs_x__batch_fsync_t *batch;
/* Notify caller we're starting to pack this shard. */
if (notify_func)
SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
scratch_pool));
+ /* Perform all fsyncs through this instance. */
+ SVN_ERR(svn_fs_x__batch_fsync_create(&batch, scratch_pool));
+
/* Some useful paths. */
- rev_pack_file_dir = svn_dirent_join(dir,
+ pack_file_dir = svn_dirent_join(dir,
apr_psprintf(scratch_pool,
"%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
shard),
scratch_pool);
- rev_shard_path = svn_dirent_join(dir,
+ shard_path = svn_dirent_join(dir,
apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard),
scratch_pool);
/* pack the revision content */
- SVN_ERR(pack_rev_shard(fs, rev_pack_file_dir, rev_shard_path,
- shard, max_files_per_dir, DEFAULT_MAX_MEM,
+ SVN_ERR(pack_rev_shard(fs, pack_file_dir, shard_path,
+ shard, max_files_per_dir, DEFAULT_MAX_MEM, batch,
cancel_func, cancel_baton, scratch_pool));
/* pack the revprops in an equivalent way */
- revprops_pack_file_dir = svn_dirent_join(dir,
- apr_psprintf(scratch_pool,
- "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
- shard),
- scratch_pool);
- revprops_shard_path = svn_dirent_join(dir,
- apr_psprintf(scratch_pool, "%" APR_INT64_T_FMT, shard),
- scratch_pool);
-
SVN_ERR(svn_fs_x__pack_revprops_shard(fs,
- revprops_pack_file_dir,
- revprops_shard_path,
+ pack_file_dir,
+ shard_path,
shard, max_files_per_dir,
(int)(0.9 * max_pack_size),
- compression_level,
+ compression_level, batch,
cancel_func, cancel_baton,
scratch_pool));
@@ -2130,8 +2111,11 @@ pack_shard(const char *dir,
scratch_pool));
ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir);
+ /* Ensure that packed file is written to disk.*/
+ SVN_ERR(svn_fs_x__batch_fsync_run(batch, scratch_pool));
+
/* Finally, remove the existing shard directories. */
- SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
+ SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
cancel_func, cancel_baton, scratch_pool));
/* Notify caller we're starting to pack this shard. */
@@ -2142,6 +2126,34 @@ pack_shard(const char *dir,
return SVN_NO_ERROR;
}
+/* Read the youngest rev and the first non-packed rev info for FS from disk.
+ Set *FULLY_PACKED when there is no completed unpacked shard.
+ Use SCRATCH_POOL for temporary allocations.
+ */
+static svn_error_t *
+get_pack_status(svn_boolean_t *fully_packed,
+ svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ apr_int64_t completed_shards;
+ svn_revnum_t youngest;
+
+ SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, fs,
+ scratch_pool));
+
+ SVN_ERR(svn_fs_x__youngest_rev(&youngest, fs, scratch_pool));
+ completed_shards = (youngest + 1) / ffd->max_files_per_dir;
+
+ /* See if we've already completed all possible shards thus far. */
+ if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
+ *fully_packed = TRUE;
+ else
+ *fully_packed = FALSE;
+
+ return SVN_NO_ERROR;
+}
+
typedef struct pack_baton_t
{
svn_fs_t *fs;
@@ -2174,21 +2186,24 @@ pack_body(void *baton,
svn_fs_x__data_t *ffd = pb->fs->fsap_data;
apr_int64_t completed_shards;
apr_int64_t i;
- svn_revnum_t youngest;
apr_pool_t *iterpool;
const char *data_path;
+ svn_boolean_t fully_packed;
- /* If we aren't using sharding, we can't do any packing, so quit. */
- SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, pb->fs,
- scratch_pool));
+ /* Since another process might have already packed the repo,
+ we need to re-read the pack status. */
+ SVN_ERR(get_pack_status(&fully_packed, pb->fs, scratch_pool));
+ if (fully_packed)
+ {
+ if (pb->notify_func)
+ (*pb->notify_func)(pb->notify_baton,
+ ffd->min_unpacked_rev / ffd->max_files_per_dir,
+ svn_fs_pack_notify_noop, scratch_pool);
- SVN_ERR(svn_fs_x__youngest_rev(&youngest, pb->fs, scratch_pool));
- completed_shards = (youngest + 1) / ffd->max_files_per_dir;
-
- /* See if we've already completed all possible shards thus far. */
- if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
- return SVN_NO_ERROR;
+ return SVN_NO_ERROR;
+ }
+ completed_shards = (ffd->youngest_rev_cache + 1) / ffd->max_files_per_dir;
data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, scratch_pool);
iterpool = svn_pool_create(scratch_pool);
@@ -2224,6 +2239,23 @@ svn_fs_x__pack(svn_fs_t *fs,
apr_pool_t *scratch_pool)
{
pack_baton_t pb = { 0 };
+ svn_boolean_t fully_packed;
+
+ /* Is there we even anything to do?. */
+ SVN_ERR(get_pack_status(&fully_packed, fs, scratch_pool));
+ if (fully_packed)
+ {
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+
+ if (notify_func)
+ (*notify_func)(notify_baton,
+ ffd->min_unpacked_rev / ffd->max_files_per_dir,
+ svn_fs_pack_notify_noop, scratch_pool);
+
+ return SVN_NO_ERROR;
+ }
+
+ /* Lock the repo and start the pack process. */
pb.fs = fs;
pb.notify_func = notify_func;
pb.notify_baton = notify_baton;
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c Fri Apr 29 18:38:53 2016
@@ -22,6 +22,7 @@
#include "recovery.h"
+#include "svn_dirent_uri.h"
#include "svn_hash.h"
#include "svn_pools.h"
#include "private/svn_string_private.h"
@@ -106,6 +107,86 @@ recover_get_largest_revision(svn_fs_t *f
return SVN_NO_ERROR;
}
+/* Delete all files and sub-directories (recursively) of DIR_PATH but
+ leave DIR_PATH itself in place. Use SCRATCH_POOL for temporaries. */
+static svn_error_t *
+clear_directory(const char *dir_path,
+ apr_pool_t *scratch_pool)
+{
+ apr_hash_t *dirents;
+ apr_hash_index_t *hi;
+ apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+
+ SVN_ERR(svn_io_get_dirents3(&dirents, dir_path, TRUE, scratch_pool,
+ scratch_pool));
+
+ for (hi = apr_hash_first(scratch_pool, dirents);
+ hi;
+ hi = apr_hash_next(hi))
+ {
+ const char *path;
+ const char *name;
+ svn_dirent_t *dirent;
+
+ svn_pool_clear(iterpool);
+ apr_hash_this(hi, (const void **)&name, NULL, (void **)&dirent);
+
+ path = svn_dirent_join(dir_path, name, iterpool);
+ if (dirent->kind == svn_node_dir)
+ SVN_ERR(svn_io_remove_dir2(path, TRUE, NULL, NULL, iterpool));
+ else
+ SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
+ }
+
+ svn_pool_destroy(iterpool);
+
+ return SVN_NO_ERROR;
+}
+
+/* Delete all uncommitted transaction data from FS.
+ Use SCRATCH_POOL for temporaries. */
+static svn_error_t *
+discard_transactions(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_x__data_t *ffd = fs->fsap_data;
+ svn_fs_x__shared_data_t *ffsd = ffd->shared;
+
+ /* In case this FS has been opened more than once in this process,
+ we should purge their shared transaction data as well. We do the
+ same as abort_txn would, except that we don't expect all txn files
+ to be complete on disk. */
+ while (ffsd->txns)
+ {
+ svn_fs_x__shared_txn_data_t *txn = ffsd->txns;
+ ffsd->txns = txn->next;
+
+ svn_pool_destroy(txn->pool);
+ }
+
+ /* Remove anything from the transaction folders. */
+ SVN_ERR(clear_directory(svn_fs_x__path_txns_dir(fs, scratch_pool),
+ scratch_pool));
+ SVN_ERR(clear_directory(svn_fs_x__path_txn_proto_revs(fs, scratch_pool),
+ scratch_pool));
+
+ return SVN_NO_ERROR;
+}
+
+/* Reset txn-current in FS. Use SCRATCH_POOL for temporaries. */
+static svn_error_t *
+reset_txn_number(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ const char *initial_txn = "0\n";
+ SVN_ERR(svn_io_write_atomic2(svn_fs_x__path_txn_current(fs, scratch_pool),
+ initial_txn, strlen(initial_txn),
+ svn_fs_x__path_uuid(fs, scratch_pool),
+ FALSE, scratch_pool));
+
+ return SVN_NO_ERROR;
+}
+
/* Baton used for recover_body below. */
typedef struct recover_baton_t {
svn_fs_t *fs;
@@ -134,7 +215,13 @@ recover_body(void *baton,
/* The admin may have created a plain copy of this repo before attempting
to recover it (hotcopy may or may not work with corrupted repos).
Bump the instance ID. */
- SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, scratch_pool));
+ SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, TRUE, scratch_pool));
+
+ /* Because transactions are not resilient against system crashes,
+ any existing transaction is suspect (and would probably not be
+ reopened anyway). Get rid of those. */
+ SVN_ERR(discard_transactions(fs, scratch_pool));
+ SVN_ERR(reset_txn_number(fs, scratch_pool));
/* We need to know the largest revision in the filesystem. */
SVN_ERR(recover_get_largest_revision(fs, &max_rev, scratch_pool));
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c Fri Apr 29 18:38:53 2016
@@ -417,8 +417,8 @@ svn_fs_x__reps_add_base(svn_fs_x__reps_b
apr_size_t idx;
SVN_ERR(svn_fs_x__get_contents(&stream, builder->fs, rep, FALSE,
scratch_pool));
- SVN_ERR(svn_string_from_stream(&contents, stream, scratch_pool,
- scratch_pool));
+ SVN_ERR(svn_string_from_stream2(&contents, stream, SVN__STREAM_CHUNK_SIZE,
+ scratch_pool));
SVN_ERR(svn_fs_x__reps_add(&idx, builder, contents));
base.revision = svn_fs_x__get_revnum(rep->id.change_set);
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_x/rev_file.c Fri Apr 29 18:38:53 2016
@@ -331,7 +331,7 @@ auto_read_footer(svn_fs_x__revision_file
&file->p2l_info.start,
&file->p2l_info.checksum,
footer, file->file_info.start_revision,
- file->pool));
+ filesize - footer_length - 1, file->pool));
file->l2p_info.end = file->p2l_info.start;
file->p2l_info.end = filesize - footer_length - 1;
}
@@ -490,8 +490,8 @@ svn_fs_x__rev_file_offset(apr_off_t *off
svn_fs_x__revision_file_t *file)
{
SVN_ERR(auto_open(file));
- return svn_error_trace(svn_fs_x__get_file_offset(offset, file->file,
- file->pool));
+ return svn_error_trace(svn_io_file_get_offset(offset, file->file,
+ file->pool));
}
svn_error_t *