You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by br...@apache.org on 2015/01/03 15:00:44 UTC
svn commit: r1649205 [8/30] - in /subversion/branches/authzperf: ./ build/
build/ac-macros/ notes/ subversion/bindings/ctypes-python/
subversion/bindings/cxxhl/
subversion/bindings/javahl/tests/org/apache/subversion/javahl/
subversion/bindings/swig/ su...
Modified: subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_fs/verify.c Sat Jan 3 14:00:41 2015
@@ -22,6 +22,7 @@
#include "svn_sorts.h"
#include "svn_checksum.h"
+#include "svn_time.h"
#include "private/svn_subr_private.h"
#include "verify.h"
@@ -159,6 +160,93 @@ verify_rep_cache(svn_fs_t *fs,
return SVN_NO_ERROR;
}
+/* Verify that the MD5 checksum of the data between offsets START and END
+ * in FILE matches the EXPECTED checksum. If there is a mismatch use the
+ * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC
+ * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */
+static svn_error_t *
+verify_index_checksum(apr_file_t *file,
+ const char *name,
+ apr_off_t start,
+ apr_off_t end,
+ svn_checksum_t *expected,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool)
+{
+ unsigned char buffer[SVN__STREAM_CHUNK_SIZE];
+ apr_off_t size = end - start;
+ svn_checksum_t *actual;
+ svn_checksum_ctx_t *context
+ = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool);
+
+ /* Calculate the index checksum. */
+ SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool));
+ while (size > 0)
+ {
+ apr_size_t to_read = size > sizeof(buffer)
+ ? sizeof(buffer)
+ : (apr_size_t)size;
+ SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL,
+ scratch_pool));
+ SVN_ERR(svn_checksum_update(context, buffer, to_read));
+ size -= to_read;
+
+ if (cancel_func)
+ SVN_ERR(cancel_func(cancel_baton));
+ }
+
+ SVN_ERR(svn_checksum_final(&actual, context, scratch_pool));
+
+ /* Verify that it matches the expected checksum. */
+ if (!svn_checksum_match(expected, actual))
+ {
+ const char *file_name;
+
+ SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool));
+ SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool,
+ _("%s checksum mismatch in file %s"),
+ name, file_name));
+ }
+
+ return SVN_NO_ERROR;
+}
+
+/* Verify the MD5 checksums of the index data in the rev / pack file
+ * containing revision START in FS. If given, invoke CANCEL_FUNC with
+ * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary
+ * allocations.
+ */
+static svn_error_t *
+verify_index_checksums(svn_fs_t *fs,
+ svn_revnum_t start,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool)
+{
+ svn_fs_fs__revision_file_t *rev_file;
+
+ /* Open the rev / pack file and read the footer */
+ SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start,
+ scratch_pool, scratch_pool));
+ SVN_ERR(svn_fs_fs__auto_read_footer(rev_file));
+
+ /* Verify the index contents against the checksum from the footer. */
+ SVN_ERR(verify_index_checksum(rev_file->file, "L2P index",
+ rev_file->l2p_offset, rev_file->p2l_offset,
+ rev_file->l2p_checksum,
+ cancel_func, cancel_baton, scratch_pool));
+ SVN_ERR(verify_index_checksum(rev_file->file, "P2L index",
+ rev_file->p2l_offset, rev_file->footer_offset,
+ rev_file->p2l_checksum,
+ cancel_func, cancel_baton, scratch_pool));
+
+ /* Done. */
+ SVN_ERR(svn_fs_fs__close_revision_file(rev_file));
+
+ return SVN_NO_ERROR;
+}
+
/* Verify that for all log-to-phys index entries for revisions START to
* START + COUNT-1 in FS there is a consistent entry in the phys-to-log
* index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular
@@ -182,7 +270,8 @@ compare_l2p_to_p2l_index(svn_fs_t *fs,
iterpool));
/* determine the range of items to check for each revision */
- SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool));
+ SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool,
+ iterpool));
/* check all items in all revisions if the given range */
for (i = 0; i < max_ids->nelts; ++i)
@@ -205,10 +294,11 @@ compare_l2p_to_p2l_index(svn_fs_t *fs,
/* find the corresponding P2L entry */
SVN_ERR(svn_fs_fs__p2l_entry_lookup(&p2l_entry, fs, rev_file,
- revision, offset, iterpool));
+ revision, offset, iterpool,
+ iterpool));
if (p2l_entry == NULL)
- return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+ return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
NULL,
_("p2l index entry not found for "
"PHYS %s returned by "
@@ -218,7 +308,7 @@ compare_l2p_to_p2l_index(svn_fs_t *fs,
if ( p2l_entry->item.number != k
|| p2l_entry->item.revision != revision)
- return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+ return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
NULL,
_("p2l index info LOG r%ld:i%ld"
" does not match "
@@ -282,9 +372,9 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
/* get all entries for the current block */
SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start,
offset, ffd->p2l_page_size,
- iterpool));
+ iterpool, iterpool));
if (entries->nelts == 0)
- return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION,
+ return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION,
NULL,
_("p2l does not cover offset %s"
" for revision %ld"),
@@ -301,7 +391,24 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
= &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t);
/* check all sub-items for consist entries in the L2P index */
- if (entry->type != SVN_FS_FS__ITEM_TYPE_UNUSED)
+ if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED)
+ {
+ /* There is no L2P entry for unused rev file sections.
+ * And its P2L index data is hardly ever used. But we
+ * should still check whether someone tempered with it. */
+ if ( entry->item.revision != SVN_INVALID_REVNUM
+ && ( entry->item.revision < start
+ || entry->item.revision >= start + count))
+ return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
+ NULL,
+ _("Empty P2L entry for PHYS %s "
+ "refers to revision %ld outside "
+ "the rev / pack file (%ld-%ld)"),
+ apr_off_t_toa(pool, entry->offset),
+ (long)entry->item.number,
+ start, start + count - 1);
+ }
+ else
{
apr_off_t l2p_offset;
SVN_ERR(svn_fs_fs__item_offset(&l2p_offset, fs, rev_file,
@@ -309,7 +416,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
entry->item.number, iterpool));
if (l2p_offset != entry->offset)
- return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+ return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
NULL,
_("l2p index entry PHYS %s"
"does not match p2l index value "
@@ -416,7 +523,7 @@ expected_checksum(apr_file_t *file,
SVN_ERR(svn_io_file_name_get(&file_name, file, pool));
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
- _("Checksum mismatch item at offset %s of "
+ _("Checksum mismatch in item at offset %s of "
"length %s bytes in file %s"),
apr_off_t_toa(pool, entry->offset),
apr_off_t_toa(pool, entry->size), file_name);
@@ -511,7 +618,7 @@ compare_p2l_to_rev(svn_fs_t *fs,
pool));
if (rev_file->l2p_offset != max_offset)
- return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT, NULL,
+ return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL,
_("File size of %s for revision r%ld does "
"not match p2l index size of %s"),
apr_off_t_toa(pool, rev_file->l2p_offset), start,
@@ -532,7 +639,12 @@ compare_p2l_to_rev(svn_fs_t *fs,
/* get all entries for the current block */
SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start,
offset, ffd->p2l_page_size,
- iterpool));
+ iterpool, iterpool));
+
+ /* The above might have moved the file pointer.
+ * Ensure we actually start reading at OFFSET. */
+ SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size,
+ NULL, offset, iterpool));
/* process all entries (and later continue with the next block) */
for (i = 0; i < entries->nelts; ++i)
@@ -550,7 +662,7 @@ compare_p2l_to_rev(svn_fs_t *fs,
/* p2l index must cover all rev / pack file offsets exactly once */
if (entry->offset != offset)
- return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_INCONSISTENT,
+ return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
NULL,
_("p2l index entry for revision r%ld"
" is non-contiguous between offsets "
@@ -566,7 +678,7 @@ compare_p2l_to_rev(svn_fs_t *fs,
if (entry->offset != max_offset)
SVN_ERR(read_all_nul(rev_file->file, entry->size, pool));
}
- else if (entry->fnv1_checksum)
+ else
{
if (entry->size < STREAM_THRESHOLD)
SVN_ERR(expected_buffered_checksum(rev_file->file, entry,
@@ -591,14 +703,47 @@ compare_p2l_to_rev(svn_fs_t *fs,
return SVN_NO_ERROR;
}
-static svn_revnum_t
-packed_base_rev(svn_fs_t *fs, svn_revnum_t rev)
+/* Verify that the revprops of the revisions START to END in FS can be
+ * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals.
+ *
+ * The values of START and END have already been auto-selected and
+ * verified.
+ */
+static svn_error_t *
+verify_revprops(svn_fs_t *fs,
+ svn_revnum_t start,
+ svn_revnum_t end,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool)
{
- fs_fs_data_t *ffd = fs->fsap_data;
+ svn_revnum_t revision;
+ apr_pool_t *iterpool = svn_pool_create(pool);
+
+ for (revision = start; revision < end; ++revision)
+ {
+ svn_string_t *date;
+ apr_time_t timetemp;
- return rev < ffd->min_unpacked_rev
- ? rev - (rev % ffd->max_files_per_dir)
- : rev;
+ svn_pool_clear(iterpool);
+
+ /* Access the svn:date revprop.
+ * This implies parsing all revprops for that revision. */
+ SVN_ERR(svn_fs_fs__revision_prop(&date, fs, revision,
+ SVN_PROP_REVISION_DATE, iterpool));
+
+ /* The time stamp is the only revprop that, if given, needs to
+ * have a valid content. */
+ if (date)
+ SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool));
+
+ if (cancel_func)
+ SVN_ERR(cancel_func(cancel_baton));
+ }
+
+ svn_pool_destroy(iterpool);
+
+ return SVN_NO_ERROR;
}
static svn_revnum_t
@@ -609,22 +754,24 @@ pack_size(svn_fs_t *fs, svn_revnum_t rev
return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1;
}
-/* Verify that the log-to-phys indexes and phys-to-log indexes are
- * consistent with each other. The function signature is similar to
- * svn_fs_fs__verify.
+/* Verify that on-disk representation has not been tempered with (in a way
+ * that leaves the repository in a corrupted state). This compares log-to-
+ * phys with phys-to-log indexes, verifies the low-level checksums and
+ * checks that all revprops are available. The function signature is
+ * similar to svn_fs_fs__verify.
*
* The values of START and END have already been auto-selected and
* verified. You may call this for format7 or higher repos.
*/
static svn_error_t *
-verify_index_consistency(svn_fs_t *fs,
- svn_revnum_t start,
- svn_revnum_t end,
- svn_fs_progress_notify_func_t notify_func,
- void *notify_baton,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *pool)
+verify_f7_metadata_consistency(svn_fs_t *fs,
+ svn_revnum_t start,
+ svn_revnum_t end,
+ svn_fs_progress_notify_func_t notify_func,
+ void *notify_baton,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool)
{
fs_fs_data_t *ffd = fs->fsap_data;
svn_revnum_t revision, next_revision;
@@ -635,7 +782,7 @@ verify_index_consistency(svn_fs_t *fs,
svn_error_t *err = SVN_NO_ERROR;
svn_revnum_t count = pack_size(fs, revision);
- svn_revnum_t pack_start = packed_base_rev(fs, revision);
+ svn_revnum_t pack_start = svn_fs_fs__packed_base_rev(fs, revision);
svn_revnum_t pack_end = pack_start + count;
svn_pool_clear(iterpool);
@@ -643,9 +790,14 @@ verify_index_consistency(svn_fs_t *fs,
if (notify_func && (pack_start % ffd->max_files_per_dir == 0))
notify_func(pack_start, notify_baton, iterpool);
+ /* Check for external corruption to the indexes. */
+ err = verify_index_checksums(fs, pack_start, cancel_func,
+ cancel_baton, iterpool);
+
/* two-way index check */
- err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start,
- cancel_func, cancel_baton, iterpool);
+ if (!err)
+ err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start,
+ cancel_func, cancel_baton, iterpool);
if (!err)
err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start,
cancel_func, cancel_baton, iterpool);
@@ -655,6 +807,11 @@ verify_index_consistency(svn_fs_t *fs,
err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start,
cancel_func, cancel_baton, iterpool);
+ /* ensure that revprops are available and accessible */
+ if (!err)
+ err = verify_revprops(fs, pack_start, pack_end,
+ cancel_func, cancel_baton, iterpool);
+
/* concurrent packing is one of the reasons why verification may fail.
Make sure, we operate on up-to-date information. */
if (err)
@@ -668,7 +825,7 @@ verify_index_consistency(svn_fs_t *fs,
/* We could simply assign revision here but the code below is
more intuitive to maintainers. */
- next_revision = packed_base_rev(fs, revision);
+ next_revision = svn_fs_fs__packed_base_rev(fs, revision);
}
else
{
@@ -705,11 +862,10 @@ svn_fs_fs__verify(svn_fs_t *fs,
/* log/phys index consistency. We need to check them first to make
sure we can access the rev / pack files in format7. */
- if (svn_fs_fs__use_log_addressing(fs, end))
- SVN_ERR(verify_index_consistency(fs,
- MAX(start, ffd->min_log_addressing_rev),
- end, notify_func, notify_baton,
- cancel_func, cancel_baton, pool));
+ if (svn_fs_fs__use_log_addressing(fs))
+ SVN_ERR(verify_f7_metadata_consistency(fs, start, end,
+ notify_func, notify_baton,
+ cancel_func, cancel_baton, pool));
/* rep cache consistency */
if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
Modified: subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c?rev=1649205&r1=1649204&r2=1649205&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c (original)
+++ subversion/branches/authzperf/subversion/libsvn_fs_util/fs-util.c Sat Jan 3 14:00:41 2015
@@ -199,6 +199,7 @@ svn_fs__path_change_create_internal(cons
change->node_rev_id = node_rev_id;
change->change_kind = change_kind;
change->mergeinfo_mod = svn_tristate_unknown;
+ change->copyfrom_rev = SVN_INVALID_REVNUM;
return change;
}
Propchange: subversion/branches/authzperf/subversion/libsvn_fs_x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 3 14:00:41 2015
@@ -20,6 +20,7 @@
/subversion/branches/fsfs-lock-many/subversion/libsvn_fs_x:1571740-1577217
/subversion/branches/fsfs-pack/subversion/libsvn_fs_x:873717-874575
/subversion/branches/fsx/subversion/libsvn_fs_x:1507845-1508040
+/subversion/branches/fsx-id/subversion/libsvn_fs_x:1645603-1649011
/subversion/branches/gnome-keyring/subversion/libsvn_fs_x:870558-871410
/subversion/branches/gpg-agent-password-store/subversion/libsvn_fs_x:1005036-1150766
/subversion/branches/gtest_addition/subversion/libsvn_fs_x:1452117-1502138
@@ -50,6 +51,7 @@
/subversion/branches/issue-4194-dev/subversion/libsvn_fs_x:1410507-1414880
/subversion/branches/javahl-ra/subversion/libsvn_fs_x:991978-1494640
/subversion/branches/kwallet/subversion/libsvn_fs_x:870785-871314
+/subversion/branches/log-addressing/subversion/libsvn_fs_fs:1531982
/subversion/branches/log-addressing/subversion/libsvn_fs_x:1511324
/subversion/branches/log-g-performance/subversion/libsvn_fs_x:870941-871032
/subversion/branches/merge-skips-obstructions/subversion/libsvn_fs_x:874525-874615
@@ -62,6 +64,8 @@
/subversion/branches/reintegrate-improvements/subversion/libsvn_fs_x:873853-874164
/subversion/branches/remote-only-status/subversion/libsvn_fs_x:1581845-1586090
/subversion/branches/revprop-cache/subversion/libsvn_fs_x:1298521-1326293
+/subversion/branches/revprop-caching-ng/subversion/libsvn_fs_fs:1619782-1620595
+/subversion/branches/revprop-caching-ng/subversion/libsvn_fs_x:1620597
/subversion/branches/revprop-packing/subversion/libsvn_fs_x:1143907,1143971,1143997,1144017,1144499,1144568,1146145
/subversion/branches/subtree-mergeinfo/subversion/libsvn_fs_x:876734-878766
/subversion/branches/svn-mergeinfo-enhancements/subversion/libsvn_fs_x:870119-870195,870197-870288
@@ -82,5 +86,5 @@
/subversion/branches/verify-at-commit/subversion/libsvn_fs_x:1462039-1462408
/subversion/branches/verify-keep-going/subversion/libsvn_fs_x:1439280-1492639,1546002-1546110
/subversion/branches/wc-collate-path/subversion/libsvn_fs_x:1402685-1480384
-/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500
-/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914,1613053-1621376
+/subversion/trunk/subversion/libsvn_fs_fs:1415133-1596500,1596567,1597414,1597989,1598273,1599140,1600872,1601633,1603485-1603487,1603499,1603605,1604128,1604188,1604413-1604414,1604416-1604417,1604421,1604442,1604700,1604717,1604720,1604726,1604755,1604794,1604802,1604824,1604836,1604844,1604902-1604903,1604911,1604925,1604933,1604947,1605059-1605060,1605064-1605065,1605068,1605071-1605073,1605075,1605123,1605188-1605189,1605191,1605197,1605444,1605633,1606132,1606142,1606144,1606514,1606526,1606528,1606551,1606554,1606564,1606598-1606599,1606656,1606658,1606662,1606744,1606840,1607085,1607572,1612407,1612810,1613339,1613872,1614611,1615348,1615351-1615352,1615356,1616338-1616339,1616613,1617586,1617688,1618138,1618151,1618153,1618226,1618641,1618653,1618662,1619068,1619358,1619413,1619769,1619774,1620602,1620909,1620912,1620928,1620930,1621275,1621635,1622931,1622937,1622942,1622946,1622959-1622960,1622963,1622987,1623007,1623368,1623373,1623377,1623379,1623381,1623398,1623402,162
4011,1624265,1624512,1626246,1626871,1626873,1626886,1627497-1627498,1627502,1627947-1627949,1627966,1628083,1628093,1628158-1628159,1628161,1628392-1628393,1628415,1628427,1628676,1628738,1628762,1628764,1629854-1629855,1629857,1629865,1629873,1629875,1629879,1630067,1630070,1631049-1631051,1631075,1631115,1631171,1631180,1631185-1631186,1631196-1631197,1631239-1631240,1631548,1631550,1631563,1631567,1631588,1631598,1632646,1632776,1632849,1632851-1632853,1632856-1632857,1632868,1632908,1632926,1633232,1633617-1633618,1634872,1634875,1634879-1634880,1634920,1636478,1636483,1636629,1636644,1637184,1637186,1637330,1637358,1637363,1637393,1639319,1639322,1639335,1639348,1639352,1639355,1639358,1639414,1639419,1639426,1639430,1639436,1639440,1639549,1640061-1640062,1640197,1640915,1640966,1641013,1643139,1643233,1645567,1646021,1646712,1646716,1647537,1647540-1647541,1647820,1647905,1648230,1648238,1648241-1648243,1648253,1648272,1648532,1648537-1648539,1648542,1648591,1648612
+/subversion/trunk/subversion/libsvn_fs_x:1414756-1509914,1613053-1649188