You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2014/10/14 00:25:01 UTC
svn commit: r1631588 - in /subversion/trunk/subversion: libsvn_fs_fs/
tests/cmdline/
Author: stefan2
Date: Mon Oct 13 22:25:01 2014
New Revision: 1631588
URL: http://svn.apache.org/r1631588
Log:
Finalize the FSFS f7 format by adding checksums to the P2L and L2P index
data itself. Now, all of the rev / pack files can be verified.
We simply wrap the streams through which we write the two indexes with
MD5 generator wrappers and put the results into the footer next to the
index offsets. Updating generators, parsers and writers is pretty
straight forward.
The checksum verification code will be added in the next patch.
* subversion/libsvn_fs_fs/index.h
(svn_fs_fs__l2p_index_append,
svn_fs_fs__p2l_index_append): Return the index content checksum as well.
* subversion/libsvn_fs_fs/index.c
(svn_fs_fs__l2p_index_append,
svn_fs_fs__p2l_index_append): Use simple stream wrappers to automatically
calculate the MD5 checksums.
* subversion/libsvn_fs_fs/transaction.c
(svn_fs_fs__add_index_data): Receive the index checksums and pass
them on to the serialization code.
* subversion/libsvn_fs_fs/low_level.h
(svn_fs_fs__parse_footer): Update signature to return the index
checksums as well.
(svn_fs_fs__unparse_footer): Update signature to accept the index
checksums, too.
* subversion/libsvn_fs_fs/low_level.c
(svn_fs_fs__parse_footer): Update the footer reader.
(svn_fs_fs__unparse_footer): Update the footer writer.
* subversion/libsvn_fs_fs/rev_file.h
(svn_fs_fs__revision_file_t): Add members for the index checksums.
* subversion/libsvn_fs_fs/rev_file.c
(init_revision_file): Update struct initialization code.
(svn_fs_fs__auto_read_footer): Receive the checksums as well.
* subversion/libsvn_fs_fs/structure
(Revision file format): Update the description of the file footer.
* subversion/tests/cmdline/svnadmin_tests.py
(set_changed_path_list): Update to preserve the two checksums
in the footer as well.
Modified:
subversion/trunk/subversion/libsvn_fs_fs/index.c
subversion/trunk/subversion/libsvn_fs_fs/index.h
subversion/trunk/subversion/libsvn_fs_fs/low_level.c
subversion/trunk/subversion/libsvn_fs_fs/low_level.h
subversion/trunk/subversion/libsvn_fs_fs/rev_file.c
subversion/trunk/subversion/libsvn_fs_fs/rev_file.h
subversion/trunk/subversion/libsvn_fs_fs/structure
subversion/trunk/subversion/libsvn_fs_fs/transaction.c
subversion/trunk/subversion/tests/cmdline/svnadmin_tests.py
Modified: subversion/trunk/subversion/libsvn_fs_fs/index.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/index.c?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/index.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/index.c Mon Oct 13 22:25:01 2014
@@ -762,10 +762,12 @@ svn_fs_fs__l2p_proto_index_add_entry(apr
}
svn_error_t *
-svn_fs_fs__l2p_index_append(svn_fs_t *fs,
+svn_fs_fs__l2p_index_append(svn_checksum_t **checksum,
+ svn_fs_t *fs,
apr_file_t *index_file,
const char *proto_file_name,
svn_revnum_t revision,
+ apr_pool_t * result_pool,
apr_pool_t *scratch_pool)
{
fs_fs_data_t *ffd = fs->fsap_data;
@@ -888,7 +890,11 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs
page_counts->nelts);
/* open target stream. */
- stream = svn_stream_from_aprfile2(index_file, TRUE, local_pool);
+ stream = svn_stream_checksummed2(svn_stream_from_aprfile2(index_file, TRUE,
+ local_pool),
+ NULL, checksum, svn_checksum_md5, FALSE,
+ result_pool);
+
/* write header info */
SVN_ERR(svn_stream_puts(stream, L2P_STREAM_PREFIX));
@@ -913,7 +919,7 @@ svn_fs_fs__l2p_index_append(svn_fs_t *fs
SVN_ERR(stream_write_encoded(stream, value));
}
- /* append page contents */
+ /* append page contents and implicitly close STREAM */
SVN_ERR(svn_stream_copy3(svn_stream__from_spillbuf(buffer, local_pool),
stream, NULL, NULL, local_pool));
@@ -1951,10 +1957,12 @@ svn_fs_fs__p2l_proto_index_next_offset(a
}
svn_error_t *
-svn_fs_fs__p2l_index_append(svn_fs_t *fs,
+svn_fs_fs__p2l_index_append(svn_checksum_t **checksum,
+ svn_fs_t *fs,
apr_file_t *index_file,
const char *proto_file_name,
svn_revnum_t revision,
+ apr_pool_t *result_pool,
apr_pool_t *scratch_pool)
{
fs_fs_data_t *ffd = fs->fsap_data;
@@ -2083,7 +2091,10 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs
= svn_spillbuf__get_size(buffer) - last_buffer_size;
/* Open target stream. */
- stream = svn_stream_from_aprfile2(index_file, TRUE, local_pool);
+ stream = svn_stream_checksummed2(svn_stream_from_aprfile2(index_file, TRUE,
+ local_pool),
+ NULL, checksum, svn_checksum_md5, FALSE,
+ result_pool);
/* write the start revision, file size and page size */
SVN_ERR(svn_stream_puts(stream, P2L_STREAM_PREFIX));
@@ -2099,7 +2110,7 @@ svn_fs_fs__p2l_index_append(svn_fs_t *fs
SVN_ERR(stream_write_encoded(stream, value));
}
- /* append page contents */
+ /* append page contents and implicitly close STREAM */
SVN_ERR(svn_stream_copy3(svn_stream__from_spillbuf(buffer, local_pool),
stream, NULL, NULL, local_pool));
Modified: subversion/trunk/subversion/libsvn_fs_fs/index.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/index.h?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/index.h (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/index.h Mon Oct 13 22:25:01 2014
@@ -105,13 +105,18 @@ svn_fs_fs__l2p_proto_index_add_entry(apr
/* Use the proto index file stored at PROTO_FILE_NAME, construct the final
* log-to-phys index and append it to INDEX_FILE. The first revision will
* be REVISION, entries to the next revision will be assigned to REVISION+1
- * and so forth. Use SCRATCH_POOL for temporary allocations.
+ * and so forth.
+ *
+ * Return the MD5 checksum of the on-disk index data in *CHECKSUM, allocated
+ * in RESULT_POOL. Use SCRATCH_POOL for temporary allocations.
*/
svn_error_t *
-svn_fs_fs__l2p_index_append(svn_fs_t *fs,
+svn_fs_fs__l2p_index_append(svn_checksum_t **checksum,
+ svn_fs_t *fs,
apr_file_t *index_file,
const char *proto_file_name,
svn_revnum_t revision,
+ apr_pool_t *result_pool,
apr_pool_t *scratch_pool);
/* Open / create a phys-to-log index file with the full file path name
@@ -145,13 +150,17 @@ svn_fs_fs__p2l_proto_index_next_offset(a
/* Use the proto index file stored at PROTO_FILE_NAME, construct the final
* phys-to-log index and append it to INDEX_FILE. Entries without a valid
* revision will be assigned to the REVISION given here.
- * Use SCRATCH_POOL for temporary allocations.
+ *
+ * Return the MD5 checksum of the on-disk index data in *CHECKSUM, allocated
+ * in RESULT_POOL. Use SCRATCH_POOL for temporary allocations.
*/
svn_error_t *
-svn_fs_fs__p2l_index_append(svn_fs_t *fs,
+svn_fs_fs__p2l_index_append(svn_checksum_t **checksum,
+ svn_fs_t *fs,
apr_file_t *index_file,
const char *proto_file_name,
svn_revnum_t revision,
+ apr_pool_t *result_pool,
apr_pool_t *scratch_pool);
/* Use the phys-to-log mapping files in FS to build a list of entries
Modified: subversion/trunk/subversion/libsvn_fs_fs/low_level.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/low_level.c?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/low_level.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/low_level.c Mon Oct 13 22:25:01 2014
@@ -191,38 +191,71 @@ svn_fs_fs__unparse_revision_trailer(apr_
svn_error_t *
svn_fs_fs__parse_footer(apr_off_t *l2p_offset,
+ svn_checksum_t **l2p_checksum,
apr_off_t *p2l_offset,
+ svn_checksum_t **p2l_checksum,
svn_stringbuf_t *footer,
- svn_revnum_t rev)
+ svn_revnum_t rev,
+ apr_pool_t *result_pool)
{
apr_int64_t val;
+ char *last_str = footer->data;
- /* Split the footer into the 2 number strings. */
- char *seperator = strchr(footer->data, ' ');
- if (!seperator)
- return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
- _("Revision file (r%ld) has corrupt footer"),
- rev);
- *seperator = '\0';
+ /* Get the L2P offset. */
+ const char *str = svn_cstring_tokenize(" ", &last_str);
+ if (str == NULL)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+ _("Invalid revision footer"));
- /* Convert offset values. */
- SVN_ERR(svn_cstring_atoi64(&val, footer->data));
+ SVN_ERR(svn_cstring_atoi64(&val, str));
*l2p_offset = (apr_off_t)val;
- SVN_ERR(svn_cstring_atoi64(&val, seperator + 1));
+
+ /* Get the L2P checksum. */
+ str = svn_cstring_tokenize(" ", &last_str);
+ if (str == NULL)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+ _("Invalid revision footer"));
+
+ SVN_ERR(svn_checksum_parse_hex(l2p_checksum, svn_checksum_md5, str,
+ result_pool));
+
+ /* Get the P2L offset. */
+ str = svn_cstring_tokenize(" ", &last_str);
+ if (str == NULL)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+ _("Invalid revision footer"));
+
+ SVN_ERR(svn_cstring_atoi64(&val, str));
*p2l_offset = (apr_off_t)val;
+ /* Get the P2L checksum. */
+ str = svn_cstring_tokenize(" ", &last_str);
+ if (str == NULL)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+ _("Invalid revision footer"));
+
+ SVN_ERR(svn_checksum_parse_hex(p2l_checksum, svn_checksum_md5, str,
+ result_pool));
+
return SVN_NO_ERROR;
}
svn_stringbuf_t *
svn_fs_fs__unparse_footer(apr_off_t l2p_offset,
+ svn_checksum_t *l2p_checksum,
apr_off_t p2l_offset,
- apr_pool_t *result_pool)
+ svn_checksum_t *p2l_checksum,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool)
{
return svn_stringbuf_createf(result_pool,
- "%" APR_OFF_T_FMT " %" APR_OFF_T_FMT,
+ "%" APR_OFF_T_FMT " %s %" APR_OFF_T_FMT " %s",
l2p_offset,
- p2l_offset);
+ svn_checksum_to_cstring(l2p_checksum,
+ scratch_pool),
+ p2l_offset,
+ svn_checksum_to_cstring(p2l_checksum,
+ scratch_pool));
}
/* Read the next entry in the changes record from file FILE and store
Modified: subversion/trunk/subversion/libsvn_fs_fs/low_level.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/low_level.h?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/low_level.h (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/low_level.h Mon Oct 13 22:25:01 2014
@@ -64,25 +64,35 @@ svn_fs_fs__unparse_revision_trailer(apr_
/* Given the format 7+ revision / pack FOOTER, parse it destructively
* and return the start offsets of the index data in *L2P_OFFSET and
- * *P2L_OFFSET, respectively.
+ * *P2L_OFFSET, respectively. Also, return the expected checksums in
+ * in *L2P_CHECKSUM and *P2L_CHECKSUM.
*
* Note that REV is only used to construct nicer error objects that
- * mention this revision.
+ * mention this revision. Allocate the checksums in RESULT_POOL.
*/
svn_error_t *
svn_fs_fs__parse_footer(apr_off_t *l2p_offset,
+ svn_checksum_t **l2p_checksum,
apr_off_t *p2l_offset,
+ svn_checksum_t **p2l_checksum,
svn_stringbuf_t *footer,
- svn_revnum_t rev);
+ svn_revnum_t rev,
+ apr_pool_t *result_pool);
-/* Given the offset of the L2P index data in L2P_OFFSET and the offset of
- * the P2L index data in P2L_OFFSET, return the corresponding format 7+
- * revision / pack file footer. Allocate it in RESULT_POOL.
+/* Given the offset of the L2P index data in L2P_OFFSET, the content
+ * checksum in L2P_CHECKSUM and the offset plus checksum of the P2L
+ * index data in P2L_OFFSET and P2L_CHECKSUM.
+ *
+ * Return the corresponding format 7+ revision / pack file footer.
+ * Allocate it in RESULT_POOL and use SCRATCH_POOL for temporary.
*/
svn_stringbuf_t *
svn_fs_fs__unparse_footer(apr_off_t l2p_offset,
+ svn_checksum_t *l2p_checksum,
apr_off_t p2l_offset,
- apr_pool_t *result_pool);
+ svn_checksum_t *p2l_checksum,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool);
/* Read all the changes from STREAM and store them in *CHANGES,
allocated in RESULT_POOL. Do temporary allocations in SCRATCH_POOL. */
Modified: subversion/trunk/subversion/libsvn_fs_fs/rev_file.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/rev_file.c?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/rev_file.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/rev_file.c Mon Oct 13 22:25:01 2014
@@ -50,7 +50,9 @@ init_revision_file(svn_fs_fs__revision_f
file->l2p_stream = NULL;
file->block_size = ffd->block_size;
file->l2p_offset = -1;
+ file->l2p_checksum = NULL;
file->p2l_offset = -1;
+ file->p2l_checksum = NULL;
file->footer_offset = -1;
file->pool = pool;
}
@@ -254,8 +256,10 @@ svn_fs_fs__auto_read_footer(svn_fs_fs__r
footer->data[footer->len] = '\0';
/* Extract index locations. */
- SVN_ERR(svn_fs_fs__parse_footer(&file->l2p_offset, &file->p2l_offset,
- footer, file->start_revision));
+ SVN_ERR(svn_fs_fs__parse_footer(&file->l2p_offset, &file->l2p_checksum,
+ &file->p2l_offset, &file->p2l_checksum,
+ footer, file->start_revision,
+ file->pool));
file->footer_offset = filesize - footer_length - 1;
}
Modified: subversion/trunk/subversion/libsvn_fs_fs/rev_file.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/rev_file.h?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/rev_file.h (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/rev_file.h Mon Oct 13 22:25:01 2014
@@ -72,11 +72,19 @@ typedef struct svn_fs_fs__revision_file_
* has not been called, yet. */
apr_off_t l2p_offset;
+ /* MD5 checksum on the whole on-disk representation of the L2P index.
+ * NULL if svn_fs_fs__auto_read_footer has not been called, yet. */
+ svn_checksum_t *l2p_checksum;
+
/* Offset within FILE at which the L2P index ends and the P2L index
* data starts. Greater than L2P_OFFSET. -1 if svn_fs_fs__auto_read_footer
* has not been called, yet. */
apr_off_t p2l_offset;
+ /* MD5 checksum on the whole on-disk representation of the P2L index.
+ * NULL if svn_fs_fs__auto_read_footer has not been called, yet. */
+ svn_checksum_t *p2l_checksum;
+
/* Offset within FILE at which the P2L index ends and the footer starts.
* Greater than P2L_OFFSET. -1 if svn_fs_fs__auto_read_footer has not
* been called, yet. */
Modified: subversion/trunk/subversion/libsvn_fs_fs/structure
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/structure?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/structure (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/structure Mon Oct 13 22:25:01 2014
@@ -615,11 +615,14 @@ lines containing "\n<root-offset> <cp-of
the offset of the root directory node revision and <cp-offset> is the
offset of the changed-path data.
-In logical addressing mode, the revision footer is pair of offsets,
-separated by a space and the whole is terminated a single byte. That
-byte contains the length (as plain 8 bit value) of the footer excluding
-the length byte. The first offset is the start of the log-to-phys index,
-the other is the start offset of the phys-to-log index.
+In logical addressing mode, the revision footer has the form
+
+ <l2p offset> <l2p checksum> <p2l offset> <p2l checksum><terminal byte>
+
+The terminal byte contains the length (as plain 8 bit value) of the footer
+excluding that length byte. The first offset is the start of the log-to-
+phys index, followed by the digest of the MD5 checksum over its content.
+The other pair gives the same of for the phys-to-log index.
All numbers in the rev file format are unsigned and are represented as
ASCII decimal.
Modified: subversion/trunk/subversion/libsvn_fs_fs/transaction.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_fs/transaction.c?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_fs/transaction.c (original)
+++ subversion/trunk/subversion/libsvn_fs_fs/transaction.c Mon Oct 13 22:25:01 2014
@@ -3588,20 +3588,25 @@ svn_fs_fs__add_index_data(svn_fs_t *fs,
apr_off_t p2l_offset;
svn_stringbuf_t *footer;
unsigned char footer_length;
+ svn_checksum_t *l2p_checksum;
+ svn_checksum_t *p2l_checksum;
/* Append the actual index data to the pack file. */
l2p_offset = 0;
SVN_ERR(svn_io_file_seek(file, APR_END, &l2p_offset, pool));
- SVN_ERR(svn_fs_fs__l2p_index_append(fs, file, l2p_proto_index, revision,
- pool));
+ SVN_ERR(svn_fs_fs__l2p_index_append(&l2p_checksum, fs, file,
+ l2p_proto_index, revision,
+ pool, pool));
p2l_offset = 0;
SVN_ERR(svn_io_file_seek(file, APR_END, &p2l_offset, pool));
- SVN_ERR(svn_fs_fs__p2l_index_append(fs, file, p2l_proto_index, revision,
- pool));
+ SVN_ERR(svn_fs_fs__p2l_index_append(&p2l_checksum, fs, file,
+ p2l_proto_index, revision,
+ pool, pool));
/* Append footer. */
- footer = svn_fs_fs__unparse_footer(l2p_offset, p2l_offset, pool);
+ footer = svn_fs_fs__unparse_footer(l2p_offset, l2p_checksum,
+ p2l_offset, p2l_checksum, pool, pool);
SVN_ERR(svn_io_file_write_full(file, footer->data, footer->len, NULL,
pool));
Modified: subversion/trunk/subversion/tests/cmdline/svnadmin_tests.py
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/tests/cmdline/svnadmin_tests.py?rev=1631588&r1=1631587&r2=1631588&view=diff
==============================================================================
--- subversion/trunk/subversion/tests/cmdline/svnadmin_tests.py (original)
+++ subversion/trunk/subversion/tests/cmdline/svnadmin_tests.py Mon Oct 13 22:25:01 2014
@@ -392,7 +392,9 @@ def set_changed_path_list(sbox, revision
footer_length = ord(contents[length-1]);
footer = contents[length - footer_length - 1:length-1]
l2p_offset = long(footer.split(' ')[0])
- p2l_offset = long(footer.split(' ')[1])
+ l2p_checksum = footer.split(' ')[1]
+ p2l_offset = long(footer.split(' ')[2])
+ p2l_checksum = footer.split(' ')[3]
idx = FSFS_Index(sbox, revision)
(offset, item_len, item_type) = idx.get_item(1)
@@ -405,7 +407,8 @@ def set_changed_path_list(sbox, revision
file_len = body_len + len(changes) + 1
p2l_offset += file_len - l2p_offset
- header = str(file_len) + ' ' + str(p2l_offset)
+ header = str(file_len) + ' ' + l2p_checksum + ' ' \
+ + str(p2l_offset) + ' ' + p2l_checksum
header += chr(len(header))
header = '\n' + indexes + header