You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2013/03/28 10:33:10 UTC
svn commit: r1462007 - in /subversion/branches/fsfs-format7/subversion:
libsvn_fs_fs/string_table.c libsvn_fs_fs/string_table.h
tests/libsvn_fs_fs/string-table-test.c
Author: stefan2
Date: Thu Mar 28 09:33:09 2013
New Revision: 1462007
URL: http://svn.apache.org/r1462007
Log:
On the fsfs-format7 branch: Use the new Packed Data API to serialize /
deserialize string tables. Duplicate the string table tests such that
the second variant will write & read the data before checking it.
* subversion/libsvn_fs_fs/string_table.h
(svn_fs_fs__string_table_builder_estimate_size,
svn_fs_fs__write_string_table,
svn_fs_fs__read_string_table): declare new private API
* subversion/libsvn_fs_fs/string_table.c
(builder_table_t): track total size of long string data
(string_sub_table_t): use the right element data type
(svn_fs_fs__string_table_builder_add): maintain new tacking info
(svn_fs_fs__string_table_builder_estimate_size): implement
(create_table): formatting fix
(svn_fs_fs__write_string_table,
svn_fs_fs__read_string_table): implement using the new packed data API
* subversion/tests/libsvn_fs_fs/string-table-test.c
(store_and_load_table): new utility function
(create_empty_table,
short_string_table,
large_string_table,
many_strings_table): rename to *_body and add do_load_store flag;
add tests with orig. names that call the bodies
(store_load_short_string_table,
store_load_large_string_table,
store_load_empty_table,
store_load_many_strings_table): new tests calling the same body
(test_funcs): register new tests
Modified:
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.c
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.h
subversion/branches/fsfs-format7/subversion/tests/libsvn_fs_fs/string-table-test.c
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.c?rev=1462007&r1=1462006&r2=1462007&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.c Thu Mar 28 09:33:09 2013
@@ -27,6 +27,8 @@
#include "svn_sorts.h"
#include "private/svn_string_private.h"
#include "private/svn_subr_private.h"
+#include "private/svn_delta_private.h"
+#include "private/svn_packed_data.h"
#include "string_table.h"
@@ -61,6 +63,7 @@ typedef struct builder_table_t
apr_array_header_t *short_strings;
apr_array_header_t *long_strings;
apr_hash_t *long_string_dict;
+ apr_size_t long_string_size;
} builder_table_t;
struct string_table_builder_t
@@ -79,7 +82,7 @@ typedef struct string_header_t
typedef struct string_sub_table_t
{
- char *data;
+ const char *data;
apr_size_t data_size;
string_header_t *short_strings;
@@ -297,6 +300,8 @@ svn_fs_fs__string_table_builder_add(stri
APR_ARRAY_PUSH(table->long_strings, svn_string_t) = item;
apr_hash_set(table->long_string_dict, string, len,
(void*)(apr_uintptr_t)table->long_strings->nelts);
+
+ table->long_string_size += len;
}
else
{
@@ -332,9 +337,40 @@ svn_fs_fs__string_table_builder_add(stri
return result;
}
+apr_size_t
+svn_fs_fs__string_table_builder_estimate_size(string_table_builder_t *builder)
+{
+ apr_size_t total = 0;
+ int i;
+
+ for (i = 0; i < builder->tables->nelts; ++i)
+ {
+ builder_table_t *table
+ = APR_ARRAY_IDX(builder->tables, i, builder_table_t*);
+
+ /* total number of chars to store,
+ * 8 bytes per short string table entry
+ * 4 bytes per long string table entry
+ * some static overhead */
+ apr_size_t table_size
+ = MAX_DATA_SIZE - table->max_data_size
+ + table->long_string_size
+ + table->short_strings->nelts * 8
+ + table->long_strings->nelts * 4
+ + 10;
+
+ total += table_size;
+ }
+
+ /* ZIP compression should give us a 50% reduction.
+ * add some static overhead */
+ return 200 + total / 2;
+
+}
+
static void
create_table(string_sub_table_t *target,
- builder_table_t* source,
+ builder_table_t *source,
apr_pool_t *pool,
apr_pool_t *scratch_pool)
{
@@ -516,3 +552,163 @@ svn_fs_fs__string_table_copy_string(char
return 0;
}
+
+svn_error_t *
+svn_fs_fs__write_string_table(svn_stream_t *stream,
+ const string_table_t *table,
+ apr_pool_t *pool)
+{
+ apr_size_t i, k;
+
+ svn__packed_data_root_t *root = svn__packed_data_create_root(pool);
+
+ svn__packed_int_stream_t *table_sizes
+ = svn__packed_create_int_stream(root, FALSE, FALSE);
+ svn__packed_int_stream_t *small_strings_headers
+ = svn__packed_create_int_stream(root, FALSE, FALSE);
+ svn__packed_byte_stream_t *large_strings
+ = svn__packed_create_bytes_stream(root);
+ svn__packed_byte_stream_t *small_strings_data
+ = svn__packed_create_bytes_stream(root);
+
+ svn__packed_create_int_substream(small_strings_headers, TRUE, FALSE);
+ svn__packed_create_int_substream(small_strings_headers, FALSE, FALSE);
+ svn__packed_create_int_substream(small_strings_headers, TRUE, FALSE);
+ svn__packed_create_int_substream(small_strings_headers, FALSE, FALSE);
+
+ /* number of sub-tables */
+
+ svn__packed_add_uint(table_sizes, table->size);
+
+ /* all short-string char data sizes */
+
+ for (i = 0; i < table->size; ++i)
+ svn__packed_add_uint(table_sizes,
+ table->sub_tables[i].short_string_count);
+
+ for (i = 0; i < table->size; ++i)
+ svn__packed_add_uint(table_sizes,
+ table->sub_tables[i].long_string_count);
+
+ /* all strings */
+
+ for (i = 0; i < table->size; ++i)
+ {
+ string_sub_table_t *sub_table = &table->sub_tables[i];
+ svn__packed_add_bytes(small_strings_data,
+ sub_table->data,
+ sub_table->data_size);
+
+ for (k = 0; k < sub_table->short_string_count; ++k)
+ {
+ string_header_t *string = &sub_table->short_strings[k];
+
+ svn__packed_add_uint(small_strings_headers, string->head_string);
+ svn__packed_add_uint(small_strings_headers, string->head_length);
+ svn__packed_add_uint(small_strings_headers, string->tail_start);
+ svn__packed_add_uint(small_strings_headers, string->tail_length);
+ }
+
+ for (k = 0; k < sub_table->long_string_count; ++k)
+ svn__packed_add_bytes(large_strings,
+ sub_table->long_strings[k].data,
+ sub_table->long_strings[k].len + 1);
+ }
+
+ /* write to target stream */
+
+ SVN_ERR(svn__packed_data_write(stream, root, pool));
+
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_fs__read_string_table(string_table_t **table_p,
+ svn_stream_t *stream,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool)
+{
+ apr_size_t i, k;
+
+ string_table_t *table = apr_palloc(result_pool, sizeof(*table));
+
+ svn__packed_data_root_t *root;
+ svn__packed_int_stream_t *table_sizes;
+ svn__packed_byte_stream_t *large_strings;
+ svn__packed_byte_stream_t *small_strings_data;
+ svn__packed_int_stream_t *headers;
+
+ SVN_ERR(svn__packed_data_read(&root, stream, result_pool, scratch_pool));
+ table_sizes = svn__packed_first_int_stream(root);
+ headers = svn__packed_next_int_stream(table_sizes);
+ large_strings = svn__packed_first_byte_stream(root);
+ small_strings_data = svn__packed_next_byte_stream(large_strings);
+
+ /* create sub-tables */
+
+ table->size = svn__packed_get_uint(table_sizes);
+ table->sub_tables = apr_pcalloc(result_pool,
+ table->size * sizeof(*table->sub_tables));
+
+ /* read short strings */
+
+ for (i = 0; i < table->size; ++i)
+ {
+ string_sub_table_t *sub_table = &table->sub_tables[i];
+
+ sub_table->short_string_count = svn__packed_get_uint(table_sizes);
+ if (sub_table->short_string_count)
+ {
+ sub_table->short_strings
+ = apr_pcalloc(result_pool, sub_table->short_string_count
+ * sizeof(*sub_table->short_strings));
+
+ /* read short string headers */
+
+ for (k = 0; k < sub_table->short_string_count; ++k)
+ {
+ string_header_t *string = &sub_table->short_strings[k];
+
+ string->head_string = svn__packed_get_uint(headers);
+ string->head_length = svn__packed_get_uint(headers);
+ string->tail_start = svn__packed_get_uint(headers);
+ string->tail_length = svn__packed_get_uint(headers);
+ }
+ }
+
+ sub_table->data = svn__packed_get_bytes(small_strings_data,
+ &sub_table->data_size);
+ }
+
+ /* read long strings */
+
+ for (i = 0; i < table->size; ++i)
+ {
+ /* initialize long string table */
+ string_sub_table_t *sub_table = &table->sub_tables[i];
+
+ sub_table->long_string_count = svn__packed_get_uint(table_sizes);
+ if (sub_table->long_string_count)
+ {
+ sub_table->long_strings
+ = apr_pcalloc(result_pool, sub_table->long_string_count
+ * sizeof(*sub_table->long_strings));
+
+ /* read long strings */
+
+ for (k = 0; k < sub_table->long_string_count; ++k)
+ {
+ svn_string_t *string = &sub_table->long_strings[k];
+ string->data = svn__packed_get_bytes(large_strings,
+ &string->len);
+ string->len--;
+ }
+ }
+ }
+
+ /* done */
+
+ *table_p = table;
+
+ return SVN_NO_ERROR;
+}
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.h?rev=1462007&r1=1462006&r2=1462007&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.h (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/string_table.h Thu Mar 28 09:33:09 2013
@@ -60,6 +60,13 @@ svn_fs_fs__string_table_builder_add(stri
const char *string,
apr_size_t len);
+/* Return an estimate for the on-disk size of the resulting string table.
+ * The estimate may err in both directions but tends to overestimate the
+ * space requirements for larger tables.
+ */
+apr_size_t
+svn_fs_fs__string_table_builder_estimate_size(string_table_builder_t *builder);
+
/* From the given BUILDER object, create a string table object allocated
* in POOL that contains all strings previously added to BUILDER.
*/
@@ -92,6 +99,24 @@ svn_fs_fs__string_table_copy_string(char
const string_table_t *table,
apr_size_t index);
+/* Write a serialized representation of the string table TABLE to STREAM.
+ * Use POOL for temporary allocations.
+ */
+svn_error_t *
+svn_fs_fs__write_string_table(svn_stream_t *stream,
+ const string_table_t *table,
+ apr_pool_t *pool);
+
+/* Read the serialized string table representation from STREAM and return
+ * the resulting runtime representation in *TABLE_P. Allocate it in
+ * RESULT_POOL and use SCRATCH_POOL for temporary allocations.
+ */
+svn_error_t *
+svn_fs_fs__read_string_table(string_table_t **table_p,
+ svn_stream_t *stream,
+ apr_pool_t *result_pool,
+ apr_pool_t *scratch_pool);
+
#ifdef __cplusplus
}
#endif /* __cplusplus */
Modified: subversion/branches/fsfs-format7/subversion/tests/libsvn_fs_fs/string-table-test.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/tests/libsvn_fs_fs/string-table-test.c?rev=1462007&r1=1462006&r2=1462007&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/tests/libsvn_fs_fs/string-table-test.c (original)
+++ subversion/branches/fsfs-format7/subversion/tests/libsvn_fs_fs/string-table-test.c Thu Mar 28 09:33:09 2013
@@ -82,7 +82,27 @@ generate_string(apr_uint64_t key, apr_si
}
static svn_error_t *
-create_empty_table(apr_pool_t *pool)
+store_and_load_table(string_table_t **table, apr_pool_t *pool)
+{
+ svn_stringbuf_t *stream_buffer = svn_stringbuf_create_empty(pool);
+ svn_stream_t *stream;
+
+ stream = svn_stream_from_stringbuf(stream_buffer, pool);
+ SVN_ERR(svn_fs_fs__write_string_table(stream, *table, pool));
+ SVN_ERR(svn_stream_close(stream));
+
+ *table = NULL;
+
+ stream = svn_stream_from_stringbuf(stream_buffer, pool);
+ SVN_ERR(svn_fs_fs__read_string_table(table, stream, pool, pool));
+ SVN_ERR(svn_stream_close(stream));
+
+ return SVN_NO_ERROR;
+}
+
+static svn_error_t *
+create_empty_table_body(svn_boolean_t do_load_store,
+ apr_pool_t *pool)
{
string_table_builder_t *builder
= svn_fs_fs__string_table_builder_create(pool);
@@ -90,13 +110,18 @@ create_empty_table(apr_pool_t *pool)
= svn_fs_fs__string_table_create(builder, pool);
SVN_TEST_STRING_ASSERT(svn_fs_fs__string_table_get(table, 0, pool), "");
+
+ if (do_load_store)
+ SVN_ERR(store_and_load_table(&table, pool));
+
SVN_TEST_ASSERT(svn_fs_fs__string_table_copy_string(NULL, 0, table, 0) == 0);
return SVN_NO_ERROR;
}
static svn_error_t *
-short_string_table(apr_pool_t *pool)
+short_string_table_body(svn_boolean_t do_load_store,
+ apr_pool_t *pool)
{
apr_size_t indexes[STRING_COUNT] = { 0 };
@@ -109,6 +134,8 @@ short_string_table(apr_pool_t *pool)
indexes[i] = svn_fs_fs__string_table_builder_add(builder, basic_strings[i], 0);
table = svn_fs_fs__string_table_create(builder, pool);
+ if (do_load_store)
+ SVN_ERR(store_and_load_table(&table, pool));
SVN_TEST_ASSERT(indexes[2] == indexes[6]);
for (i = 0; i < STRING_COUNT; ++i)
@@ -144,7 +171,8 @@ short_string_table(apr_pool_t *pool)
}
static svn_error_t *
-large_string_table(apr_pool_t *pool)
+large_string_table_body(svn_boolean_t do_load_store,
+ apr_pool_t *pool)
{
enum { COUNT = 10 };
@@ -163,6 +191,9 @@ large_string_table(apr_pool_t *pool)
}
table = svn_fs_fs__string_table_create(builder, pool);
+ if (do_load_store)
+ SVN_ERR(store_and_load_table(&table, pool));
+
for (i = 0; i < COUNT; ++i)
{
char long_buffer[73000 + 1000 * COUNT] = { 0 };
@@ -193,7 +224,8 @@ large_string_table(apr_pool_t *pool)
}
static svn_error_t *
-many_strings_table(apr_pool_t *pool)
+many_strings_table_body(svn_boolean_t do_load_store,
+ apr_pool_t *pool)
{
/* cause multiple sub-tables to be created */
enum { COUNT = 1000 };
@@ -213,6 +245,9 @@ many_strings_table(apr_pool_t *pool)
}
table = svn_fs_fs__string_table_create(builder, pool);
+ if (do_load_store)
+ SVN_ERR(store_and_load_table(&table, pool));
+
for (i = 0; i < COUNT; ++i)
{
char long_buffer[23000] = { 0 };
@@ -243,6 +278,55 @@ many_strings_table(apr_pool_t *pool)
return SVN_NO_ERROR;
}
+static svn_error_t *
+create_empty_table(apr_pool_t *pool)
+{
+ return svn_error_trace(create_empty_table_body(FALSE, pool));
+}
+
+static svn_error_t *
+short_string_table(apr_pool_t *pool)
+{
+ return svn_error_trace(short_string_table_body(FALSE, pool));
+}
+
+static svn_error_t *
+large_string_table(apr_pool_t *pool)
+{
+ return svn_error_trace(large_string_table_body(FALSE, pool));
+}
+
+static svn_error_t *
+many_strings_table(apr_pool_t *pool)
+{
+ return svn_error_trace(many_strings_table_body(FALSE, pool));
+}
+
+static svn_error_t *
+store_load_short_string_table(apr_pool_t *pool)
+{
+ return svn_error_trace(short_string_table_body(TRUE, pool));
+}
+
+static svn_error_t *
+store_load_large_string_table(apr_pool_t *pool)
+{
+ return svn_error_trace(large_string_table_body(TRUE, pool));
+}
+
+static svn_error_t *
+store_load_empty_table(apr_pool_t *pool)
+{
+ return svn_error_trace(create_empty_table_body(TRUE, pool));
+}
+
+static svn_error_t *
+store_load_many_strings_table(apr_pool_t *pool)
+{
+ return svn_error_trace(many_strings_table_body(TRUE, pool));
+}
+
+
/* ------------------------------------------------------------------------ */
/* The test table. */
@@ -258,5 +342,13 @@ struct svn_test_descriptor_t test_funcs[
"string table with large strings only"),
SVN_TEST_PASS2(many_strings_table,
"string table with many strings"),
+ SVN_TEST_PASS2(store_load_empty_table,
+ "store and load an empty string table"),
+ SVN_TEST_PASS2(store_load_short_string_table,
+ "store and load table with short strings only"),
+ SVN_TEST_PASS2(store_load_large_string_table,
+ "store and load table with large strings only"),
+ SVN_TEST_PASS2(store_load_many_strings_table,
+ "store and load string table with many strings"),
SVN_TEST_NULL
};