You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2013/01/18 00:44:59 UTC
svn commit: r1434970 - in
/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs: fs.h fs_fs.c
index.c index.h pack.c transaction.c
Author: stefan2
Date: Thu Jan 17 23:44:58 2013
New Revision: 1434970
URL: http://svn.apache.org/viewvc?rev=1434970&view=rev
Log:
On the fsfs-format7 branch: make page sizes configurable.
As a result, we must allow for index page sizes to vary from
one index file to the other, i.e. need to store them.
* subversion/libsvn_fs_fs/fs.h
(CONFIG_SECTION_IO,
CONFIG_OPTION_BLOCK_SIZE,
CONFIG_OPTION_L2P_PAGE_SIZE,
CONFIG_OPTION_P2L_PAGE_SIZE): define new options
(fs_fs_data_t): declare new page / block size members
* subversion/libsvn_fs_fs/fs_fs.c
(read_config): read new options from config file
(write_config): extent config file template
(write_revision_zero): store page sizes within index files
* subversion/libsvn_fs_fs/index.h
(svn_fs_fs__l2p_index_create,
svn_fs_fs__p2l_index_create): add fs as parameter
* subversion/libsvn_fs_fs/index.c
(l2p_index_header_t,
p2l_index_header_t): add page sizes to index header structs
(svn_fs_fs__l2p_index_create,
svn_fs_fs__p2l_index_create): take page sizes form fs settings;
write them to index files
(get_l2p_header,
get_p2l_header): read page size from index file
(l2p_index_lookup): varible page size; more efficient lookup
(get_p2l_page,
svn_fs_fs__p2l_index_lookup): use page size provided in index header
* subversion/libsvn_fs_fs/pack.c
(pack_rev_shard): adapt to API change
* subversion/libsvn_fs_fs/transaction.c
(commit_body): ditto
Modified:
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs_fs.c
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.c
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/transaction.c
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h?rev=1434970&r1=1434969&r2=1434970&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h Thu Jan 17 23:44:58 2013
@@ -105,6 +105,10 @@ extern "C" {
#define CONFIG_SECTION_PACKED_REVPROPS "packed-revprops"
#define CONFIG_OPTION_REVPROP_PACK_SIZE "revprop-pack-size"
#define CONFIG_OPTION_COMPRESS_PACKED_REVPROPS "compress-packed-revprops"
+#define CONFIG_SECTION_IO "io"
+#define CONFIG_OPTION_BLOCK_SIZE "block-size"
+#define CONFIG_OPTION_L2P_PAGE_SIZE "l2p-page-size"
+#define CONFIG_OPTION_P2L_PAGE_SIZE "p2l-page-size"
/* The format number of this filesystem.
This is independent of the repository format number, and
@@ -250,6 +254,15 @@ typedef struct fs_fs_data_t
layouts) or zero (for linear layouts). */
int max_files_per_dir;
+ /* Rev / pack file read granularity. */
+ apr_int64_t block_size;
+
+ /* Capacity in entries of log-to-phys index pages */
+ apr_int64_t l2p_page_size;
+
+ /* Rev / pack file granularity covered by phys-to-log index pages */
+ apr_int64_t p2l_page_size;
+
/* The revision that was youngest, last time we checked. */
svn_revnum_t youngest_rev_cache;
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs_fs.c?rev=1434970&r1=1434969&r2=1434970&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs_fs.c Thu Jan 17 23:44:58 2013
@@ -314,6 +314,32 @@ read_config(fs_fs_data_t *ffd,
ffd->compress_packed_revprops = FALSE;
}
+ if (ffd->format >= SVN_FS_FS__MIN_LOG_ADDRESSING_FORMAT)
+ {
+ SVN_ERR(svn_config_get_int64(ffd->config, &ffd->block_size,
+ CONFIG_SECTION_IO,
+ CONFIG_OPTION_BLOCK_SIZE,
+ 64));
+ SVN_ERR(svn_config_get_int64(ffd->config, &ffd->l2p_page_size,
+ CONFIG_SECTION_IO,
+ CONFIG_OPTION_L2P_PAGE_SIZE,
+ 0x2000));
+ SVN_ERR(svn_config_get_int64(ffd->config, &ffd->p2l_page_size,
+ CONFIG_SECTION_IO,
+ CONFIG_OPTION_P2L_PAGE_SIZE,
+ 64));
+
+ ffd->block_size *= 0x400;
+ ffd->p2l_page_size *= 0x400;
+ }
+ else
+ {
+ /* should be irrelevant but we initialize them anyway */
+ ffd->block_size = 0x1000;
+ ffd->l2p_page_size = 0x2000;
+ ffd->p2l_page_size = 0x1000;
+ }
+
return SVN_NO_ERROR;
}
@@ -448,6 +474,58 @@ write_config(svn_fs_t *fs,
"### unless you often modify revprops after packing." NL
"### Compressing packed revprops is disabled by default." NL
"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false" NL
+"" NL
+"[" CONFIG_SECTION_IO "]" NL
+"### Parameters in this section control the data access granularity in" NL
+"### format 7 repositories and later. The defaults should translate into" NL
+"### decent performance over a wide range of setups." NL
+"###" NL
+"### When a specific piece of information needs to be read from disk, a" NL
+"### data block is being read at once and its contents are being cached." NL
+"### If the repository is being stored on a RAID, the block size should" NL
+"### be either 50% or 100% of RAID block size / granularity. Also, your" NL
+"### file system (clusters) should be properly aligned and sized. In that" NL
+"### setup, each access will hit only one disk (minimizes I/O load) but" NL
+"### uses all the data provided by the disk in a single access." NL
+"### For SSD-based storage systems, slightly lower values around 16 kB" NL
+"### may improve latency while still maximizing throughput." NL
+"### Can be changed at any time but must be a power of 2." NL
+"### block-size is 64 kBytes by default." NL
+"# " CONFIG_OPTION_BLOCK_SIZE " = 64" NL
+"###" NL
+"### The log-to-phys index maps data item numbers to offsets within the" NL
+"### rev or pack file. A revision typically contains 2 .. 5 such items" NL
+"### per changed path. For each revision, at least one page is being" NL
+"### allocated in the l2p index with unused parts resulting in no wasted" NL
+"### space." NL
+"### Changing this parameter only affects larger revisions with thousands" NL
+"### of changed paths. A smaller value means that more pages need to be" NL
+"### allocated for such revisions, increasing the size of the page table" NL
+"### meaning it takes longer to read that table (once). Access to each" NL
+"### page is then faster because less data has to read. So, if you have" NL
+"### several extremely large revisions (approaching 1 mio changes), think" NL
+"### about increasing this setting. Reducing the value will rarely result" NL
+"### in a net speedup." NL
+"### This is an expert setting. Any non-zero value is possible." NL
+"### l2p-page-size is 8192 entries by default." NL
+"# " CONFIG_OPTION_L2P_PAGE_SIZE " = 8192" NL
+"###" NL
+"### The phys-to-log index maps positions within the rev or pack file to" NL
+"### to data items, i.e. describes what piece of information is being" NL
+"### stored at that particular offset. The index describes the rev file" NL
+"### in chunks (pages) and keeps a global list of all those pages. Large" NL
+"### pages mean a shorter page table but a larger per-page description of" NL
+"### data items in it. The latency sweetspot depends on the change size" NL
+"### distribution but is relatively wide." NL
+"### If the repository contains very large files, i.e. individual changes" NL
+"### of tens of MB each, increasing the page size will shorten the index" NL
+"### file at the expense of a slightly increased latency in sections with" NL
+"### smaller changes." NL
+"### For practical reasons, this should match block-size. Differing" NL
+"### values are perfectly legal but may result in some processing overhead." NL
+"### Must be a power of 2." NL
+"### p2l-page-size is 64 kBytes by default." NL
+"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 64" NL
;
#undef NL
return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool),
@@ -850,23 +928,23 @@ write_revision_zero(svn_fs_t *fs)
{
const char *path = path_l2p_index(fs, 0, fs->pool);
SVN_ERR(svn_io_file_create2(path,
- "\0\1\1\1" /* rev 0, single page */
+ "\0\1\x80\x40\1\1" /* rev 0, single page */
"\4\4" /* page size: count, bytes */
"\0\x6b\x12\1", /* phys offsets + 1 */
- 10,
+ 12,
fs->pool));
SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool));
path = path_p2l_index(fs, 0, fs->pool);
SVN_ERR(svn_io_file_create2(path,
"\0"
- "\1\x13"
+ "\x80\x80\4\1\x13"
"\0"
"\x11\1\0\3"
"\x59\2\0\2"
"\1\3\0\1"
"\x95\xff\3\0\0\0",
- 22,
+ 25,
fs->pool));
SVN_ERR(svn_io_set_file_read_only(path, FALSE, fs->pool));
}
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.c?rev=1434970&r1=1434969&r2=1434970&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.c Thu Jan 17 23:44:58 2013
@@ -65,6 +65,9 @@ typedef struct l2p_index_header_t
/* number of revisions covered */
apr_size_t revision_count;
+ /* (max) number of entries per page */
+ apr_size_t page_size;
+
/* pointers into PAGE_TABLE that mark the first page of the respective
* revision. PAGE_TABLES[REVISION_COUNT] points to the end of PAGE_TABLE.
*/
@@ -106,6 +109,9 @@ typedef struct p2l_index_header_t
/* first revision covered by the index (and rev file) */
svn_revnum_t first_revision;
+ /* number of bytes in the rev files covered by each p2l page */
+ apr_uint64_t page_size;
+
/* number of pages / clusters in that rev file */
apr_size_t page_count;
@@ -194,11 +200,13 @@ encode_uint(unsigned char *p, apr_uint64
}
svn_error_t *
-svn_fs_fs__l2p_index_create(const char *file_name,
+svn_fs_fs__l2p_index_create(svn_fs_t *fs,
+ const char *file_name,
const char *proto_file_name,
svn_revnum_t revision,
apr_pool_t *pool)
{
+ fs_fs_data_t *ffd = fs->fsap_data;
apr_file_t *proto_index = NULL;
int i;
apr_uint64_t entry;
@@ -253,9 +261,9 @@ svn_fs_fs__l2p_index_create(const char *
for (i = 0; i < offsets->nelts; i = k)
{
/* 1 page with up to 8k entries */
- int entry_count = offsets->nelts - i < 0x2000
+ int entry_count = offsets->nelts - i < ffd->l2p_page_size
? offsets->nelts
- : 0x2000;
+ : ffd->l2p_page_size;
apr_size_t last_buffer_size = svn_spillbuf__get_size(buffer);
for (k = i; k < i + entry_count; ++k)
@@ -305,6 +313,9 @@ svn_fs_fs__l2p_index_create(const char *
encode_uint(encoded, page_counts->nelts),
NULL, local_pool));
SVN_ERR(svn_io_file_write_full(index_file, encoded,
+ encode_uint(encoded, ffd->l2p_page_size),
+ NULL, local_pool));
+ SVN_ERR(svn_io_file_write_full(index_file, encoded,
encode_uint(encoded, page_sizes->nelts),
NULL, local_pool));
@@ -396,6 +407,8 @@ get_l2p_header(l2p_index_header_t **head
SVN_ERR(read_number(&value, file, pool));
result->revision_count = (int)value;
SVN_ERR(read_number(&value, file, pool));
+ result->page_size = (apr_size_t)value;
+ SVN_ERR(read_number(&value, file, pool));
page_count = (apr_size_t)value;
/* allocate the page tables */
@@ -485,15 +498,17 @@ get_l2p_page(l2p_index_page_t **page,
* Use POOL for allocations.
*/
static svn_error_t *
-svn_fs_fs__l2p_index_lookup(apr_off_t *offset,
- svn_fs_t *fs,
- svn_revnum_t revision,
- apr_uint64_t item_index,
- apr_pool_t *pool)
+l2p_index_lookup(apr_off_t *offset,
+ svn_fs_t *fs,
+ svn_revnum_t revision,
+ apr_uint64_t item_index,
+ apr_pool_t *pool)
{
l2p_index_header_t *header = NULL;
l2p_index_page_t *page = NULL;
l2_index_page_table_entry_t *entry, *first_entry, *last_entry;
+ apr_size_t page_no;
+ apr_uint32_t page_offset;
/* read index master data structure */
SVN_ERR(get_l2p_header(&header, fs, revision, pool));
@@ -503,18 +518,41 @@ svn_fs_fs__l2p_index_lookup(apr_off_t *o
_("Revision %ld not covered by item index"),
revision);
- /* iterate to the relevant page (fast enough for even 1 mio items / rev) */
- first_entry = header->page_tables[revision - header->first_revision];
- last_entry = header->page_tables[revision + 1 - header->first_revision];
- for (entry = first_entry; entry < last_entry; ++entry)
- if (entry->entry_count > item_index)
- break;
- else
- item_index -= entry->entry_count;
+ /* select the relevant page */
+ if (item_index < header->page_size)
+ {
+ /* most revs fit well into a single page */
+ page_offset = (apr_size_t)item_index;
+ page_no = 0;
+ entry = header->page_tables[revision - header->first_revision];
+ }
+ else
+ {
+ /* all pages are of the same size and full, except for the last one */
+ page_offset = (apr_size_t)(item_index % header->page_size);
+ page_no = (apr_uint32_t)(item_index / header->page_size);
+
+ /* range of pages for this rev */
+ first_entry = header->page_tables[revision - header->first_revision];
+ last_entry = header->page_tables[revision + 1 - header->first_revision];
+
+ if (last_entry - first_entry > page_no)
+ {
+ entry = first_entry + page_no;
+ }
+ else
+ {
+ /* limit page index to the valid range */
+ entry = last_entry - 1;
+
+ /* cause index overflow further down the road */
+ page_offset = header->page_size;
+ }
+ }
/* read the relevant page */
SVN_ERR(get_l2p_page(&page, fs, header->first_revision, entry, pool));
- if (page->entry_count <= item_index)
+ if (page->entry_count <= page_offset)
return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL,
_("Item index %" APR_UINT64_T_FMT
" too large in revision %ld"),
@@ -596,17 +634,19 @@ svn_fs_fs__p2l_proto_index_add_entry(apr
}
svn_error_t *
-svn_fs_fs__p2l_index_create(const char *file_name,
+svn_fs_fs__p2l_index_create(svn_fs_t *fs,
+ const char *file_name,
const char *proto_file_name,
svn_revnum_t revision,
apr_pool_t *pool)
{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ apr_uint64_t page_size = ffd->p2l_page_size;
apr_file_t *proto_index = NULL;
int i;
svn_boolean_t eof = FALSE;
apr_file_t *index_file;
unsigned char encoded[ENCODED_INT_LENGTH];
- apr_uint64_t page_size = 0x10000;
apr_uint64_t last_entry_end = 0;
apr_uint64_t last_page_end = 0;
@@ -701,10 +741,13 @@ svn_fs_fs__p2l_index_create(const char *
| APR_CREATE | APR_TRUNCATE | APR_BUFFERED,
APR_OS_DEFAULT, local_pool));
- /* write the start revision */
+ /* write the start revision and page size */
SVN_ERR(svn_io_file_write_full(index_file, encoded,
encode_uint(encoded, revision),
NULL, local_pool));
+ SVN_ERR(svn_io_file_write_full(index_file, encoded,
+ encode_uint(encoded, page_size),
+ NULL, local_pool));
/* write the page table (actually, the sizes of each page description) */
SVN_ERR(svn_io_file_write_full(index_file, encoded,
@@ -756,6 +799,8 @@ get_p2l_header(p2l_index_header_t **head
SVN_ERR(read_number(&value, file, pool));
result->first_revision = (svn_revnum_t)value;
SVN_ERR(read_number(&value, file, pool));
+ result->page_size = value;
+ SVN_ERR(read_number(&value, file, pool));
result->page_count = (apr_size_t)value;
result->offsets
= apr_pcalloc(pool, (result->page_count + 1) * sizeof(*result->offsets));
@@ -771,7 +816,7 @@ get_p2l_header(p2l_index_header_t **head
/* correct the offset values */
offset = 0;
SVN_ERR(svn_io_file_seek(file, SEEK_CUR, &offset, pool));
- for (i = 0; i < result->page_count; ++i)
+ for (i = 0; i <= result->page_count; ++i)
result->offsets[i] += offset;
SVN_ERR(svn_io_file_close(file, pool));
@@ -823,13 +868,13 @@ get_p2l_page(apr_array_header_t **entrie
apr_off_t start_offset,
apr_off_t next_offset,
apr_off_t page_start,
+ apr_uint64_t page_size,
apr_pool_t *pool)
{
apr_uint64_t value;
apr_array_header_t *result
= apr_array_make(pool, 16, sizeof(svn_fs_fs__p2l_entry_t));
apr_off_t item_offset;
- apr_uint64_t page_size = 0x10000;
apr_off_t offset;
/* open index and navigate to page start */
@@ -875,10 +920,11 @@ svn_fs_fs__p2l_index_lookup(apr_array_he
apr_pool_t *pool)
{
p2l_index_header_t *header = NULL;
- apr_uint64_t page_size = 0x10000;
- apr_size_t page_no = offset / page_size;
+ apr_size_t page_no;
SVN_ERR(get_p2l_header(&header, fs, revision, pool));
+ page_no = offset / header->page_size;
+
if (header->page_count <= page_no)
return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_OVERFLOW , NULL,
_("Offset %" APR_OFF_T_FMT
@@ -888,7 +934,8 @@ svn_fs_fs__p2l_index_lookup(apr_array_he
SVN_ERR(get_p2l_page(entries, fs, header->first_revision,
header->offsets[page_no],
header->offsets[page_no + 1],
- (apr_off_t)(page_no * page_size), pool));
+ (apr_off_t)(page_no * header->page_size),
+ header->page_size, pool));
return SVN_NO_ERROR;
}
@@ -920,8 +967,7 @@ svn_fs_fs__item_offset(apr_off_t *offset
if (txn_id)
SVN_ERR(l2p_proto_index_lookup(offset, fs, txn_id, item_index, pool));
else
- SVN_ERR(svn_fs_fs__l2p_index_lookup(offset, fs, revision,
- item_index, pool));
+ SVN_ERR(l2p_index_lookup(offset, fs, revision, item_index, pool));
return SVN_NO_ERROR;
}
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h?rev=1434970&r1=1434969&r2=1434970&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h Thu Jan 17 23:44:58 2013
@@ -100,7 +100,8 @@ svn_fs_fs__l2p_proto_index_add_entry(apr
* and so forth. Use POOL for allocations.
*/
svn_error_t *
-svn_fs_fs__l2p_index_create(const char *file_name,
+svn_fs_fs__l2p_index_create(svn_fs_t *fs,
+ const char *file_name,
const char *proto_file_name,
svn_revnum_t revision,
apr_pool_t *pool);
@@ -130,7 +131,8 @@ svn_fs_fs__p2l_proto_index_add_entry(apr
* Use POOL for allocations.
*/
svn_error_t *
-svn_fs_fs__p2l_index_create(const char *file_name,
+svn_fs_fs__p2l_index_create(svn_fs_t *fs,
+ const char *file_name,
const char *proto_file_name,
svn_revnum_t revision,
apr_pool_t *pool);
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c?rev=1434970&r1=1434969&r2=1434970&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c Thu Jan 17 23:44:58 2013
@@ -268,10 +268,10 @@ pack_rev_shard(svn_fs_t *fs,
SVN_ERR(svn_io_file_close(proto_p2l_index, iterpool));
/* Create the actual index files*/
- SVN_ERR(svn_fs_fs__l2p_index_create(l2p_index_path,
+ SVN_ERR(svn_fs_fs__l2p_index_create(fs, l2p_index_path,
proto_l2p_index_path,
start_rev, iterpool));
- SVN_ERR(svn_fs_fs__p2l_index_create(p2l_index_path,
+ SVN_ERR(svn_fs_fs__p2l_index_create(fs, p2l_index_path,
proto_p2l_index_path,
start_rev, iterpool));
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/transaction.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/transaction.c?rev=1434970&r1=1434969&r2=1434970&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/transaction.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/transaction.c Thu Jan 17 23:44:58 2013
@@ -3107,13 +3107,15 @@ commit_body(void *baton, apr_pool_t *poo
{
/* Convert the index files from the proto format into their form
in their final location */
- SVN_ERR(svn_fs_fs__l2p_index_create(path_l2p_index(cb->fs, new_rev,
+ SVN_ERR(svn_fs_fs__l2p_index_create(cb->fs,
+ path_l2p_index(cb->fs, new_rev,
pool),
path_l2p_proto_index(cb->fs,
cb->txn->id,
pool),
new_rev, pool));
- SVN_ERR(svn_fs_fs__p2l_index_create(path_p2l_index(cb->fs, new_rev,
+ SVN_ERR(svn_fs_fs__p2l_index_create(cb->fs,
+ path_p2l_index(cb->fs, new_rev,
pool),
path_p2l_proto_index(cb->fs,
cb->txn->id,