You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2013/06/07 12:51:49 UTC
svn commit: r1490579 - in
/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs: cached_data.c
caching.c fs.h index.h low_level.h pack.c
Author: stefan2
Date: Fri Jun 7 10:51:48 2013
New Revision: 1490579
URL: http://svn.apache.org/r1490579
Log:
On the fsfs-format7 branch: add caches & read support for representation
containers. Begin writing them for properties. This has not been tested
very well, yet.
* subversion/libsvn_fs_fs/index.h
(SVN_FS_FS__ITEM_TYPE_REPS_CONT): declare new container item type
* subversion/libsvn_fs_fs/low_level.h
(svn_fs_fs__rep_type_t): add enum value for representations in containers
* subversion/libsvn_fs_fs/fs.h
(fs_fs_data_t): add cache for representation containers
* subversion/libsvn_fs_fs/caching.c
(svn_fs_fs__initialize_caches): init the new cache
* subversion/libsvn_fs_fs/cached_data.c
(dgb__log_access): teach tracing output to handle the new container type
(rep_state_t): support representations that don't have headers
(create_rep_state_body): return a special rep_state for containered items
(build_rep_list,
get_contents): handle reps are similarly to plaintext
(read_container_window): new function to read a window from a container
(get_combined_window): call it for containered reps
(block_read_reps_container): implement a new block reader
(block_read): call it
* subversion/libsvn_fs_fs/pack.c
(write_reps_container,
write_property_containers): new functions stuffing prop repesenations
into representation containers
(pack_range): call them
Modified:
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/cached_data.c
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/caching.c
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/low_level.h
subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/cached_data.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/cached_data.c?rev=1490579&r1=1490578&r2=1490579&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/cached_data.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/cached_data.c Fri Jun 7 10:51:48 2013
@@ -36,6 +36,7 @@
#include "index.h"
#include "changes.h"
#include "noderevs.h"
+#include "reps.h"
#include "../libsvn_fs/fs-loader.h"
@@ -161,7 +162,8 @@ dgb__log_access(svn_fs_t *fs,
/* merge the sub-item number with the container type */
if ( entry->type == SVN_FS_FS__ITEM_TYPE_CHANGES_CONT
- || entry->type == SVN_FS_FS__ITEM_TYPE_NODEREVS_CONT)
+ || entry->type == SVN_FS_FS__ITEM_TYPE_NODEREVS_CONT
+ || entry->type == SVN_FS_FS__ITEM_TYPE_REPS_CONT)
type = apr_psprintf(scratch_pool, "%s%-3d", type, sub_item);
}
@@ -623,7 +625,9 @@ typedef struct rep_state_t
svn_revnum_t revision;
/* representation's item index in REVISION */
apr_uint64_t item_index;
- /* length of the header at the start of the rep */
+ /* length of the header at the start of the rep.
+ 0 iff this is rep is stored in a container
+ (i.e. does not have a header) */
apr_size_t header_size;
apr_off_t start; /* The starting offset for the raw
svndiff/plaintext data minus header.
@@ -705,15 +709,43 @@ create_rep_state_body(rep_state_t **rep_
}
else
{
+ /* we will need the on-disk location for non-txn reps */
+ apr_off_t offset;
+ apr_uint32_t sub_item;
+ if (! svn_fs_fs__id_txn_used(&rep->txn_id))
+ SVN_ERR(svn_fs_fs__item_offset(&offset, &sub_item,
+ fs, rep->revision, NULL,
+ rep->item_index, pool));
+
+ /* is rep stored in some star-deltified container? */
+ if (! svn_fs_fs__id_txn_used(&rep->txn_id)
+ && ffd->format >= SVN_FS_FS__MIN_LOG_ADDRESSING_FORMAT)
+ {
+ svn_boolean_t in_container = TRUE;
+ if (sub_item == 0)
+ {
+ svn_fs_fs__p2l_entry_t *entry;
+ SVN_ERR(svn_fs_fs__p2l_entry_lookup(&entry, fs, rep->revision,
+ offset, pool));
+ in_container = entry->type == SVN_FS_FS__ITEM_TYPE_REPS_CONT;
+ }
+
+ if (in_container)
+ {
+ /* construct a container rep header */
+ *rep_header = apr_pcalloc(pool, sizeof(**rep_header));
+ (*rep_header)->type = svn_fs_fs__rep_container;
+
+ /* exit to caller */
+ *rep_state = rs;
+ return SVN_NO_ERROR;
+ }
+ }
+
if (reuse_shared_file)
{
/* ... we can re-use the same, already open file object
*/
- apr_off_t offset;
- apr_uint32_t sub_item;
- SVN_ERR(svn_fs_fs__item_offset(&offset, &sub_item,
- fs, rep->revision, NULL,
- rep->item_index, pool));
SVN_ERR_ASSERT(sub_item == 0);
SVN_ERR(aligned_seek(fs, (*shared_file)->file, NULL, offset, pool));
@@ -1201,7 +1233,10 @@ build_rep_list(apr_array_header_t **list
SVN_ERR(create_rep_state(&rs, &rep_header, &shared_file,
&rep, fs, pool));
- if (!svn_fs_fs__id_txn_used(&rep.txn_id))
+ /* for txn reps and containered reps, there won't be a cached
+ * combined window */
+ if (!svn_fs_fs__id_txn_used(&rep.txn_id)
+ && rep_header->type != svn_fs_fs__rep_container)
SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, pool));
if (is_cached)
@@ -1215,9 +1250,11 @@ build_rep_list(apr_array_header_t **list
return SVN_NO_ERROR;
}
- if (rep_header->type == svn_fs_fs__rep_plain)
+ if (rep_header->type == svn_fs_fs__rep_plain
+ || rep_header->type == svn_fs_fs__rep_container)
{
- /* This is a plaintext, so just return the current rep_state. */
+ /* This is a plaintext or container item, so just return the
+ current rep_state. */
*src_state = rs;
return SVN_NO_ERROR;
}
@@ -1457,6 +1494,42 @@ read_plain_window(svn_stringbuf_t **nwin
return SVN_NO_ERROR;
}
+/* Read the whole representation RS and return it in *NWIN. */
+static svn_error_t *
+read_container_window(svn_stringbuf_t **nwin,
+ rep_state_t *rs,
+ apr_pool_t *pool)
+{
+ svn_fs_fs__rep_extractor_t *extractor = NULL;
+ svn_fs_t *fs = rs->file->fs;
+ fs_fs_data_t *ffd = fs->fsap_data;
+ pair_cache_key_t key;
+
+ SVN_ERR(auto_set_start_offset(rs, pool));
+ key.revision = packed_base_rev(fs, rs->revision);
+ key.second = rs->start;
+
+ /* already in cache? */
+ if (ffd->reps_container_cache)
+ {
+ svn_boolean_t is_cached = FALSE;
+ SVN_ERR(svn_cache__get((void**)&extractor, &is_cached,
+ ffd->reps_container_cache, &key, pool));
+ }
+
+ /* read from disk, if necessary */
+ if (extractor == NULL)
+ {
+ SVN_ERR(auto_open_shared_file(rs->file));
+ SVN_ERR(block_read((void **)&extractor, fs, rs->revision,
+ rs->item_index, rs->file->file, pool, pool));
+ }
+
+ SVN_ERR(svn_fs_fs__extractor_drive(nwin, extractor, pool, pool));
+
+ return SVN_NO_ERROR;
+}
+
/* Get the undeltified window that is a result of combining all deltas
from the current desired representation identified in *RB with its
base representation. Store the window in *RESULT. */
@@ -1504,8 +1577,13 @@ get_combined_window(svn_stringbuf_t **re
Note that BUF / SOURCE may only be NULL in the first iteration. */
source = buf;
if (source == NULL && rb->src_state != NULL)
- SVN_ERR(read_plain_window(&source, rb->src_state, window->sview_len,
- pool));
+ {
+ if (rs->header_size == 0)
+ SVN_ERR(read_container_window(&source, rb->src_state, pool));
+ else
+ SVN_ERR(read_plain_window(&source, rb->src_state,
+ window->sview_len, pool));
+ }
/* Combine this window with the current one. */
new_pool = svn_pool_create(rb->pool);
@@ -1742,12 +1820,16 @@ get_contents(struct rep_read_baton *rb,
rep_state_t *rs;
/* Special case for when there are no delta reps, only a plain
- text. */
- if (rb->rs_list->nelts == 0)
+ text or containered text. */
+ if (rb->rs_list->nelts == 0 && rb->buf == NULL)
{
copy_len = remaining;
rs = rb->src_state;
+ /* reps in containers don't have a header */
+ if (rs->header_size == 0 && rb->base_window == NULL)
+ SVN_ERR(read_container_window(&rb->base_window, rs, rb->pool));
+
if (rb->base_window != NULL)
{
/* We got the desired rep directly from the cache.
@@ -2742,6 +2824,52 @@ block_read_noderevs_container(node_revis
}
static svn_error_t *
+block_read_reps_container(svn_fs_fs__rep_extractor_t **extractor,
+ svn_fs_t *fs,
+ apr_file_t *file,
+ svn_stream_t *file_stream,
+ svn_fs_fs__p2l_entry_t* entry,
+ apr_uint32_t sub_item,
+ svn_boolean_t must_read,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ svn_fs_fs__reps_t *container;
+ svn_stream_t *stream;
+ pair_cache_key_t key;
+
+ key.revision = packed_base_rev(fs, entry->items[0].revision);
+ key.second = entry->offset;
+
+ /* already in cache? */
+ if (!must_read && ffd->reps_container_cache)
+ {
+ svn_boolean_t is_cached = FALSE;
+ SVN_ERR(svn_cache__has_key(&is_cached, ffd->reps_container_cache,
+ &key, pool));
+ if (is_cached)
+ return SVN_NO_ERROR;
+ }
+
+ SVN_ERR(auto_select_stream(&stream, fs, file, file_stream, entry, pool));
+
+ /* read noderevs from revision file */
+
+ SVN_ERR(svn_fs_fs__read_reps_container(&container, stream, pool, pool));
+
+ /* extract requested data */
+
+ if (must_read)
+ SVN_ERR(svn_fs_fs__reps_get(extractor, fs, container, sub_item, pool));
+
+ if (ffd->noderevs_container_cache)
+ SVN_ERR(svn_cache__set(ffd->reps_container_cache, &key, container,
+ pool));
+
+ return SVN_NO_ERROR;
+}
+
+static svn_error_t *
block_read(void **result,
svn_fs_t *fs,
svn_revnum_t revision,
@@ -2853,6 +2981,14 @@ block_read(void **result,
is_result, pool));
break;
+ case SVN_FS_FS__ITEM_TYPE_REPS_CONT:
+ SVN_ERR(block_read_reps_container
+ ((svn_fs_fs__rep_extractor_t **)&item,
+ fs, revision_file, stream,
+ entry, wanted_sub_item,
+ is_result, pool));
+ break;
+
default:
break;
}
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/caching.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/caching.c?rev=1490579&r1=1490578&r2=1490579&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/caching.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/caching.c Fri Jun 7 10:51:48 2013
@@ -29,6 +29,7 @@
#include "changes.h"
#include "noderevs.h"
#include "temp_serializer.h"
+#include "reps.h"
#include "../libsvn_fs/fs-loader.h"
#include "svn_config.h"
@@ -650,6 +651,19 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
fs,
no_handler,
fs->pool));
+ SVN_ERR(create_cache(&(ffd->reps_container_cache),
+ NULL,
+ membuffer,
+ 0, 0, /* Do not use inprocess cache */
+ svn_fs_fs__serialize_reps_container,
+ svn_fs_fs__deserialize_reps_container,
+ sizeof(pair_cache_key_t),
+ apr_pstrcat(pool, prefix, "REPSCNT",
+ (char *)NULL),
+ 0,
+ fs,
+ no_handler,
+ fs->pool));
SVN_ERR(create_cache(&(ffd->l2p_header_cache),
NULL,
@@ -708,6 +722,7 @@ svn_fs_fs__initialize_caches(svn_fs_t *f
{
ffd->noderevs_container_cache = NULL;
ffd->changes_container_cache = NULL;
+ ffd->reps_container_cache = NULL;
ffd->l2p_header_cache = NULL;
ffd->l2p_page_cache = NULL;
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h?rev=1490579&r1=1490578&r2=1490579&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/fs.h Fri Jun 7 10:51:48 2013
@@ -352,6 +352,10 @@ typedef struct fs_fs_data_t
the key is a (pack file revision, file offset) pair */
svn_cache__t *changes_container_cache;
+ /* Cache for star-delta / representation containers;
+ the key is a (pack file revision, file offset) pair */
+ svn_cache__t *reps_container_cache;
+
/* Cache for svn_fs_fs__rep_header_t objects; the key is a
(revision, item index) pair */
svn_cache__t *rep_header_cache;
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h?rev=1490579&r1=1490578&r2=1490579&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/index.h Fri Jun 7 10:51:48 2013
@@ -49,6 +49,8 @@
#define SVN_FS_FS__ITEM_TYPE_CHANGES_CONT 8 /* item is a changes container */
#define SVN_FS_FS__ITEM_TYPE_NODEREVS_CONT 9 /* item is a noderevs container */
+#define SVN_FS_FS__ITEM_TYPE_REPS_CONT 10 /* item is a representations
+ container */
/* (user visible) entry in the phys-to-log index. It describes a section
* of some packed / non-packed rev file as containing a specific item.
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/low_level.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/low_level.h?rev=1490579&r1=1490578&r2=1490579&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/low_level.h (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/low_level.h Fri Jun 7 10:51:48 2013
@@ -98,7 +98,10 @@ typedef enum svn_fs_fs__rep_type_t
svn_fs_fs__rep_self_delta,
/* this is a DELTA representation against some base representation */
- svn_fs_fs__rep_delta
+ svn_fs_fs__rep_delta,
+
+ /* this is a representation in a star-delta container */
+ svn_fs_fs__rep_container
} svn_fs_fs__rep_type_t;
/* This structure is used to hold the information stored in a representation
Modified: subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c?rev=1490579&r1=1490578&r2=1490579&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c (original)
+++ subversion/branches/fsfs-format7/subversion/libsvn_fs_fs/pack.c Fri Jun 7 10:51:48 2013
@@ -26,6 +26,7 @@
#include "svn_sorts.h"
#include "private/svn_temp_serializer.h"
#include "private/svn_subr_private.h"
+#include "private/svn_string_private.h"
#include "fs_fs.h"
#include "pack.h"
@@ -37,6 +38,7 @@
#include "cached_data.h"
#include "changes.h"
#include "noderevs.h"
+#include "reps.h"
#include "../libsvn_fs/fs-loader.h"
@@ -1354,6 +1356,146 @@ write_changes_containers(pack_context_t
return SVN_NO_ERROR;
}
+/* Finalize CONTAINER and write it to CONTEXT's pack file.
+ * Append an P2L entry containing the given SUB_ITEMS to NEW_ENTRIES.
+ * Use POOL for temporary allocations.
+ */
+static svn_error_t *
+write_reps_container(pack_context_t *context,
+ svn_fs_fs__reps_builder_t *container,
+ apr_array_header_t *sub_items,
+ apr_array_header_t *new_entries,
+ apr_pool_t *pool)
+{
+ apr_off_t offset = 0;
+ svn_fs_fs__p2l_entry_t container_entry;
+
+ svn_stream_t *pack_stream
+ = svn_stream_from_aprfile2(context->pack_file, TRUE, pool);
+
+ SVN_ERR(svn_fs_fs__write_reps_container(pack_stream, container, pool));
+ SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool));
+
+ container_entry.offset = context->pack_offset;
+ container_entry.size = offset - container_entry.offset;
+ container_entry.type = SVN_FS_FS__ITEM_TYPE_REPS_CONT;
+ container_entry.item_count = sub_items->nelts;
+ container_entry.items = (svn_fs_fs__id_part_t *)sub_items->elts;
+
+ context->pack_offset = offset;
+ APR_ARRAY_PUSH(new_entries, svn_fs_fs__p2l_entry_t *)
+ = svn_fs_fs__p2l_entry_dup(&container_entry, context->info_pool);
+
+ SVN_ERR(svn_fs_fs__p2l_proto_index_add_entry
+ (context->proto_p2l_index, &container_entry, pool));
+
+ return SVN_NO_ERROR;
+}
+
+/* Read the (property) representations identified by svn_fs_fs__p2l_entry_t
+ * elements in ENTRIES from TEMP_FILE, aggregate them and write them into
+ * CONTEXT->PACK_FILE. Use POOL for temporary allocations.
+ */
+static svn_error_t *
+write_property_containers(pack_context_t *context,
+ apr_array_header_t *entries,
+ apr_file_t *temp_file,
+ apr_pool_t *pool)
+{
+ apr_pool_t *iterpool = svn_pool_create(pool);
+ apr_pool_t *container_pool = svn_pool_create(pool);
+ int i;
+
+ apr_ssize_t block_left = get_block_left(context);
+
+ svn_fs_fs__reps_builder_t *container
+ = svn_fs_fs__reps_builder_create(context->fs, container_pool);
+ apr_array_header_t *sub_items
+ = apr_array_make(pool, 64, sizeof(svn_fs_fs__id_part_t));
+ apr_array_header_t *new_entries
+ = apr_array_make(context->info_pool, 16, entries->elt_size);
+ svn_stream_t *temp_stream
+ = svn_stream_from_aprfile2(temp_file, TRUE, pool);
+
+ /* copy all items in strict order */
+ for (i = entries->nelts-1; i >= 0; --i)
+ {
+ representation_t representation = { 0 };
+ svn_stringbuf_t *contents;
+ svn_stream_t *stream;
+ apr_size_t list_index;
+ svn_fs_fs__p2l_entry_t *entry
+ = APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t *);
+
+ if ((block_left < entry->size) && sub_items->nelts)
+ {
+ block_left = get_block_left(context)
+ - svn_fs_fs__reps_estimate_size(container);
+ }
+
+ if ((block_left < entry->size) && sub_items->nelts)
+ {
+ SVN_ERR(write_reps_container(context, container, sub_items,
+ new_entries, iterpool));
+
+ apr_array_clear(sub_items);
+ svn_pool_clear(container_pool);
+ container = svn_fs_fs__reps_builder_create(context->fs,
+ container_pool);
+ block_left = get_block_left(context);
+ }
+
+ /* still enough space in current block? */
+ if (block_left < entry->size)
+ {
+ SVN_ERR(auto_pad_block(context, iterpool));
+ block_left = get_block_left(context);
+ }
+
+ assert(entry->item_count == 1);
+ representation.revision = entry->items[0].revision;
+ representation.item_index = entry->items[0].number;
+ svn_fs_fs__id_txn_reset(&representation.txn_id);
+
+ /* select the change list in the source file, parse it and add it to
+ * the container */
+ SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset,
+ iterpool));
+ SVN_ERR(svn_fs_fs__get_representation_length(&representation.size,
+ &representation.expanded_size,
+ context->fs, temp_file,
+ temp_stream, entry, iterpool));
+ SVN_ERR(svn_fs_fs__get_contents(&stream, context->fs, &representation,
+ iterpool));
+ contents = svn_stringbuf_create_ensure(representation.expanded_size,
+ iterpool);
+ contents->len = representation.expanded_size;
+
+ /* The representation is immutable. Read it normally. */
+ SVN_ERR(svn_stream_read(stream, contents->data, &contents->len));
+ SVN_ERR(svn_stream_close(stream));
+
+ list_index = svn_fs_fs__reps_add(container,
+ svn_stringbuf__morph_into_string(contents));
+ SVN_ERR_ASSERT(list_index == sub_items->nelts);
+ block_left -= entry->size;
+
+ APR_ARRAY_PUSH(sub_items, svn_fs_fs__id_part_t) = entry->items[0];
+
+ svn_pool_clear(iterpool);
+ }
+
+ if (sub_items->nelts)
+ SVN_ERR(write_reps_container(context, container, sub_items,
+ new_entries, iterpool));
+
+ *entries = *new_entries;
+ svn_pool_destroy(iterpool);
+ svn_pool_destroy(container_pool);
+
+ return SVN_NO_ERROR;
+}
+
/* Append all entries of svn_fs_fs__p2l_entry_t * array TO_APPEND to
* svn_fs_fs__p2l_entry_t * array DEST.
*/
@@ -1587,11 +1729,11 @@ pack_range(pack_context_t *context,
SVN_ERR(write_changes_containers(context, context->changes,
context->changes_file, revpool));
svn_pool_clear(revpool);
- SVN_ERR(copy_items_from_temp(context, context->file_props,
- context->file_props_file, revpool));
+ SVN_ERR(write_property_containers(context, context->file_props,
+ context->file_props_file, revpool));
svn_pool_clear(revpool);
- SVN_ERR(copy_items_from_temp(context, context->dir_props,
- context->dir_props_file, revpool));
+ SVN_ERR(write_property_containers(context, context->dir_props,
+ context->dir_props_file, revpool));
svn_pool_clear(revpool);
SVN_ERR(copy_items_from_temp(context, context->reps,
context->reps_file, revpool));