You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2013/07/12 18:34:14 UTC
svn commit: r1502605 - in
/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs: fs.c
fs_fs.c fs_fs.h pack.c pack.h
Author: stefan2
Date: Fri Jul 12 16:34:13 2013
New Revision: 1502605
URL: http://svn.apache.org/r1502605
Log:
On the fsfs-improvements branch: Classic refactoring moving general
packing logic from fs_fs.c to the new pack.* pair of files. Revprop
packing will be bundled with the revprop logic later and remains in
fs_fs.c for now. We need to expose some of it as private API, though.
* subversion/libsvn_fs_fs/pack.h
(svn_fs_fs__pack,
svn_fs_fs__get_packed_offset): define private API
* subversion/libsvn_fs_fs/pack.c
(pack_baton): struct moved here from fs_fs.c
(pack_phys_addressed,
pack_rev_shard,
pack_shard,
pack_body): code moved here from fs_fs.c
(svn_fs_fs__get_packed_offset,
svn_fs_fs__pack): rename / move here from fs_fs.c
* subversion/libsvn_fs_fs/fs_fs.h
(svn_fs_fs__pack): remove here
(svn_fs_fs__pack_revprops_shard,
svn_fs_fs__delete_revprops_shard): make part of our private API
* subversion/libsvn_fs_fs/fs_fs.c
(upgrade_pack_revprops,
upgrade_cleanup_pack_revprops,
open_and_seek_revision,
get_root_changes_offset,
create_rep_state_body): update callers
(svn_fs_fs__pack_revprops_shard,
svn_fs_fs__delete_revprops_shard): add "svn_fs_fs__" prefix;
drop forward declaration from .c file
(get_packed_offset,
pack_rev_shard,
pack_shard,
pack_baton
pack_body,
svn_fs_fs__pack): code moved to pack.c
* subversion/libsvn_fs_fs/fs.c
(): add #include
Added:
subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.c
subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.h
Modified:
subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs.c
subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.c
subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.h
Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs.c?rev=1502605&r1=1502604&r2=1502605&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs.c (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs.c Fri Jul 12 16:34:13 2013
@@ -38,6 +38,7 @@
#include "tree.h"
#include "lock.h"
#include "id.h"
+#include "pack.h"
#include "rep-cache.h"
#include "svn_private_config.h"
#include "private/svn_fs_util.h"
Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.c?rev=1502605&r1=1502604&r2=1502605&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.c Fri Jul 12 16:34:13 2013
@@ -58,6 +58,7 @@
#include "fs_fs.h"
#include "id.h"
#include "low_level.h"
+#include "pack.h"
#include "rep-cache.h"
#include "temp_serializer.h"
#include "util.h"
@@ -1346,27 +1347,6 @@ create_file_ignore_eexist(const char *fi
return svn_error_trace(err);
}
-/* forward declarations */
-
-static svn_error_t *
-pack_revprops_shard(const char *pack_file_dir,
- const char *shard_path,
- apr_int64_t shard,
- int max_files_per_dir,
- apr_off_t max_pack_size,
- int compression_level,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *scratch_pool);
-
-static svn_error_t *
-delete_revprops_shard(const char *shard_path,
- apr_int64_t shard,
- int max_files_per_dir,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *scratch_pool);
-
/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev.
*
* NOTE: Keep the old non-packed shards around until after the format bump.
@@ -1411,11 +1391,14 @@ upgrade_pack_revprops(svn_fs_t *fs,
apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
iterpool);
- SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
- shard, ffd->max_files_per_dir,
- (int)(0.9 * ffd->revprop_pack_size),
- compression_level,
- cancel_func, cancel_baton, iterpool));
+ SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir,
+ revprops_shard_path,
+ shard,
+ ffd->max_files_per_dir,
+ (int)(0.9 * ffd->revprop_pack_size),
+ compression_level,
+ cancel_func, cancel_baton,
+ iterpool));
if (notify_func)
SVN_ERR(notify_func(notify_baton, shard,
svn_fs_upgrade_pack_revprops, iterpool));
@@ -1462,9 +1445,10 @@ upgrade_cleanup_pack_revprops(svn_fs_t *
revprops_shard_path = svn_dirent_join(revsprops_dir,
apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
iterpool);
- SVN_ERR(delete_revprops_shard(revprops_shard_path,
- shard, ffd->max_files_per_dir,
- cancel_func, cancel_baton, iterpool));
+ SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path,
+ shard, ffd->max_files_per_dir,
+ cancel_func, cancel_baton,
+ iterpool));
if (notify_func)
SVN_ERR(notify_func(notify_baton, shard,
svn_fs_upgrade_cleanup_revprops, iterpool));
@@ -1865,71 +1849,6 @@ open_pack_or_rev_file(apr_file_t **file,
return svn_error_trace(err);
}
-/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
- Use POOL for temporary allocations. */
-static svn_error_t *
-get_packed_offset(apr_off_t *rev_offset,
- svn_fs_t *fs,
- svn_revnum_t rev,
- apr_pool_t *pool)
-{
- fs_fs_data_t *ffd = fs->fsap_data;
- svn_stream_t *manifest_stream;
- svn_boolean_t is_cached;
- svn_revnum_t shard;
- apr_int64_t shard_pos;
- apr_array_header_t *manifest;
- apr_pool_t *iterpool;
-
- shard = rev / ffd->max_files_per_dir;
-
- /* position of the shard within the manifest */
- shard_pos = rev % ffd->max_files_per_dir;
-
- /* fetch exactly that element into *rev_offset, if the manifest is found
- in the cache */
- SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached,
- ffd->packed_offset_cache, &shard,
- svn_fs_fs__get_sharded_offset, &shard_pos,
- pool));
-
- if (is_cached)
- return SVN_NO_ERROR;
-
- /* Open the manifest file. */
- SVN_ERR(svn_stream_open_readonly(&manifest_stream,
- svn_fs_fs__path_rev_packed(fs, rev,
- PATH_MANIFEST,
- pool),
- pool, pool));
-
- /* While we're here, let's just read the entire manifest file into an array,
- so we can cache the entire thing. */
- iterpool = svn_pool_create(pool);
- manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t));
- while (1)
- {
- svn_boolean_t eof;
- apr_int64_t val;
-
- svn_pool_clear(iterpool);
- SVN_ERR(svn_fs_fs__read_number_from_stream(&val, &eof, manifest_stream,
- iterpool));
- if (eof)
- break;
-
- APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val;
- }
- svn_pool_destroy(iterpool);
-
- *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir,
- apr_off_t);
-
- /* Close up shop and cache the array. */
- SVN_ERR(svn_stream_close(manifest_stream));
- return svn_cache__set(ffd->packed_offset_cache, &shard, manifest, pool);
-}
-
/* Open the revision file for revision REV in filesystem FS and store
the newly opened file in FILE. Seek to location OFFSET before
returning. Perform temporary allocations in POOL. */
@@ -1950,7 +1869,7 @@ open_and_seek_revision(apr_file_t **file
{
apr_off_t rev_offset;
- SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
+ SVN_ERR(svn_fs_fs__get_packed_offset(&rev_offset, fs, rev, pool));
offset += rev_offset;
}
@@ -2295,7 +2214,7 @@ get_root_changes_offset(apr_off_t *root_
if (svn_fs_fs__is_packed_rev(fs, rev)
&& ((rev + 1) % ffd->max_files_per_dir != 0))
{
- SVN_ERR(get_packed_offset(&offset, fs, rev + 1, pool));
+ SVN_ERR(svn_fs_fs__get_packed_offset(&offset, fs, rev + 1, pool));
seek_relative = APR_SET;
}
else
@@ -2306,7 +2225,7 @@ get_root_changes_offset(apr_off_t *root_
/* Offset of the revision from the start of the pack file, if applicable. */
if (svn_fs_fs__is_packed_rev(fs, rev))
- SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
+ SVN_ERR(svn_fs_fs__get_packed_offset(&rev_offset, fs, rev, pool));
else
rev_offset = 0;
@@ -3775,7 +3694,7 @@ create_rep_state_body(struct rep_state *
/* ... we can re-use the same, already open file object
*/
apr_off_t offset;
- SVN_ERR(get_packed_offset(&offset, fs, rep->revision, pool));
+ SVN_ERR(svn_fs_fs__get_packed_offset(&offset, fs, rep->revision, pool));
offset += rep->offset;
SVN_ERR(svn_io_file_seek(*file_hint, APR_SET, &offset, pool));
@@ -8843,87 +8762,6 @@ svn_fs_fs__begin_txn(svn_fs_txn_t **txn_
}
-/****** Packing FSFS shards *********/
-
-/* Pack the revision SHARD containing exactly MAX_FILES_PER_DIR revisions
- * from SHARD_PATH into the PACK_FILE_DIR, using POOL for allocations.
- * CANCEL_FUNC and CANCEL_BATON are what you think they are.
- *
- * If for some reason we detect a partial packing already performed, we
- * remove the pack file and start again.
- */
-static svn_error_t *
-pack_rev_shard(const char *pack_file_dir,
- const char *shard_path,
- apr_int64_t shard,
- int max_files_per_dir,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *pool)
-{
- const char *pack_file_path, *manifest_file_path;
- svn_stream_t *pack_stream, *manifest_stream;
- svn_revnum_t start_rev, end_rev, rev;
- apr_off_t next_offset;
- apr_pool_t *iterpool;
-
- /* Some useful paths. */
- pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
- manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool);
-
- /* Remove any existing pack file for this shard, since it is incomplete. */
- SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
- pool));
-
- /* Create the new directory and pack and manifest files. */
- SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, pool));
- SVN_ERR(svn_stream_open_writable(&pack_stream, pack_file_path, pool,
- pool));
- SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path,
- pool, pool));
-
- start_rev = (svn_revnum_t) (shard * max_files_per_dir);
- end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
- next_offset = 0;
- iterpool = svn_pool_create(pool);
-
- /* Iterate over the revisions in this shard, squashing them together. */
- for (rev = start_rev; rev <= end_rev; rev++)
- {
- svn_stream_t *rev_stream;
- apr_finfo_t finfo;
- const char *path;
-
- svn_pool_clear(iterpool);
-
- /* Get the size of the file. */
- path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
- iterpool);
- SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
-
- /* Update the manifest. */
- SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%" APR_OFF_T_FMT
- "\n", next_offset));
- next_offset += finfo.size;
-
- /* Copy all the bits from the rev file to the end of the pack file. */
- SVN_ERR(svn_stream_open_readonly(&rev_stream, path, iterpool, iterpool));
- SVN_ERR(svn_stream_copy3(rev_stream, svn_stream_disown(pack_stream,
- iterpool),
- cancel_func, cancel_baton, iterpool));
- }
-
- SVN_ERR(svn_stream_close(manifest_stream));
- SVN_ERR(svn_stream_close(pack_stream));
- SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
- SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, iterpool));
- SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool));
-
- svn_pool_destroy(iterpool);
-
- return SVN_NO_ERROR;
-}
-
/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH
* to the pack file at PACK_FILE_NAME in PACK_FILE_DIR.
*
@@ -9023,16 +8861,16 @@ copy_revprops(const char *pack_file_dir,
* CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary
* allocations are done in SCRATCH_POOL.
*/
-static svn_error_t *
-pack_revprops_shard(const char *pack_file_dir,
- const char *shard_path,
- apr_int64_t shard,
- int max_files_per_dir,
- apr_off_t max_pack_size,
- int compression_level,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *scratch_pool)
+svn_error_t *
+svn_fs_fs__pack_revprops_shard(const char *pack_file_dir,
+ const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ apr_off_t max_pack_size,
+ int compression_level,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool)
{
const char *manifest_file_path, *pack_filename = NULL;
svn_stream_t *manifest_stream;
@@ -9133,13 +8971,13 @@ pack_revprops_shard(const char *pack_fil
* CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary
* allocations are done in SCRATCH_POOL.
*/
-static svn_error_t *
-delete_revprops_shard(const char *shard_path,
- apr_int64_t shard,
- int max_files_per_dir,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *scratch_pool)
+svn_error_t *
+svn_fs_fs__delete_revprops_shard(const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool)
{
if (shard == 0)
{
@@ -9168,225 +9006,6 @@ delete_revprops_shard(const char *shard_
return SVN_NO_ERROR;
}
-/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and
- * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL
- * for allocations. REVPROPS_DIR will be NULL if revprop packing is not
- * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that
- * case.
- *
- * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly
- * NOTIFY_FUNC and NOTIFY_BATON.
- *
- * If for some reason we detect a partial packing already performed, we
- * remove the pack file and start again.
- */
-static svn_error_t *
-pack_shard(const char *revs_dir,
- const char *revsprops_dir,
- svn_fs_t *fs,
- apr_int64_t shard,
- int max_files_per_dir,
- apr_off_t max_pack_size,
- int compression_level,
- svn_fs_pack_notify_t notify_func,
- void *notify_baton,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *pool)
-{
- const char *rev_shard_path, *rev_pack_file_dir;
- const char *revprops_shard_path, *revprops_pack_file_dir;
-
- /* Notify caller we're starting to pack this shard. */
- if (notify_func)
- SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
- pool));
-
- /* Some useful paths. */
- rev_pack_file_dir = svn_dirent_join(revs_dir,
- apr_psprintf(pool,
- "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
- shard),
- pool);
- rev_shard_path = svn_dirent_join(revs_dir,
- apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
- pool);
-
- /* pack the revision content */
- SVN_ERR(pack_rev_shard(rev_pack_file_dir, rev_shard_path,
- shard, max_files_per_dir,
- cancel_func, cancel_baton, pool));
-
- /* if enabled, pack the revprops in an equivalent way */
- if (revsprops_dir)
- {
- revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
- apr_psprintf(pool,
- "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
- shard),
- pool);
- revprops_shard_path = svn_dirent_join(revsprops_dir,
- apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
- pool);
-
- SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
- shard, max_files_per_dir,
- (int)(0.9 * max_pack_size),
- compression_level,
- cancel_func, cancel_baton, pool));
- }
-
- /* Update the min-unpacked-rev file to reflect our newly packed shard.
- * (This doesn't update ffd->min_unpacked_rev. That will be updated by
- * update_min_unpacked_rev() when necessary.) */
- SVN_ERR(svn_fs_fs__write_revnum_file(fs,
- (svn_revnum_t)((shard + 1) * max_files_per_dir),
- pool));
-
- /* Finally, remove the existing shard directories.
- * For revprops, clean up older obsolete shards as well as they might
- * have been left over from an interrupted FS upgrade. */
- SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
- cancel_func, cancel_baton, pool));
- if (revsprops_dir)
- {
- svn_node_kind_t kind = svn_node_dir;
- apr_int64_t to_cleanup = shard;
- do
- {
- SVN_ERR(delete_revprops_shard(revprops_shard_path,
- to_cleanup, max_files_per_dir,
- cancel_func, cancel_baton, pool));
-
- /* If the previous shard exists, clean it up as well.
- Don't try to clean up shard 0 as it we can't tell quickly
- whether it actually needs cleaning up. */
- revprops_shard_path = svn_dirent_join(revsprops_dir,
- apr_psprintf(pool, "%" APR_INT64_T_FMT, --to_cleanup),
- pool);
- SVN_ERR(svn_io_check_path(revprops_shard_path, &kind, pool));
- }
- while (kind == svn_node_dir && to_cleanup > 0);
- }
-
- /* Notify caller we're starting to pack this shard. */
- if (notify_func)
- SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_end,
- pool));
-
- return SVN_NO_ERROR;
-}
-
-struct pack_baton
-{
- svn_fs_t *fs;
- svn_fs_pack_notify_t notify_func;
- void *notify_baton;
- svn_cancel_func_t cancel_func;
- void *cancel_baton;
-};
-
-
-/* The work-horse for svn_fs_fs__pack, called with the FS write lock.
- This implements the svn_fs_fs__with_write_lock() 'body' callback
- type. BATON is a 'struct pack_baton *'.
-
- WARNING: if you add a call to this function, please note:
- The code currently assumes that any piece of code running with
- the write-lock set can rely on the ffd->min_unpacked_rev and
- ffd->min_unpacked_revprop caches to be up-to-date (and, by
- extension, on not having to use a retry when calling
- svn_fs_fs__path_rev_absolute() and friends). If you add a call
- to this function, consider whether you have to call
- update_min_unpacked_rev().
- See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith
- */
-static svn_error_t *
-pack_body(void *baton,
- apr_pool_t *pool)
-{
- struct pack_baton *pb = baton;
- fs_fs_data_t ffd = {0};
- apr_int64_t completed_shards;
- apr_int64_t i;
- svn_revnum_t youngest;
- apr_pool_t *iterpool;
- const char *rev_data_path;
- const char *revprops_data_path = NULL;
-
- /* read repository settings */
- SVN_ERR(read_format(&ffd.format, &ffd.max_files_per_dir,
- path_format(pb->fs, pool), pool));
- SVN_ERR(check_format(ffd.format));
- SVN_ERR(read_config(&ffd, pb->fs->path, pool));
-
- /* If the repository isn't a new enough format, we don't support packing.
- Return a friendly error to that effect. */
- if (ffd.format < SVN_FS_FS__MIN_PACKED_FORMAT)
- return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
- _("FSFS format (%d) too old to pack; please upgrade the filesystem."),
- ffd.format);
-
- /* If we aren't using sharding, we can't do any packing, so quit. */
- if (!ffd.max_files_per_dir)
- return SVN_NO_ERROR;
-
- SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&ffd.min_unpacked_rev, pb->fs,
- pool));
-
- SVN_ERR(get_youngest(&youngest, pb->fs->path, pool));
- completed_shards = (youngest + 1) / ffd.max_files_per_dir;
-
- /* See if we've already completed all possible shards thus far. */
- if (ffd.min_unpacked_rev == (completed_shards * ffd.max_files_per_dir))
- return SVN_NO_ERROR;
-
- rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool);
- if (ffd.format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
- revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR,
- pool);
-
- iterpool = svn_pool_create(pool);
- for (i = ffd.min_unpacked_rev / ffd.max_files_per_dir;
- i < completed_shards;
- i++)
- {
- svn_pool_clear(iterpool);
-
- if (pb->cancel_func)
- SVN_ERR(pb->cancel_func(pb->cancel_baton));
-
- SVN_ERR(pack_shard(rev_data_path, revprops_data_path,
- pb->fs, i, ffd.max_files_per_dir,
- ffd.revprop_pack_size,
- ffd.compress_packed_revprops
- ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
- : SVN_DELTA_COMPRESSION_LEVEL_NONE,
- pb->notify_func, pb->notify_baton,
- pb->cancel_func, pb->cancel_baton, iterpool));
- }
-
- svn_pool_destroy(iterpool);
- return SVN_NO_ERROR;
-}
-
-svn_error_t *
-svn_fs_fs__pack(svn_fs_t *fs,
- svn_fs_pack_notify_t notify_func,
- void *notify_baton,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *pool)
-{
- struct pack_baton pb = { 0 };
- pb.fs = fs;
- pb.notify_func = notify_func;
- pb.notify_baton = notify_baton;
- pb.cancel_func = cancel_func;
- pb.cancel_baton = cancel_baton;
- return svn_fs_fs__with_write_lock(fs, pack_body, &pb, pool);
-}
-
/** Verifying. **/
Modified: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.h?rev=1502605&r1=1502604&r2=1502605&view=diff
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.h (original)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/fs_fs.h Fri Jul 12 16:34:13 2013
@@ -445,6 +445,46 @@ svn_error_t *svn_fs_fs__revision_prop(sv
const char *propname,
apr_pool_t *pool);
+/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev.
+ *
+ * NOTE: Keep the old non-packed shards around until after the format bump.
+ * Otherwise, re-running upgrade will drop the packed revprop shard but
+ * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after
+ * the bump.
+ *
+ * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are
+ * used in the usual way. Temporary allocations are done in SCRATCH_POOL.
+ */
+svn_error_t *
+svn_fs_fs__pack_revprops_shard(const char *pack_file_dir,
+ const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ apr_off_t max_pack_size,
+ int compression_level,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool);
+
+/* In the filesystem FS, remove all non-packed revprop shards up to
+ * min_unpacked_rev. Temporary allocations are done in SCRATCH_POOL.
+ *
+ * NOTIFY_FUNC and NOTIFY_BATON as well as CANCEL_FUNC and CANCEL_BATON are
+ * used in the usual way. Cancellation is supported in the sense that we
+ * will cleanly abort the operation. However, there will be remnant shards
+ * that must be removed manually.
+ *
+ * See upgrade_pack_revprops for more info.
+ */
+svn_error_t *
+svn_fs_fs__delete_revprops_shard(const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool);
+
+
/* Change, add, or delete a property on a revision REV in filesystem
FS. NAME gives the name of the property, and value, if non-NULL,
gives the new contents of the property. If value is NULL, then the
@@ -545,18 +585,4 @@ svn_fs_fs__initialize_txn_caches(svn_fs_
void
svn_fs_fs__reset_txn_caches(svn_fs_t *fs);
-/* Possibly pack the repository at PATH. This just take full shards, and
- combines all the revision files into a single one, with a manifest header.
- Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support.
-
- Existing filesystem references need not change. */
-svn_error_t *
-svn_fs_fs__pack(svn_fs_t *fs,
- svn_fs_pack_notify_t notify_func,
- void *notify_baton,
- svn_cancel_func_t cancel_func,
- void *cancel_baton,
- apr_pool_t *pool);
-
-
#endif
Added: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.c?rev=1502605&view=auto
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.c (added)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.c Fri Jul 12 16:34:13 2013
@@ -0,0 +1,444 @@
+/* pack.c --- FSFS shard packing functionality
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+#include <assert.h>
+
+#include "svn_pools.h"
+#include "svn_dirent_uri.h"
+#include "svn_sorts.h"
+#include "private/svn_temp_serializer.h"
+#include "private/svn_subr_private.h"
+#include "private/svn_string_private.h"
+
+#include "fs_fs.h"
+#include "pack.h"
+#include "util.h"
+#include "id.h"
+#include "low_level.h"
+
+#include "../libsvn_fs/fs-loader.h"
+
+#include "svn_private_config.h"
+#include "temp_serializer.h"
+
+/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
+ Use POOL for temporary allocations. */
+svn_error_t *
+svn_fs_fs__get_packed_offset(apr_off_t *rev_offset,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ svn_stream_t *manifest_stream;
+ svn_boolean_t is_cached;
+ svn_revnum_t shard;
+ apr_int64_t shard_pos;
+ apr_array_header_t *manifest;
+ apr_pool_t *iterpool;
+
+ shard = rev / ffd->max_files_per_dir;
+
+ /* position of the shard within the manifest */
+ shard_pos = rev % ffd->max_files_per_dir;
+
+ /* fetch exactly that element into *rev_offset, if the manifest is found
+ in the cache */
+ SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached,
+ ffd->packed_offset_cache, &shard,
+ svn_fs_fs__get_sharded_offset, &shard_pos,
+ pool));
+
+ if (is_cached)
+ return SVN_NO_ERROR;
+
+ /* Open the manifest file. */
+ SVN_ERR(svn_stream_open_readonly(&manifest_stream,
+ svn_fs_fs__path_rev_packed(fs, rev,
+ PATH_MANIFEST,
+ pool),
+ pool, pool));
+
+ /* While we're here, let's just read the entire manifest file into an array,
+ so we can cache the entire thing. */
+ iterpool = svn_pool_create(pool);
+ manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t));
+ while (1)
+ {
+ svn_boolean_t eof;
+ apr_int64_t val;
+
+ svn_pool_clear(iterpool);
+ SVN_ERR(svn_fs_fs__read_number_from_stream(&val, &eof, manifest_stream,
+ iterpool));
+ if (eof)
+ break;
+
+ APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val;
+ }
+ svn_pool_destroy(iterpool);
+
+ *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir,
+ apr_off_t);
+
+ /* Close up shop and cache the array. */
+ SVN_ERR(svn_stream_close(manifest_stream));
+ return svn_cache__set(ffd->packed_offset_cache, &shard, manifest, pool);
+}
+
+/* Packing logic: Simply concatenate all revision contents.
+ *
+ * Pack the revision shard starting at SHARD_REV containing exactly
+ * MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
+ * using POOL for allocations. CANCEL_FUNC and CANCEL_BATON are what you
+ * think they are.
+ */
+static svn_error_t *
+pack_phys_addressed(const char *pack_file_dir,
+ const char *shard_path,
+ svn_revnum_t start_rev,
+ int max_files_per_dir,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool)
+{
+ const char *pack_file_path, *manifest_file_path;
+ svn_stream_t *pack_stream, *manifest_stream;
+ svn_revnum_t end_rev, rev;
+ apr_off_t next_offset;
+ apr_pool_t *iterpool;
+
+ /* Some useful paths. */
+ pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
+ manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool);
+
+ /* Create the new directory and pack file. */
+ SVN_ERR(svn_stream_open_writable(&pack_stream, pack_file_path, pool,
+ pool));
+
+ /* Create the manifest file. */
+ SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path,
+ pool, pool));
+
+ end_rev = start_rev + max_files_per_dir - 1;
+ next_offset = 0;
+ iterpool = svn_pool_create(pool);
+
+ /* Iterate over the revisions in this shard, squashing them together. */
+ for (rev = start_rev; rev <= end_rev; rev++)
+ {
+ svn_stream_t *rev_stream;
+ apr_finfo_t finfo;
+ const char *path;
+
+ svn_pool_clear(iterpool);
+
+ /* Get the size of the file. */
+ path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
+ iterpool);
+ SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
+
+ /* build manifest */
+ SVN_ERR(svn_stream_printf(manifest_stream, iterpool,
+ "%" APR_OFF_T_FMT "\n", next_offset));
+ next_offset += finfo.size;
+
+ /* Copy all the bits from the rev file to the end of the pack file. */
+ SVN_ERR(svn_stream_open_readonly(&rev_stream, path, iterpool, iterpool));
+ SVN_ERR(svn_stream_copy3(rev_stream, svn_stream_disown(pack_stream,
+ iterpool),
+ cancel_func, cancel_baton, iterpool));
+ }
+
+ /* disallow write access to the manifest file */
+ SVN_ERR(svn_stream_close(manifest_stream));
+ SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool));
+
+ SVN_ERR(svn_stream_close(pack_stream));
+
+ svn_pool_destroy(iterpool);
+
+ return SVN_NO_ERROR;
+}
+
+/* In filesystem FS, pack the revision SHARD containing exactly
+ * MAX_FILES_PER_DIR revisions from SHARD_PATH into the PACK_FILE_DIR,
+ * using POOL for allocations. Try to limit the amount of temporary
+ * memory needed to MAX_MEM bytes. CANCEL_FUNC and CANCEL_BATON are what
+ * you think they are.
+ *
+ * If for some reason we detect a partial packing already performed, we
+ * remove the pack file and start again.
+ *
+ * The actual packing will be done in a format-specific sub-function.
+ */
+static svn_error_t *
+pack_rev_shard(svn_fs_t *fs,
+ const char *pack_file_dir,
+ const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ apr_size_t max_mem,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool)
+{
+ const char *pack_file_path;
+ svn_revnum_t shard_rev = (svn_revnum_t) (shard * max_files_per_dir);
+
+ /* Some useful paths. */
+ pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
+
+ /* Remove any existing pack file for this shard, since it is incomplete. */
+ SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
+ pool));
+
+ /* Create the new directory and pack file. */
+ SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, pool));
+
+ /* Index information files */
+ SVN_ERR(pack_phys_addressed(pack_file_dir, shard_path, shard_rev,
+ max_files_per_dir, cancel_func,
+ cancel_baton, pool));
+
+ SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, pool));
+ SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, pool));
+
+ return SVN_NO_ERROR;
+}
+
+/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and
+ * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL
+ * for allocations. REVPROPS_DIR will be NULL if revprop packing is not
+ * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that
+ * case.
+ *
+ * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly
+ * NOTIFY_FUNC and NOTIFY_BATON.
+ *
+ * If for some reason we detect a partial packing already performed, we
+ * remove the pack file and start again.
+ */
+static svn_error_t *
+pack_shard(const char *revs_dir,
+ const char *revsprops_dir,
+ svn_fs_t *fs,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ apr_off_t max_pack_size,
+ int compression_level,
+ svn_fs_pack_notify_t notify_func,
+ void *notify_baton,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ const char *rev_shard_path, *rev_pack_file_dir;
+ const char *revprops_shard_path, *revprops_pack_file_dir;
+
+ /* Notify caller we're starting to pack this shard. */
+ if (notify_func)
+ SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
+ pool));
+
+ /* Some useful paths. */
+ rev_pack_file_dir = svn_dirent_join(revs_dir,
+ apr_psprintf(pool,
+ "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
+ shard),
+ pool);
+ rev_shard_path = svn_dirent_join(revs_dir,
+ apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
+ pool);
+
+ /* pack the revision content */
+ SVN_ERR(pack_rev_shard(fs, rev_pack_file_dir, rev_shard_path,
+ shard, max_files_per_dir, 64 * 1024 * 1024,
+ cancel_func, cancel_baton, pool));
+
+ /* if enabled, pack the revprops in an equivalent way */
+ if (revsprops_dir)
+ {
+ revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
+ apr_psprintf(pool,
+ "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
+ shard),
+ pool);
+ revprops_shard_path = svn_dirent_join(revsprops_dir,
+ apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
+ pool);
+
+ SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir,
+ revprops_shard_path,
+ shard, max_files_per_dir,
+ (int)(0.9 * max_pack_size),
+ compression_level,
+ cancel_func, cancel_baton,
+ pool));
+ }
+
+ /* Update the min-unpacked-rev file to reflect our newly packed shard. */
+ SVN_ERR(svn_fs_fs__write_revnum_file(fs,
+ (svn_revnum_t)((shard + 1) * max_files_per_dir),
+ pool));
+ ffd->min_unpacked_rev = (svn_revnum_t)((shard + 1) * max_files_per_dir);
+
+ /* Finally, remove the existing shard directories.
+ * For revprops, clean up older obsolete shards as well as they might
+ * have been left over from an interrupted FS upgrade. */
+ SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
+ cancel_func, cancel_baton, pool));
+ if (revsprops_dir)
+ {
+ svn_node_kind_t kind = svn_node_dir;
+ apr_int64_t to_cleanup = shard;
+ do
+ {
+ SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path,
+ to_cleanup,
+ max_files_per_dir,
+ cancel_func,
+ cancel_baton,
+ pool));
+
+ /* If the previous shard exists, clean it up as well.
+ Don't try to clean up shard 0 as it we can't tell quickly
+ whether it actually needs cleaning up. */
+ revprops_shard_path = svn_dirent_join(revsprops_dir,
+ apr_psprintf(pool, "%" APR_INT64_T_FMT, --to_cleanup),
+ pool);
+ SVN_ERR(svn_io_check_path(revprops_shard_path, &kind, pool));
+ }
+ while (kind == svn_node_dir && to_cleanup > 0);
+ }
+
+ /* Notify caller we're starting to pack this shard. */
+ if (notify_func)
+ SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_end,
+ pool));
+
+ return SVN_NO_ERROR;
+}
+
+struct pack_baton
+{
+ svn_fs_t *fs;
+ svn_fs_pack_notify_t notify_func;
+ void *notify_baton;
+ svn_cancel_func_t cancel_func;
+ void *cancel_baton;
+};
+
+
+/* The work-horse for svn_fs_fs__pack, called with the FS write lock.
+ This implements the svn_fs_fs__with_write_lock() 'body' callback
+ type. BATON is a 'struct pack_baton *'.
+
+ WARNING: if you add a call to this function, please note:
+ The code currently assumes that any piece of code running with
+ the write-lock set can rely on the ffd->min_unpacked_rev and
+ ffd->min_unpacked_revprop caches to be up-to-date (and, by
+ extension, on not having to use a retry when calling
+ svn_fs_fs__path_rev_absolute() and friends). If you add a call
+ to this function, consider whether you have to call
+ update_min_unpacked_rev().
+ See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith
+ */
+static svn_error_t *
+pack_body(void *baton,
+ apr_pool_t *pool)
+{
+ struct pack_baton *pb = baton;
+ fs_fs_data_t *ffd = pb->fs->fsap_data;
+ apr_int64_t completed_shards;
+ apr_int64_t i;
+ svn_revnum_t youngest;
+ apr_pool_t *iterpool;
+ const char *rev_data_path;
+ const char *revprops_data_path = NULL;
+
+ /* If the repository isn't a new enough format, we don't support packing.
+ Return a friendly error to that effect. */
+ if (ffd->format < SVN_FS_FS__MIN_PACKED_FORMAT)
+ return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
+ _("FSFS format (%d) too old to pack; please upgrade the filesystem."),
+ ffd->format);
+
+ /* If we aren't using sharding, we can't do any packing, so quit. */
+ if (!ffd->max_files_per_dir)
+ return SVN_NO_ERROR;
+
+ SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&ffd->min_unpacked_rev, pb->fs,
+ pool));
+
+ SVN_ERR(svn_fs_fs__youngest_rev(&youngest, pb->fs, pool));
+ completed_shards = (youngest + 1) / ffd->max_files_per_dir;
+
+ /* See if we've already completed all possible shards thus far. */
+ if (ffd->min_unpacked_rev == (completed_shards * ffd->max_files_per_dir))
+ return SVN_NO_ERROR;
+
+ rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool);
+ if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
+ revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR,
+ pool);
+
+ iterpool = svn_pool_create(pool);
+ for (i = ffd->min_unpacked_rev / ffd->max_files_per_dir;
+ i < completed_shards;
+ i++)
+ {
+ svn_pool_clear(iterpool);
+
+ if (pb->cancel_func)
+ SVN_ERR(pb->cancel_func(pb->cancel_baton));
+
+ SVN_ERR(pack_shard(rev_data_path, revprops_data_path,
+ pb->fs, i, ffd->max_files_per_dir,
+ ffd->revprop_pack_size,
+ ffd->compress_packed_revprops
+ ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
+ : SVN_DELTA_COMPRESSION_LEVEL_NONE,
+ pb->notify_func, pb->notify_baton,
+ pb->cancel_func, pb->cancel_baton, iterpool));
+ }
+
+ svn_pool_destroy(iterpool);
+ return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_fs_fs__pack(svn_fs_t *fs,
+ svn_fs_pack_notify_t notify_func,
+ void *notify_baton,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool)
+{
+ struct pack_baton pb = { 0 };
+ pb.fs = fs;
+ pb.notify_func = notify_func;
+ pb.notify_baton = notify_baton;
+ pb.cancel_func = cancel_func;
+ pb.cancel_baton = cancel_baton;
+ return svn_fs_fs__with_write_lock(fs, pack_body, &pb, pool);
+}
Added: subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.h?rev=1502605&view=auto
==============================================================================
--- subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.h (added)
+++ subversion/branches/fsfs-improvements/subversion/libsvn_fs_fs/pack.h Fri Jul 12 16:34:13 2013
@@ -0,0 +1,53 @@
+/* pack.h : interface FSFS pack functionality
+ *
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ */
+
+#ifndef SVN_LIBSVN_FS__PACK_H
+#define SVN_LIBSVN_FS__PACK_H
+
+#include "fs.h"
+
+/* Possibly pack the repository at PATH. This just take full shards, and
+ combines all the revision files into a single one, with a manifest header.
+ If given, NOTIFY_FUNC will be called with NOTIFY_BATON to report progress.
+ Use optional CANCEL_FUNC/CANCEL_BATON for cancellation support.
+
+ Existing filesystem references need not change. */
+svn_error_t *
+svn_fs_fs__pack(svn_fs_t *fs,
+ svn_fs_pack_notify_t notify_func,
+ void *notify_baton,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *pool);
+
+/*
+ * For the packed revision @a rev in @a fs, determine the offset within
+ * the revision pack file and return it in @a rev_offset. Use @a pool for
+ * allocations.
+ */
+svn_error_t *
+svn_fs_fs__get_packed_offset(apr_off_t *rev_offset,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_pool_t *pool);
+
+#endif