You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by br...@apache.org on 2012/08/16 12:18:03 UTC
svn commit: r1373783 [15/50] - in /subversion/branches/compressed-pristines:
./ build/ build/ac-macros/ build/generator/ build/generator/templates/
build/win32/ contrib/client-side/emacs/ contrib/client-side/svn-push/
contrib/client-side/svnmerge/ cont...
Modified: subversion/branches/compressed-pristines/subversion/libsvn_fs_fs/fs_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/subversion/libsvn_fs_fs/fs_fs.c?rev=1373783&r1=1373782&r2=1373783&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/subversion/libsvn_fs_fs/fs_fs.c (original)
+++ subversion/branches/compressed-pristines/subversion/libsvn_fs_fs/fs_fs.c Thu Aug 16 10:17:48 2012
@@ -59,7 +59,10 @@
#include "rep-cache.h"
#include "temp_serializer.h"
+#include "private/svn_string_private.h"
#include "private/svn_fs_util.h"
+#include "private/svn_subr_private.h"
+#include "private/svn_delta_private.h"
#include "../libsvn_fs/fs-loader.h"
#include "svn_private_config.h"
@@ -92,6 +95,17 @@
Values < 1 disable deltification. */
#define SVN_FS_FS_MAX_DELTIFICATION_WALK 1023
+/* Give writing processes 10 seconds to replace an existing revprop
+ file with a new one. After that time, we assume that the writing
+ process got aborted and that we have re-read revprops. */
+#define REVPROP_CHANGE_TIMEOUT 10 * 1000000
+
+/* The following are names of atomics that will be used to communicate
+ * revprop updates across all processes on this machine. */
+#define ATOMIC_REVPROP_GENERATION "rev-prop-generation"
+#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout"
+#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics"
+
/* Following are defines that specify the textual elements of the
native filesystem directories and revision files. */
@@ -179,13 +193,12 @@ is_packed_rev(svn_fs_t *fs, svn_revnum_t
static svn_boolean_t
is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev)
{
-#if 0
fs_fs_data_t *ffd = fs->fsap_data;
- return (rev < ffd->min_unpacked_revprop);
-#else
- return FALSE;
-#endif
+ /* rev 0 will not be packed */
+ return (rev < ffd->min_unpacked_rev)
+ && (rev != 0)
+ && (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT);
}
static const char *
@@ -225,6 +238,12 @@ path_lock(svn_fs_t *fs, apr_pool_t *pool
}
static const char *
+path_revprop_generation(svn_fs_t *fs, apr_pool_t *pool)
+{
+ return svn_dirent_join(fs->path, PATH_REVPROP_GENERATION, pool);
+}
+
+static const char *
path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind,
apr_pool_t *pool)
{
@@ -234,7 +253,8 @@ path_rev_packed(svn_fs_t *fs, svn_revnum
assert(is_packed_rev(fs, rev));
return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR,
- apr_psprintf(pool, "%ld.pack",
+ apr_psprintf(pool,
+ "%ld" PATH_EXT_PACKED_SHARD,
rev / ffd->max_files_per_dir),
kind, NULL);
}
@@ -284,7 +304,7 @@ svn_fs_fs__path_rev_absolute(const char
}
else
{
- *path = path_rev_packed(fs, rev, "pack", pool);
+ *path = path_rev_packed(fs, rev, PATH_PACKED, pool);
}
return SVN_NO_ERROR;
@@ -303,6 +323,18 @@ path_revprops_shard(svn_fs_t *fs, svn_re
}
static const char *
+path_revprops_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ assert(ffd->max_files_per_dir);
+ return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR,
+ apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD,
+ rev / ffd->max_files_per_dir),
+ NULL);
+}
+
+static const char *
path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
{
fs_fs_data_t *ffd = fs->fsap_data;
@@ -870,25 +902,39 @@ get_file_offset(apr_off_t *offset_p, apr
}
-/* Check that BUF, a nul-terminated buffer of text from format file PATH,
+/* Check that BUF, a nul-terminated buffer of text from file PATH,
contains only digits at OFFSET and beyond, raising an error if not.
+ TITLE contains a user-visible description of the file, usually the
+ short file name.
Uses POOL for temporary allocation. */
static svn_error_t *
-check_format_file_buffer_numeric(const char *buf, apr_off_t offset,
- const char *path, apr_pool_t *pool)
+check_file_buffer_numeric(const char *buf, apr_off_t offset,
+ const char *path, const char *title,
+ apr_pool_t *pool)
{
const char *p;
for (p = buf + offset; *p; p++)
if (!svn_ctype_isdigit(*p))
return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
- _("Format file '%s' contains unexpected non-digit '%c' within '%s'"),
- svn_dirent_local_style(path, pool), *p, buf);
+ _("%s file '%s' contains unexpected non-digit '%c' within '%s'"),
+ title, svn_dirent_local_style(path, pool), *p, buf);
return SVN_NO_ERROR;
}
+/* Check that BUF, a nul-terminated buffer of text from format file PATH,
+ contains only digits at OFFSET and beyond, raising an error if not.
+
+ Uses POOL for temporary allocation. */
+static svn_error_t *
+check_format_file_buffer_numeric(const char *buf, apr_off_t offset,
+ const char *path, apr_pool_t *pool)
+{
+ return check_file_buffer_numeric(buf, offset, path, "Format", pool);
+}
+
/* Read the format number and maximum number of files per directory
from PATH and return them in *PFORMAT and *MAX_FILES_PER_DIR
respectively.
@@ -902,12 +948,12 @@ read_format(int *pformat, int *max_files
const char *path, apr_pool_t *pool)
{
svn_error_t *err;
- apr_file_t *file;
- char buf[80];
- apr_size_t len;
+ svn_stream_t *stream;
+ svn_stringbuf_t *content;
+ svn_stringbuf_t *buf;
+ svn_boolean_t eos = FALSE;
- err = svn_io_file_open(&file, path, APR_READ | APR_BUFFERED,
- APR_OS_DEFAULT, pool);
+ err = svn_stringbuf_from_file2(&content, path, pool);
if (err && APR_STATUS_IS_ENOENT(err->apr_err))
{
/* Treat an absent format file as format 1. Do not try to
@@ -925,62 +971,54 @@ read_format(int *pformat, int *max_files
}
SVN_ERR(err);
- len = sizeof(buf);
- err = svn_io_read_length_line(file, buf, &len, pool);
- if (err && APR_STATUS_IS_EOF(err->apr_err))
+ stream = svn_stream_from_stringbuf(content, pool);
+ SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool));
+ if (buf->len == 0 && eos)
{
/* Return a more useful error message. */
- svn_error_clear(err);
return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
_("Can't read first line of format file '%s'"),
svn_dirent_local_style(path, pool));
}
- SVN_ERR(err);
/* Check that the first line contains only digits. */
- SVN_ERR(check_format_file_buffer_numeric(buf, 0, path, pool));
- SVN_ERR(svn_cstring_atoi(pformat, buf));
+ SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, pool));
+ SVN_ERR(svn_cstring_atoi(pformat, buf->data));
/* Set the default values for anything that can be set via an option. */
*max_files_per_dir = 0;
/* Read any options. */
- while (1)
+ while (!eos)
{
- len = sizeof(buf);
- err = svn_io_read_length_line(file, buf, &len, pool);
- if (err && APR_STATUS_IS_EOF(err->apr_err))
- {
- /* No more options; that's okay. */
- svn_error_clear(err);
- break;
- }
- SVN_ERR(err);
+ SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool));
+ if (buf->len == 0)
+ break;
if (*pformat >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT &&
- strncmp(buf, "layout ", 7) == 0)
+ strncmp(buf->data, "layout ", 7) == 0)
{
- if (strcmp(buf+7, "linear") == 0)
+ if (strcmp(buf->data + 7, "linear") == 0)
{
*max_files_per_dir = 0;
continue;
}
- if (strncmp(buf+7, "sharded ", 8) == 0)
+ if (strncmp(buf->data + 7, "sharded ", 8) == 0)
{
/* Check that the argument is numeric. */
- SVN_ERR(check_format_file_buffer_numeric(buf, 15, path, pool));
- SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf + 15));
+ SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path, pool));
+ SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15));
continue;
}
}
return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
_("'%s' contains invalid filesystem format option '%s'"),
- svn_dirent_local_style(path, pool), buf);
+ svn_dirent_local_style(path, pool), buf->data);
}
- return svn_io_file_close(file, pool);
+ return SVN_NO_ERROR;
}
/* Write the format number and maximum number of files per directory
@@ -1064,15 +1102,17 @@ svn_fs_fs__fs_supports_mergeinfo(svn_fs_
return ffd->format >= SVN_FS_FS__MIN_MERGEINFO_FORMAT;
}
+/* Read the configuration information of the file system at FS_PATH
+ * and set the respective values in FFD. Use POOL for allocations.
+ */
static svn_error_t *
-read_config(svn_fs_t *fs,
+read_config(fs_fs_data_t *ffd,
+ const char *fs_path,
apr_pool_t *pool)
{
- fs_fs_data_t *ffd = fs->fsap_data;
-
SVN_ERR(svn_config_read2(&ffd->config,
- svn_dirent_join(fs->path, PATH_CONFIG, pool),
- FALSE, FALSE, fs->pool));
+ svn_dirent_join(fs_path, PATH_CONFIG, pool),
+ FALSE, FALSE, pool));
/* Initialize ffd->rep_sharing_allowed. */
if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
@@ -1082,7 +1122,7 @@ read_config(svn_fs_t *fs,
else
ffd->rep_sharing_allowed = FALSE;
- /* Initialize ffd->deltify_directories. */
+ /* Initialize deltification settings in ffd. */
if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT)
{
SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories,
@@ -1110,6 +1150,28 @@ read_config(svn_fs_t *fs,
ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION;
}
+ /* Initialize revprop packing settings in ffd. */
+ if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
+ {
+ SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops,
+ CONFIG_SECTION_PACKED_REVPROPS,
+ CONFIG_OPTION_COMPRESS_PACKED_REVPROPS,
+ FALSE));
+ SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size,
+ CONFIG_SECTION_PACKED_REVPROPS,
+ CONFIG_OPTION_REVPROP_PACK_SIZE,
+ ffd->compress_packed_revprops
+ ? 0x100
+ : 0x40));
+
+ ffd->revprop_pack_size *= 1024;
+ }
+ else
+ {
+ ffd->revprop_pack_size = 0x10000;
+ ffd->compress_packed_revprops = FALSE;
+ }
+
return SVN_NO_ERROR;
}
@@ -1179,7 +1241,7 @@ write_config(svn_fs_t *fs,
"### In rarely read repositories, the I/O overhead may be significant as" NL
"### cache hit rates will most likely be low" NL
"### directory deltification is disabled by default." NL
-"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = true" NL
+"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = false" NL
"###" NL
"### The following parameter enables deltification for properties on files" NL
"### and directories. Overall, this is a minor tuning option but can save" NL
@@ -1187,7 +1249,7 @@ write_config(svn_fs_t *fs,
"### properties. You should not activate this if rep-sharing has been" NL
"### disabled." NL
"### property deltification is disabled by default." NL
-"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = true" NL
+"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = false" NL
"###" NL
"### During commit, the server may need to walk the whole change history of" NL
"### of a given node to find a suitable deltification base. This linear" NL
@@ -1219,6 +1281,31 @@ write_config(svn_fs_t *fs,
"### exclusive use of skip-deltas (as in pre-1.8)." NL
"### For 1.8, the default value is 16; earlier versions use 1." NL
"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL
+"" NL
+"[" CONFIG_SECTION_PACKED_REVPROPS "]" NL
+"### This parameter controls the size (in kBytes) of packed revprop files." NL
+"### Revprops of consecutive revisions will be concatenated into a single" NL
+"### file up to but not exceeding the threshold given here. However, each" NL
+"### pack file may be much smaller and revprops of a single revision may be" NL
+"### much larger than the limit set here. The threshold will be applied" NL
+"### before optional compression takes place." NL
+"### Large values will reduce disk space usage at the expense of increased" NL
+"### latency and CPU usage reading and changing individual revprops. They" NL
+"### become an advantage when revprop caching has been enabled because a" NL
+"### lot of data can be read in one go. Values smaller than 4 kByte will" NL
+"### not improve latency any further and quickly render revprop packing" NL
+"### ineffective." NL
+"### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL
+"### pack files and 256 kBytes when compression has been enabled." NL
+"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL
+"###" NL
+"### To save disk space, packed revprop files may be compressed. Standard" NL
+"### revprops tend to allow for very effective compression. Reading and" NL
+"### even more so writing, become significantly more CPU intensive. With" NL
+"### revprop caching enabled, the overhead can be offset by reduced I/O" NL
+"### unless you often modify revprops after packing." NL
+"### Compressing packed revprops is disabled by default." NL
+"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false" NL
;
#undef NL
return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool),
@@ -1282,7 +1369,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char
limit = sizeof(buf);
SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool));
- ffd->uuid = apr_pstrdup(fs->pool, buf);
+ fs->uuid = apr_pstrdup(fs->pool, buf);
SVN_ERR(svn_io_file_close(uuid_file, pool));
@@ -1291,7 +1378,7 @@ svn_fs_fs__open(svn_fs_t *fs, const char
SVN_ERR(update_min_unpacked_rev(fs, pool));
/* Read the configuration file. */
- SVN_ERR(read_config(fs, pool));
+ SVN_ERR(read_config(ffd, fs->path, pool));
return get_youngest(&(ffd->youngest_rev_cache), path, pool);
}
@@ -1311,6 +1398,85 @@ create_file_ignore_eexist(const char *fi
return svn_error_trace(err);
}
+/* forward declarations */
+
+static svn_error_t *
+pack_revprops_shard(const char *pack_file_dir,
+ const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ apr_off_t max_pack_size,
+ int compression_level,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool);
+
+static svn_error_t *
+delete_revprops_shard(const char *shard_path,
+ apr_int64_t shard,
+ int max_files_per_dir,
+ svn_cancel_func_t cancel_func,
+ void *cancel_baton,
+ apr_pool_t *scratch_pool);
+
+/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev.
+ * Use SCRATCH_POOL for temporary allocations.
+ */
+static svn_error_t *
+upgrade_pack_revprops(svn_fs_t *fs,
+ apr_pool_t *scratch_pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ const char *revprops_shard_path;
+ const char *revprops_pack_file_dir;
+ apr_int64_t shard;
+ apr_int64_t first_unpacked_shard
+ = ffd->min_unpacked_rev / ffd->max_files_per_dir;
+
+ apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+ const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
+ scratch_pool);
+ int compression_level = ffd->compress_packed_revprops
+ ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
+ : SVN_DELTA_COMPRESSION_LEVEL_NONE;
+
+ /* first, pack all revprops shards to match the packed revision shards */
+ for (shard = 0; shard < first_unpacked_shard; ++shard)
+ {
+ revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
+ apr_psprintf(iterpool,
+ "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
+ shard),
+ iterpool);
+ revprops_shard_path = svn_dirent_join(revsprops_dir,
+ apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
+ iterpool);
+
+ SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
+ shard, ffd->max_files_per_dir,
+ (int)(0.9 * ffd->revprop_pack_size),
+ compression_level,
+ NULL, NULL, iterpool));
+ svn_pool_clear(iterpool);
+ }
+
+ /* delete the non-packed revprops shards afterwards */
+ for (shard = 0; shard < first_unpacked_shard; ++shard)
+ {
+ revprops_shard_path = svn_dirent_join(revsprops_dir,
+ apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
+ iterpool);
+ SVN_ERR(delete_revprops_shard(revprops_shard_path,
+ shard, ffd->max_files_per_dir,
+ NULL, NULL, iterpool));
+ svn_pool_clear(iterpool);
+ }
+
+ svn_pool_destroy(iterpool);
+
+ return SVN_NO_ERROR;
+}
+
static svn_error_t *
upgrade_body(void *baton, apr_pool_t *pool)
{
@@ -1369,6 +1535,12 @@ upgrade_body(void *baton, apr_pool_t *po
if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool));
+ /* If the file system supports revision packing but not revprop packing,
+ pack the revprops up to the point that revision data has been packed. */
+ if ( format >= SVN_FS_FS__MIN_PACKED_FORMAT
+ && format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
+ SVN_ERR(upgrade_pack_revprops(fs, pool));
+
/* Bump the format file. */
return write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, max_files_per_dir,
TRUE, pool);
@@ -1382,7 +1554,7 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_poo
}
-/* SVN_ERR-like macros for dealing with recoverable errors on mutable files
+/* Functions for dealing with recoverable errors on mutable files
*
* Revprops, current, and txn-current files are mutable; that is, they
* change as part of normal fsfs operation, in constrat to revs files, or
@@ -1415,94 +1587,82 @@ svn_fs_fs__upgrade(svn_fs_t *fs, apr_poo
*
** Solution
*
- * Wrap opens and reads of such files with RETRY_RECOVERABLE and
- * closes with IGNORE_RECOVERABLE. Call these macros within a loop of
- * RECOVERABLE_RETRY_COUNT iterations (though, realistically, the
- * second try will succeed). Make sure you put a break statement
- * after the close, at the end of your loop. Immediately after your
- * loop, return err if err.
- *
- * You must initialize err to SVN_NO_ERROR and filehandle to NULL, as
- * these macros do not.
+ * Try open and read of such files in try_stringbuf_from_file(). Call
+ * this function within a loop of RECOVERABLE_RETRY_COUNT iterations
+ * (though, realistically, the second try will succeed).
*/
#define RECOVERABLE_RETRY_COUNT 10
+/* Read the file at PATH and return its content in *CONTENT. *CONTENT will
+ * not be modified unless the whole file was read successfully.
+ *
+ * ESTALE, EIO and ENOENT will not cause this function to return an error
+ * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate
+ * missing files (ENOENT) there.
+ *
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+try_stringbuf_from_file(svn_stringbuf_t **content,
+ svn_boolean_t *missing,
+ const char *path,
+ svn_boolean_t last_attempt,
+ apr_pool_t *pool)
+{
+ svn_error_t *err = svn_stringbuf_from_file2(content, path, pool);
+ if (missing)
+ *missing = FALSE;
+
+ if (err)
+ {
+ *content = NULL;
+
+ if (APR_STATUS_IS_ENOENT(err->apr_err))
+ {
+ if (!last_attempt)
+ {
+ svn_error_clear(err);
+ if (missing)
+ *missing = TRUE;
+ return SVN_NO_ERROR;
+ }
+ }
#ifdef ESTALE
-/* Do not use do-while due to the embedded 'continue'. */
-#define RETRY_RECOVERABLE(err, filehandle, expr) \
- if (1) { \
- svn_error_clear(err); \
- err = (expr); \
- if (err) \
- { \
- apr_status_t _e = APR_TO_OS_ERROR(err->apr_err); \
- if ((_e == ESTALE) || (_e == EIO) || (_e == ENOENT)) { \
- if (NULL != filehandle) \
- (void)apr_file_close(filehandle); \
- continue; \
- } \
- return svn_error_trace(err); \
- } \
- } else
-#define IGNORE_RECOVERABLE(err, expr) \
- if (1) { \
- svn_error_clear(err); \
- err = (expr); \
- if (err) \
- { \
- apr_status_t _e = APR_TO_OS_ERROR(err->apr_err); \
- if ((_e != ESTALE) && (_e != EIO)) \
- return svn_error_trace(err); \
- } \
- } else
-#else
-#define RETRY_RECOVERABLE(err, filehandle, expr) SVN_ERR(expr)
-#define IGNORE_RECOVERABLE(err, expr) SVN_ERR(expr)
+ else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE
+ || APR_TO_OS_ERROR(err->apr_err) == EIO)
+ {
+ if (!last_attempt)
+ {
+ svn_error_clear(err);
+ return SVN_NO_ERROR;
+ }
+ }
#endif
+ }
-/* Long enough to hold: "<svn_revnum_t> <node id> <copy id>\0"
- * 19 bytes for svn_revnum_t (room for 32 or 64 bit values)
- * + 2 spaces
- * + 26 bytes for each id (these are actually unbounded, so we just
- * have to pick something; 2^64 is 13 bytes in base-36)
- * + 1 terminating null
- */
-#define CURRENT_BUF_LEN 48
+ return svn_error_trace(err);
+}
/* Read the 'current' file FNAME and store the contents in *BUF.
Allocations are performed in POOL. */
static svn_error_t *
-read_current(const char *fname, char **buf, apr_pool_t *pool)
+read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *pool)
{
- apr_file_t *revision_file = NULL;
- apr_size_t len;
int i;
- svn_error_t *err = SVN_NO_ERROR;
- apr_pool_t *iterpool;
-
- *buf = apr_palloc(pool, CURRENT_BUF_LEN);
- iterpool = svn_pool_create(pool);
- for (i = 0; i < RECOVERABLE_RETRY_COUNT; i++)
- {
- svn_pool_clear(iterpool);
+ *content = NULL;
- RETRY_RECOVERABLE(err, revision_file,
- svn_io_file_open(&revision_file, fname,
- APR_READ | APR_BUFFERED,
- APR_OS_DEFAULT, iterpool));
-
- len = CURRENT_BUF_LEN;
- RETRY_RECOVERABLE(err, revision_file,
- svn_io_read_length_line(revision_file,
- *buf, &len, iterpool));
- IGNORE_RECOVERABLE(err, svn_io_file_close(revision_file, iterpool));
+ for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i)
+ SVN_ERR(try_stringbuf_from_file(content, NULL,
+ fname, i + 1 < RECOVERABLE_RETRY_COUNT,
+ pool));
- break;
- }
- svn_pool_destroy(iterpool);
+ if (!*content)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ _("Can't read '%s'"),
+ svn_dirent_local_style(fname, pool));
- return svn_error_trace(err);
+ return SVN_NO_ERROR;
}
/* Find the youngest revision in a repository at path FS_PATH and
@@ -1513,12 +1673,11 @@ get_youngest(svn_revnum_t *youngest_p,
const char *fs_path,
apr_pool_t *pool)
{
- char *buf;
-
- SVN_ERR(read_current(svn_dirent_join(fs_path, PATH_CURRENT, pool),
- &buf, pool));
+ svn_stringbuf_t *buf;
+ SVN_ERR(read_content(&buf, svn_dirent_join(fs_path, PATH_CURRENT, pool),
+ pool));
- *youngest_p = SVN_STR_TO_REV(buf);
+ *youngest_p = SVN_STR_TO_REV(buf->data);
return SVN_NO_ERROR;
}
@@ -1677,22 +1836,30 @@ open_pack_or_rev_file(apr_file_t **file,
err = svn_io_file_open(file, path,
APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
- if (err && APR_STATUS_IS_ENOENT(err->apr_err)
- && ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
+ if (err && APR_STATUS_IS_ENOENT(err->apr_err))
{
- /* Could not open the file. This may happen if the
- * file once existed but got packed later. */
- svn_error_clear(err);
+ if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
+ {
+ /* Could not open the file. This may happen if the
+ * file once existed but got packed later. */
+ svn_error_clear(err);
- /* if that was our 2nd attempt, leave it at that. */
- if (retry)
- return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
- _("No such revision %ld"), rev);
+ /* if that was our 2nd attempt, leave it at that. */
+ if (retry)
+ return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
+ _("No such revision %ld"), rev);
- /* We failed for the first time. Refresh cache & retry. */
- SVN_ERR(update_min_unpacked_rev(fs, pool));
+ /* We failed for the first time. Refresh cache & retry. */
+ SVN_ERR(update_min_unpacked_rev(fs, pool));
- retry = TRUE;
+ retry = TRUE;
+ }
+ else
+ {
+ svn_error_clear(err);
+ return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
+ _("No such revision %ld"), rev);
+ }
}
else
{
@@ -1704,6 +1871,41 @@ open_pack_or_rev_file(apr_file_t **file,
return svn_error_trace(err);
}
+/* Reads a line from STREAM and converts it to a 64 bit integer to be
+ * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave
+ * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS"
+ * error return.
+ * SCRATCH_POOL is used for temporary allocations.
+ */
+static svn_error_t *
+read_number_from_stream(apr_int64_t *result,
+ svn_boolean_t *hit_eof,
+ svn_stream_t *stream,
+ apr_pool_t *scratch_pool)
+{
+ svn_stringbuf_t *sb;
+ svn_boolean_t eof;
+ svn_error_t *err;
+
+ SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool));
+ if (hit_eof)
+ *hit_eof = eof;
+ else
+ if (eof)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF"));
+
+ if (!eof)
+ {
+ err = svn_cstring_atoi64(result, sb->data);
+ if (err)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
+ _("Number '%s' invalid or too large"),
+ sb->data);
+ }
+
+ return SVN_NO_ERROR;
+}
+
/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
Use POOL for temporary allocations. */
static svn_error_t *
@@ -1737,7 +1939,8 @@ get_packed_offset(apr_off_t *rev_offset,
/* Open the manifest file. */
SVN_ERR(svn_stream_open_readonly(&manifest_stream,
- path_rev_packed(fs, rev, "manifest", pool),
+ path_rev_packed(fs, rev, PATH_MANIFEST,
+ pool),
pool, pool));
/* While we're here, let's just read the entire manifest file into an array,
@@ -1746,21 +1949,14 @@ get_packed_offset(apr_off_t *rev_offset,
manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t));
while (1)
{
- svn_stringbuf_t *sb;
svn_boolean_t eof;
apr_int64_t val;
- svn_error_t *err;
svn_pool_clear(iterpool);
- SVN_ERR(svn_stream_readline(manifest_stream, &sb, "\n", &eof, iterpool));
+ SVN_ERR(read_number_from_stream(&val, &eof, manifest_stream, iterpool));
if (eof)
break;
- err = svn_cstring_atoi64(&val, sb->data);
- if (err)
- return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
- _("Manifest offset '%s' too large"),
- sb->data);
APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val;
}
svn_pool_destroy(iterpool);
@@ -2359,7 +2555,7 @@ svn_fs_fs__write_noderev(svn_stream_t *o
noderev->copyroot_path));
if (noderev->is_fresh_txn_root)
- SVN_ERR(svn_stream_printf(outfile, pool, HEADER_FRESHTXNRT ": y\n"));
+ SVN_ERR(svn_stream_puts(outfile, HEADER_FRESHTXNRT ": y\n"));
if (include_mergeinfo)
{
@@ -2369,10 +2565,10 @@ svn_fs_fs__write_noderev(svn_stream_t *o
noderev->mergeinfo_count));
if (noderev->has_mergeinfo)
- SVN_ERR(svn_stream_printf(outfile, pool, HEADER_MINFO_HERE ": y\n"));
+ SVN_ERR(svn_stream_puts(outfile, HEADER_MINFO_HERE ": y\n"));
}
- return svn_stream_printf(outfile, pool, "\n");
+ return svn_stream_puts(outfile, "\n");
}
svn_error_t *
@@ -2775,107 +2971,1187 @@ svn_fs_fs__rev_get_root(svn_fs_id_t **ro
return SVN_NO_ERROR;
}
-/* Set the revision property list of revision REV in filesystem FS to
- PROPLIST. Use POOL for temporary allocations. */
+/* Revprop caching management.
+ *
+ * Mechanism:
+ * ----------
+ *
+ * Revprop caching needs to be activated and will be deactivated for the
+ * respective FS instance if the necessary infrastructure could not be
+ * initialized. In deactivated mode, there is almost no runtime overhead
+ * associated with revprop caching. As long as no revprops are being read
+ * or changed, revprop caching imposes no overhead.
+ *
+ * When activated, we cache revprops using (revision, generation) pairs
+ * as keys with the generation being incremented upon every revprop change.
+ * Since the cache is process-local, the generation needs to be tracked
+ * for at least as long as the process lives but may be reset afterwards.
+ *
+ * To track the revprop generation, we use two-layer approach. On the lower
+ * level, we use named atomics to have a system-wide consistent value for
+ * the current revprop generation. However, those named atomics will only
+ * remain valid for as long as at least one process / thread in the system
+ * accesses revprops in the respective repository. The underlying shared
+ * memory gets cleaned up afterwards.
+ *
+ * On the second level, we will use a persistent file to track the latest
+ * revprop generation. It will be written upon each revprop change but
+ * only be read if we are the first process to initialize the named atomics
+ * with that value.
+ *
+ * The overhead for the second and following accesses to revprops is
+ * almost zero on most systems.
+ *
+ *
+ * Tech aspects:
+ * -------------
+ *
+ * A problem is that we need to provide a globally available file name to
+ * back the SHM implementation on OSes that need it. We can only assume
+ * write access to some file within the respective repositories. Because
+ * a given server process may access thousands of repositories during its
+ * lifetime, keeping the SHM data alive for all of them is also not an
+ * option.
+ *
+ * So, we store the new revprop generation on disk as part of each
+ * setrevprop call, i.e. this write will be serialized and the write order
+ * be guaranteed by the repository write lock.
+ *
+ * The only racy situation occurs when the data is being read again by two
+ * processes concurrently but in that situation, the first process to
+ * finish that procedure is guaranteed to be the only one that initializes
+ * the SHM data. Since even writers will first go through that
+ * initialization phase, they will never operate on stale data.
+ */
+
+/* Read revprop generation as stored on disk for repository FS. The result
+ * is returned in *CURRENT. Default to 2 if no such file is available.
+ */
static svn_error_t *
-set_revision_proplist(svn_fs_t *fs,
- svn_revnum_t rev,
- apr_hash_t *proplist,
- apr_pool_t *pool)
+read_revprop_generation_file(apr_int64_t *current,
+ svn_fs_t *fs,
+ apr_pool_t *pool)
{
- SVN_ERR(ensure_revision_exists(fs, rev, pool));
+ svn_error_t *err;
+ apr_file_t *file;
+ char buf[80];
+ apr_size_t len;
+ const char *path = path_revprop_generation(fs, pool);
- /* if (1); null condition for easier merging to revprop-packing */
+ err = svn_io_file_open(&file, path,
+ APR_READ | APR_BUFFERED,
+ APR_OS_DEFAULT, pool);
+ if (err && APR_STATUS_IS_ENOENT(err->apr_err))
{
- const char *final_path = path_revprops(fs, rev, pool);
- const char *tmp_path;
- const char *perms_reference;
- svn_stream_t *stream;
-
- /* ### do we have a directory sitting around already? we really shouldn't
- ### have to get the dirname here. */
- SVN_ERR(svn_stream_open_unique(&stream, &tmp_path,
- svn_dirent_dirname(final_path, pool),
- svn_io_file_del_none, pool, pool));
- SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
- SVN_ERR(svn_stream_close(stream));
+ svn_error_clear(err);
+ *current = 2;
+
+ return SVN_NO_ERROR;
+ }
+ SVN_ERR(err);
+
+ len = sizeof(buf);
+ SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
+
+ /* Check that the first line contains only digits. */
+ SVN_ERR(check_file_buffer_numeric(buf, 0, path,
+ "Revprop Generation", pool));
+ SVN_ERR(svn_cstring_atoi64(current, buf));
+
+ return svn_io_file_close(file, pool);
+}
+
+/* Write the CURRENT revprop generation to disk for repository FS.
+ */
+static svn_error_t *
+write_revprop_generation_file(svn_fs_t *fs,
+ apr_int64_t current,
+ apr_pool_t *pool)
+{
+ apr_file_t *file;
+ const char *tmp_path;
+
+ char buf[SVN_INT64_BUFFER_SIZE];
+ apr_size_t len = svn__i64toa(buf, current);
+ buf[len] = '\n';
+
+ SVN_ERR(svn_io_open_unique_file3(&file, &tmp_path, fs->path,
+ svn_io_file_del_none, pool, pool));
+ SVN_ERR(svn_io_file_write_full(file, buf, len + 1, NULL, pool));
+ SVN_ERR(svn_io_file_close(file, pool));
+
+ return move_into_place(tmp_path, path_revprop_generation(fs, pool),
+ tmp_path, pool);
+}
+
+/* Make sure the revprop_namespace member in FS is set. */
+static svn_error_t *
+ensure_revprop_namespace(svn_fs_t *fs)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ return ffd->revprop_namespace == NULL
+ ? svn_atomic_namespace__create(&ffd->revprop_namespace,
+ svn_dirent_join(fs->path,
+ ATOMIC_REVPROP_NAMESPACE,
+ fs->pool),
+ fs->pool)
+ : SVN_NO_ERROR;
+}
+
+/* Make sure the revprop_generation member in FS is set and, if necessary,
+ * initialized with the latest value stored on disk.
+ */
+static svn_error_t *
+ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
- /* We use the rev file of this revision as the perms reference,
- because when setting revprops for the first time, the revprop
- file won't exist and therefore can't serve as its own reference.
- (Whereas the rev file should already exist at this point.) */
- SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool));
- SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool));
+ SVN_ERR(ensure_revprop_namespace(fs));
+ if (ffd->revprop_generation == NULL)
+ {
+ apr_int64_t current = 0;
+
+ SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation,
+ ffd->revprop_namespace,
+ ATOMIC_REVPROP_GENERATION,
+ TRUE));
+
+ /* If the generation is at 0, we just created a new namespace
+ * (it would be at least 2 otherwise). Read the lastest generation
+ * from disk and if we are the first one to initialize the atomic
+ * (i.e. is still 0), set it to the value just gotten.
+ */
+ SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation));
+ if (current == 0)
+ {
+ SVN_ERR(read_revprop_generation_file(¤t, fs, pool));
+ SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0,
+ ffd->revprop_generation));
+ }
}
return SVN_NO_ERROR;
}
+/* Make sure the revprop_timeout member in FS is set. */
static svn_error_t *
-revision_proplist(apr_hash_t **proplist_p,
- svn_fs_t *fs,
- svn_revnum_t rev,
- apr_pool_t *pool)
+ensure_revprop_timeout(svn_fs_t *fs)
{
- apr_hash_t *proplist;
+ fs_fs_data_t *ffd = fs->fsap_data;
- SVN_ERR(ensure_revision_exists(fs, rev, pool));
+ SVN_ERR(ensure_revprop_namespace(fs));
+ return ffd->revprop_timeout == NULL
+ ? svn_named_atomic__get(&ffd->revprop_timeout,
+ ffd->revprop_namespace,
+ ATOMIC_REVPROP_TIMEOUT,
+ TRUE)
+ : SVN_NO_ERROR;
+}
+
+/* Test whether revprop cache and necessary infrastructure are
+ available in FS. */
+static svn_boolean_t
+has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ svn_error_t *error;
+
+ /* is the cache (still) enabled? */
+ if (ffd->revprop_cache == NULL)
+ return FALSE;
- /* if (1); null condition for easier merging to revprop-packing */
+ /* is it efficient? */
+ if (!svn_named_atomic__is_efficient())
{
- apr_file_t *revprop_file = NULL;
- svn_error_t *err = SVN_NO_ERROR;
- int i;
- apr_pool_t *iterpool;
+ /* access to it would be quite slow
+ * -> disable the revprop cache for good
+ */
+ ffd->revprop_cache = NULL;
+ return FALSE;
+ }
- proplist = apr_hash_make(pool);
- iterpool = svn_pool_create(pool);
- for (i = 0; i < RECOVERABLE_RETRY_COUNT; i++)
- {
- svn_pool_clear(iterpool);
+ /* try to access our SHM-backed infrastructure */
+ error = ensure_revprop_generation(fs, pool);
+ if (error)
+ {
+ /* failure -> disable revprop cache for good */
- /* Clear err here rather than after finding a recoverable error so
- * we can return that error on the last iteration of the loop. */
- svn_error_clear(err);
- err = svn_io_file_open(&revprop_file, path_revprops(fs, rev,
- iterpool),
- APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
- iterpool);
- if (err)
- {
- if (APR_STATUS_IS_ENOENT(err->apr_err))
- {
- svn_error_clear(err);
- return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
- _("No such revision %ld"), rev);
- }
-#ifdef ESTALE
- else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE
- || APR_TO_OS_ERROR(err->apr_err) == EIO
- || APR_TO_OS_ERROR(err->apr_err) == ENOENT)
- continue;
-#endif
- return svn_error_trace(err);
- }
+ svn_error_clear(error);
+ ffd->revprop_cache = NULL;
- SVN_ERR(svn_hash__clear(proplist, iterpool));
- RETRY_RECOVERABLE(err, revprop_file,
- svn_hash_read2(proplist,
- svn_stream_from_aprfile2(
- revprop_file, TRUE, iterpool),
- SVN_HASH_TERMINATOR, pool));
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+/* Read the current revprop generation and return it in *GENERATION.
+ Also, detect aborted / crashed writers and recover from that.
+ Use the access object in FS to set the shared mem values. */
+static svn_error_t *
+read_revprop_generation(apr_int64_t *generation,
+ svn_fs_t *fs,
+ apr_pool_t *pool)
+{
+ apr_int64_t current = 0;
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ /* read the current revprop generation number */
+ SVN_ERR(ensure_revprop_generation(fs, pool));
+ SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation));
+
+ /* is an unfinished revprop write under the way? */
+ if (current % 2)
+ {
+ apr_int64_t timeout = 0;
+
+ /* read timeout for the write operation */
+ SVN_ERR(ensure_revprop_timeout(fs));
+ SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout));
+
+ /* has the writer process been aborted,
+ * i.e. has the timeout been reached?
+ */
+ if (apr_time_now() > timeout)
+ {
+ /* Cause everyone to re-read revprops upon their next access.
+ * Keep in mind that we may not be the only one trying to do it.
+ */
+ while (current % 2)
+ SVN_ERR(svn_named_atomic__add(¤t,
+ 1,
+ ffd->revprop_generation));
+ }
+ }
+
+ /* return the value we just got */
+ *generation = current;
+ return SVN_NO_ERROR;
+}
+
+/* Set the revprop generation to the next odd number to indicate that
+ there is a revprop write process under way. If that times out,
+ readers shall recover from that state & re-read revprops.
+ Use the access object in FS to set the shared mem value. */
+static svn_error_t *
+begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool)
+{
+ apr_int64_t current;
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ /* set the timeout for the write operation */
+ SVN_ERR(ensure_revprop_timeout(fs));
+ SVN_ERR(svn_named_atomic__write(NULL,
+ apr_time_now() + REVPROP_CHANGE_TIMEOUT,
+ ffd->revprop_timeout));
+
+ /* set the revprop generation to an odd value to indicate
+ * that a write is in progress
+ */
+ SVN_ERR(ensure_revprop_generation(fs, pool));
+ do
+ {
+ SVN_ERR(svn_named_atomic__add(¤t,
+ 1,
+ ffd->revprop_generation));
+ }
+ while (current % 2 == 0);
+
+ return SVN_NO_ERROR;
+}
+
+/* Set the revprop generation to the next even number to indicate that
+ a) readers shall re-read revprops, and
+ b) the write process has been completed (no recovery required)
+ Use the access object in FS to set the shared mem value. */
+static svn_error_t *
+end_revprop_change(svn_fs_t *fs, apr_pool_t *pool)
+{
+ apr_int64_t current = 1;
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ /* set the revprop generation to an even value to indicate
+ * that a write has been completed
+ */
+ SVN_ERR(ensure_revprop_generation(fs, pool));
+ do
+ {
+ SVN_ERR(svn_named_atomic__add(¤t,
+ 1,
+ ffd->revprop_generation));
+ }
+ while (current % 2);
+
+ /* Save the latest generation to disk. FS is currently in a "locked"
+ * state such that we can be sure the be the only ones to write that
+ * file.
+ */
+ return write_revprop_generation_file(fs, current, pool);
+}
+
+/* Container for all data required to access the packed revprop file
+ * for a given REVISION. This structure will be filled incrementally
+ * by read_pack_revprops() its sub-routines.
+ */
+typedef struct packed_revprops_t
+{
+ /* revision number to read (not necessarily the first in the pack) */
+ svn_revnum_t revision;
+
+ /* current revprop generation. Used when populating the revprop cache */
+ apr_int64_t generation;
+
+ /* the actual revision properties */
+ apr_hash_t *properties;
+
+ /* their size when serialized to a single string
+ * (as found in PACKED_REVPROPS) */
+ apr_size_t serialized_size;
+
+
+ /* name of the pack file (without folder path) */
+ const char *filename;
+
+ /* packed shard folder path */
+ const char *folder;
+
+ /* sum of values in SIZES */
+ apr_size_t total_size;
+
+ /* first revision in the pack */
+ svn_revnum_t start_revision;
+
+ /* size of the revprops in PACKED_REVPROPS */
+ apr_array_header_t *sizes;
+
+ /* offset of the revprops in PACKED_REVPROPS */
+ apr_array_header_t *offsets;
+
+
+ /* concatenation of the serialized representation of all revprops
+ * in the pack, i.e. the pack content without header and compression */
+ svn_stringbuf_t *packed_revprops;
+
+ /* content of the manifest.
+ * Maps long(rev - START_REVISION) to const char* pack file name */
+ apr_array_header_t *manifest;
+} packed_revprops_t;
+
+/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
+ * Also, put them into the revprop cache, if activated, for future use.
+ * Three more parameters are being used to update the revprop cache: FS is
+ * our file system, the revprops belong to REVISION and the global revprop
+ * GENERATION is used as well.
+ *
+ * The returned hash will be allocated in POOL, SCRATCH_POOL is being used
+ * for temporary allocations.
+ */
+static svn_error_t *
+parse_revprop(apr_hash_t **properties,
+ svn_fs_t *fs,
+ svn_revnum_t revision,
+ apr_int64_t generation,
+ svn_string_t *content,
+ apr_pool_t *pool,
+ apr_pool_t *scratch_pool)
+{
+ svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
+ *properties = apr_hash_make(pool);
+
+ SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool));
+ if (has_revprop_cache(fs, pool))
+ {
+ const char *key;
+ fs_fs_data_t *ffd = fs->fsap_data;
+
+ key = svn_fs_fs__combine_two_numbers(revision, generation,
+ scratch_pool);
+ SVN_ERR(svn_cache__set(ffd->revprop_cache, key, *properties,
+ scratch_pool));
+ }
+
+ return SVN_NO_ERROR;
+}
+
+/* Read the non-packed revprops for revision REV in FS, put them into the
+ * revprop cache if activated and return them in *PROPERTIES. GENERATION
+ * is the current revprop generation.
+ *
+ * If the data could not be read due to an otherwise recoverable error,
+ * leave *PROPERTIES unchanged. No error will be returned in that case.
+ *
+ * Allocations will be done in POOL.
+ */
+static svn_error_t *
+read_non_packed_revprop(apr_hash_t **properties,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_int64_t generation,
+ apr_pool_t *pool)
+{
+ svn_stringbuf_t *content = NULL;
+ apr_pool_t *iterpool = svn_pool_create(pool);
+ svn_boolean_t missing = FALSE;
+ int i;
+
+ for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i)
+ {
+ svn_pool_clear(iterpool);
+ SVN_ERR(try_stringbuf_from_file(&content,
+ &missing,
+ path_revprops(fs, rev, iterpool),
+ i + 1 < RECOVERABLE_RETRY_COUNT,
+ iterpool));
+ }
+
+ if (content)
+ SVN_ERR(parse_revprop(properties, fs, rev, generation,
+ svn_stringbuf__morph_into_string(content),
+ pool, iterpool));
+
+ svn_pool_clear(iterpool);
+
+ return SVN_NO_ERROR;
+}
+
+/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
+ * members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
+ */
+static svn_error_t *
+get_revprop_packname(svn_fs_t *fs,
+ packed_revprops_t *revprops,
+ apr_pool_t *pool,
+ apr_pool_t *scratch_pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ svn_stringbuf_t *content = NULL;
+ const char *manifest_file_path;
+ int idx;
+
+ /* read content of the manifest file */
+ revprops->folder = path_revprops_pack_shard(fs, revprops->revision, pool);
+ manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
+
+ SVN_ERR(read_content(&content, manifest_file_path, pool));
+
+ /* parse the manifest. Every line is a file name */
+ revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir,
+ sizeof(const char*));
+ while (content->data)
+ {
+ APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data;
+ content->data = strchr(content->data, '\n');
+ if (content->data)
+ {
+ *content->data = 0;
+ content->data++;
+ }
+ }
+
+ /* Index for our revision. Rev 0 is excluded from the first shard. */
+ idx = (int)(revprops->revision % ffd->max_files_per_dir);
+ if (revprops->revision < ffd->max_files_per_dir)
+ --idx;
+
+ if (revprops->manifest->nelts <= idx)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ _("Packed revprop manifest for rev %ld too "
+ "small"), revprops->revision);
+
+ /* Now get the file name */
+ revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
+
+ return SVN_NO_ERROR;
+}
+
+/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
+ * fill the START_REVISION, SIZES, OFFSETS members. Also, make
+ * PACKED_REVPROPS point to the first serialized revprop.
+ *
+ * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
+ * well as the SERIALIZED_SIZE member. If revprop caching has been
+ * enabled, parse all revprops in the pack and cache them.
+ */
+static svn_error_t *
+parse_packed_revprops(svn_fs_t *fs,
+ packed_revprops_t *revprops,
+ apr_pool_t *pool,
+ apr_pool_t *scratch_pool)
+{
+ svn_stream_t *stream;
+ apr_int64_t first_rev, count, i;
+ apr_off_t offset;
+ const char *header_end;
+ apr_pool_t *iterpool = svn_pool_create(scratch_pool);
+
+ /* decompress (even if the data is only "stored", there is still a
+ * length header to remove) */
+ svn_string_t *compressed
+ = svn_stringbuf__morph_into_string(revprops->packed_revprops);
+ svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool);
+ SVN_ERR(svn__decompress(compressed, uncompressed, 0x1000000));
+
+ /* read first revision number and number of revisions in the pack */
+ stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
+ SVN_ERR(read_number_from_stream(&first_rev, NULL, stream, iterpool));
+ SVN_ERR(read_number_from_stream(&count, NULL, stream, iterpool));
+
+ /* make PACKED_REVPROPS point to the first char after the header.
+ * This is where the serialized revprops are. */
+ header_end = strstr(uncompressed->data, "\n\n");
+ if (header_end == NULL)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+ _("Header end not found"));
+
+ offset = header_end - uncompressed->data + 2;
+
+ revprops->packed_revprops = svn_stringbuf_create_empty(pool);
+ revprops->packed_revprops->data = uncompressed->data + offset;
+ revprops->packed_revprops->len = uncompressed->len - offset;
+ revprops->packed_revprops->blocksize = uncompressed->blocksize - offset;
+
+ /* STREAM still points to the first entry in the sizes list.
+ * Init / construct REVPROPS members. */
+ revprops->start_revision = (svn_revnum_t)first_rev;
+ revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
+ revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
+
+ /* Now parse, revision by revision, the size and content of each
+ * revisions' revprops. */
+ for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i)
+ {
+ apr_int64_t size;
+ svn_string_t serialized;
+ apr_hash_t *properties;
+ svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
+
+ /* read & check the serialized size */
+ SVN_ERR(read_number_from_stream(&size, NULL, stream, iterpool));
+ if (size + offset > revprops->packed_revprops->len)
+ return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
+ _("Packed revprop size exceeds pack file size"));
+
+ /* Parse this revprops list, if necessary */
+ serialized.data = revprops->packed_revprops->data + offset;
+ serialized.len = (apr_size_t)size;
+
+ if (revision == revprops->revision)
+ {
+ SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
+ revprops->generation, &serialized,
+ pool, iterpool));
+ revprops->serialized_size = serialized.len;
+ }
+ else
+ {
+ /* If revprop caching is enabled, parse any revprops.
+ * They will get cached as a side-effect of this. */
+ if (has_revprop_cache(fs, pool))
+ SVN_ERR(parse_revprop(&properties, fs, revision,
+ revprops->generation, &serialized,
+ iterpool, iterpool));
+ }
+
+ /* fill REVPROPS data structures */
+ APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
+ APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
+ revprops->total_size += serialized.len;
+
+ offset += serialized.len;
+
+ svn_pool_clear(iterpool);
+ }
+
+ return SVN_NO_ERROR;
+}
+
+/* In filesystem FS, read the packed revprops for revision REV into
+ * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled.
+ * Allocate data in POOL.
+ */
+static svn_error_t *
+read_pack_revprop(packed_revprops_t **revprops,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_int64_t generation,
+ apr_pool_t *pool)
+{
+ apr_pool_t *iterpool = svn_pool_create(pool);
+ svn_boolean_t missing = FALSE;
+ svn_error_t *err;
+ packed_revprops_t *result;
+ int i;
+
+ /* someone insisted that REV is packed. Double-check if necessary */
+ if (!is_packed_revprop(fs, rev))
+ SVN_ERR(update_min_unpacked_rev(fs, iterpool));
+
+ if (!is_packed_revprop(fs, rev))
+ return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
+ _("No such packed revision %ld"), rev);
+
+ /* initialize the result data structure */
+ result = apr_pcalloc(pool, sizeof(*result));
+ result->revision = rev;
+ result->generation = generation;
+
+ /* try to read the packed revprops. This may require retries if we have
+ * concurrent writers. */
+ for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i)
+ {
+ const char *file_path;
+
+ /* there might have been concurrent writes.
+ * Re-read the manifest and the pack file.
+ */
+ SVN_ERR(get_revprop_packname(fs, result, pool, iterpool));
+ file_path = svn_dirent_join(result->folder,
+ result->filename,
+ iterpool);
+ SVN_ERR(try_stringbuf_from_file(&result->packed_revprops,
+ &missing,
+ file_path,
+ i + 1 < RECOVERABLE_RETRY_COUNT,
+ pool));
+
+ /* If we could not find the file, there was a write.
+ * So, we should refresh our revprop generation info as well such
+ * that others may find data we will put into the cache. They would
+ * consider it outdated, otherwise.
+ */
+ if (missing && has_revprop_cache(fs, pool))
+ SVN_ERR(read_revprop_generation(&result->generation, fs, pool));
+
+ svn_pool_clear(iterpool);
+ }
+
+ /* the file content should be available now */
+ if (!result->packed_revprops)
+ return svn_error_createf(SVN_ERR_FS_PACKED_REPPROP_READ_FAILURE, NULL,
+ _("Failed to read revprop pack file for rev %ld"), rev);
+
+ /* parse it. RESULT will be complete afterwards. */
+ err = parse_packed_revprops(fs, result, pool, iterpool);
+ svn_pool_destroy(iterpool);
+ if (err)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
+ _("Revprop pack file for rev %ld is corrupt"), rev);
+
+ *revprops = result;
+
+ return SVN_NO_ERROR;
+}
+
+/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
+ *
+ * Allocations will be done in POOL.
+ */
+static svn_error_t *
+get_revision_proplist(apr_hash_t **proplist_p,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ apr_int64_t generation = 0;
+
+ /* not found, yet */
+ *proplist_p = NULL;
+
+ /* should they be available at all? */
+ SVN_ERR(ensure_revision_exists(fs, rev, pool));
+
+ /* Try cache lookup first. */
+ if (has_revprop_cache(fs, pool))
+ {
+ svn_boolean_t is_cached;
+ const char *key;
+
+ SVN_ERR(read_revprop_generation(&generation, fs, pool));
+
+ key = svn_fs_fs__combine_two_numbers(rev, generation, pool);
+ SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached,
+ ffd->revprop_cache, key, pool));
+ if (is_cached)
+ return SVN_NO_ERROR;
+ }
+
+ /* if REV had not been packed when we began, try reading it from the
+ * non-packed shard. If that fails, we will fall through to packed
+ * shard reads. */
+ if (!is_packed_revprop(fs, rev))
+ SVN_ERR(read_non_packed_revprop(proplist_p, fs, rev, generation,
+ pool));
+
+ /* if revprop packing is available and we have not read the revprops, yet,
+ * try reading them from a packed shard. If that fails, REV is most
+ * likely invalid (or its revprops highly contested). */
+ if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
+ {
+ packed_revprops_t *packed_revprops;
+ SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool));
+ *proplist_p = packed_revprops->properties;
+ }
+
+ /* The revprops should have been there. Did we get them? */
+ if (!*proplist_p)
+ return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
+ _("Could not read revprops for revision %ld"),
+ rev);
+
+ return SVN_NO_ERROR;
+}
+
+/* Serialize the revision property list PROPLIST of revision REV in
+ * filesystem FS to a non-packed file. Return the name of that temporary
+ * file in *TMP_PATH and the file path that it must be moved to in
+ * *FINAL_PATH.
+ *
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+write_non_packed_revprop(const char **final_path,
+ const char **tmp_path,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_hash_t *proplist,
+ apr_pool_t *pool)
+{
+ svn_stream_t *stream;
+ *final_path = path_revprops(fs, rev, pool);
+
+ /* ### do we have a directory sitting around already? we really shouldn't
+ ### have to get the dirname here. */
+ SVN_ERR(svn_stream_open_unique(&stream, tmp_path,
+ svn_dirent_dirname(*final_path, pool),
+ svn_io_file_del_none, pool, pool));
+ SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
+ SVN_ERR(svn_stream_close(stream));
+
+ return SVN_NO_ERROR;
+}
+
+/* After writing the new revprop file(s), call this function to move the
+ * file at TMP_PATH to FINAL_PATH and give it the permissions from
+ * PERMS_REFERENCE.
+ *
+ * If indicated in BUMP_GENERATION, increase FS' revprop generation.
+ * Finally, delete all the temporary files given in FILES_TO_DELETE.
+ * The latter may be NULL.
+ *
+ * Use POOL for temporary allocations.
+ */
+static svn_error_t *
+switch_to_new_revprop(svn_fs_t *fs,
+ const char *final_path,
+ const char *tmp_path,
+ const char *perms_reference,
+ apr_array_header_t *files_to_delete,
+ svn_boolean_t bump_generation,
+ apr_pool_t *pool)
+{
+ /* Now, we may actually be replacing revprops. Make sure that all other
+ threads and processes will know about this. */
+ if (bump_generation)
+ SVN_ERR(begin_revprop_change(fs, pool));
+
+ SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool));
+
+ /* Indicate that the update (if relevant) has been completed. */
+ if (bump_generation)
+ SVN_ERR(end_revprop_change(fs, pool));
+
+ /* Clean up temporary files, if necessary. */
+ if (files_to_delete)
+ {
+ apr_pool_t *iterpool = svn_pool_create(pool);
+ int i;
+
+ for (i = 0; i < files_to_delete->nelts; ++i)
+ {
+ const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*);
+ SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
+ svn_pool_clear(iterpool);
+ }
+
+ svn_pool_destroy(iterpool);
+ }
+ return SVN_NO_ERROR;
+}
+
+/* Write a pack file header to STREAM that starts at revision START_REVISION
+ * and contains the indexes [START,END) of SIZES.
+ */
+static svn_error_t *
+serialize_revprops_header(svn_stream_t *stream,
+ svn_revnum_t start_revision,
+ apr_array_header_t *sizes,
+ int start,
+ int end,
+ apr_pool_t *pool)
+{
+ apr_pool_t *iterpool = svn_pool_create(pool);
+ int i;
+
+ SVN_ERR_ASSERT(start < end);
+
+ /* start revision and entry count */
+ SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision));
+ SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start));
+
+ /* the sizes array */
+ for (i = start; i < end; ++i)
+ {
+ apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t);
+ SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n",
+ size));
+ }
+
+ /* the double newline char indicates the end of the header */
+ SVN_ERR(svn_stream_printf(stream, iterpool, "\n"));
+
+ svn_pool_clear(iterpool);
+ return SVN_NO_ERROR;
+}
+
+/* Writes the a pack file to FILE_STREAM. It copies the serialized data
+ * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX.
+ *
+ * The data for the latter is taken from NEW_SERIALIZED. Note, that
+ * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is
+ * taken in that case but only a subset of the old data will be copied.
+ *
+ * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size.
+ * POOL is used for temporary allocations.
+ */
+static svn_error_t *
+repack_revprops(svn_fs_t *fs,
+ packed_revprops_t *revprops,
+ int start,
+ int end,
+ int changed_index,
+ svn_stringbuf_t *new_serialized,
+ apr_off_t new_total_size,
+ svn_stream_t *file_stream,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ svn_stream_t *stream;
+ int i;
+
+ /* create data empty buffers and the stream object */
+ svn_stringbuf_t *uncompressed
+ = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool);
+ svn_stringbuf_t *compressed
+ = svn_stringbuf_create_empty(pool);
+ stream = svn_stream_from_stringbuf(uncompressed, pool);
+
+ /* write the header*/
+ SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start,
+ revprops->sizes, start, end, pool));
+
+ /* append the serialized revprops */
+ for (i = start; i < end; ++i)
+ if (i == changed_index)
+ {
+ SVN_ERR(svn_stream_write(stream,
+ new_serialized->data,
+ &new_serialized->len));
+ }
+ else
+ {
+ apr_size_t size
+ = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t);
+ apr_size_t offset
+ = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t);
+
+ SVN_ERR(svn_stream_write(stream,
+ revprops->packed_revprops->data + offset,
+ &size));
+ }
+
+ /* flush the stream buffer (if any) to our underlying data buffer */
+ SVN_ERR(svn_stream_close(stream));
+
+ /* compress / store the data */
+ SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed),
+ compressed,
+ ffd->compress_packed_revprops
+ ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
+ : SVN_DELTA_COMPRESSION_LEVEL_NONE));
+
+ /* finally, write the content to the target stream and close it */
+ SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len));
+ SVN_ERR(svn_stream_close(file_stream));
+
+ return SVN_NO_ERROR;
+}
+
+/* Allocate a new pack file name for the revisions at index [START,END)
+ * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE,
+ * auto-create that array if necessary. Return an open file stream to
+ * the new file in *STREAM allocated in POOL.
+ */
+static svn_error_t *
+repack_stream_open(svn_stream_t **stream,
+ svn_fs_t *fs,
+ packed_revprops_t *revprops,
+ int start,
+ int end,
+ apr_array_header_t **files_to_delete,
+ apr_pool_t *pool)
+{
+ apr_int64_t tag;
+ const char *tag_string;
+ svn_string_t *new_filename;
+ int i;
+ apr_file_t *file;
+
+ /* get the old (= current) file name and enlist it for later deletion */
+ const char *old_filename
+ = APR_ARRAY_IDX(revprops->manifest, start, const char*);
+
+ if (*files_to_delete == NULL)
+ *files_to_delete = apr_array_make(pool, 3, sizeof(const char*));
+
+ APR_ARRAY_PUSH(*files_to_delete, const char*)
+ = svn_dirent_join(revprops->folder, old_filename, pool);
+
+ /* increase the tag part, i.e. the counter after the dot */
+ tag_string = strchr(old_filename, '.');
+ if (tag_string == NULL)
+ return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
+ _("Packed file '%s' misses a tag"),
+ old_filename);
+
+ SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1));
+ new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT,
+ revprops->start_revision + start,
+ ++tag);
+
+ /* update the manifest to point to the new file */
+ for (i = start; i < end; ++i)
+ APR_ARRAY_IDX(revprops->manifest, i, const char*) = new_filename->data;
+
+ /* create a file stream for the new file */
+ SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder,
+ new_filename->data,
+ pool),
+ APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
+ *stream = svn_stream_from_aprfile2(file, FALSE, pool);
+
+ return SVN_NO_ERROR;
+}
+
+/* For revision REV in filesystem FS, set the revision properties to
+ * PROPLIST. Return a new file in *TMP_PATH that the caller shall move
+ * to *FINAL_PATH to make the change visible. Files to be deleted will
+ * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated.
+ * Use POOL for allocations.
+ */
+static svn_error_t *
+write_packed_revprop(const char **final_path,
+ const char **tmp_path,
+ apr_array_header_t **files_to_delete,
+ svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_hash_t *proplist,
+ apr_pool_t *pool)
+{
+ fs_fs_data_t *ffd = fs->fsap_data;
+ packed_revprops_t *revprops;
+ apr_int64_t generation = 0;
+ svn_stream_t *stream;
+ svn_stringbuf_t *serialized;
+ apr_off_t new_total_size;
+ int changed_index;
+
+ /* read the current revprop generation. This value will not change
+ * while we hold the global write lock to this FS. */
+ if (has_revprop_cache(fs, pool))
+ SVN_ERR(read_revprop_generation(&generation, fs, pool));
+
+ /* read contents of the current pack file */
+ SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool));
+
+ /* serialize the new revprops */
+ serialized = svn_stringbuf_create_empty(pool);
+ stream = svn_stream_from_stringbuf(serialized, pool);
+ SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
+ SVN_ERR(svn_stream_close(stream));
+
+ /* calculate the size of the new data */
+ changed_index = (int)(rev - revprops->start_revision);
+ new_total_size = revprops->total_size - revprops->serialized_size
+ + serialized->len
+ + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
+
+ APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len;
+
+ /* can we put the new data into the same pack as the before? */
+ if ( new_total_size < ffd->revprop_pack_size
+ || revprops->sizes->nelts == 1)
+ {
+ /* simply replace the old pack file with new content as we do it
+ * in the non-packed case */
+
+ *final_path = svn_dirent_join(revprops->folder, revprops->filename,
+ pool);
+ SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder,
+ svn_io_file_del_none, pool, pool));
+ SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts,
+ changed_index, serialized, new_total_size,
+ stream, pool));
+ }
+ else
+ {
+ /* split the pack file into two of roughly equal size */
+ int right_count, left_count, i;
+
+ int left = 0;
+ int right = revprops->sizes->nelts - 1;
+ apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
+ apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
+
+ /* let left and right side grow such that their size difference
+ * is minimal after each step. */
+ while (left <= right)
+ if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
+ < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t))
+ {
+ left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
+ + SVN_INT64_BUFFER_SIZE;
+ ++left;
+ }
+ else
+ {
+ right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)
+ + SVN_INT64_BUFFER_SIZE;
+ --right;
+ }
+
+ /* since the items need much less than SVN_INT64_BUFFER_SIZE
+ * bytes to represent their length, the split may not be optimal */
+ left_count = left;
+ right_count = revprops->sizes->nelts - left;
+
+ /* if new_size is large, one side may exceed the pack size limit.
+ * In that case, split before and after the modified revprop.*/
+ if ( left_size > ffd->revprop_pack_size
+ || right_size > ffd->revprop_pack_size)
+ {
+ left_count = changed_index;
+ right_count = revprops->sizes->nelts - left_count - 1;
+ }
+
+ /* write the new, split files */
+ if (left_count)
+ {
+ SVN_ERR(repack_stream_open(&stream, fs, revprops, 0,
+ left_count, files_to_delete, pool));
+ SVN_ERR(repack_revprops(fs, revprops, 0, left_count,
+ changed_index, serialized, new_total_size,
+ stream, pool));
+ }
+
+ if (left_count + right_count < revprops->sizes->nelts)
+ {
+ SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index,
+ changed_index + 1, files_to_delete,
+ pool));
+ SVN_ERR(repack_revprops(fs, revprops, changed_index,
+ changed_index + 1,
+ changed_index, serialized, new_total_size,
+ stream, pool));
+ }
+
+ if (right_count)
+ {
+ SVN_ERR(repack_stream_open(&stream, fs, revprops,
+ revprops->sizes->nelts - right_count,
+ revprops->sizes->nelts,
+ files_to_delete, pool));
+ SVN_ERR(repack_revprops(fs, revprops,
+ revprops->sizes->nelts - right_count,
+ revprops->sizes->nelts, changed_index,
+ serialized, new_total_size, stream,
+ pool));
+ }
+
+ /* write the new manifest */
+ *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
+ SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder,
+ svn_io_file_del_none, pool, pool));
+
+ for (i = 0; i < revprops->manifest->nelts; ++i)
+ {
+ const char *filename = APR_ARRAY_IDX(revprops->manifest, i,
+ const char*);
+ SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename));
+ }
+
+ SVN_ERR(svn_stream_close(stream));
+ }
+
+ return SVN_NO_ERROR;
+}
+
+/* Set the revision property list of revision REV in filesystem FS to
+ PROPLIST. Use POOL for temporary allocations. */
+static svn_error_t *
+set_revision_proplist(svn_fs_t *fs,
+ svn_revnum_t rev,
+ apr_hash_t *proplist,
+ apr_pool_t *pool)
+{
+ svn_boolean_t is_packed;
+ svn_boolean_t bump_generation = FALSE;
+ const char *final_path;
+ const char *tmp_path;
+ const char *perms_reference;
+ apr_array_header_t *files_to_delete = NULL;
- IGNORE_RECOVERABLE(err, svn_io_file_close(revprop_file, iterpool));
+ SVN_ERR(ensure_revision_exists(fs, rev, pool));
- break;
+ /* this info will not change while we hold the global FS write lock */
+ is_packed = is_packed_revprop(fs, rev);
+
+ /* Test whether revprops already exist for this revision.
+ * Only then will we need to bump the revprop generation. */
+ if (has_revprop_cache(fs, pool))
+ {
+ if (is_packed)
+ {
+ bump_generation = TRUE;
+ }
+ else
+ {
+ svn_node_kind_t kind;
+ SVN_ERR(svn_io_check_path(path_revprops(fs, rev, pool), &kind,
+ pool));
+ bump_generation = kind != svn_node_none;
}
-
- if (err)
- return svn_error_trace(err);
- svn_pool_destroy(iterpool);
}
- *proplist_p = proplist;
+ /* Serialize the new revprop data */
+ if (is_packed)
+ SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete,
+ fs, rev, proplist, pool));
+ else
+ SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
+ fs, rev, proplist, pool));
+
+ /* We use the rev file of this revision as the perms reference,
+ * because when setting revprops for the first time, the revprop
+ * file won't exist and therefore can't serve as its own reference.
+ * (Whereas the rev file should already exist at this point.)
+ */
+ SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool));
+
+ /* Now, switch to the new revprop data. */
+ SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference,
+ files_to_delete, bump_generation, pool));
return SVN_NO_ERROR;
}
@@ -2886,7 +4162,7 @@ svn_fs_fs__revision_proplist(apr_hash_t
svn_revnum_t rev,
apr_pool_t *pool)
{
- SVN_ERR(revision_proplist(proplist_p, fs, rev, pool));
+ SVN_ERR(get_revision_proplist(proplist_p, fs, rev, pool));
return SVN_NO_ERROR;
}
@@ -3139,7 +4415,10 @@ set_cached_window(svn_txdelta_window_t *
if (rs->window_cache)
{
/* store the window and the first offset _past_ it */
- svn_fs_fs__txdelta_cached_window_t cached_window = { window, rs->off };
+ svn_fs_fs__txdelta_cached_window_t cached_window;
+
+ cached_window.window = window;
+ cached_window.end_offset = rs->off;
/* but key it with the start offset because that is the known state
* when we will look it up */
@@ -3457,7 +4736,8 @@ get_contents(struct rep_read_baton *rb,
char *buf,
apr_size_t *len)
{
- apr_size_t copy_len, remaining = *len, offset;
+ apr_size_t copy_len, remaining = *len;
+ apr_off_t offset;
char *cur = buf;
struct rep_state *rs;
@@ -3735,7 +5015,11 @@ svn_fs_fs__get_file_delta_stream(svn_txd
else
source_stream = svn_stream_empty(pool);
SVN_ERR(read_representation(&target_stream, fs, target->data_rep, pool));
- svn_txdelta(stream_p, source_stream, target_stream, pool);
+
+ /* Because source and target stream will already verify their content,
+ * there is no need to do this once more. In particular if the stream
+ * content is being fetched from cache. */
+ svn_txdelta2(stream_p, source_stream, target_stream, FALSE, pool);
return SVN_NO_ERROR;
}
@@ -3796,7 +5080,14 @@ unparse_dir_entries(apr_hash_t **str_ent
{
apr_hash_index_t *hi;
- *str_entries_p = apr_hash_make(pool);
+ /* For now, we use a our own hash function to ensure that we get a
+ * (largely) stable order when serializing the data. It also gives
+ * us some performance improvement.
+ *
+ * ### TODO ###
+ * Use some sorted or other fixed order data container.
+ */
+ *str_entries_p = svn_hash__make(pool);
for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi))
{
@@ -4487,20 +5778,37 @@ fetch_all_changes(apr_hash_t *changed_pa
{
apr_hash_index_t *hi;
+ /* a potential child path must contain at least 2 more chars
+ (the path separator plus at least one char for the name).
+ Also, we should not assume that all paths have been normalized
+ i.e. some might have trailing path separators.
+ */
+ apr_ssize_t change_path_len = strlen(change->path);
+ apr_ssize_t min_child_len = change_path_len == 0
+ ? 1
+ : change->path[change_path_len-1] == '/'
+ ? change_path_len + 1
+ : change_path_len + 2;
+
+ /* CAUTION: This is the inner loop of an O(n^2) algorithm.
+ The number of changes to process may be >> 1000.
+ Therefore, keep the inner loop as tight as possible.
+ */
for (hi = apr_hash_first(iterpool, changed_paths);
hi;
hi = apr_hash_next(hi))
{
/* KEY is the path. */
- const char *path = svn__apr_hash_index_key(hi);
- apr_ssize_t klen = svn__apr_hash_index_klen(hi);
-
- /* If we come across our own path, ignore it. */
- if (strcmp(change->path, path) == 0)
- continue;
-
- /* If we come across a child of our path, remove it. */
- if (svn_dirent_is_child(change->path, path, iterpool))
+ const void *path;
+ apr_ssize_t klen;
+ apr_hash_this(hi, &path, &klen, NULL);
+
+ /* If we come across a child of our path, remove it.
+ Call svn_dirent_is_child only if there is a chance that
+ this is actually a sub-path.
+ */
+ if ( klen >= min_child_len
+ && svn_dirent_is_child(change->path, path, iterpool))
apr_hash_set(changed_paths, path, klen, NULL);
}
}
@@ -4618,40 +5926,17 @@ get_and_increment_txn_key_body(void *bat
{
struct get_and_increment_txn_key_baton *cb = baton;
const char *txn_current_filename = path_txn_current(cb->fs, pool);
- apr_file_t *txn_current_file = NULL;
const char *tmp_filename;
char next_txn_id[MAX_KEY_SIZE+3];
- svn_error_t *err = SVN_NO_ERROR;
- apr_pool_t *iterpool;
apr_size_t len;
- int i;
-
- cb->txn_id = apr_palloc(cb->pool, MAX_KEY_SIZE);
-
- iterpool = svn_pool_create(pool);
- for (i = 0; i < RECOVERABLE_RETRY_COUNT; ++i)
- {
- svn_pool_clear(iterpool);
- RETRY_RECOVERABLE(err, txn_current_file,
- svn_io_file_open(&txn_current_file,
- txn_current_filename,
- APR_READ | APR_BUFFERED,
- APR_OS_DEFAULT, iterpool));
- len = MAX_KEY_SIZE;
- RETRY_RECOVERABLE(err, txn_current_file,
- svn_io_read_length_line(txn_current_file,
- cb->txn_id,
- &len,
- iterpool));
- IGNORE_RECOVERABLE(err, svn_io_file_close(txn_current_file,
- iterpool));
-
- break;
- }
- SVN_ERR(err);
+ svn_stringbuf_t *buf;
+ SVN_ERR(read_content(&buf, txn_current_filename, cb->pool));
- svn_pool_destroy(iterpool);
+ /* remove trailing newlines */
+ svn_stringbuf_strip_whitespace(buf);
+ cb->txn_id = buf->data;
+ len = buf->len;
/* Increment the key and add a trailing \n to the string so the
txn-current file has a newline in it. */
@@ -5141,7 +6426,10 @@ svn_fs_fs__set_entry(svn_fs_t *fs,
/* build parameters: (name, new entry) pair */
const char *key =
svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data;
- replace_baton_t baton = {name, NULL};
+ replace_baton_t baton;
+
+ baton.name = name;
+ baton.new_entry = NULL;
if (id)
{
@@ -5607,7 +6895,7 @@ rep_write_contents_close(void *baton)
else
{
/* Write out our cosmetic end marker. */
- SVN_ERR(svn_stream_printf(b->rep_stream, b->pool, "ENDREP\n"));
+ SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n"));
b->noderev->data_rep = rep;
}
@@ -5736,8 +7024,10 @@ get_next_revision_ids(const char **node_
{
char *buf;
char *str;
+ svn_stringbuf_t *content;
- SVN_ERR(read_current(svn_fs_fs__path_current(fs, pool), &buf, pool));
+ SVN_ERR(read_content(&content, svn_fs_fs__path_current(fs, pool), pool));
+ buf = content->data;
str = svn_cstring_tokenize(" ", &buf);
if (! str)
@@ -5796,7 +7086,7 @@ write_hash_handler(void *baton,
well as SHA1 in REP. If rep sharing has been enabled and REPS_HASH
is not NULL, it will be used in addition to the on-disk cache to find
earlier reps with the same content. When such existing reps can be
- found, we will truncate the one just written from the file and return
+ found, we will truncate the one just written from the file and return
the existing rep. Perform temporary allocations in POOL. */
static svn_error_t *
write_hash_rep(representation_t *rep,
@@ -5822,7 +7112,7 @@ write_hash_rep(representation_t *rep,
stream = svn_stream_create(whb, pool);
svn_stream_set_write(stream, write_hash_handler);
- SVN_ERR(svn_stream_printf(whb->stream, pool, "PLAIN\n"));
+ SVN_ERR(svn_stream_puts(whb->stream, "PLAIN\n"));
SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool));
@@ -5845,7 +7135,7 @@ write_hash_rep(representation_t *rep,
else
{
/* Write out our cosmetic end marker. */
- SVN_ERR(svn_stream_printf(whb->stream, pool, "ENDREP\n"));
+ SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n"));
/* update the representation */
rep->size = whb->size;
@@ -5953,7 +7243,7 @@ write_hash_delta_rep(representation_t *r
{
/* Write out our cosmetic end marker. */
SVN_ERR(get_file_offset(&rep_end, file, pool));
- SVN_ERR(svn_stream_printf(file_stream, pool, "ENDREP\n"));
+ SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n"));
/* update the representation */
rep->expanded_size = whb->size;
@@ -6085,16 +7375,23 @@ write_final_rev(const svn_fs_id_t **new_
{
apr_pool_t *subpool;
apr_hash_t *entries, *str_entries;
- apr_hash_index_t *hi;
+ apr_array_header_t *sorted_entries;
+ int i;
/* This is a directory. Write out all the children first. */
subpool = svn_pool_create(pool);
SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, pool));
-
- for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi))
+ /* For the sake of the repository administrator sort the entries
+ so that the final file is deterministic and repeatable,
+ however the rest of the FSFS code doesn't require any
+ particular order here. */
+ sorted_entries = svn_sort__hash(entries, svn_sort_compare_items_lexically,
+ pool);
+ for (i = 0; i < sorted_entries->nelts; ++i)
{
- svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi);
+ svn_fs_dirent_t *dirent = APR_ARRAY_IDX(sorted_entries, i,
+ svn_sort__item_t).value;
svn_pool_clear(subpool);
SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id,
@@ -6264,19 +7561,25 @@ write_final_changed_path_info(apr_off_t
{
apr_hash_t *changed_paths;
apr_off_t offset;
- apr_hash_index_t *hi;
apr_pool_t *iterpool = svn_pool_create(pool);
fs_fs_data_t *ffd = fs->fsap_data;
svn_boolean_t include_node_kinds =
ffd->format >= SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT;
+ apr_array_header_t *sorted_changed_paths;
+ int i;
SVN_ERR(get_file_offset(&offset, file, pool));
SVN_ERR(svn_fs_fs__txn_changes_fetch(&changed_paths, fs, txn_id, pool));
+ /* For the sake of the repository administrator sort the changes so
+ that the final file is deterministic and repeatable, however the
+ rest of the FSFS code doesn't require any particular order here. */
+ sorted_changed_paths = svn_sort__hash(changed_paths,
+ svn_sort_compare_items_lexically, pool);
/* Iterate through the changed paths one at a time, and convert the
temporary node-id into a permanent one for each change entry. */
[... 810 lines stripped ...]