You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2013/02/04 21:48:13 UTC
svn commit: r1442344 [38/39] - in /subversion/branches/fsfs-format7: ./
build/ build/ac-macros/ build/generator/ build/generator/templates/
build/win32/ contrib/client-side/emacs/
contrib/server-side/fsfsfixer/fixer/ contrib/server-side/svncutter/ doc/...
Modified: subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.c (original)
+++ subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.c Mon Feb 4 20:48:05 2013
@@ -174,7 +174,8 @@ sbox_wc_add(svn_test__sandbox_t *b, cons
parent_abspath = svn_dirent_dirname(path, b->pool);
SVN_ERR(svn_wc__acquire_write_lock(NULL, b->wc_ctx, parent_abspath, FALSE,
b->pool, b->pool));
- SVN_ERR(svn_wc_add_from_disk(b->wc_ctx, path, NULL, NULL, b->pool));
+ SVN_ERR(svn_wc_add_from_disk2(b->wc_ctx, path, NULL /*props*/,
+ NULL, NULL, b->pool));
SVN_ERR(svn_wc__release_write_lock(b->wc_ctx, parent_abspath, b->pool));
return SVN_NO_ERROR;
}
@@ -286,17 +287,30 @@ sbox_wc_exclude(svn_test__sandbox_t *b,
}
svn_error_t *
-sbox_wc_commit(svn_test__sandbox_t *b, const char *path)
+sbox_wc_commit_ex(svn_test__sandbox_t *b,
+ apr_array_header_t *targets,
+ svn_depth_t depth)
{
svn_client_ctx_t *ctx;
+ SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
+ ctx->wc_ctx = b->wc_ctx;
+ return svn_client_commit6(targets, depth,
+ FALSE /* keep_locks */,
+ FALSE /* keep_changelist */,
+ TRUE /* commit_as_operations */,
+ TRUE /* include_file_externals */,
+ FALSE /* include_dir_externals */,
+ NULL, NULL, NULL, NULL, ctx, b->pool);
+}
+
+svn_error_t *
+sbox_wc_commit(svn_test__sandbox_t *b, const char *path)
+{
apr_array_header_t *targets = apr_array_make(b->pool, 1,
sizeof(const char *));
APR_ARRAY_PUSH(targets, const char *) = sbox_wc_path(b, path);
- SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
- return svn_client_commit5(targets, svn_depth_infinity,
- FALSE, FALSE, TRUE, /* keep locks/cl's/use_ops*/
- NULL, NULL, NULL, NULL, ctx, b->pool);
+ return sbox_wc_commit_ex(b, targets, svn_depth_infinity);
}
svn_error_t *
@@ -313,30 +327,52 @@ sbox_wc_update(svn_test__sandbox_t *b, c
APR_ARRAY_PUSH(paths, const char *) = sbox_wc_path(b, path);
SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
+ ctx->wc_ctx = b->wc_ctx;
return svn_client_update4(&result_revs, paths, &revision, svn_depth_infinity,
TRUE, FALSE, FALSE, FALSE, FALSE,
ctx, b->pool);
}
svn_error_t *
-sbox_wc_resolved(svn_test__sandbox_t *b, const char *path)
+sbox_wc_switch(svn_test__sandbox_t *b, const char *url)
{
svn_client_ctx_t *ctx;
+ svn_revnum_t result_rev;
+ svn_opt_revision_t head_rev = { svn_opt_revision_head, {0} };
+ url = apr_pstrcat(b->pool, b->repos_url, url, (char*)NULL);
SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
- return svn_client_resolve(sbox_wc_path(b, path), svn_depth_infinity,
- svn_wc_conflict_choose_merged, ctx, b->pool);
+ ctx->wc_ctx = b->wc_ctx;
+ return svn_client_switch3(&result_rev, sbox_wc_path(b, ""), url,
+ &head_rev, &head_rev, svn_depth_infinity,
+ FALSE /* depth_is_sticky */,
+ TRUE /* ignore_externals */,
+ FALSE /* allow_unver_obstructions */,
+ TRUE /* ignore_ancestry */,
+ ctx, b->pool);
}
svn_error_t *
-sbox_wc_resolve(svn_test__sandbox_t *b, const char *path)
+sbox_wc_resolved(svn_test__sandbox_t *b, const char *path)
{
- svn_client_ctx_t *ctx;
+ return sbox_wc_resolve(b, path, svn_wc_conflict_choose_merged);
+}
- SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
- return svn_client_resolve(sbox_wc_path(b, path), svn_depth_infinity,
- svn_wc_conflict_choose_mine_conflict,
- ctx, b->pool);
+svn_error_t *
+sbox_wc_resolve(svn_test__sandbox_t *b, const char *path,
+ svn_wc_conflict_choice_t conflict_choice)
+{
+ SVN_ERR(svn_wc__resolve_conflicts(b->wc_ctx, sbox_wc_path(b, path),
+ svn_depth_infinity,
+ TRUE /* resolve_text */,
+ "" /* resolve_prop (ALL props) */,
+ TRUE /* resolve_tree */,
+ conflict_choice,
+ NULL, NULL, /* conflict func */
+ NULL, NULL, /* cancellation */
+ NULL, NULL, /* notification */
+ b->pool));
+ return SVN_NO_ERROR;
}
svn_error_t *
@@ -347,9 +383,16 @@ sbox_wc_move(svn_test__sandbox_t *b, con
sizeof(const char *));
SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
+ ctx->wc_ctx = b->wc_ctx;
APR_ARRAY_PUSH(paths, const char *) = sbox_wc_path(b, src);
- return svn_client_move6(paths, sbox_wc_path(b, dst),
- FALSE, FALSE, NULL, NULL, NULL, ctx, b->pool);
+ return svn_client_move7(paths, sbox_wc_path(b, dst),
+ FALSE /* move_as_child */,
+ FALSE /* make_parents */,
+ TRUE /* allow_mixed_revisions */,
+ FALSE /* metadata_only */,
+ NULL /* revprop_table */,
+ NULL, NULL, /* commit callback */
+ ctx, b->pool);
}
svn_error_t *
@@ -361,12 +404,14 @@ sbox_wc_propset(svn_test__sandbox_t *b,
svn_client_ctx_t *ctx;
apr_array_header_t *paths = apr_array_make(b->pool, 1,
sizeof(const char *));
+ svn_string_t *pval = value ? svn_string_create(value, b->pool) : NULL;
SVN_ERR(svn_client_create_context2(&ctx, NULL, b->pool));
+ ctx->wc_ctx = b->wc_ctx;
APR_ARRAY_PUSH(paths, const char *) = sbox_wc_path(b, path);
- return svn_client_propset_local(name, svn_string_create(value, b->pool),
- paths, svn_depth_empty, TRUE, NULL, ctx,
- b->pool);
+ return svn_client_propset_local(name, pval, paths, svn_depth_empty,
+ TRUE /* skip_checks */,
+ NULL, ctx, b->pool);
}
svn_error_t *
Modified: subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.h?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.h (original)
+++ subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/utils.h Mon Feb 4 20:48:05 2013
@@ -122,21 +122,31 @@ sbox_wc_commit(svn_test__sandbox_t *b, c
/* */
svn_error_t *
+sbox_wc_commit_ex(svn_test__sandbox_t *b,
+ apr_array_header_t *targets,
+ svn_depth_t depth);
+
+/* */
+svn_error_t *
sbox_wc_update(svn_test__sandbox_t *b, const char *path, svn_revnum_t revnum);
+svn_error_t *
+sbox_wc_switch(svn_test__sandbox_t *b, const char *url);
+
/* */
svn_error_t *
sbox_wc_resolved(svn_test__sandbox_t *b, const char *path);
/* */
svn_error_t *
-sbox_wc_resolve(svn_test__sandbox_t *b, const char *path);
+sbox_wc_resolve(svn_test__sandbox_t *b, const char *path,
+ svn_wc_conflict_choice_t conflict_choice);
/* */
svn_error_t *
sbox_wc_move(svn_test__sandbox_t *b, const char *src, const char *dst);
-/* */
+/* Set property NAME to VALUE on PATH. If VALUE=NULL, delete the property. */
svn_error_t *
sbox_wc_propset(svn_test__sandbox_t *b,
const char *name,
Modified: subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/wc-queries-test.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/wc-queries-test.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/wc-queries-test.c (original)
+++ subversion/branches/fsfs-format7/subversion/tests/libsvn_wc/wc-queries-test.c Mon Feb 4 20:48:05 2013
@@ -73,6 +73,7 @@ static const int schema_statements[] =
STMT_CREATE_TARGET_PROP_CACHE,
STMT_CREATE_REVERT_LIST,
STMT_CREATE_DELETE_LIST,
+ STMT_CREATE_UPDATE_MOVE_LIST,
-1 /* final marker */
};
@@ -87,12 +88,12 @@ static const int slow_statements[] =
/* Is there a record? ### Can we somehow check for LIMIT 1? */
STMT_LOOK_FOR_WORK,
- STMT_HAS_WORKING_NODES,
/* Full temporary table read */
STMT_INSERT_ACTUAL_EMPTIES,
STMT_SELECT_REVERT_LIST_RECURSIVE,
STMT_SELECT_DELETE_LIST,
+ STMT_SELECT_UPDATE_MOVE_LIST,
/* Designed as slow to avoid penalty on other queries */
STMT_SELECT_UNREFERENCED_PRISTINES,
Modified: subversion/branches/fsfs-format7/subversion/tests/svn_test_fs.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/tests/svn_test_fs.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/tests/svn_test_fs.c (original)
+++ subversion/branches/fsfs-format7/subversion/tests/svn_test_fs.c Mon Feb 4 20:48:05 2013
@@ -213,7 +213,10 @@ svn_test__create_repos(svn_repos_t **rep
if (apr_stat(&finfo, name, APR_FINFO_TYPE, pool) == APR_SUCCESS)
{
if (finfo.filetype == APR_DIR)
- SVN_ERR(svn_repos_delete(name, pool));
+ SVN_ERR_W(svn_io_remove_dir2(name, TRUE, NULL, NULL, pool),
+ apr_psprintf(pool,
+ "cannot create repos '%s' there is already "
+ "a directory of that name", name));
else
return svn_error_createf(SVN_ERR_TEST_FAILED, NULL,
"there is already a file named '%s'", name);
Modified: subversion/branches/fsfs-format7/subversion/tests/svn_test_main.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/subversion/tests/svn_test_main.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/subversion/tests/svn_test_main.c (original)
+++ subversion/branches/fsfs-format7/subversion/tests/svn_test_main.c Mon Feb 4 20:48:05 2013
@@ -1,5 +1,5 @@
/*
- * tests-main.c: shared main() & friends for SVN test-suite programs
+ * svn_test_main.c: shared main() & friends for SVN test-suite programs
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
Modified: subversion/branches/fsfs-format7/tools/buildbot/slaves/win32-SharpSvn/svntest-cleanup.cmd
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/buildbot/slaves/win32-SharpSvn/svntest-cleanup.cmd?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/buildbot/slaves/win32-SharpSvn/svntest-cleanup.cmd (original)
+++ subversion/branches/fsfs-format7/tools/buildbot/slaves/win32-SharpSvn/svntest-cleanup.cmd Mon Feb 4 20:48:05 2013
@@ -61,6 +61,7 @@ taskkill /im fs-test.exe /f 2> nul:
taskkill /im op-depth-test.exe /f 2> nul:
taskkill /im java.exe /f 2> nul:
taskkill /im perl.exe /f 2> nul:
+taskkill /im mspdbsrv.exe /f 2> nul:
IF EXIST "%TESTDIR%\tests\subversion\tests\cmdline\httpd\" (
rmdir /s /q "%TESTDIR%\tests\subversion\tests\cmdline\httpd"
)
Modified: subversion/branches/fsfs-format7/tools/client-side/svn-bench/cl.h
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/client-side/svn-bench/cl.h?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/client-side/svn-bench/cl.h (original)
+++ subversion/branches/fsfs-format7/tools/client-side/svn-bench/cl.h Mon Feb 4 20:48:05 2013
@@ -149,21 +149,6 @@ svn_error_t *
svn_cl__check_cancel(void *baton);
-/* Print to stdout a hash that maps property names (char *) to property
- values (svn_string_t *). The names are assumed to be in UTF-8 format;
- the values are either in UTF-8 (the special Subversion props) or
- plain binary values.
-
- If OUT is not NULL, then write to it rather than stdout.
-
- If NAMES_ONLY is true, print just names, else print names and
- values. */
-svn_error_t *
-svn_cl__print_prop_hash(svn_stream_t *out,
- apr_hash_t *prop_hash,
- svn_boolean_t names_only,
- apr_pool_t *pool);
-
/*** Notification functions to display results on the terminal. */
Modified: subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-list-cmd.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-list-cmd.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-list-cmd.c (original)
+++ subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-list-cmd.c Mon Feb 4 20:48:05 2013
@@ -48,7 +48,7 @@ struct print_baton {
svn_client_ctx_t *ctx;
};
-/* This implements the svn_client_list_func_t API, printing a single
+/* This implements the svn_client_list_func2_t API, printing a single
directory entry in text format. */
static svn_error_t *
print_dirent(void *baton,
@@ -56,6 +56,8 @@ print_dirent(void *baton,
const svn_dirent_t *dirent,
const svn_lock_t *lock,
const char *abs_path,
+ const char *external_parent_url,
+ const char *external_target,
apr_pool_t *pool)
{
struct print_baton *pb = baton;
@@ -123,11 +125,12 @@ svn_cl__null_list(apr_getopt_t *os,
SVN_ERR(svn_opt_parse_path(&peg_revision, &truepath, target,
subpool));
- err = svn_client_list2(truepath, &peg_revision,
+ err = svn_client_list3(truepath, &peg_revision,
&(opt_state->start_revision),
opt_state->depth,
dirent_fields,
opt_state->verbose,
+ FALSE, /* include externals */
print_dirent,
&pb, ctx, subpool);
Modified: subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-log-cmd.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-log-cmd.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-log-cmd.c (original)
+++ subversion/branches/fsfs-format7/tools/client-side/svn-bench/null-log-cmd.c Mon Feb 4 20:48:05 2013
@@ -134,7 +134,7 @@ svn_cl__null_log(apr_getopt_t *os,
svn_cl__opt_state_t *opt_state = ((svn_cl__cmd_baton_t *) baton)->opt_state;
svn_client_ctx_t *ctx = ((svn_cl__cmd_baton_t *) baton)->ctx;
apr_array_header_t *targets;
- struct log_receiver_baton lb;
+ struct log_receiver_baton lb = { 0 };
const char *target;
int i;
apr_array_header_t *revprops;
Propchange: subversion/branches/fsfs-format7/tools/dev/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Mon Feb 4 20:48:05 2013
@@ -1 +1,4 @@
+.libs
+fsfs-access-map
+fsfs-reorg
logdata.py
Modified: subversion/branches/fsfs-format7/tools/dev/benchmarks/suite1/benchmark.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/dev/benchmarks/suite1/benchmark.py?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/dev/benchmarks/suite1/benchmark.py (original)
+++ subversion/branches/fsfs-format7/tools/dev/benchmarks/suite1/benchmark.py Mon Feb 4 20:48:05 2013
@@ -1306,4 +1306,4 @@ if __name__ == '__main__':
usage()
else:
- usage('Unknown command argument: %s' % cmd)
+ usage('Unknown subcommand argument: %s' % cmd)
Modified: subversion/branches/fsfs-format7/tools/dev/contribulyze.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/dev/contribulyze.py?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/dev/contribulyze.py (original)
+++ subversion/branches/fsfs-format7/tools/dev/contribulyze.py Mon Feb 4 20:48:05 2013
@@ -511,8 +511,13 @@ class LogMessage(object):
log_separator = '-' * 72 + '\n'
log_header_re = re.compile\
('^(r[0-9]+) \| ([^|]+) \| ([^|]+) \| ([0-9]+)[^0-9]')
-field_re = re.compile('^(Patch|Review(ed)?|Suggested|Found|Inspired) by:\s*\S.*$')
-field_aliases = { 'Reviewed' : 'Review' }
+field_re = re.compile(
+ '^(Patch|Review(ed)?|Suggested|Found|Inspired|Tested|Reported) by:'
+ '\s*\S.*$')
+field_aliases = {
+ 'Reviewed' : 'Review',
+ 'Reported' : 'Found',
+}
parenthetical_aside_re = re.compile('^\s*\(.*\)\s*$')
def graze(input):
Propchange: subversion/branches/fsfs-format7/tools/dist/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Mon Feb 4 20:48:05 2013
@@ -1,3 +1,4 @@
prefix
tempdir
deploy
+*.pyc
Modified: subversion/branches/fsfs-format7/tools/dist/backport.pl
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/dist/backport.pl?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/dist/backport.pl (original)
+++ subversion/branches/fsfs-format7/tools/dist/backport.pl Mon Feb 4 20:48:05 2013
@@ -110,10 +110,6 @@ $SVNq up
$SVNq merge $mergeargs
$VIM -e -s -n -N -i NONE -u NONE -c '/$pattern/normal! dap' -c wq $STATUS
if $WET_RUN; then
- if [ -n "\$PRINT_SOMETHING_BETWEEN_PROMPTS" ]; then
- # hack for pw-driver.pl to see some output between prompts
- head -n1 $logmsg_filename
- fi
$SVNq commit -F $logmsg_filename
else
echo "Committing:"
@@ -233,6 +229,9 @@ sub main {
die "A vim with the +ex_extra feature is required"
if `${VIM} --version` !~ /[+]ex_extra/;
+ # ### TODO: need to run 'revert' here
+ # ### TODO: both here and in merge(), unlink files that previous merges added
+ die "Local mods to STATUS file $STATUS" if `$SVN status -q $STATUS`;
@ARGV = $STATUS;
# Skip most of the file
Propchange: subversion/branches/fsfs-format7/tools/dist/make-deps-tarball.sh
------------------------------------------------------------------------------
Merged /subversion/trunk/tools/dist/make-deps-tarball.sh:r1414758-1442089
Modified: subversion/branches/fsfs-format7/tools/dist/release.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/dist/release.py?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/dist/release.py (original)
+++ subversion/branches/fsfs-format7/tools/dist/release.py Mon Feb 4 20:48:05 2013
@@ -34,6 +34,9 @@
# It'd be kind of nice to use the Subversion python bindings in this script,
# but people.apache.org doesn't currently have them installed
+# Futures (Python 2.5 compatibility)
+from __future__ import with_statement
+
# Stuff we need
import os
import re
@@ -88,6 +91,7 @@ secure_repos = 'https://svn.apache.org/r
dist_repos = 'https://dist.apache.org/repos/dist'
dist_dev_url = dist_repos + '/dev/subversion'
dist_release_url = dist_repos + '/release/subversion'
+KEYS = 'https://people.apache.org/keys/group/subversion.asc'
extns = ['zip', 'tar.gz', 'tar.bz2']
@@ -374,7 +378,10 @@ def compare_changes(repos, branch, revis
if stderr:
raise RuntimeError('svn mergeinfo failed: %s' % stderr)
if stdout:
- raise RuntimeError('CHANGES has unmerged revisions: %s' % stdout)
+ # Treat this as a warning since we are now putting entries for future
+ # minor releases in CHANGES on trunk.
+ logging.warning('CHANGES has unmerged revisions: %s' %
+ stdout.replace("\n", " "))
def roll_tarballs(args):
'Create the release artifacts.'
@@ -620,7 +627,13 @@ def write_news(args):
def get_sha1info(args, replace=False):
'Return a list of sha1 info for the release'
- sha1s = glob.glob(os.path.join(get_deploydir(args.base_dir), '*.sha1'))
+
+ if args.target:
+ target = args.target
+ else:
+ target = get_deploydir(args.base_dir)
+
+ sha1s = glob.glob(os.path.join(target, '*.sha1'))
class info(object):
pass
@@ -644,10 +657,11 @@ def get_sha1info(args, replace=False):
def write_announcement(args):
'Write the release announcement.'
sha1info = get_sha1info(args)
+ siginfo = "\n".join(get_siginfo(args, True)) + "\n"
data = { 'version' : str(args.version),
'sha1info' : sha1info,
- 'siginfo' : open('getsigs-output', 'r').read(),
+ 'siginfo' : siginfo,
'major-minor' : '%d.%d' % (args.version.major,
args.version.minor),
'major-minor-patch' : args.version.base,
@@ -682,8 +696,8 @@ def write_downloads(args):
key_start = '-----BEGIN PGP SIGNATURE-----'
fp_pattern = re.compile(r'^pub\s+(\w+\/\w+)[^\n]*\n\s+Key\sfingerprint\s=((\s+[0-9A-F]{4}){10})\nuid\s+([^<\(]+)\s')
-def check_sigs(args):
- 'Check the signatures for the release.'
+def get_siginfo(args, quiet=False):
+ 'Returns a list of signatures for the release.'
try:
import gnupg
@@ -697,13 +711,16 @@ def check_sigs(args):
target = get_deploydir(args.base_dir)
good_sigs = {}
+ fingerprints = {}
+ output = []
glob_pattern = os.path.join(target, 'subversion*-%s*.asc' % args.version)
for filename in glob.glob(glob_pattern):
text = open(filename).read()
keys = text.split(key_start)
- logging.info("Checking %d sig(s) in %s" % (len(keys[1:]), filename))
+ if not quiet:
+ logging.info("Checking %d sig(s) in %s" % (len(keys[1:]), filename))
for key in keys[1:]:
fd, fn = tempfile.mkstemp()
os.write(fd, key_start + key)
@@ -733,9 +750,30 @@ def check_sigs(args):
if l[0:7] != 'Warning' ])
fp = fp_pattern.match(gpg_output).groups()
- print(" %s [%s] with fingerprint:" % (fp[3], fp[0]))
- print(" %s" % fp[1])
+ fingerprints["%s [%s] %s" % (fp[3], fp[0], fp[1])] = fp
+ for entry in sorted(fingerprints.keys()):
+ fp = fingerprints[entry]
+ output.append(" %s [%s] with fingerprint:" % (fp[3], fp[0]))
+ output.append(" %s" % fp[1])
+
+ return output
+
+def check_sigs(args):
+ 'Check the signatures for the release.'
+
+ output = get_siginfo(args)
+ for line in output:
+ print(line)
+
+def get_keys(args):
+ 'Import the LDAP-based KEYS file to gpg'
+ # We use a tempfile because urlopen() objects don't have a .fileno()
+ with tempfile.SpooledTemporaryFile() as fd:
+ fd.write(urllib2.urlopen(KEYS).read())
+ fd.flush()
+ fd.seek(0)
+ subprocess.check_call(['gpg', '--import'], stdin=fd)
#----------------------------------------------------------------------
# Main entry point for argument parsing and handling
@@ -844,13 +882,18 @@ def main():
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
+ # write-announcement
subparser = subparsers.add_parser('write-announcement',
help='''Output to stdout template text for the emailed
release announcement.''')
subparser.set_defaults(func=write_announcement)
+ subparser.add_argument('--target',
+ help='''The full path to the directory containing
+ release artifacts.''')
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
+ # write-downloads
subparser = subparsers.add_parser('write-downloads',
help='''Output to stdout template text for the download
table for subversion.apache.org''')
@@ -858,7 +901,7 @@ def main():
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
- # The check sigs subcommand
+ # check-sigs
subparser = subparsers.add_parser('check-sigs',
help='''Output to stdout the signatures collected for this
release''')
@@ -869,6 +912,11 @@ def main():
help='''The full path to the directory containing
release artifacts.''')
+ # get-keys
+ subparser = subparsers.add_parser('get-keys',
+ help='''Import committers' public keys to ~/.gpg/''')
+ subparser.set_defaults(func=get_keys)
+
# A meta-target
subparser = subparsers.add_parser('clean',
help='''The same as the '--clean' switch, but as a
Modified: subversion/branches/fsfs-format7/tools/examples/svnshell.rb
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/examples/svnshell.rb?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/examples/svnshell.rb (original)
+++ subversion/branches/fsfs-format7/tools/examples/svnshell.rb Mon Feb 4 20:48:05 2013
@@ -125,7 +125,7 @@ class SvnShell
puts("Invalid argument for #{cmd}: #{args.join(' ')}")
end
else
- puts("Unknown command: #{cmd}")
+ puts("Unknown subcommand: #{cmd}")
puts("Try one of these commands: ", WORDS.sort.join(" "))
end
end
Modified: subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.conf.example
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.conf.example?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.conf.example (original)
+++ subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.conf.example Mon Feb 4 20:48:05 2013
@@ -146,6 +146,15 @@
#
# from_addr = %(author)s@example.com
#
+# The substitution variable "repos_basename" is provided, and is set to
+# the directory name of the repository. This can be useful to set
+# a custom subject that can be re-used in multiple repositories:
+#
+# commit_subject_prefix = [svn-%(repos_basename)s]
+#
+# For example if the repository is at /path/to/repo/project-x then
+# the subject of commit emails will be prefixed with [svn-project-x]
+#
#
# SUMMARY
#
Modified: subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.py?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.py (original)
+++ subversion/branches/fsfs-format7/tools/hook-scripts/mailer/mailer.py Mon Feb 4 20:48:05 2013
@@ -98,7 +98,10 @@ def main(pool, cmd, config_fname, repos_
if cmd == 'commit':
revision = int(cmd_args[0])
repos = Repository(repos_dir, revision, pool)
- cfg = Config(config_fname, repos, { 'author' : repos.author })
+ cfg = Config(config_fname, repos,
+ {'author': author,
+ 'repos_basename': os.path.basename(repos.repos_dir)
+ })
messenger = Commit(pool, cfg, repos)
elif cmd == 'propchange' or cmd == 'propchange2':
revision = int(cmd_args[0])
@@ -108,14 +111,20 @@ def main(pool, cmd, config_fname, repos_
repos = Repository(repos_dir, revision, pool)
# Override the repos revision author with the author of the propchange
repos.author = author
- cfg = Config(config_fname, repos, { 'author' : author })
+ cfg = Config(config_fname, repos,
+ {'author': author,
+ 'repos_basename': os.path.basename(repos.repos_dir)
+ })
messenger = PropChange(pool, cfg, repos, author, propname, action)
elif cmd == 'lock' or cmd == 'unlock':
author = cmd_args[0]
repos = Repository(repos_dir, 0, pool) ### any old revision will do
# Override the repos revision author with the author of the lock/unlock
repos.author = author
- cfg = Config(config_fname, repos, { 'author' : author })
+ cfg = Config(config_fname, repos,
+ {'author': author,
+ 'repos_basename': os.path.basename(repos.repos_dir)
+ })
messenger = Lock(pool, cfg, repos, author, cmd == 'lock')
else:
raise UnknownSubcommand(cmd)
Propchange: subversion/branches/fsfs-format7/tools/server-side/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Mon Feb 4 20:48:05 2013
@@ -1,5 +1,6 @@
.libs
svn-populate-node-origins-index
+svnauthz
svnauthz-validate
svn-rep-sharing-stats
fsfs-reorg
Modified: subversion/branches/fsfs-format7/tools/server-side/fsfs-stats.c
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/server-side/fsfs-stats.c?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/server-side/fsfs-stats.c (original)
+++ subversion/branches/fsfs-format7/tools/server-side/fsfs-stats.c Mon Feb 4 20:48:05 2013
@@ -1,4 +1,4 @@
-/* diff.c -- test driver for text diffs
+/* fsfs-stats.c -- gather size statistics on FSFS repositories
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
@@ -22,7 +22,6 @@
#include <assert.h>
-#include <sys/stat.h>
#include <apr.h>
#include <apr_general.h>
@@ -37,29 +36,40 @@
#include "svn_sorts.h"
#include "svn_delta.h"
#include "svn_hash.h"
+#include "svn_cache_config.h"
#include "private/svn_string_private.h"
#include "private/svn_subr_private.h"
#include "private/svn_dep_compat.h"
+#include "private/svn_cache.h"
#ifndef _
#define _(x) x
#endif
-#define ERROR_TAG "diff: "
+#define ERROR_TAG "fsfs-stats: "
+/* We group representations into 2x2 different kinds plus one default:
+ * [dir / file] x [text / prop]. The assignment is done by the first node
+ * that references the respective representation.
+ */
typedef enum rep_kind_t
{
+ /* The representation is _directly_ unused, i.e. not referenced by any
+ * noderev. However, some other representation may use it as delta base.
+ * null value. Should not occur in real-word repositories. */
unused_rep,
+ /* a properties on directory rep */
dir_property_rep,
+ /* a properties on file rep */
file_property_rep,
- /* a directory rep (including PLAIN / DELTA header) */
+ /* a directory rep */
dir_rep,
- /* a file rep (including PLAIN / DELTA header) */
+ /* a file rep */
file_rep
} rep_kind_t;
@@ -73,6 +83,7 @@ typedef struct representation_t
/* item length in bytes */
apr_size_t size;
+ /* item length after de-deltification */
apr_size_t expanded_size;
/* deltification base, or NULL if there is none */
@@ -80,13 +91,15 @@ typedef struct representation_t
/* revision that contains this representation
* (may be referenced by other revisions, though) */
-
svn_revnum_t revision;
+
+ /* number of nodes that reference this representation */
apr_uint32_t ref_count;
/* length of the PLAIN / DELTA line in the source file in bytes */
apr_uint16_t header_size;
+ /* classification of the representation. values of rep_kind_t */
char kind;
/* the source content has a PLAIN header, so we may simply copy the
@@ -118,9 +131,16 @@ typedef struct revision_info_t
* for non-packed revs) */
apr_size_t end;
+ /* number of directory noderevs in this revision */
apr_size_t dir_noderev_count;
+
+ /* number of file noderevs in this revision */
apr_size_t file_noderev_count;
+
+ /* total size of directory noderevs (i.e. the structs - not the rep) */
apr_size_t dir_noderev_size;
+
+ /* total size of file noderevs (i.e. the structs - not the rep) */
apr_size_t file_noderev_size;
/* all representation_t of this revision (in no particular order),
@@ -128,43 +148,48 @@ typedef struct revision_info_t
apr_array_header_t *representations;
} revision_info_t;
-/* A cached, undeltified txdelta window.
+/* Data type to identify a representation. It will be used to address
+ * cached combined (un-deltified) windows.
*/
-typedef struct window_cache_entry_t
+typedef struct window_cache_key_t
{
- /* revision containing the window */
+ /* revision of the representation */
svn_revnum_t revision;
- /* offset of the deltified window within that revision */
+ /* its offset */
apr_size_t offset;
+} window_cache_key_t;
- /* window content */
- svn_stringbuf_t *window;
-} window_cache_entry_t;
-
-/* Cache for undeltified txdelta windows. (revision, offset) will be mapped
- * directly into the ENTRIES array of INSERT_COUNT buckets (most entries
- * will be NULL).
- *
- * The cache will be cleared when USED exceeds CAPACITY.
+/* Description of one large representation. It's content will be reused /
+ * overwritten when it gets replaced by an even larger representation.
*/
-typedef struct window_cache_t
+typedef struct large_change_info_t
{
- /* fixed-size array of ENTRY_COUNT elements */
- window_cache_entry_t *entries;
+ /* size of the (deltified) representation */
+ apr_size_t size;
- /* used to allocate windows */
- apr_pool_t *pool;
+ /* revision of the representation */
+ svn_revnum_t revision;
- /* size of ENTRIES in elements */
- apr_size_t entry_count;
+ /* node path. "" for unused instances */
+ svn_stringbuf_t *path;
+} large_change_info_t;
+
+/* Container for the largest representations found so far. The capacity
+ * is fixed and entries will be inserted by reusing the last one and
+ * reshuffling the entry pointers.
+ */
+typedef struct largest_changes_t
+{
+ /* number of entries allocated in CHANGES */
+ apr_size_t count;
- /* maximum combined size of all cached windows */
- apr_size_t capacity;
+ /* size of the smallest change */
+ apr_size_t min_size;
- /* current combined size of all cached windows */
- apr_size_t used;
-} window_cache_t;
+ /* changes kept in this struct */
+ large_change_info_t **changes;
+} largest_changes_t;
/* Root data structure containing all information about a given repository.
*/
@@ -196,7 +221,10 @@ typedef struct fs_fs_t
representation_t *null_base;
/* undeltified txdelta window cache */
- window_cache_t *window_cache;
+ svn_cache__t *window_cache;
+
+ /* track the biggest contributors to repo size */
+ largest_changes_t *largest_changes;
} fs_fs_t;
/* Return the rev pack folder for revision REV in FS.
@@ -238,45 +266,55 @@ open_rev_or_pack_file(apr_file_t **file,
pool);
}
-/* Read the whole content of the file containing REV in FS and return that
- * in *CONTENT.
- */
+/* Return the length of FILE in *FILE_SIZE. Use POOL for allocations.
+*/
static svn_error_t *
-rev_or_pack_file_size(apr_off_t *file_size,
- fs_fs_t *fs,
- svn_revnum_t rev,
- apr_pool_t *pool)
+get_file_size(apr_off_t *file_size,
+ apr_file_t *file,
+ apr_pool_t *pool)
{
- apr_file_t *file;
apr_finfo_t finfo;
- SVN_ERR(open_rev_or_pack_file(&file, fs, rev, pool));
SVN_ERR(svn_io_file_info_get(&finfo, APR_FINFO_SIZE, file, pool));
- SVN_ERR(svn_io_file_close(file, pool));
*file_size = finfo.size;
return SVN_NO_ERROR;
}
-/* Get the file content of revision REVISION in FS and return it in *DATA.
- * Use SCRATCH_POOL for temporary allocations.
+/* Get the file content of revision REVISION in FS and return it in *CONTENT.
+ * Read the LEN bytes starting at file OFFSET. When provided, use FILE as
+ * packed or plain rev file.
+ * Use POOL for temporary allocations.
*/
static svn_error_t *
get_content(svn_stringbuf_t **content,
+ apr_file_t *file,
fs_fs_t *fs,
svn_revnum_t revision,
apr_off_t offset,
apr_size_t len,
apr_pool_t *pool)
{
- apr_file_t *file;
apr_pool_t * file_pool = svn_pool_create(pool);
+ apr_size_t large_buffer_size = 0x10000;
- SVN_ERR(open_rev_or_pack_file(&file, fs, revision, file_pool));
- SVN_ERR(svn_io_file_seek(file, APR_SET, &offset, pool));
+ if (file == NULL)
+ SVN_ERR(open_rev_or_pack_file(&file, fs, revision, file_pool));
*content = svn_stringbuf_create_ensure(len, pool);
(*content)->len = len;
+
+#if APR_VERSION_AT_LEAST(1,3,0)
+ /* for better efficiency use larger buffers on large reads */
+ if ( (len >= large_buffer_size)
+ && (apr_file_buffer_size_get(file) < large_buffer_size))
+ apr_file_buffer_set(file,
+ apr_palloc(apr_file_pool_get(file),
+ large_buffer_size),
+ large_buffer_size);
+#endif
+
+ SVN_ERR(svn_io_file_seek(file, APR_SET, &offset, pool));
SVN_ERR(svn_io_file_read_full2(file, (*content)->data, len,
NULL, NULL, pool));
svn_pool_destroy(file_pool);
@@ -284,89 +322,112 @@ get_content(svn_stringbuf_t **content,
return SVN_NO_ERROR;
}
-/* Return a new txdelta window cache with ENTRY_COUNT buckets in its index
- * and a the total CAPACITY given in bytes.
- * Use POOL for all cache-related allocations.
+/* In *RESULT, return the cached txdelta window stored in REPRESENTATION
+ * within FS. If that has not been found in cache, return NULL.
+ * Allocate the result in POOL.
*/
-static window_cache_t *
-create_window_cache(apr_pool_t *pool,
- apr_size_t entry_count,
- apr_size_t capacity)
-{
- window_cache_t *result = apr_pcalloc(pool, sizeof(*result));
-
- result->pool = svn_pool_create(pool);
- result->entry_count = entry_count;
- result->capacity = capacity;
- result->used = 0;
- result->entries = apr_pcalloc(pool, sizeof(*result->entries) * entry_count);
-
- return result;
-}
-
-/* Return the position within FS' window cache ENTRIES index for the given
- * (REVISION, OFFSET) pair. This is a cache-internal function.
- */
-static apr_size_t
-get_window_cache_index(fs_fs_t *fs,
- svn_revnum_t revision,
- apr_size_t offset)
+static svn_error_t *
+get_cached_window(svn_stringbuf_t **result,
+ fs_fs_t *fs,
+ representation_t *representation,
+ apr_pool_t *pool)
{
- return (revision + offset * 0xd1f3da69) % fs->window_cache->entry_count;
+ svn_boolean_t found = FALSE;
+ window_cache_key_t key;
+ key.revision = representation->revision;
+ key.offset = representation->offset;
+
+ *result = NULL;
+ return svn_error_trace(svn_cache__get((void**)result, &found,
+ fs->window_cache,
+ &key, pool));
}
-/* Return the cached txdelta window stored in REPRESENTAION within FS.
- * If that has not been found in cache, return NULL.
+/* Cache the undeltified txdelta WINDOW for REPRESENTATION within FS.
+ * Use POOL for temporaries.
*/
-static svn_stringbuf_t *
-get_cached_window(fs_fs_t *fs,
+static svn_error_t *
+set_cached_window(fs_fs_t *fs,
representation_t *representation,
+ svn_stringbuf_t *window,
apr_pool_t *pool)
{
- svn_revnum_t revision = representation->revision;
- apr_size_t offset = representation->offset;
-
- apr_size_t i = get_window_cache_index(fs, revision, offset);
- window_cache_entry_t *entry = &fs->window_cache->entries[i];
+ /* select entry */
+ window_cache_key_t key;
+ key.revision = representation->revision;
+ key.offset = representation->offset;
- return entry->offset == offset && entry->revision == revision
- ? svn_stringbuf_dup(entry->window, pool)
- : NULL;
+ return svn_error_trace(svn_cache__set(fs->window_cache, &key, window,
+ pool));
}
-/* Cache the undeltified txdelta WINDOW for REPRESENTAION within FS.
+/* Initialize the LARGEST_CHANGES member in FS with a capacity of COUNT
+ * entries. Use POOL for allocations.
*/
static void
-set_cached_window(fs_fs_t *fs,
- representation_t *representation,
- svn_stringbuf_t *window)
+initialize_largest_changes(fs_fs_t *fs,
+ apr_size_t count,
+ apr_pool_t *pool)
{
- /* select entry */
- svn_revnum_t revision = representation->revision;
- apr_size_t offset = representation->offset;
+ apr_size_t i;
+
+ fs->largest_changes = apr_pcalloc(pool, sizeof(*fs->largest_changes));
+ fs->largest_changes->count = count;
+ fs->largest_changes->min_size = 1;
+ fs->largest_changes->changes
+ = apr_palloc(pool, count * sizeof(*fs->largest_changes->changes));
- apr_size_t i = get_window_cache_index(fs, revision, offset);
- window_cache_entry_t *entry = &fs->window_cache->entries[i];
+ /* allocate *all* entries before the path stringbufs. This increases
+ * cache locality and enhances performance significantly. */
+ for (i = 0; i < count; ++i)
+ fs->largest_changes->changes[i]
+ = apr_palloc(pool, sizeof(**fs->largest_changes->changes));
- /* if the capacity is exceeded, clear the cache */
- fs->window_cache->used += window->len;
- if (fs->window_cache->used >= fs->window_cache->capacity)
+ /* now initialize them and allocate the stringbufs */
+ for (i = 0; i < count; ++i)
{
- svn_pool_clear(fs->window_cache->pool);
- memset(fs->window_cache->entries,
- 0,
- sizeof(*fs->window_cache->entries) * fs->window_cache->entry_count);
- fs->window_cache->used = window->len;
+ fs->largest_changes->changes[i]->size = 0;
+ fs->largest_changes->changes[i]->revision = SVN_INVALID_REVNUM;
+ fs->largest_changes->changes[i]->path
+ = svn_stringbuf_create_ensure(1024, pool);
}
+}
- /* set the entry to a copy of the window data */
- entry->window = svn_stringbuf_dup(window, fs->window_cache->pool);
- entry->offset = offset;
- entry->revision = revision;
+/* Update data aggregators in FS with this representation of on-disk SIZE
+ * for PATH in REVSION.
+ */
+static void
+add_change(fs_fs_t *fs,
+ apr_size_t size,
+ svn_revnum_t revision,
+ const char *path)
+{
+ if (size >= fs->largest_changes->min_size)
+ {
+ apr_size_t i;
+ large_change_info_t *info
+ = fs->largest_changes->changes[fs->largest_changes->count - 1];
+ info->size = size;
+ info->revision = revision;
+ svn_stringbuf_set(info->path, path);
+
+ /* linear insertion but not too bad since count is low and insertions
+ * near the end are more likely than close to front */
+ for (i = fs->largest_changes->count - 1; i > 0; --i)
+ if (fs->largest_changes->changes[i-1]->size >= size)
+ break;
+ else
+ fs->largest_changes->changes[i] = fs->largest_changes->changes[i-1];
+
+ fs->largest_changes->changes[i] = info;
+ fs->largest_changes->min_size
+ = fs->largest_changes->changes[fs->largest_changes->count-1]->size;
+ }
}
-/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
- Use POOL for temporary allocations. */
+/* Given rev pack PATH in FS, read the manifest file and return the offsets
+ * in *MANIFEST. Use POOL for allocations.
+ */
static svn_error_t *
read_manifest(apr_array_header_t **manifest,
fs_fs_t *fs,
@@ -409,6 +470,10 @@ read_manifest(apr_array_header_t **manif
return svn_stream_close(manifest_stream);
}
+/* Read header information for the revision stored in FILE_CONTENT (one
+ * whole revision). Return the offsets within FILE_CONTENT for the
+ * *ROOT_NODEREV, the list of *CHANGES and its len in *CHANGES_LEN.
+ * Use POOL for temporary allocations. */
static svn_error_t *
read_revision_header(apr_size_t *changes,
apr_size_t *changes_len,
@@ -447,8 +512,10 @@ read_revision_header(apr_size_t *changes
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
_("Final line in revision file missing space"));
+ /* terminate the header line */
*space = 0;
-
+
+ /* extract information */
SVN_ERR(svn_cstring_strtoui64(&val, line+1, 0, APR_SIZE_MAX, 10));
*root_noderev = (apr_size_t)val;
SVN_ERR(svn_cstring_strtoui64(&val, space+1, 0, APR_SIZE_MAX, 10));
@@ -458,6 +525,10 @@ read_revision_header(apr_size_t *changes
return SVN_NO_ERROR;
}
+/* Read the FSFS format number and sharding size from the format file at
+ * PATH and return it in *PFORMAT and *MAX_FILES_PER_DIR respectively.
+ * Use POOL for temporary allocations.
+ */
static svn_error_t *
read_format(int *pformat, int *max_files_per_dir,
const char *path, apr_pool_t *pool)
@@ -467,6 +538,7 @@ read_format(int *pformat, int *max_files
char buf[80];
apr_size_t len;
+ /* open format file and read the first line */
err = svn_io_file_open(&file, path, APR_READ | APR_BUFFERED,
APR_OS_DEFAULT, pool);
if (err && APR_STATUS_IS_ENOENT(err->apr_err))
@@ -541,21 +613,27 @@ read_format(int *pformat, int *max_files
return svn_io_file_close(file, pool);
}
+/* Read the content of the file at PATH and return it in *RESULT.
+ * Use POOL for temporary allocations.
+ */
static svn_error_t *
read_number(svn_revnum_t *result, const char *path, apr_pool_t *pool)
{
svn_stringbuf_t *content;
- apr_int64_t number;
+ apr_uint64_t number;
SVN_ERR(svn_stringbuf_from_file2(&content, path, pool));
content->data[content->len-1] = 0;
- SVN_ERR(svn_cstring_atoi64(&number, content->data));
+ SVN_ERR(svn_cstring_strtoui64(&number, content->data, 0, LONG_MAX, 10));
*result = (svn_revnum_t)number;
return SVN_NO_ERROR;
}
+/* Create *FS for the repository at PATH and read the format and size info.
+ * Use POOL for temporary allocations.
+ */
static svn_error_t *
fs_open(fs_fs_t **fs, const char *path, apr_pool_t *pool)
{
@@ -570,7 +648,8 @@ fs_open(fs_fs_t **fs, const char *path,
pool));
if (((*fs)->format != 4) && ((*fs)->format != 6))
return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, NULL);
-
+
+ /* read size (HEAD) info */
SVN_ERR(read_number(&(*fs)->min_unpacked_rev,
svn_dirent_join(path, "db/min-unpacked-rev", pool),
pool));
@@ -579,12 +658,18 @@ fs_open(fs_fs_t **fs, const char *path,
pool);
}
+/* Utility function that returns true if STRING->DATA matches KEY.
+ */
static svn_boolean_t
key_matches(svn_string_t *string, const char *key)
{
return strcmp(string->data, key) == 0;
}
+/* Comparator used for binary search comparing the absolute file offset
+ * of a representation to some other offset. DATA is a *representation_t,
+ * KEY is a pointer to an apr_size_t.
+ */
static int
compare_representation_offsets(const void *data, const void *key)
{
@@ -597,6 +682,15 @@ compare_representation_offsets(const voi
return diff > 0 ? 1 : 0;
}
+/* Find the revision_info_t object to the given REVISION in FS and return
+ * it in *REVISION_INFO. For performance reasons, we skip the lookup if
+ * the info is already provided.
+ *
+ * In that revision, look for the representation_t object for offset OFFSET.
+ * If it already exists, set *IDX to its index in *REVISION_INFO's
+ * representations list and return the representation object. Otherwise,
+ * set the index to where it must be inserted and return NULL.
+ */
static representation_t *
find_representation(int *idx,
fs_fs_t *fs,
@@ -606,7 +700,8 @@ find_representation(int *idx,
{
revision_info_t *info;
*idx = -1;
-
+
+ /* first let's find the revision */
info = revision_info ? *revision_info : NULL;
if (info == NULL || info->revision != revision)
{
@@ -617,23 +712,36 @@ find_representation(int *idx,
*revision_info = info;
}
+ /* not found -> no result */
if (info == NULL)
return NULL;
+
+ assert(revision == info->revision);
+ /* look for the representation */
*idx = svn_sort__bsearch_lower_bound(&offset,
info->representations,
compare_representation_offsets);
if (*idx < info->representations->nelts)
{
+ /* return the representation, if this is the one we were looking for */
representation_t *result
= APR_ARRAY_IDX(info->representations, *idx, representation_t *);
if (result->offset == offset)
return result;
}
+ /* not parsed, yet */
return NULL;
}
+/* Read the representation header in FILE_CONTENT at OFFSET. Return its
+ * size in *HEADER_SIZE, set *IS_PLAIN if no deltification was used and
+ * return the deltification base representation in *REPRESENTATION. If
+ * there is none, set it to NULL. Use FS to it look up.
+ *
+ * Use POOL for allocations and SCRATCH_POOL for temporaries.
+ */
static svn_error_t *
read_rep_base(representation_t **representation,
apr_size_t *header_size,
@@ -649,10 +757,12 @@ read_rep_base(representation_t **represe
svn_revnum_t revision;
apr_uint64_t temp;
+ /* identify representation header (1 line) */
const char *buffer = file_content->data + offset;
const char *line_end = strchr(buffer, '\n');
*header_size = line_end - buffer + 1;
+ /* check for PLAIN rep */
if (strncmp(buffer, "PLAIN\n", *header_size) == 0)
{
*is_plain = TRUE;
@@ -660,6 +770,7 @@ read_rep_base(representation_t **represe
return SVN_NO_ERROR;
}
+ /* check for DELTA against empty rep */
*is_plain = FALSE;
if (strncmp(buffer, "DELTA\n", *header_size) == 0)
{
@@ -671,7 +782,7 @@ read_rep_base(representation_t **represe
str = apr_pstrndup(scratch_pool, buffer, line_end - buffer);
last_str = str;
- /* We hopefully have a DELTA vs. a non-empty base revision. */
+ /* parse it. */
str = svn_cstring_tokenize(" ", &last_str);
str = svn_cstring_tokenize(" ", &last_str);
SVN_ERR(svn_revnum_parse(&revision, str, NULL));
@@ -679,10 +790,18 @@ read_rep_base(representation_t **represe
str = svn_cstring_tokenize(" ", &last_str);
SVN_ERR(svn_cstring_strtoui64(&temp, str, 0, APR_SIZE_MAX, 10));
+ /* it should refer to a rep in an earlier revision. Look it up */
*representation = find_representation(&idx, fs, NULL, revision, (apr_size_t)temp);
return SVN_NO_ERROR;
}
+/* Parse the representation reference (text: or props:) in VALUE, look
+ * it up in FS and return it in *REPRESENTATION. To be able to parse the
+ * base rep, we pass the FILE_CONTENT as well.
+ *
+ * If necessary, allocate the result in POOL; use SCRATCH_POOL for temp.
+ * allocations.
+ */
static svn_error_t *
parse_representation(representation_t **representation,
fs_fs_t *fs,
@@ -700,15 +819,20 @@ parse_representation(representation_t **
apr_uint64_t expanded_size;
int idx;
+ /* read location (revision, offset) and size */
char *c = (char *)value->data;
SVN_ERR(svn_revnum_parse(&revision, svn_cstring_tokenize(" ", &c), NULL));
SVN_ERR(svn_cstring_strtoui64(&offset, svn_cstring_tokenize(" ", &c), 0, APR_SIZE_MAX, 10));
SVN_ERR(svn_cstring_strtoui64(&size, svn_cstring_tokenize(" ", &c), 0, APR_SIZE_MAX, 10));
SVN_ERR(svn_cstring_strtoui64(&expanded_size, svn_cstring_tokenize(" ", &c), 0, APR_SIZE_MAX, 10));
+ /* look it up */
result = find_representation(&idx, fs, &revision_info, revision, (apr_size_t)offset);
if (!result)
{
+ /* not parsed, yet (probably a rep in the same revision).
+ * Create a new rep object and determine its base rep as well.
+ */
apr_size_t header_size;
svn_boolean_t is_plain;
@@ -732,8 +856,10 @@ parse_representation(representation_t **
return SVN_NO_ERROR;
}
-/* Get the file content of revision REVISION in FS and return it in *DATA.
- * Use SCRATCH_POOL for temporary allocations.
+/* Get the unprocessed (i.e. still deltified) content of REPRESENTATION in
+ * FS and return it in *CONTENT. If no NULL, FILE_CONTENT must contain
+ * the contents of the revision that also contains the representation.
+ * Use POOL for allocations.
*/
static svn_error_t *
get_rep_content(svn_stringbuf_t **content,
@@ -765,7 +891,7 @@ get_rep_content(svn_stringbuf_t **conten
offset = revision_info->offset
+ representation->offset
+ representation->header_size;
- SVN_ERR(get_content(content, fs, revision, offset,
+ SVN_ERR(get_content(content, NULL, fs, revision, offset,
representation->size, pool));
}
@@ -773,8 +899,12 @@ get_rep_content(svn_stringbuf_t **conten
}
-/* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta
- window into *NWIN. */
+/* Read the delta window contents of all windows in REPRESENTATION in FS.
+ * If no NULL, FILE_CONTENT must contain the contents of the revision that
+ * also contains the representation.
+ * Return the data as svn_txdelta_window_t* instances in *WINDOWS.
+ * Use POOL for allocations.
+ */
static svn_error_t *
read_windows(apr_array_header_t **windows,
fs_fs_t *fs,
@@ -789,13 +919,16 @@ read_windows(apr_array_header_t **window
*windows = apr_array_make(pool, 0, sizeof(svn_txdelta_window_t *));
+ /* get the whole revision content */
SVN_ERR(get_rep_content(&content, fs, representation, file_content, pool));
+ /* create a read stream and position it directly after the rep header */
content->data += 3;
content->len -= 3;
stream = svn_stream_from_stringbuf(content, pool);
SVN_ERR(svn_stream_read(stream, &version, &len));
+ /* read the windows from that stream */
while (TRUE)
{
svn_txdelta_window_t *window;
@@ -816,9 +949,12 @@ read_windows(apr_array_header_t **window
return SVN_NO_ERROR;
}
-/* Get the undeltified window that is a result of combining all deltas
- from the current desired representation identified in *RB with its
- base representation. Store the window in *RESULT. */
+/* Get the undeltified representation that is a result of combining all
+ * deltas from the current desired REPRESENTATION in FS with its base
+ * representation. If no NULL, FILE_CONTENT must contain the contents of
+ * the revision that also contains the representation. Store the result
+ * in *CONTENT. Use POOL for allocations.
+ */
static svn_error_t *
get_combined_window(svn_stringbuf_t **content,
fs_fs_t *fs,
@@ -830,23 +966,36 @@ get_combined_window(svn_stringbuf_t **co
apr_array_header_t *windows;
svn_stringbuf_t *base_content, *result;
const char *source;
- apr_pool_t *sub_pool = svn_pool_create(pool);
- apr_pool_t *iter_pool = svn_pool_create(pool);
+ apr_pool_t *sub_pool;
+ apr_pool_t *iter_pool;
+ /* special case: no un-deltification necessary */
if (representation->is_plain)
- return get_rep_content(content, fs, representation, file_content, pool);
+ {
+ SVN_ERR(get_rep_content(content, fs, representation, file_content,
+ pool));
+ SVN_ERR(set_cached_window(fs, representation, *content, pool));
+ return SVN_NO_ERROR;
+ }
- *content = get_cached_window(fs, representation, pool);
+ /* special case: data already in cache */
+ SVN_ERR(get_cached_window(content, fs, representation, pool));
if (*content)
return SVN_NO_ERROR;
+ /* read the delta windows for this representation */
+ sub_pool = svn_pool_create(pool);
+ iter_pool = svn_pool_create(pool);
SVN_ERR(read_windows(&windows, fs, representation, file_content, sub_pool));
+
+ /* fetch the / create a base content */
if (representation->delta_base && representation->delta_base->revision)
SVN_ERR(get_combined_window(&base_content, fs,
representation->delta_base, NULL, sub_pool));
else
base_content = svn_stringbuf_create_empty(sub_pool);
+ /* apply deltas */
result = svn_stringbuf_create_empty(pool);
source = base_content->data;
@@ -867,14 +1016,17 @@ get_combined_window(svn_stringbuf_t **co
svn_pool_clear(iter_pool);
}
+ /* cache result and return it */
+ SVN_ERR(set_cached_window(fs, representation, result, sub_pool));
+ *content = result;
+
svn_pool_destroy(iter_pool);
svn_pool_destroy(sub_pool);
-
- set_cached_window(fs, representation, result);
- *content = result;
+
return SVN_NO_ERROR;
}
+/* forward declaration */
static svn_error_t *
read_noderev(fs_fs_t *fs,
svn_stringbuf_t *file_content,
@@ -883,6 +1035,12 @@ read_noderev(fs_fs_t *fs,
apr_pool_t *pool,
apr_pool_t *scratch_pool);
+/* Starting at the directory in REPRESENTATION in FILE_CONTENT, read all
+ * DAG nodes, directories and representations linked in that tree structure.
+ * Store them in FS and REVISION_INFO. Also, read them only once.
+ *
+ * Use POOL for persistent allocations and SCRATCH_POOL for temporaries.
+ */
static svn_error_t *
parse_dir(fs_fs_t *fs,
svn_stringbuf_t *file_content,
@@ -898,9 +1056,11 @@ parse_dir(fs_fs_t *fs,
const char *revision_key;
apr_size_t key_len;
+ /* special case: empty dir rep */
if (representation == NULL)
return SVN_NO_ERROR;
+ /* get the directory as unparsed string */
iter_pool = svn_pool_create(scratch_pool);
text_pool = svn_pool_create(scratch_pool);
@@ -908,14 +1068,16 @@ parse_dir(fs_fs_t *fs,
text_pool));
current = text->data;
+ /* calculate some invariants */
revision_key = apr_psprintf(text_pool, "r%ld/", representation->revision);
key_len = strlen(revision_key);
- /* Translate the string dir entries into real entries. */
+ /* Parse and process all directory entries. */
while (*current != 'E')
{
char *next;
+ /* skip "K ???\n<name>\nV ???\n" lines*/
current = strchr(current, '\n');
if (current)
current = strchr(current+1, '\n');
@@ -927,11 +1089,14 @@ parse_dir(fs_fs_t *fs,
_("Corrupt directory representation in rev %ld at offset %ld"),
representation->revision,
(long)representation->offset);
-
+
+ /* iff this entry refers to a node in the same revision as this dir,
+ * recurse into that node */
*next = 0;
current = strstr(current, revision_key);
if (current)
{
+ /* recurse */
apr_uint64_t offset;
SVN_ERR(svn_cstring_strtoui64(&offset, current + key_len, 0,
@@ -949,6 +1114,13 @@ parse_dir(fs_fs_t *fs,
return SVN_NO_ERROR;
}
+/* Starting at the noderev at OFFSET in FILE_CONTENT, read all DAG nodes,
+ * directories and representations linked in that tree structure. Store
+ * them in FS and REVISION_INFO. Also, read them only once. Return the
+ * result in *NODEREV.
+ *
+ * Use POOL for persistent allocations and SCRATCH_POOL for temporaries.
+ */
static svn_error_t *
read_noderev(fs_fs_t *fs,
svn_stringbuf_t *file_content,
@@ -962,11 +1134,14 @@ read_noderev(fs_fs_t *fs,
representation_t *props = NULL;
apr_size_t start_offset = offset;
svn_boolean_t is_dir = FALSE;
+ const char *path = "???";
scratch_pool = svn_pool_create(scratch_pool);
-
+
+ /* parse the noderev line-by-line until we find an empty line */
while (1)
{
+ /* for this line, extract key and value. Ignore invalid values */
svn_string_t key;
svn_string_t value;
char *sep;
@@ -975,6 +1150,8 @@ read_noderev(fs_fs_t *fs,
line = svn_string_ncreate(start, end - start, scratch_pool);
offset += end - start + 1;
+
+ /* empty line -> end of noderev data */
if (line->len == 0)
break;
@@ -992,6 +1169,7 @@ read_noderev(fs_fs_t *fs,
value.data = sep + 2;
value.len = line->len - (key.len + 2);
+ /* translate (key, value) into noderev elements */
if (key_matches(&key, "type"))
is_dir = strcmp(value.data, "dir") == 0;
else if (key_matches(&key, "text"))
@@ -999,6 +1177,8 @@ read_noderev(fs_fs_t *fs,
SVN_ERR(parse_representation(&text, fs, file_content,
&value, revision_info,
pool, scratch_pool));
+
+ /* if we are the first to use this rep, mark it as "text rep" */
if (++text->ref_count == 1)
text->kind = is_dir ? dir_rep : file_rep;
}
@@ -1007,15 +1187,26 @@ read_noderev(fs_fs_t *fs,
SVN_ERR(parse_representation(&props, fs, file_content,
&value, revision_info,
pool, scratch_pool));
+
+ /* if we are the first to use this rep, mark it as "prop rep" */
if (++props->ref_count == 1)
props->kind = is_dir ? dir_property_rep : file_property_rep;
}
+ else if (key_matches(&key, "cpath"))
+ path = value.data;
}
+ /* record largest changes */
+ if (text && text->ref_count == 1)
+ add_change(fs, text->size, text->revision, path);
+
+ /* if this is a directory and has not been processed, yet, read and
+ * process it recursively */
if (is_dir && text && text->ref_count == 1)
SVN_ERR(parse_dir(fs, file_content, text, revision_info,
pool, scratch_pool));
+ /* update stats */
if (is_dir)
{
revision_info->dir_noderev_size += offset - start_offset;
@@ -1031,6 +1222,9 @@ read_noderev(fs_fs_t *fs,
return SVN_NO_ERROR;
}
+/* Given the unparsed changes list in CHANGES with LEN chars, return the
+ * number of changed paths encoded in it.
+ */
static apr_size_t
get_change_count(const char *changes,
apr_size_t len)
@@ -1038,19 +1232,27 @@ get_change_count(const char *changes,
apr_size_t lines = 0;
const char *end = changes + len;
+ /* line count */
for (; changes < end; ++changes)
if (*changes == '\n')
++lines;
+ /* two lines per change */
return lines / 2;
}
-static void print_progress(svn_revnum_t revision)
+/* Simple utility to print a REVISION number and make it appear immediately.
+ */
+static void
+print_progress(svn_revnum_t revision)
{
printf("%8ld", revision);
fflush(stdout);
}
+/* Read the content of the pack file staring at revision BASE and store it
+ * in FS. Use POOL for allocations.
+ */
static svn_error_t *
read_pack_file(fs_fs_t *fs,
svn_revnum_t base,
@@ -1061,19 +1263,24 @@ read_pack_file(fs_fs_t *fs,
apr_pool_t *iter_pool = svn_pool_create(local_pool);
int i;
apr_off_t file_size = 0;
+ apr_file_t *file;
const char *pack_folder = get_pack_folder(fs, base, local_pool);
+ /* parse the manifest file */
SVN_ERR(read_manifest(&manifest, fs, pack_folder, local_pool));
if (manifest->nelts != fs->max_files_per_dir)
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, NULL);
- SVN_ERR(rev_or_pack_file_size(&file_size, fs, base, pool));
+ SVN_ERR(open_rev_or_pack_file(&file, fs, base, local_pool));
+ SVN_ERR(get_file_size(&file_size, file, local_pool));
+ /* process each revision in the pack file */
for (i = 0; i < manifest->nelts; ++i)
{
apr_size_t root_node_offset;
svn_stringbuf_t *rev_content;
+ /* create the revision info for the current rev */
revision_info_t *info = apr_pcalloc(pool, sizeof(*info));
info->representations = apr_array_make(iter_pool, 4, sizeof(representation_t*));
@@ -1083,7 +1290,7 @@ read_pack_file(fs_fs_t *fs,
? APR_ARRAY_IDX(manifest, i+1 , apr_size_t)
: file_size;
- SVN_ERR(get_content(&rev_content, fs, info->revision,
+ SVN_ERR(get_content(&rev_content, file, fs, info->revision,
info->offset,
info->end - info->offset,
iter_pool));
@@ -1103,15 +1310,20 @@ read_pack_file(fs_fs_t *fs,
info->representations = apr_array_copy(pool, info->representations);
APR_ARRAY_PUSH(fs->revisions, revision_info_t*) = info;
+ /* destroy temps */
svn_pool_clear(iter_pool);
}
+ /* one more pack file processed */
print_progress(base);
- apr_pool_destroy(local_pool);
+ svn_pool_destroy(local_pool);
return SVN_NO_ERROR;
}
+/* Read the content of the file for REVSION and store its contents in FS.
+ * Use POOL for allocations.
+ */
static svn_error_t *
read_revision_file(fs_fs_t *fs,
svn_revnum_t revision,
@@ -1122,16 +1334,21 @@ read_revision_file(fs_fs_t *fs,
svn_stringbuf_t *rev_content;
revision_info_t *info = apr_pcalloc(pool, sizeof(*info));
apr_off_t file_size = 0;
+ apr_file_t *file;
- SVN_ERR(rev_or_pack_file_size(&file_size, fs, revision, pool));
+ /* read the whole pack file into memory */
+ SVN_ERR(open_rev_or_pack_file(&file, fs, revision, local_pool));
+ SVN_ERR(get_file_size(&file_size, file, local_pool));
+ /* create the revision info for the current rev */
info->representations = apr_array_make(pool, 4, sizeof(representation_t*));
info->revision = revision;
info->offset = 0;
info->end = file_size;
- SVN_ERR(get_content(&rev_content, fs, revision, 0, file_size, local_pool));
+ SVN_ERR(get_content(&rev_content, file, fs, revision, 0, file_size,
+ local_pool));
SVN_ERR(read_revision_header(&info->changes,
&info->changes_len,
@@ -1139,24 +1356,31 @@ read_revision_file(fs_fs_t *fs,
rev_content,
local_pool));
+ /* put it into our containers */
APR_ARRAY_PUSH(fs->revisions, revision_info_t*) = info;
info->change_count
= get_change_count(rev_content->data + info->changes,
info->changes_len);
+ /* parse the revision content recursively. */
SVN_ERR(read_noderev(fs, rev_content,
root_node_offset, info,
pool, local_pool));
+ /* show progress every 1000 revs or so */
if (revision % fs->max_files_per_dir == 0)
print_progress(revision);
- apr_pool_destroy(local_pool);
+ svn_pool_destroy(local_pool);
return SVN_NO_ERROR;
}
+/* Read the repository at PATH beginning with revision START_REVISION and
+ * return the result in *FS. Allocate caches with MEMSIZE bytes total
+ * capacity. Use POOL for non-cache allocations.
+ */
static svn_error_t *
read_revisions(fs_fs_t **fs,
const char *path,
@@ -1165,63 +1389,98 @@ read_revisions(fs_fs_t **fs,
apr_pool_t *pool)
{
svn_revnum_t revision;
- apr_size_t window_cache_size;
+ svn_cache_config_t cache_config = *svn_cache_config_get();
/* determine cache sizes */
if (memsize < 100)
memsize = 100;
-
- window_cache_size = memsize * 1024 * 1024;
+
+ cache_config.cache_size = memsize * 1024 * 1024;
+ svn_cache_config_set(&cache_config);
SVN_ERR(fs_open(fs, path, pool));
+ /* create data containers and caches */
(*fs)->start_revision = start_revision
- (start_revision % (*fs)->max_files_per_dir);
(*fs)->revisions = apr_array_make(pool,
(*fs)->max_revision + 1 - (*fs)->start_revision,
sizeof(revision_info_t *));
(*fs)->null_base = apr_pcalloc(pool, sizeof(*(*fs)->null_base));
- (*fs)->window_cache = create_window_cache
- (apr_allocator_owner_get
- (svn_pool_create_allocator(FALSE)),
- 10000, window_cache_size);
+ initialize_largest_changes(*fs, 64, pool);
+
+ SVN_ERR(svn_cache__create_membuffer_cache(&(*fs)->window_cache,
+ svn_cache__get_global_membuffer_cache(),
+ NULL, NULL,
+ sizeof(window_cache_key_t),
+ "", FALSE, pool));
+ /* read all packed revs */
for ( revision = start_revision
; revision < (*fs)->min_unpacked_rev
; revision += (*fs)->max_files_per_dir)
SVN_ERR(read_pack_file(*fs, revision, pool));
-
+
+ /* read non-packed revs */
for ( ; revision <= (*fs)->max_revision; ++revision)
SVN_ERR(read_revision_file(*fs, revision, pool));
return SVN_NO_ERROR;
}
+/* Compression statistics we collect over a given set of representations.
+ */
typedef struct rep_pack_stats_t
{
+ /* number of representations */
apr_int64_t count;
+
+ /* total size after deltification (i.e. on disk size) */
apr_int64_t packed_size;
+
+ /* total size after de-deltification (i.e. plain text size) */
apr_int64_t expanded_size;
+
+ /* total on-disk header size */
apr_int64_t overhead_size;
} rep_pack_stats_t;
+/* Statistics we collect over a given set of representations.
+ * We group them into shared and non-shared ("unique") reps.
+ */
typedef struct representation_stats_t
{
+ /* stats over all representations */
rep_pack_stats_t total;
+
+ /* stats over those representations with ref_count == 1 */
rep_pack_stats_t uniques;
+
+ /* stats over those representations with ref_count > 1 */
rep_pack_stats_t shared;
+ /* sum of all ref_counts */
apr_int64_t references;
+
+ /* sum of ref_count * expanded_size,
+ * i.e. total plaintext content if there was no rep sharing */
apr_int64_t expanded_size;
} representation_stats_t;
+/* Basic statistics we collect over a given set of noderevs.
+ */
typedef struct node_stats_t
{
+ /* number of noderev structs */
apr_int64_t count;
+
+ /* their total size on disk (structs only) */
apr_int64_t size;
} node_stats_t;
+/* Accumulate stats of REP in STATS.
+ */
static void
add_rep_pack_stats(rep_pack_stats_t *stats,
representation_t *rep)
@@ -1230,9 +1489,11 @@ add_rep_pack_stats(rep_pack_stats_t *sta
stats->packed_size += rep->size;
stats->expanded_size += rep->expanded_size;
- stats->overhead_size += rep->header_size + 7;
+ stats->overhead_size += rep->header_size + 7 /* ENDREP\n */;
}
+/* Accumulate stats of REP in STATS.
+ */
static void
add_rep_stats(representation_stats_t *stats,
representation_t *rep)
@@ -1247,6 +1508,9 @@ add_rep_stats(representation_stats_t *st
stats->expanded_size += rep->ref_count * rep->expanded_size;
}
+/* Print statistics for the given group of representations to console.
+ * Use POOL for allocations.
+ */
static void
print_rep_stats(representation_stats_t *stats,
apr_pool_t *pool)
@@ -1267,12 +1531,30 @@ print_rep_stats(representation_stats_t *
svn__i64toa_sep(stats->references - stats->total.count, ',', pool));
}
+/* Print the (used) contents of CHANGES. Use POOL for allocations.
+ */
+static void
+print_largest_reps(largest_changes_t *changes,
+ apr_pool_t *pool)
+{
+ apr_size_t i;
+ for (i = 0; i < changes->count && changes->changes[i]->size; ++i)
+ printf(_("%12s r%-8ld %s\n"),
+ svn__i64toa_sep(changes->changes[i]->size, ',', pool),
+ changes->changes[i]->revision,
+ changes->changes[i]->path->data);
+}
+
+/* Post-process stats for FS and print them to the console.
+ * Use POOL for allocations.
+ */
static void
print_stats(fs_fs_t *fs,
apr_pool_t *pool)
{
int i, k;
-
+
+ /* initialize stats to collect */
representation_stats_t file_rep_stats = { { 0 } };
representation_stats_t dir_rep_stats = { { 0 } };
representation_stats_t file_prop_rep_stats = { { 0 } };
@@ -1286,11 +1568,14 @@ print_stats(fs_fs_t *fs,
apr_int64_t total_size = 0;
apr_int64_t change_count = 0;
apr_int64_t change_len = 0;
-
+
+ /* aggregate info from all revisions */
for (i = 0; i < fs->revisions->nelts; ++i)
{
revision_info_t *revision = APR_ARRAY_IDX(fs->revisions, i,
revision_info_t *);
+
+ /* data gathered on a revision level */
change_count += revision->change_count;
change_len += revision->changes_len;
total_size += revision->end - revision->offset;
@@ -1303,11 +1588,14 @@ print_stats(fs_fs_t *fs,
+ revision->file_noderev_count;
total_node_stats.size += revision->dir_noderev_size
+ revision->file_noderev_size;
-
+
+ /* process representations */
for (k = 0; k < revision->representations->nelts; ++k)
{
representation_t *rep = APR_ARRAY_IDX(revision->representations,
k, representation_t *);
+
+ /* accumulate in the right bucket */
switch(rep->kind)
{
case file_rep:
@@ -1330,6 +1618,7 @@ print_stats(fs_fs_t *fs,
}
}
+ /* print results */
printf("\nGlobal statistics:\n");
printf(_("%20s bytes in %12s revisions\n"
"%20s bytes in %12s changes\n"
@@ -1386,24 +1675,30 @@ print_stats(fs_fs_t *fs,
print_rep_stats(&dir_prop_rep_stats, pool);
printf("\nFile property representation statistics:\n");
print_rep_stats(&file_prop_rep_stats, pool);
+ printf("\nLargest representations:\n");
+ print_largest_reps(fs->largest_changes, pool);
}
+/* Write tool usage info text to OSTREAM using PROGNAME as a prefix and
+ * POOL for allocations.
+ */
static void
print_usage(svn_stream_t *ostream, const char *progname,
apr_pool_t *pool)
{
svn_error_clear(svn_stream_printf(ostream, pool,
"\n"
- "Usage: %s <repo> <cachesize>\n"
+ "Usage: %s <repo> [cachesize]\n"
"\n"
"Read the repository at local path <repo> starting at revision 0,\n"
"count statistical information and write that data to stdout.\n"
- "Use up to <cachesize> MB of memory for caching. This does not include\n"
+ "Use up to [cachesize] MB of memory for caching. This does not include\n"
"temporary representation of the repository structure, i.e. the actual\n"
- "memory may be considerably higher.\n",
+ "memory may be considerably higher. If not given, defaults to 100 MB.\n",
progname));
}
+/* linear control flow */
int main(int argc, const char *argv[])
{
apr_pool_t *pool;
@@ -1411,7 +1706,7 @@ int main(int argc, const char *argv[])
svn_error_t *svn_err;
const char *repo_path = NULL;
svn_revnum_t start_revision = 0;
- apr_size_t memsize = 0;
+ apr_size_t memsize = 100;
apr_uint64_t temp = 0;
fs_fs_t *fs;
@@ -1427,22 +1722,26 @@ int main(int argc, const char *argv[])
return 2;
}
- if (argc != 3)
+ if (argc < 2 || argc > 3)
{
print_usage(ostream, argv[0], pool);
return 2;
}
- svn_err = svn_cstring_strtoui64(&temp, argv[2], 0, APR_SIZE_MAX, 10);
- if (svn_err)
+ if (argc == 3)
{
- print_usage(ostream, argv[0], pool);
- svn_error_clear(svn_err);
- return 2;
+ svn_err = svn_cstring_strtoui64(&temp, argv[2], 0, APR_SIZE_MAX, 10);
+ if (svn_err)
+ {
+ print_usage(ostream, argv[0], pool);
+ svn_error_clear(svn_err);
+ return 2;
+ }
+
+ memsize = (apr_size_t)temp;
}
- memsize = (apr_size_t)temp;
- repo_path = argv[1];
+ repo_path = svn_dirent_canonicalize(argv[1], pool);
start_revision = 0;
printf("Reading revisions\n");
Propchange: subversion/branches/fsfs-format7/tools/server-side/fsfs-stats.c
------------------------------------------------------------------------------
svn:eol-style = native
Modified: subversion/branches/fsfs-format7/tools/server-side/svnpubsub/README.txt
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/server-side/svnpubsub/README.txt?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/server-side/svnpubsub/README.txt (original)
+++ subversion/branches/fsfs-format7/tools/server-side/svnpubsub/README.txt Mon Feb 4 20:48:05 2013
@@ -1,21 +1,3 @@
-### write a README
-
-
-TODO:
-- bulk update at startup time to avoid backlog warnings
-- switch to host:port format in config file
-- fold BDEC into Daemon
-- fold WorkingCopy._get_match() into __init__
-- remove wc_ready(). assume all WorkingCopy instances are usable.
- place the instances into .watch at creation. the .update_applies()
- just returns if the wc is disabled (eg. could not find wc dir)
-- figure out way to avoid the ASF-specific PRODUCTION_RE_FILTER
- (a base path exclusion list should work for the ASF)
-- add support for SIGHUP to reread the config and reinitialize working copies
-- joes will write documentation for svnpubsub as these items become fulfilled
-- make LOGLEVEL configurable
-
-
Installation instructions:
1. Set up an svnpubsub service.
@@ -39,4 +21,4 @@ Installation instructions:
3. Set up svnpubsub clients.
(eg svnwcsub.py, svnpubsub/client.py,
- 'curl -i http://${hostname}:2069/commits/json')
+ 'curl -sN http://${hostname}:2069/commits')
Modified: subversion/branches/fsfs-format7/tools/server-side/svnpubsub/commit-hook.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/server-side/svnpubsub/commit-hook.py?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/server-side/svnpubsub/commit-hook.py (original)
+++ subversion/branches/fsfs-format7/tools/server-side/svnpubsub/commit-hook.py Mon Feb 4 20:48:05 2013
@@ -42,26 +42,28 @@ def svncmd_uuid(repo):
def svncmd_info(repo, revision):
cmd = "%s info -r %s %s" % (SVNLOOK, revision, repo)
p = svncmd(cmd)
- data = p.stdout.read().strip().split("\n")
+ data = p.stdout.read().split("\n")
#print data
- return {'author': data[0],
- 'date': data[1],
- 'log': "\n".join(data[3:])}
+ return {'author': data[0].strip(),
+ 'date': data[1].strip(),
+ 'log': "\n".join(data[3:]).strip()}
-def svncmd_dirs(repo, revision):
- cmd = "%s dirs-changed -r %s %s" % (SVNLOOK, revision, repo)
+def svncmd_changed(repo, revision):
+ cmd = "%s changed -r %s %s" % (SVNLOOK, revision, repo)
p = svncmd(cmd)
- dirs = []
+ changed = {}
while True:
line = p.stdout.readline()
if not line:
break
- dirs.append(line.strip())
- return dirs
+ line = line.strip()
+ (flags, filename) = (line[0:3], line[4:])
+ changed[filename] = {'flags': flags}
+ return changed
def do_put(body):
opener = urllib2.build_opener(urllib2.HTTPHandler)
- request = urllib2.Request("http://%s:%d/dirs-changed" %(HOST, PORT), data=body)
+ request = urllib2.Request("http://%s:%d/commits" %(HOST, PORT), data=body)
request.add_header('Content-Type', 'application/json')
request.get_method = lambda: 'PUT'
url = opener.open(request)
@@ -70,16 +72,17 @@ def do_put(body):
def main(repo, revision):
revision = revision.lstrip('r')
i = svncmd_info(repo, revision)
- data = {'revision': int(revision),
- 'dirs_changed': [],
- 'repos': svncmd_uuid(repo),
- 'author': i['author'],
+ data = {'type': 'svn',
+ 'format': 1,
+ 'id': int(revision),
+ 'changed': {},
+ 'repository': svncmd_uuid(repo),
+ 'committer': i['author'],
'log': i['log'],
'date': i['date'],
}
- data['dirs_changed'].extend(svncmd_dirs(repo, revision))
+ data['changed'].update(svncmd_changed(repo, revision))
body = json.dumps(data)
- #print body
do_put(body)
if __name__ == "__main__":
Modified: subversion/branches/fsfs-format7/tools/server-side/svnpubsub/daemonize.py
URL: http://svn.apache.org/viewvc/subversion/branches/fsfs-format7/tools/server-side/svnpubsub/daemonize.py?rev=1442344&r1=1442343&r2=1442344&view=diff
==============================================================================
--- subversion/branches/fsfs-format7/tools/server-side/svnpubsub/daemonize.py (original)
+++ subversion/branches/fsfs-format7/tools/server-side/svnpubsub/daemonize.py Mon Feb 4 20:48:05 2013
@@ -50,11 +50,11 @@ class Daemon(object):
def daemonize_exit(self):
try:
result = self.daemonize()
- except (ChildFailed, DaemonFailed), e:
+ except (ChildFailed, DaemonFailed) as e:
# duplicate the exit code
sys.exit(e.code)
except (ChildTerminatedAbnormally, ChildForkFailed,
- DaemonTerminatedAbnormally, DaemonForkFailed), e:
+ DaemonTerminatedAbnormally, DaemonForkFailed) as e:
sys.stderr.write('ERROR: %s\n' % e)
sys.exit(1)
except ChildResumedIncorrectly:
@@ -74,7 +74,7 @@ class Daemon(object):
# fork off a child that can detach itself from this process.
try:
pid = os.fork()
- except OSError, e:
+ except OSError as e:
raise ChildForkFailed(e.errno, e.strerror)
if pid > 0:
@@ -113,7 +113,7 @@ class Daemon(object):
# perform the second fork
try:
pid = os.fork()
- except OSError, e:
+ except OSError as e:
raise DaemonForkFailed(e.errno, e.strerror)
if pid > 0: