You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by ju...@apache.org on 2013/06/04 22:56:33 UTC
svn commit: r1489602 [3/3] - in /subversion/branches/move-tracking-1: ./
build/generator/ contrib/hook-scripts/ notes/
subversion/bindings/swig/perl/libsvn_swig_perl/
subversion/bindings/swig/perl/native/ subversion/bindings/swig/ruby/test/
subversion/...
Modified: subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl (original)
+++ subversion/branches/move-tracking-1/tools/dev/build-svn-deps-win.pl Tue Jun 4 20:56:23 2013
@@ -74,6 +74,8 @@ our $CMAKE = 'cmake';
our $NMAKE = 'nmake';
# Use the .com version so we get output, the .exe doesn't produce any output
our $DEVENV = 'devenv.com';
+our $VCUPGRADE = 'vcupgrade';
+our $PYTHON = 'python';
# Versions of the dependencies we will use
# Change these if you want but these are known to work with
@@ -87,7 +89,8 @@ our $OPENSSL_VER = '1.0.1e';
our $PCRE_VER = '8.32';
our $BDB_VER = '5.3.21';
our $SQLITE_VER = '3071602';
-our $SERF_VER = '1.2.0';
+our $SERF_VER = '1.2.1';
+our $NEON_VER = '0.29.6';
# Sources for files to download
our $AWK_URL = 'http://www.cs.princeton.edu/~bwk/btl.mirror/awk95.exe';
@@ -101,6 +104,8 @@ our $PCRE_URL;
our $BDB_URL;
our $SQLITE_URL;
our $SERF_URL;
+our $NEON_URL;
+our $PROJREF_URL = 'https://downloads.redhoundsoftware.com/blog/ProjRef.py';
# Location of the already downloaded file.
# by default these are undefined and set by the downloader.
@@ -118,6 +123,8 @@ our $PCRE_FILE;
our $BDB_FILE;
our $SQLITE_FILE;
our $SERF_FILE;
+our $NEON_FILE;
+our $PROJREF_FILE;
# Various directories we use
our $TOPDIR = Cwd::cwd(); # top of our tree
@@ -125,6 +132,11 @@ our $INSTDIR; # where we install to
our $BLDDIR; # directory where we actually build
our $SRCDIR; # directory where we store package files
+# Some other options
+our $VS_VER;
+our $NEON;
+our $SVN_VER = '1.8.x';
+
# Utility function to remove dots from a string
sub remove_dots {
my $in = shift;
@@ -143,6 +155,16 @@ sub set_default {
}
}
+sub set_svn_ver_defaults {
+ my ($svn_major, $svn_minor, $svn_patch) = $SVN_VER =~ /^(\d+)\.(\d+)\.(.+)$/;
+
+ if ($svn_major > 1 or ($svn_major == 1 and $svn_minor >= 8)) {
+ $NEON=0 unless defined($NEON);
+ } else {
+ $NEON=1 unless defined($NEON);
+ }
+}
+
# Any variables with defaults that reference other values
# should be set here. This defers setting of the default until runtime in these cases.
sub set_defaults {
@@ -156,9 +178,11 @@ sub set_defaults {
set_default(\$BDB_URL, "http://download.oracle.com/berkeley-db/db-5.3.21.zip");
set_default(\$SQLITE_URL, "http://www.sqlite.org/2013/sqlite-amalgamation-$SQLITE_VER.zip");
set_default(\$SERF_URL, "http://serf.googlecode.com/files/serf-$SERF_VER.zip");
+ set_default(\$NEON_URL, "http://www.webdav.org/neon/neon-$NEON_VER.tar.gz");
set_default(\$INSTDIR, $TOPDIR);
set_default(\$BLDDIR, "$TOPDIR\\build");
set_default(\$SRCDIR, "$TOPDIR\\sources");
+ set_svn_ver_defaults();
}
#################################
@@ -277,6 +301,27 @@ sub modify_file_in_place {
close(OUT);
}
+sub check_vs_ver {
+ return if defined($VS_VER);
+
+ # using the vcupgrade command here because it has a consistent name and version
+ # numbering across versions including express versions.
+ my $help_output = `"$VCUPGRADE" /?`;
+ my ($major_version) = $help_output =~ /Version (\d+)\./s;
+
+ if (defined($major_version)) {
+ if ($major_version eq '11') {
+ $VS_VER = '2012';
+ return;
+ } elsif ($major_version eq '10') {
+ $VS_VER = '2010';
+ return;
+ }
+ }
+
+ die("Visual Studio Version Not Supported");
+}
+
##################
# TREE STRUCTURE #
##################
@@ -363,6 +408,10 @@ sub download_dependencies {
unless(-x "$BINDIR\\awk.exe") { # skip the copy if it exists
copy_or_die($AWK_FILE, "$BINDIR\\awk.exe");
}
+ download_file($PROJREF_URL, "$SRCDIR\\ProjRef.py", \$PROJREF_FILE);
+ unless(-x "$BINDIR\\ProjRef.py") { # skip the copy if it exists
+ copy_or_die($PROJREF_FILE, $BINDIR);
+ }
download_file($BDB_URL, "$SRCDIR\\db.zip", \$BDB_FILE);
download_file($ZLIB_URL, "$SRCDIR\\zlib.zip", \$ZLIB_FILE);
download_file($OPENSSL_URL, "$SRCDIR\\openssl.tar.gz", \$OPENSSL_FILE);
@@ -373,6 +422,7 @@ sub download_dependencies {
download_file($PCRE_URL, "$SRCDIR\\pcre.zip", \$PCRE_FILE);
download_file($SQLITE_URL, "$SRCDIR\\sqlite-amalgamation.zip", \$SQLITE_FILE);
download_file($SERF_URL, "$SRCDIR\\serf.zip", \$SERF_FILE);
+ download_file($NEON_URL, "$SRCDIR\\neon.tar.gz", \$NEON_FILE) if defined($NEON);
}
##############
@@ -437,6 +487,8 @@ sub extract_dependencies {
"$INSTDIR\\sqlite-amalgamation");
extract_file($SERF_FILE, $INSTDIR,
"$INSTDIR\\serf-$SERF_VER", "$INSTDIR\\serf");
+ extract_file($NEON_FILE, $INSTDIR,
+ "$INSTDIR\\neon-$NEON_VER", "$INSTDIR\\neon") if defined($NEON);
}
#########
@@ -507,6 +559,8 @@ sub build_openssl {
# Visual Studio whining about its backup step.
sub upgrade_solution {
my $file = shift;
+ my $interactive = shift;
+ my $flags = "";
my ($basename, $directories) = fileparse($file, qr/\.[^.]*$/);
my $sln = $directories . $basename . '.sln';
@@ -519,7 +573,15 @@ sub upgrade_solution {
close(SLN);
}
print "Upgrading $file (this may take a while)\n";
- system_or_die("Failure upgrading $file", qq("$DEVENV" $file /Upgrade));
+ $flags = " /Upgrade" unless $interactive;
+ system_or_die("Failure upgrading $file", qq("$DEVENV" "$file"$flags));
+ if ($interactive) {
+ print "Can't do automatic upgrade, doing interactive upgrade\n";
+ print "IDE will load, choose to convert all projects, exit the IDE and\n";
+ print "save the resulting solution file\n\n";
+ print "Press Enter to Continue\n";
+ <>;
+ }
}
# Run the lineends.pl script
@@ -629,6 +691,9 @@ sub httpd_enable_bdb {
sub build_httpd {
chdir_or_die($HTTPD);
+ my $vs_2012 = $VS_VER eq '2012';
+ my $vs_2010 = $VS_VER eq '2010';
+
# I don't think cvtdsp.pl is necessary with Visual Studio 2012
# but it shouldn't hurt anything either. Including it allows
# for the possibility that this may work for older Visual Studio
@@ -636,20 +701,10 @@ sub build_httpd {
system_or_die("Failure converting DSP files",
qq("$PERL" srclib\\apr\\build\\cvtdsp.pl -2005));
- upgrade_solution('Apache.dsw');
+ upgrade_solution('Apache.dsw', $vs_2010);
httpd_enable_bdb();
httpd_fix_makefile('Makefile.win');
- # Turn off pre-compiled headers for apr-iconv to avoid:
- # LNK2011: http://msdn.microsoft.com/en-us/library/3ay26wa2(v=vs.110).aspx
- disable_pch('srclib\apr-iconv\build\modules.mk.win');
-
- # ApacheMonitor build fails due a duplicate manifest, turn off
- # GenerateManifest
- insert_property_group('support\win32\ApacheMonitor.vcxproj',
- '<GenerateManifest>false</GenerateManifest>',
- '.dupman');
-
# Modules and support projects randomly fail due to an error about the
# CL.read.1.tlog file already existing. This is really because of the
# intermediate dirs being shared between modules, but for the time being
@@ -660,25 +715,40 @@ sub build_httpd {
}
}, 'modules', 'support');
- # The APR libraries have projects named libapr but produce output named libapr-1
- # The problem with this is in newer versions of Visual Studio TargetName defaults
- # to the project name and not the basename of the output. Since the PDB file
- # is named based on the TargetName the pdb file ends up being named libapr.pdb
- # instead of libapr-1.pdb. The below call fixes this by explicitly providing
- # a TargetName definition and shuts up some warnings about this problem as well.
- # Without this fix the install fails when it tries to copy libapr-1.pdb.
- # See this thread for details of the changes:
- # http://social.msdn.microsoft.com/Forums/en-US/vcprerelease/thread/3c03e730-6a0e-4ee4-a0d6-6a5c3ce4343c
- find(sub {
- return unless (/\.vcxproj$/);
- my $output_file = get_output_file($_);
- return unless (defined($output_file));
- my ($project_name) = fileparse($_, qr/\.[^.]*$/);
- my ($old_style_target_name) = fileparse($output_file, qr/\.[^.]*$/);
- return if ($old_style_target_name eq $project_name);
- insert_property_group($_,
- "<TargetName>$old_style_target_name</TargetName>", '.torig');
- }, "$SRCLIB\\apr", "$SRCLIB\\apr-util", "$SRCLIB\\apr-iconv");
+ if ($vs_2012) {
+ # Turn off pre-compiled headers for apr-iconv to avoid:
+ # LNK2011: http://msdn.microsoft.com/en-us/library/3ay26wa2(v=vs.110).aspx
+ disable_pch('srclib\apr-iconv\build\modules.mk.win');
+
+ # ApacheMonitor build fails due a duplicate manifest, turn off
+ # GenerateManifest
+ insert_property_group('support\win32\ApacheMonitor.vcxproj',
+ '<GenerateManifest>false</GenerateManifest>',
+ '.dupman');
+
+ # The APR libraries have projects named libapr but produce output named libapr-1
+ # The problem with this is in newer versions of Visual Studio TargetName defaults
+ # to the project name and not the basename of the output. Since the PDB file
+ # is named based on the TargetName the pdb file ends up being named libapr.pdb
+ # instead of libapr-1.pdb. The below call fixes this by explicitly providing
+ # a TargetName definition and shuts up some warnings about this problem as well.
+ # Without this fix the install fails when it tries to copy libapr-1.pdb.
+ # See this thread for details of the changes:
+ # http://social.msdn.microsoft.com/Forums/en-US/vcprerelease/thread/3c03e730-6a0e-4ee4-a0d6-6a5c3ce4343c
+ find(sub {
+ return unless (/\.vcxproj$/);
+ my $output_file = get_output_file($_);
+ return unless (defined($output_file));
+ my ($project_name) = fileparse($_, qr/\.[^.]*$/);
+ my ($old_style_target_name) = fileparse($output_file, qr/\.[^.]*$/);
+ return if ($old_style_target_name eq $project_name);
+ insert_property_group($_,
+ "<TargetName>$old_style_target_name</TargetName>", '.torig');
+ }, "$SRCLIB\\apr", "$SRCLIB\\apr-util", "$SRCLIB\\apr-iconv");
+ } elsif ($vs_2010) {
+ system_or_die("Failed fixing project guid references",
+ qq("$PYTHON" "$BINDIR\\ProjRef.py" -i Apache.sln"));
+ }
# If you're looking here it's possible that something went
# wrong with the httpd build. Debugging it can be a bit of a pain
@@ -774,6 +844,9 @@ sub main {
Vars::set_defaults();
set_paths();
+ # Determine the Visual Studio Version and die if not supported.
+ check_vs_ver();
+
# change directory to our TOPDIR before running any commands
# the variable assignment might have changed it.
chdir_or_die($TOPDIR);
Modified: subversion/branches/move-tracking-1/tools/dist/backport.pl
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dist/backport.pl?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/dist/backport.pl (original)
+++ subversion/branches/move-tracking-1/tools/dist/backport.pl Tue Jun 4 20:56:23 2013
@@ -123,7 +123,9 @@ fi
$SVN diff > $backupfile
cp STATUS STATUS.$$
$SVNq revert -R .
-mv STATUS.$$ STATUS
+if $MAY_COMMIT ; then
+ mv STATUS.$$ STATUS
+fi
$SVNq up
$SVNq merge $mergeargs
if [ "`$SVN status -q | wc -l`" -eq 1 ]; then
@@ -161,9 +163,9 @@ else
fi
EOF
- open SHELL, '|-', qw#/bin/sh# or die $!;
+ open SHELL, '|-', qw#/bin/sh# or die "$! (in '$entry{header}')";
print SHELL $script;
- close SHELL or warn "$0: sh($?): $!";
+ close SHELL or warn "$0: sh($?): $! (in '$entry{header}')";
unlink $backupfile if -z $backupfile;
unlink $logmsg_filename unless $? or $!;
Modified: subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh (original)
+++ subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh Tue Jun 4 20:56:23 2013
@@ -25,7 +25,7 @@ APR=apr-1.4.6
APR_UTIL=apr-util-1.4.1
NEON=neon-0.29.6
SERF=serf-0.3.1
-ZLIB=zlib-1.2.7
+ZLIB=zlib-1.2.8
SQLITE_VERSION=3071400
SQLITE=sqlite-amalgamation-$SQLITE_VERSION
@@ -58,13 +58,13 @@ create_deps() {
fi
wget -qnc http://webdav.org/neon/$NEON.tar.gz
wget -qnc http://serf.googlecode.com/files/$SERF.tar.bz2
- wget -qnc http://www.zlib.net/$ZLIB.tar.bz2
+ wget -qnc http://www.zlib.net/$ZLIB.tar.gz
wget -qnc http://www.sqlite.org/$SQLITE.zip
mkdir $BASEDIR/unix-dependencies
cd $BASEDIR/unix-dependencies
tar zxf $TEMPDIR/$NEON.tar.gz
- tar jxf $TEMPDIR/$ZLIB.tar.bz2
+ tar zxf $TEMPDIR/$ZLIB.tar.gz
tar jxf $TEMPDIR/$SERF.tar.bz2
unzip -q $TEMPDIR/$SQLITE.zip
mv $NEON neon
@@ -80,7 +80,7 @@ create_deps() {
mkdir $BASEDIR/win32-dependencies
cd $BASEDIR/win32-dependencies
tar zxf $TEMPDIR/$NEON.tar.gz
- tar jxf $TEMPDIR/$ZLIB.tar.bz2
+ tar zxf $TEMPDIR/$ZLIB.tar.gz
tar jxf $TEMPDIR/$SERF.tar.bz2
unzip -q $TEMPDIR/$SQLITE.zip
mv $NEON neon
Propchange: subversion/branches/move-tracking-1/tools/dist/make-deps-tarball.sh
------------------------------------------------------------------------------
Merged /subversion/trunk/tools/dist/make-deps-tarball.sh:r1485307-1489600
Modified: subversion/branches/move-tracking-1/tools/dist/release.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/dist/release.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/dist/release.py (original)
+++ subversion/branches/move-tracking-1/tools/dist/release.py Tue Jun 4 20:56:23 2013
@@ -66,6 +66,22 @@ except ImportError:
import ezt
+try:
+ subprocess.check_output
+except AttributeError:
+ def check_output(cmd):
+ proc = subprocess.Popen(['svn', 'list', dist_dev_url],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ (stdout, stderr) = proc.communicate()
+ rc = proc.wait()
+ if rc or stderr:
+ logging.error('%r failed with stderr %r', cmd, stderr)
+ raise subprocess.CalledProcessError(rc, cmd)
+ return stdout
+ subprocess.check_output = check_output
+ del check_output
+
# Our required / recommended release tool versions by release branch
tool_versions = {
'trunk' : {
@@ -104,7 +120,7 @@ extns = ['zip', 'tar.gz', 'tar.bz2']
# Utility functions
class Version(object):
- regex = re.compile('(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
+ regex = re.compile(r'(\d+).(\d+).(\d+)(?:-(?:(rc|alpha|beta)(\d+)))?')
def __init__(self, ver_str):
# Special case the 'trunk-nightly' version
@@ -160,7 +176,7 @@ class Version(object):
else:
return self.pre_num < that.pre_num
- def __str(self):
+ def __str__(self):
if self.pre:
if self.pre == 'nightly':
return 'nightly'
@@ -173,11 +189,7 @@ class Version(object):
def __repr__(self):
- return "Version('%s')" % self.__str()
-
- def __str__(self):
- return self.__str()
-
+ return "Version(%s)" % repr(str(self))
def get_prefix(base_dir):
return os.path.join(base_dir, 'prefix')
@@ -188,6 +200,13 @@ def get_tempdir(base_dir):
def get_deploydir(base_dir):
return os.path.join(base_dir, 'deploy')
+def get_target(args):
+ "Return the location of the artifacts"
+ if args.target:
+ return args.target
+ else:
+ return get_deploydir(args.base_dir)
+
def get_tmpldir():
return os.path.join(os.path.abspath(sys.path[0]), 'templates')
@@ -199,8 +218,7 @@ def get_tmplfile(filename):
return urllib2.urlopen(repos + '/trunk/tools/dist/templates/' + filename)
def get_nullfile():
- # This is certainly not cross platform
- return open('/dev/null', 'w')
+ return open(os.path.devnull, 'w')
def run_script(verbose, script):
if verbose:
@@ -376,12 +394,7 @@ def compare_changes(repos, branch, revis
mergeinfo_cmd = ['svn', 'mergeinfo', '--show-revs=eligible',
repos + '/trunk/CHANGES',
repos + '/' + branch + '/' + 'CHANGES']
- proc = subprocess.Popen(mergeinfo_cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (stdout, stderr) = proc.communicate()
- rc = proc.wait()
- if stderr:
- raise RuntimeError('svn mergeinfo failed: %s' % stderr)
+ stdout = subprocess.check_output(mergeinfo_cmd)
if stdout:
# Treat this as a warning since we are now putting entries for future
# minor releases in CHANGES on trunk.
@@ -468,15 +481,11 @@ def sign_candidates(args):
def sign_file(filename):
asc_file = open(filename + '.asc', 'a')
logging.info("Signing %s" % filename)
- proc = subprocess.Popen(['gpg', '-ba', '-o', '-', filename],
- stdout=asc_file)
- proc.wait()
+ proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename],
+ stdout=asc_file)
asc_file.close()
- if args.target:
- target = args.target
- else:
- target = get_deploydir(args.base_dir)
+ target = get_target(args)
for e in extns:
filename = os.path.join(target, 'subversion-%s.%s' % (args.version, e))
@@ -493,17 +502,17 @@ def sign_candidates(args):
def post_candidates(args):
'Post candidate artifacts to the dist development directory.'
+ target = get_target(args)
+
logging.info('Importing tarballs to %s' % dist_dev_url)
svn_cmd = ['svn', 'import', '-m',
'Add %s candidate release artifacts' % args.version.base,
'--auto-props', '--config-option',
'config:auto-props:*.asc=svn:eol-style=native;svn:mime-type=text/plain',
- get_deploydir(args.base_dir), dist_dev_url]
+ target, dist_dev_url]
if (args.username):
svn_cmd += ['--username', args.username]
- proc = subprocess.Popen(svn_cmd)
- (stdout, stderr) = proc.communicate()
- proc.wait()
+ subprocess.check_call(svn_cmd)
#----------------------------------------------------------------------
# Create tag
@@ -518,6 +527,7 @@ def create_tag(args):
else:
branch = secure_repos + '/branches/%d.%d.x' % (args.version.major,
args.version.minor)
+ target = get_target(args)
tag = secure_repos + '/tags/' + str(args.version)
@@ -526,14 +536,63 @@ def create_tag(args):
if (args.username):
svnmucc_cmd += ['--username', args.username]
svnmucc_cmd += ['cp', str(args.revnum), branch, tag]
- svnmucc_cmd += ['put', os.path.join(get_deploydir(args.base_dir),
- 'svn_version.h.dist' + '-' +
+ svnmucc_cmd += ['put', os.path.join(target, 'svn_version.h.dist' + '-' +
str(args.version)),
tag + '/subversion/include/svn_version.h']
# don't redirect stdout/stderr since svnmucc might ask for a password
- proc = subprocess.Popen(svnmucc_cmd)
- proc.wait()
+ subprocess.check_call(svnmucc_cmd)
+
+ if not args.version.is_prerelease():
+ logging.info('Bumping revisions on the branch')
+ def replace_in_place(fd, startofline, flat, spare):
+ """In file object FD, replace FLAT with SPARE in the first line
+ starting with STARTOFLINE."""
+
+ fd.seek(0, os.SEEK_SET)
+ lines = fd.readlines()
+ for i, line in enumerate(lines):
+ if line.startswith(startofline):
+ lines[i] = line.replace(flat, spare)
+ break
+ else:
+ raise RuntimeError('Definition of %r not found' % startofline)
+
+ fd.seek(0, os.SEEK_SET)
+ fd.writelines(lines)
+ fd.truncate() # for current callers, new value is never shorter.
+
+ new_version = Version('%d.%d.%d' %
+ (args.version.major, args.version.minor,
+ args.version.patch + 1))
+
+ def file_object_for(relpath):
+ fd = tempfile.NamedTemporaryFile()
+ url = branch + '/' + relpath
+ fd.url = url
+ subprocess.check_call(['svn', 'cat', '%s@%d' % (url, args.revnum)],
+ stdout=fd)
+ return fd
+
+ svn_version_h = file_object_for('subversion/include/svn_version.h')
+ replace_in_place(svn_version_h, '#define SVN_VER_PATCH ',
+ str(args.version.patch), str(new_version.patch))
+
+ STATUS = file_object_for('STATUS')
+ replace_in_place(STATUS, 'Status of ',
+ str(args.version), str(new_version))
+
+ svn_version_h.seek(0, os.SEEK_SET)
+ STATUS.seek(0, os.SEEK_SET)
+ subprocess.check_call(['svnmucc', '-r', str(args.revnum),
+ '-m', 'Post-release housekeeping: '
+ 'bump the %s branch to %s.'
+ % (branch.split('/')[-1], str(new_version)),
+ 'put', svn_version_h.name, svn_version_h.url,
+ 'put', STATUS.name, STATUS.url,
+ ])
+ del svn_version_h
+ del STATUS
#----------------------------------------------------------------------
# Clean dist
@@ -541,13 +600,7 @@ def create_tag(args):
def clean_dist(args):
'Clean the distribution directory of all but the most recent artifacts.'
- proc = subprocess.Popen(['svn', 'list', dist_release_url],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (stdout, stderr) = proc.communicate()
- proc.wait()
- if stderr:
- raise RuntimeError(stderr)
+ stdout = subprocess.check_output(['svn', 'list', dist_release_url])
filenames = stdout.split('\n')
tar_gz_archives = []
@@ -576,8 +629,7 @@ def clean_dist(args):
svnmucc_cmd += ['rm', dist_release_url + '/' + filename]
# don't redirect stdout/stderr since svnmucc might ask for a password
- proc = subprocess.Popen(svnmucc_cmd)
- proc.wait()
+ subprocess.check_call(svnmucc_cmd)
#----------------------------------------------------------------------
# Move to dist
@@ -585,13 +637,7 @@ def clean_dist(args):
def move_to_dist(args):
'Move candidate artifacts to the distribution directory.'
- proc = subprocess.Popen(['svn', 'list', dist_dev_url],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (stdout, stderr) = proc.communicate()
- proc.wait()
- if stderr:
- raise RuntimeError(stderr)
+ stdout = subprocess.check_output(['svn', 'list', dist_dev_url])
filenames = []
for entry in stdout.split('\n'):
@@ -609,8 +655,7 @@ def move_to_dist(args):
# don't redirect stdout/stderr since svnmucc might ask for a password
logging.info('Moving release artifacts to %s' % dist_release_url)
- proc = subprocess.Popen(svnmucc_cmd)
- proc.wait()
+ subprocess.check_call(svnmucc_cmd)
#----------------------------------------------------------------------
# Write announcements
@@ -637,10 +682,7 @@ def write_news(args):
def get_sha1info(args, replace=False):
'Return a list of sha1 info for the release'
- if args.target:
- target = args.target
- else:
- target = get_deploydir(args.base_dir)
+ target = get_target(args)
sha1s = glob.glob(os.path.join(target, 'subversion*-%s*.sha1' % args.version))
@@ -714,10 +756,7 @@ def get_siginfo(args, quiet=False):
import _gnupg as gnupg
gpg = gnupg.GPG()
- if args.target:
- target = args.target
- else:
- target = get_deploydir(args.base_dir)
+ target = get_target(args)
good_sigs = {}
fingerprints = {}
@@ -848,6 +887,9 @@ def main():
help='''The release label, such as '1.7.0-alpha1'.''')
subparser.add_argument('--username',
help='''Username for ''' + dist_repos + '''.''')
+ subparser.add_argument('--target',
+ help='''The full path to the directory containing
+ release artifacts.''')
# Setup the parser for the create-tag subcommand
subparser = subparsers.add_parser('create-tag',
@@ -861,6 +903,9 @@ def main():
help='''The branch to base the release on.''')
subparser.add_argument('--username',
help='''Username for ''' + secure_repos + '''.''')
+ subparser.add_argument('--target',
+ help='''The full path to the directory containing
+ release artifacts.''')
# The clean-dist subcommand
subparser = subparsers.add_parser('clean-dist',
Modified: subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py (original)
+++ subversion/branches/move-tracking-1/tools/hook-scripts/mailer/mailer.py Tue Jun 4 20:56:23 2013
@@ -236,16 +236,30 @@ class MailedOutput(OutputBase):
and self.reply_to[2] == ']':
self.reply_to = self.reply_to[3:]
+ def _rfc2047_encode(self, hdr):
+ # Return the result of splitting HDR into tokens (on space
+ # characters), encoding (per RFC2047) each token as necessary, and
+ # slapping 'em back to together again.
+ from email.Header import Header
+
+ def _maybe_encode_header(hdr_token):
+ try:
+ hdr_token.encode('ascii')
+ return hdr_token
+ except UnicodeError:
+ return Header(hdr_token, 'utf-8').encode()
+
+ return ' '.join(map(_maybe_encode_header, hdr.split()))
+
def mail_headers(self, group, params):
from email import Utils
- subject = self.make_subject(group, params)
- try:
- subject.encode('ascii')
- except UnicodeError:
- from email.Header import Header
- subject = Header(subject, 'utf-8').encode()
- hdrs = 'From: %s\n' \
- 'To: %s\n' \
+
+ subject = self._rfc2047_encode(self.make_subject(group, params))
+ from_hdr = self._rfc2047_encode(self.from_addr)
+ to_hdr = self._rfc2047_encode(', '.join(self.to_addrs))
+
+ hdrs = 'From: %s\n' \
+ 'To: %s\n' \
'Subject: %s\n' \
'Date: %s\n' \
'Message-ID: %s\n' \
@@ -256,7 +270,7 @@ class MailedOutput(OutputBase):
'X-Svn-Commit-Author: %s\n' \
'X-Svn-Commit-Revision: %d\n' \
'X-Svn-Commit-Repository: %s\n' \
- % (self.from_addr, ', '.join(self.to_addrs), subject,
+ % (from_hdr, to_hdr, subject,
Utils.formatdate(), Utils.make_msgid(), group,
self.repos.author or 'no_author', self.repos.rev,
os.path.basename(self.repos.repos_dir))
Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py (original)
+++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/commit-hook.py Tue Jun 4 20:56:23 2013
@@ -19,8 +19,10 @@
SVNLOOK="/usr/local/svn-install/current/bin/svnlook"
#SVNLOOK="/usr/local/bin/svnlook"
+HOST="127.0.0.1"
+PORT=2069
+
import sys
-import subprocess
try:
import simplejson as json
except ImportError:
@@ -28,35 +30,32 @@ except ImportError:
import urllib2
-HOST="127.0.0.1"
-PORT=2069
-
-def svncmd(cmd):
- return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+import svnpubsub.util
-def svncmd_uuid(repo):
- cmd = "%s uuid %s" % (SVNLOOK, repo)
- p = svncmd(cmd)
- return p.stdout.read().strip()
-
-def svncmd_info(repo, revision):
- cmd = "%s info -r %s %s" % (SVNLOOK, revision, repo)
- p = svncmd(cmd)
- data = p.stdout.read().split("\n")
+def svnlook(cmd, **kwargs):
+ args = [SVNLOOK] + cmd
+ return svnpubsub.util.check_output(args, **kwargs)
+
+def svnlook_uuid(repo):
+ cmd = ["uuid", "--", repo]
+ return svnlook(cmd).strip()
+
+def svnlook_info(repo, revision):
+ cmd = ["info", "-r", revision, "--", repo]
+ data = svnlook(cmd, universal_newlines=True).split("\n")
#print data
return {'author': data[0].strip(),
'date': data[1].strip(),
'log': "\n".join(data[3:]).strip()}
-def svncmd_changed(repo, revision):
- cmd = "%s changed -r %s %s" % (SVNLOOK, revision, repo)
- p = svncmd(cmd)
+def svnlook_changed(repo, revision):
+ cmd = ["changed", "-r", revision, "--", repo]
+ lines = svnlook(cmd, universal_newlines=True).split("\n")
changed = {}
- while True:
- line = p.stdout.readline()
- if not line:
- break
+ for line in lines:
line = line.strip()
+ if not line:
+ continue
(flags, filename) = (line[0:3], line[4:])
changed[filename] = {'flags': flags}
return changed
@@ -71,23 +70,23 @@ def do_put(body):
def main(repo, revision):
revision = revision.lstrip('r')
- i = svncmd_info(repo, revision)
+ i = svnlook_info(repo, revision)
data = {'type': 'svn',
'format': 1,
'id': int(revision),
'changed': {},
- 'repository': svncmd_uuid(repo),
+ 'repository': svnlook_uuid(repo),
'committer': i['author'],
'log': i['log'],
'date': i['date'],
}
- data['changed'].update(svncmd_changed(repo, revision))
+ data['changed'].update(svnlook_changed(repo, revision))
body = json.dumps(data)
do_put(body)
if __name__ == "__main__":
- if len(sys.argv) != 3:
- print "invalid args"
- sys.exit(0)
+ if len(sys.argv) not in (3, 4):
+ sys.stderr.write("invalid args\n")
+ sys.exit(1)
- main(sys.argv[1], sys.argv[2])
+ main(*sys.argv[1:3])
Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py (original)
+++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/client.py Tue Jun 4 20:56:23 2013
@@ -62,7 +62,8 @@ class SvnpubsubClientException(Exception
class Client(asynchat.async_chat):
- def __init__(self, url, commit_callback, event_callback):
+ def __init__(self, url, commit_callback, event_callback,
+ metadata_callback = None):
asynchat.async_chat.__init__(self)
self.last_activity = time.time()
@@ -82,7 +83,8 @@ class Client(asynchat.async_chat):
self.event_callback = event_callback
- self.parser = JSONRecordHandler(commit_callback, event_callback)
+ self.parser = JSONRecordHandler(commit_callback, event_callback,
+ metadata_callback)
# Wait for the end of headers. Then we start parsing JSON.
self.set_terminator(b'\r\n\r\n')
@@ -126,36 +128,50 @@ class Client(asynchat.async_chat):
self.ibuffer.append(data)
+class Notification(object):
+ def __init__(self, data):
+ self.__dict__.update(data)
+
+class Commit(Notification):
+ KIND = 'COMMIT'
+
+class Metadata(Notification):
+ KIND = 'METADATA'
+
+
class JSONRecordHandler:
- def __init__(self, commit_callback, event_callback):
+ def __init__(self, commit_callback, event_callback, metadata_callback):
self.commit_callback = commit_callback
self.event_callback = event_callback
+ self.metadata_callback = metadata_callback
+
+ EXPECTED_VERSION = 1
def feed(self, record):
obj = json.loads(record)
if 'svnpubsub' in obj:
actual_version = obj['svnpubsub'].get('version')
- EXPECTED_VERSION = 1
- if actual_version != EXPECTED_VERSION:
- raise SvnpubsubClientException("Unknown svnpubsub format: %r != %d"
- % (actual_format, expected_format))
+ if actual_version != self.EXPECTED_VERSION:
+ raise SvnpubsubClientException(
+ "Unknown svnpubsub format: %r != %d"
+ % (actual_version, self.EXPECTED_VERSION))
self.event_callback('version', obj['svnpubsub']['version'])
elif 'commit' in obj:
commit = Commit(obj['commit'])
self.commit_callback(commit)
elif 'stillalive' in obj:
self.event_callback('ping', obj['stillalive'])
-
-
-class Commit(object):
- def __init__(self, commit):
- self.__dict__.update(commit)
+ elif 'metadata' in obj and self.metadata_callback:
+ metadata = Metadata(obj['metadata'])
+ self.metadata_callback(metadata)
class MultiClient(object):
- def __init__(self, urls, commit_callback, event_callback):
+ def __init__(self, urls, commit_callback, event_callback,
+ metadata_callback = None):
self.commit_callback = commit_callback
self.event_callback = event_callback
+ self.metadata_callback = metadata_callback
# No target time, as no work to do
self.target_time = 0
@@ -185,9 +201,15 @@ class MultiClient(object):
def _add_channel(self, url):
# Simply instantiating the client will install it into the global map
# for processing in the main event loop.
- Client(url,
- functools.partial(self.commit_callback, url),
- functools.partial(self._reconnect, url))
+ if self.metadata_callback:
+ Client(url,
+ functools.partial(self.commit_callback, url),
+ functools.partial(self._reconnect, url),
+ functools.partial(self.metadata_callback, url))
+ else:
+ Client(url,
+ functools.partial(self.commit_callback, url),
+ functools.partial(self._reconnect, url))
def _check_stale(self):
now = time.time()
Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py (original)
+++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnpubsub/server.py Tue Jun 4 20:56:23 2013
@@ -28,17 +28,24 @@
# Currently supports both XML and JSON serialization.
#
# Example Sub clients:
-# curl -sN http://127.0.0.1:2069/commits
-# curl -sN http://127.0.0.1:2069/commits/svn/*
-# curl -sN http://127.0.0.1:2069/commits/svn
-# curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68
-# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
+# curl -sN http://127.0.0.1:2069/commits
+# curl -sN 'http://127.0.0.1:2069/commits/svn/*'
+# curl -sN http://127.0.0.1:2069/commits/svn
+# curl -sN 'http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68'
+# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
#
-# URL is built into 2 parts:
-# /commits/${optional_type}/${optional_repository}
+# curl -sN http://127.0.0.1:2069/metadata
+# curl -sN 'http://127.0.0.1:2069/metadata/svn/*'
+# curl -sN http://127.0.0.1:2069/metadata/svn
+# curl -sN 'http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68'
+# curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68
#
-# If the type is included in the URL, you will only get commits of that type.
-# The type can be * and then you will receive commits of any type.
+# URLs are constructed from 3 parts:
+# /${notification}/${optional_type}/${optional_repository}
+#
+# Notifications can be sent for commits or metadata (e.g., revprop) changes.
+# If the type is included in the URL, you will only get notifications of that type.
+# The type can be * and then you will receive notifications of any type.
#
# If the repository is included in the URL, you will only receive
# messages about that repository. The repository can be * and then you
@@ -71,7 +78,7 @@ from twisted.python import log
import time
-class Commit:
+class Notification(object):
def __init__(self, r):
self.__dict__.update(r)
if not self.check_value('repository'):
@@ -86,7 +93,16 @@ class Commit:
def check_value(self, k):
return hasattr(self, k) and self.__dict__[k]
- def render_commit(self):
+ def render(self):
+ raise NotImplementedError
+
+ def render_log(self):
+ raise NotImplementedError
+
+class Commit(Notification):
+ KIND = 'COMMIT'
+
+ def render(self):
obj = {'commit': {}}
obj['commit'].update(self.__dict__)
return json.dumps(obj)
@@ -96,20 +112,32 @@ class Commit:
paths_changed = " %d paths changed" % len(self.changed)
except:
paths_changed = ""
- return "%s:%s repo '%s' id '%s'%s" % (self.type,
- self.format,
- self.repository,
- self.id,
- paths_changed)
+ return "commit %s:%s repo '%s' id '%s'%s" % (
+ self.type, self.format, self.repository, self.id,
+ paths_changed)
+
+class Metadata(Notification):
+ KIND = 'METADATA'
+
+ def render(self):
+ obj = {'metadata': {}}
+ obj['metadata'].update(self.__dict__)
+ return json.dumps(obj)
+
+ def render_log(self):
+ return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % (
+ self.type, self.format, self.repository, self.id,
+ self.revprop['name'])
HEARTBEAT_TIME = 15
class Client(object):
- def __init__(self, pubsub, r, type, repository):
+ def __init__(self, pubsub, r, kind, type, repository):
self.pubsub = pubsub
r.notifyFinish().addErrback(self.finished)
self.r = r
+ self.kind = kind
self.type = type
self.repository = repository
self.alive = True
@@ -123,11 +151,14 @@ class Client(object):
except ValueError:
pass
- def interested_in(self, commit):
- if self.type and self.type != commit.type:
+ def interested_in(self, notification):
+ if self.kind != notification.KIND:
return False
- if self.repository and self.repository != commit.repository:
+ if self.type and self.type != notification.type:
+ return False
+
+ if self.repository and self.repository != notification.repository:
return False
return True
@@ -152,7 +183,10 @@ class Client(object):
self.r.write(str(input))
def write_start(self):
- self.r.setHeader('content-type', 'application/json')
+ # TODO: use application/x-* or vnd.* - see
+ # Message-ID: <CA...@mail.gmail.com>
+ # on May 2013
+ self.r.setHeader('content-type', 'application/octet-stream')
self.write('{"svnpubsub": {"version": 1}}\n\0')
def write_heartbeat(self):
@@ -163,6 +197,13 @@ class SvnPubSub(resource.Resource):
isLeaf = True
clients = []
+ __notification_uri_map = {'commits': Commit.KIND,
+ 'metadata': Metadata.KIND}
+
+ def __init__(self, notification_class):
+ resource.Resource.__init__(self)
+ self.__notification_class = notification_class
+
def cc(self):
return len(self.clients)
@@ -182,6 +223,11 @@ class SvnPubSub(resource.Resource):
request.setResponseCode(400)
return "Invalid path\n"
+ kind = self.__notification_uri_map.get(uri[1], None)
+ if kind is None:
+ request.setResponseCode(400)
+ return "Invalid path\n"
+
if uri_len >= 3:
type = uri[2]
@@ -194,17 +240,18 @@ class SvnPubSub(resource.Resource):
if repository == '*':
repository = None
- c = Client(self, request, type, repository)
+ c = Client(self, request, kind, type, repository)
self.clients.append(c)
c.start()
return twisted.web.server.NOT_DONE_YET
- def notifyAll(self, commit):
- data = commit.render_commit()
+ def notifyAll(self, notification):
+ data = notification.render()
- log.msg("COMMIT: %s (%d clients)" % (commit.render_log(), self.cc()))
+ log.msg("%s: %s (%d clients)"
+ % (notification.KIND, notification.render_log(), self.cc()))
for client in self.clients:
- if client.interested_in(commit):
+ if client.interested_in(notification):
client.write_data(data)
def render_PUT(self, request):
@@ -217,19 +264,23 @@ class SvnPubSub(resource.Resource):
#import pdb;pdb.set_trace()
#print "input: %s" % (input)
try:
- c = json.loads(input)
- commit = Commit(c)
+ data = json.loads(input)
+ notification = self.__notification_class(data)
except ValueError as e:
request.setResponseCode(400)
- log.msg("COMMIT: failed due to: %s" % str(e))
- return str(e)
- self.notifyAll(commit)
+ errstr = str(e)
+ log.msg("%s: failed due to: %s" % (notification.KIND, errstr))
+ return errstr
+ self.notifyAll(notification)
return "Ok"
+
def svnpubsub_server():
root = resource.Resource()
- s = SvnPubSub()
- root.putChild("commits", s)
+ c = SvnPubSub(Commit)
+ m = SvnPubSub(Metadata)
+ root.putChild('commits', c)
+ root.putChild('metadata', m)
return server.Site(root)
if __name__ == "__main__":
Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py (original)
+++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/svnwcsub.py Tue Jun 4 20:56:23 2013
@@ -69,18 +69,7 @@ except ImportError:
import daemonize
import svnpubsub.client
-
-# check_output() is only available in Python 2.7. Allow us to run with
-# earlier versions
-try:
- check_output = subprocess.check_output
-except AttributeError:
- def check_output(args, env): # note: we only use these two args
- pipe = subprocess.Popen(args, stdout=subprocess.PIPE, env=env)
- output, _ = pipe.communicate()
- if pipe.returncode:
- raise subprocess.CalledProcessError(pipe.returncode, args)
- return output
+import svnpubsub.util
assert hasattr(subprocess, 'check_call')
def check_call(*args, **kwds):
@@ -103,7 +92,7 @@ def check_call(*args, **kwds):
def svn_info(svnbin, env, path):
"Run 'svn info' on the target path, returning a dict of info data."
args = [svnbin, "info", "--non-interactive", "--", path]
- output = check_output(args, env=env).strip()
+ output = svnpubsub.util.check_output(args, env=env).strip()
info = { }
for line in output.split('\n'):
idx = line.index(':')
Modified: subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py
URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py?rev=1489602&r1=1489601&r2=1489602&view=diff
==============================================================================
--- subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py (original)
+++ subversion/branches/move-tracking-1/tools/server-side/svnpubsub/watcher.py Tue Jun 4 20:56:23 2013
@@ -35,6 +35,9 @@ def _commit(url, commit):
print('COMMIT: from %s' % url)
pprint.pprint(vars(commit), indent=2)
+def _metadata(url, metadata):
+ print('METADATA: from %s' % url)
+ pprint.pprint(vars(metadata), indent=2)
def _event(url, event_name, event_arg):
if event_arg:
@@ -44,7 +47,7 @@ def _event(url, event_name, event_arg):
def main(urls):
- mc = svnpubsub.client.MultiClient(urls, _commit, _event)
+ mc = svnpubsub.client.MultiClient(urls, _commit, _event, _metadata)
mc.run_forever()