You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by rh...@apache.org on 2018/11/07 12:30:11 UTC
svn commit: r1846002 [43/44] - in /subversion/branches/ra-git: ./ build/
build/ac-macros/ build/generator/ build/generator/swig/
build/generator/templates/ build/generator/util/ build/win32/
contrib/client-side/ contrib/client-side/svn_load_dirs/ contr...
Modified: subversion/branches/ra-git/tools/dist/release.py
URL: http://svn.apache.org/viewvc/subversion/branches/ra-git/tools/dist/release.py?rev=1846002&r1=1846001&r2=1846002&view=diff
==============================================================================
--- subversion/branches/ra-git/tools/dist/release.py (original)
+++ subversion/branches/ra-git/tools/dist/release.py Wed Nov 7 12:30:06 2018
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-#
+# python: coding=utf-8
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -51,6 +51,9 @@ import operator
import itertools
import subprocess
import argparse # standard in Python 2.7
+import io
+
+import backport.status
# Find ezt, using Subversion's copy, if there isn't one on the system.
try:
@@ -61,58 +64,61 @@ except ImportError:
sys.path.append(ezt_path)
import ezt
+ sys.path.remove(ezt_path)
-try:
- subprocess.check_output
-except AttributeError:
- def check_output(cmd):
- proc = subprocess.Popen(['svn', 'list', dist_dev_url],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (stdout, stderr) = proc.communicate()
- rc = proc.wait()
- if rc or stderr:
- logging.error('%r failed with stderr %r', cmd, stderr)
- raise subprocess.CalledProcessError(rc, cmd)
- return stdout
- subprocess.check_output = check_output
- del check_output
-
# Our required / recommended release tool versions by release branch
tool_versions = {
'trunk' : {
- 'autoconf' : '2.69',
- 'libtool' : '2.4.6',
- 'swig' : '2.0.12',
+ 'autoconf' : ['2.69',
+ '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
+ 'libtool' : ['2.4.6',
+ 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
+ 'swig' : ['3.0.12',
+ '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'],
},
- '1.9' : {
- 'autoconf' : '2.69',
- 'libtool' : '2.4.6',
- 'swig' : '2.0.12'
+ '1.11' : {
+ 'autoconf' : ['2.69',
+ '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
+ 'libtool' : ['2.4.6',
+ 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
+ 'swig' : ['3.0.12',
+ '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'],
},
- '1.8' : {
- 'autoconf' : '2.69',
- 'libtool' : '2.4.3',
- 'swig' : '2.0.9',
+ '1.10' : {
+ 'autoconf' : ['2.69',
+ '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
+ 'libtool' : ['2.4.6',
+ 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
+ 'swig' : ['3.0.12',
+ '7cf9f447ae7ed1c51722efc45e7f14418d15d7a1e143ac9f09a668999f4fc94d'],
},
- '1.7' : {
- 'autoconf' : '2.68',
- 'libtool' : '2.4.3',
- 'swig' : '2.0.4',
+ '1.9' : {
+ 'autoconf' : ['2.69',
+ '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
+ 'libtool' : ['2.4.6',
+ 'e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3'],
+ 'swig' : ['2.0.12',
+ '65e13f22a60cecd7279c59882ff8ebe1ffe34078e85c602821a541817a4317f7'],
},
- '1.6' : {
- 'autoconf' : '2.64',
- 'libtool' : '1.5.26',
- 'swig' : '1.3.36',
+ '1.8' : {
+ 'autoconf' : ['2.69',
+ '954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969'],
+ 'libtool' : ['2.4.3',
+ '36b4881c1843d7585de9c66c4c3d9a067ed3a3f792bc670beba21f5a4960acdf'],
+ 'swig' : ['2.0.9',
+ '586954000d297fafd7e91d1ad31089cc7e249f658889d11a44605d3662569539'],
},
}
# The version that is our current recommended release
-recommended_release = '1.8'
+# ### TODO: derive this from svn_version.h; see ../../build/getversion.py
+recommended_release = '1.11'
+# For clean-dist, a whitelist of artifacts to keep, by version.
+supported_release_lines = frozenset({"1.9", "1.10", "1.11", "1.12"})
# Some constants
-repos = 'http://svn.apache.org/repos/asf/subversion'
+repos = 'https://svn.apache.org/repos/asf/subversion'
secure_repos = 'https://svn.apache.org/repos/asf/subversion'
dist_repos = 'https://dist.apache.org/repos/dist'
dist_dev_url = dist_repos + '/dev/subversion'
@@ -173,6 +179,39 @@ class Version(object):
else:
return 'supported-releases'
+ def get_ver_tags(self, revnum):
+ # These get substituted into svn_version.h
+ ver_tag = ''
+ ver_numtag = ''
+ if self.pre == 'alpha':
+ ver_tag = '" (Alpha %d)"' % self.pre_num
+ ver_numtag = '"-alpha%d"' % self.pre_num
+ elif self.pre == 'beta':
+ ver_tag = '" (Beta %d)"' % self.pre_num
+ ver_numtag = '"-beta%d"' % self.pre_num
+ elif self.pre == 'rc':
+ ver_tag = '" (Release Candidate %d)"' % self.pre_num
+ ver_numtag = '"-rc%d"' % self.pre_num
+ elif self.pre == 'nightly':
+ ver_tag = '" (Nightly Build r%d)"' % revnum
+ ver_numtag = '"-nightly-r%d"' % revnum
+ else:
+ ver_tag = '" (r%d)"' % revnum
+ ver_numtag = '""'
+ return (ver_tag, ver_numtag)
+
+ def __serialize(self):
+ return (self.major, self.minor, self.patch, self.pre, self.pre_num)
+
+ def __eq__(self, that):
+ return self.__serialize() == that.__serialize()
+
+ def __ne__(self, that):
+ return self.__serialize() != that.__serialize()
+
+ def __hash__(self):
+ return hash(self.__serialize())
+
def __lt__(self, that):
if self.major < that.major: return True
if self.major > that.major: return False
@@ -194,6 +233,7 @@ class Version(object):
return self.pre_num < that.pre_num
def __str__(self):
+ "Return an SVN_VER_NUMBER-formatted string, or 'nightly'."
if self.pre:
if self.pre == 'nightly':
return 'nightly'
@@ -214,6 +254,18 @@ def get_prefix(base_dir):
def get_tempdir(base_dir):
return os.path.join(base_dir, 'tempdir')
+def get_workdir(base_dir):
+ return os.path.join(get_tempdir(base_dir), 'working')
+
+# The name of this directory is also used to name the tarball and for
+# the root of paths within the tarball, e.g. subversion-1.9.5 or
+# subversion-nightly-r1800000
+def get_exportdir(base_dir, version, revnum):
+ if version.pre != 'nightly':
+ return os.path.join(get_tempdir(base_dir), 'subversion-'+str(version))
+ return os.path.join(get_tempdir(base_dir),
+ 'subversion-%s-r%d' % (version, revnum))
+
def get_deploydir(base_dir):
return os.path.join(base_dir, 'deploy')
@@ -237,21 +289,45 @@ def get_tmplfile(filename):
def get_nullfile():
return open(os.path.devnull, 'w')
-def run_script(verbose, script):
+def run_script(verbose, script, hide_stderr=False):
+ stderr = None
if verbose:
stdout = None
- stderr = None
else:
stdout = get_nullfile()
- stderr = subprocess.STDOUT
+ if hide_stderr:
+ stderr = get_nullfile()
for l in script.split('\n'):
subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)
-def download_file(url, target):
+def download_file(url, target, checksum):
response = urllib2.urlopen(url)
- target_file = open(target, 'w')
+ target_file = open(target, 'w+')
target_file.write(response.read())
+ target_file.seek(0)
+ m = hashlib.sha256()
+ m.update(target_file.read())
+ target_file.close()
+ checksum2 = m.hexdigest()
+ if checksum != checksum2:
+ raise RuntimeError("Checksum mismatch for '%s': "\
+ "downloaded: '%s'; expected: '%s'" % \
+ (target, checksum, checksum2))
+
+#----------------------------------------------------------------------
+# ezt helpers
+
+# In ezt, «[if-any foo]» is true when «data['foo'] == False»,
+# hence, provide this constant for readability.
+ezt_False = ""
+
+# And this constant for symmetry.
+ezt_True = True
+
+# And this for convenience.
+def ezt_bool(boolean_value):
+ return ezt_True if boolean_value else ezt_False
#----------------------------------------------------------------------
# Cleaning up the environment
@@ -298,7 +374,7 @@ class RollDep(object):
logging.info('Using existing %s.tar.gz' % self._filebase)
else:
logging.info('Fetching %s' % self._filebase)
- download_file(self._url, tarball)
+ download_file(self._url, tarball, self._checksum)
# Extract tarball
tarfile.open(tarball).extractall(tempdir)
@@ -315,12 +391,13 @@ class RollDep(object):
class AutoconfDep(RollDep):
- def __init__(self, base_dir, use_existing, verbose, autoconf_ver):
+ def __init__(self, base_dir, use_existing, verbose, autoconf_ver, checksum):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'autoconf'
self._filebase = 'autoconf-' + autoconf_ver
self._autoconf_ver = autoconf_ver
- self._url = 'http://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase
+ self._url = 'https://ftp.gnu.org/gnu/autoconf/%s.tar.gz' % self._filebase
+ self._checksum = checksum
def have_usable(self):
output = self._test_version(['autoconf', '-V'])
@@ -335,12 +412,13 @@ class AutoconfDep(RollDep):
class LibtoolDep(RollDep):
- def __init__(self, base_dir, use_existing, verbose, libtool_ver):
+ def __init__(self, base_dir, use_existing, verbose, libtool_ver, checksum):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'libtool'
self._filebase = 'libtool-' + libtool_ver
self._libtool_ver = libtool_ver
- self._url = 'http://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase
+ self._url = 'https://ftp.gnu.org/gnu/libtool/%s.tar.gz' % self._filebase
+ self._checksum = checksum
def have_usable(self):
output = self._test_version(['libtool', '--version'])
@@ -362,14 +440,16 @@ class LibtoolDep(RollDep):
class SwigDep(RollDep):
- def __init__(self, base_dir, use_existing, verbose, swig_ver, sf_mirror):
+ def __init__(self, base_dir, use_existing, verbose, swig_ver, checksum,
+ sf_mirror):
RollDep.__init__(self, base_dir, use_existing, verbose)
self.label = 'swig'
self._filebase = 'swig-' + swig_ver
self._swig_ver = swig_ver
- self._url = 'http://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
+ self._url = 'https://sourceforge.net/projects/swig/files/swig/%(swig)s/%(swig)s.tar.gz/download?use_mirror=%(sf_mirror)s' % \
{ 'swig' : self._filebase,
'sf_mirror' : sf_mirror }
+ self._checksum = checksum
self._extra_configure_flags = '--without-pcre'
def have_usable(self):
@@ -396,11 +476,14 @@ def build_env(args):
raise
autoconf = AutoconfDep(args.base_dir, args.use_existing, args.verbose,
- tool_versions[args.version.branch]['autoconf'])
+ tool_versions[args.version.branch]['autoconf'][0],
+ tool_versions[args.version.branch]['autoconf'][1])
libtool = LibtoolDep(args.base_dir, args.use_existing, args.verbose,
- tool_versions[args.version.branch]['libtool'])
+ tool_versions[args.version.branch]['libtool'][0],
+ tool_versions[args.version.branch]['libtool'][1])
swig = SwigDep(args.base_dir, args.use_existing, args.verbose,
- tool_versions[args.version.branch]['swig'],
+ tool_versions[args.version.branch]['swig'][0],
+ tool_versions[args.version.branch]['swig'][1],
args.sf_mirror)
# iterate over our rolling deps, and build them if needed
@@ -448,13 +531,24 @@ def check_copyright_year(repos, branch,
check_file('NOTICE')
check_file('subversion/libsvn_subr/version.c')
+def replace_lines(path, actions):
+ with open(path, 'r') as old_content:
+ lines = old_content.readlines()
+ with open(path, 'w') as new_content:
+ for line in lines:
+ for start, pattern, repl in actions:
+ if line.startswith(start):
+ line = re.sub(pattern, repl, line)
+ new_content.write(line)
+
def roll_tarballs(args):
'Create the release artifacts.'
- if args.branch:
- branch = args.branch
- else:
- branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor)
+ if not args.branch:
+ args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor)
+
+ branch = args.branch # shorthand
+ branch = branch.rstrip('/') # canonicalize for later comparisons
logging.info('Rolling release %s from branch %s@%d' % (args.version,
branch, args.revnum))
@@ -463,11 +557,14 @@ def roll_tarballs(args):
# Ensure we've got the appropriate rolling dependencies available
autoconf = AutoconfDep(args.base_dir, False, args.verbose,
- tool_versions[args.version.branch]['autoconf'])
+ tool_versions[args.version.branch]['autoconf'][0],
+ tool_versions[args.version.branch]['autoconf'][1])
libtool = LibtoolDep(args.base_dir, False, args.verbose,
- tool_versions[args.version.branch]['libtool'])
+ tool_versions[args.version.branch]['libtool'][0],
+ tool_versions[args.version.branch]['libtool'][1])
swig = SwigDep(args.base_dir, False, args.verbose,
- tool_versions[args.version.branch]['swig'], None)
+ tool_versions[args.version.branch]['swig'][0],
+ tool_versions[args.version.branch]['swig'][1], None)
for dep in [autoconf, libtool, swig]:
if not dep.have_usable():
@@ -484,40 +581,167 @@ def roll_tarballs(args):
os.mkdir(get_deploydir(args.base_dir))
- # For now, just delegate to dist.sh to create the actual artifacts
- extra_args = ''
- if args.version.is_prerelease():
- if args.version.pre == 'nightly':
- extra_args = '-nightly'
+ logging.info('Preparing working copy source')
+ shutil.rmtree(get_workdir(args.base_dir), True)
+ run_script(args.verbose, 'svn checkout %s %s'
+ % (repos + '/' + branch + '@' + str(args.revnum),
+ get_workdir(args.base_dir)))
+
+ # Exclude stuff we don't want in the tarball, it will not be present
+ # in the exported tree.
+ exclude = ['contrib', 'notes']
+ if branch != 'trunk':
+ exclude += ['STATUS']
+ if args.version.minor < 7:
+ exclude += ['packages', 'www']
+ cwd = os.getcwd()
+ os.chdir(get_workdir(args.base_dir))
+ run_script(args.verbose,
+ 'svn update --set-depth exclude %s' % " ".join(exclude))
+ os.chdir(cwd)
+
+ if args.patches:
+ # Assume patches are independent and can be applied in any
+ # order, no need to sort.
+ majmin = '%d.%d' % (args.version.major, args.version.minor)
+ for name in os.listdir(args.patches):
+ if name.find(majmin) != -1 and name.endswith('patch'):
+ logging.info('Applying patch %s' % name)
+ run_script(args.verbose,
+ '''svn patch %s %s'''
+ % (os.path.join(args.patches, name),
+ get_workdir(args.base_dir)))
+
+ # Massage the new version number into svn_version.h.
+ ver_tag, ver_numtag = args.version.get_ver_tags(args.revnum)
+ replacements = [('#define SVN_VER_TAG',
+ '".*"', ver_tag),
+ ('#define SVN_VER_NUMTAG',
+ '".*"', ver_numtag),
+ ('#define SVN_VER_REVISION',
+ '[0-9][0-9]*', str(args.revnum))]
+ if args.version.pre != 'nightly':
+ # SVN_VER_PATCH might change for security releases, e.g., when
+ # releasing 1.9.7 from the magic revision of 1.9.6.
+ #
+ # ### Would SVN_VER_MAJOR / SVN_VER_MINOR ever change?
+ # ### Note that SVN_VER_MINOR is duplicated in some places, see
+ # ### <https://subversion.apache.org/docs/community-guide/releasing.html#release-branches>
+ replacements += [('#define SVN_VER_MAJOR',
+ '[0-9][0-9]*', str(args.version.major)),
+ ('#define SVN_VER_MINOR',
+ '[0-9][0-9]*', str(args.version.minor)),
+ ('#define SVN_VER_PATCH',
+ '[0-9][0-9]*', str(args.version.patch))]
+ replace_lines(os.path.join(get_workdir(args.base_dir),
+ 'subversion', 'include', 'svn_version.h'),
+ replacements)
+
+ # Basename for export and tarballs, e.g. subversion-1.9.5 or
+ # subversion-nightly-r1800000
+ exportdir = get_exportdir(args.base_dir, args.version, args.revnum)
+ basename = os.path.basename(exportdir)
+
+ def export(windows):
+ shutil.rmtree(exportdir, True)
+ if windows:
+ eol_style = "--native-eol CRLF"
else:
- extra_args = '-%s %d' % (args.version.pre, args.version.pre_num)
- # Build Unix last to leave Unix-style svn_version.h for tagging
- logging.info('Buildling Windows tarballs')
- run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
- % (sys.path[0], args.version.base, branch, args.revnum,
- extra_args) )
- logging.info('Building UNIX tarballs')
- run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
- % (sys.path[0], args.version.base, branch, args.revnum,
- extra_args) )
+ eol_style = "--native-eol LF"
+ run_script(args.verbose, "svn export %s %s %s"
+ % (eol_style, get_workdir(args.base_dir), exportdir))
+
+ def transform_sql():
+ for root, dirs, files in os.walk(exportdir):
+ for fname in files:
+ if fname.endswith('.sql'):
+ run_script(args.verbose,
+ 'python build/transform_sql.py %s/%s %s/%s'
+ % (root, fname, root, fname[:-4] + '.h'))
+
+ def clean_autom4te():
+ for root, dirs, files in os.walk(get_workdir(args.base_dir)):
+ for dname in dirs:
+ if dname.startswith('autom4te') and dname.endswith('.cache'):
+ shutil.rmtree(os.path.join(root, dname))
+
+ logging.info('Building Windows tarballs')
+ export(windows=True)
+ os.chdir(exportdir)
+ transform_sql()
+ # Can't use the po-update.sh in the Windows export since it has CRLF
+ # line endings and won't run, so use the one in the working copy.
+ run_script(args.verbose,
+ '%s/tools/po/po-update.sh pot' % get_workdir(args.base_dir))
+ os.chdir(cwd)
+ clean_autom4te() # dist.sh does it but pointless on Windows?
+ os.chdir(get_tempdir(args.base_dir))
+ run_script(args.verbose,
+ 'zip -q -r %s %s' % (basename + '.zip', basename))
+ os.chdir(cwd)
+
+ logging.info('Building Unix tarballs')
+ export(windows=False)
+ os.chdir(exportdir)
+ transform_sql()
+ run_script(args.verbose,
+ '''tools/po/po-update.sh pot
+ ./autogen.sh --release''',
+ hide_stderr=True) # SWIG is noisy
+ os.chdir(cwd)
+ clean_autom4te() # dist.sh does it but probably pointless
+
+ # Do not use tar, it's probably GNU tar which produces tar files
+ # that are not compliant with POSIX.1 when including filenames
+ # longer than 100 chars. Platforms without a tar that understands
+ # the GNU tar extension will not be able to extract the resulting
+ # tar file. Use pax to produce POSIX.1 tar files.
+ #
+ # Use the gzip -n flag - this prevents it from storing the
+ # original name of the .tar file, and far more importantly, the
+ # mtime of the .tar file, in the produced .tar.gz file. This is
+ # important, because it makes the gzip encoding reproducable by
+ # anyone else who has an similar version of gzip, and also uses
+ # "gzip -9n". This means that committers who want to GPG-sign both
+ # the .tar.gz and the .tar.bz2 can download the .tar.bz2 (which is
+ # smaller), and locally generate an exact duplicate of the
+ # official .tar.gz file. This metadata is data on the temporary
+ # uncompressed tarball itself, not any of its contents, so there
+ # will be no effect on end-users.
+ os.chdir(get_tempdir(args.base_dir))
+ run_script(args.verbose,
+ '''pax -x ustar -w -f %s %s
+ bzip2 -9fk %s
+ gzip -9nf %s'''
+ % (basename + '.tar', basename,
+ basename + '.tar',
+ basename + '.tar'))
+ os.chdir(cwd)
# Move the results to the deploy directory
logging.info('Moving artifacts and calculating checksums')
for e in extns:
- if args.version.pre == 'nightly':
- filename = 'subversion-nightly.%s' % e
- else:
- filename = 'subversion-%s.%s' % (args.version, e)
-
- shutil.move(filename, get_deploydir(args.base_dir))
- filename = os.path.join(get_deploydir(args.base_dir), filename)
- m = hashlib.sha1()
- m.update(open(filename, 'r').read())
- open(filename + '.sha1', 'w').write(m.hexdigest())
-
- shutil.move('svn_version.h.dist',
- get_deploydir(args.base_dir) + '/' + 'svn_version.h.dist'
- + '-' + str(args.version))
+ filename = basename + '.' + e
+ filepath = os.path.join(get_tempdir(args.base_dir), filename)
+ shutil.move(filepath, get_deploydir(args.base_dir))
+ filepath = os.path.join(get_deploydir(args.base_dir), filename)
+ if args.version < Version("1.11.0-alpha1"):
+ # 1.10 and earlier generate *.sha1 files for compatibility reasons.
+ # They are deprecated, however, so we don't publicly link them in
+ # the announcements any more.
+ m = hashlib.sha1()
+ m.update(open(filepath, 'r').read())
+ open(filepath + '.sha1', 'w').write(m.hexdigest())
+ m = hashlib.sha512()
+ m.update(open(filepath, 'r').read())
+ open(filepath + '.sha512', 'w').write(m.hexdigest())
+
+ # Nightlies do not get tagged so do not need the header
+ if args.version.pre != 'nightly':
+ shutil.copy(os.path.join(get_workdir(args.base_dir),
+ 'subversion', 'include', 'svn_version.h'),
+ os.path.join(get_deploydir(args.base_dir),
+ 'svn_version.h.dist-%s' % str(args.version)))
# And we're done!
@@ -530,8 +754,12 @@ def sign_candidates(args):
def sign_file(filename):
asc_file = open(filename + '.asc', 'a')
logging.info("Signing %s" % filename)
- proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename],
- stdout=asc_file)
+ if args.userid:
+ proc = subprocess.check_call(['gpg', '-ba', '-u', args.userid,
+ '-o', '-', filename], stdout=asc_file)
+ else:
+ proc = subprocess.check_call(['gpg', '-ba', '-o', '-', filename],
+ stdout=asc_file)
asc_file.close()
target = get_target(args)
@@ -554,8 +782,9 @@ def post_candidates(args):
target = get_target(args)
logging.info('Importing tarballs to %s' % dist_dev_url)
+ ver = str(args.version)
svn_cmd = ['svn', 'import', '-m',
- 'Add %s candidate release artifacts' % args.version.base,
+ 'Add Subversion %s candidate release artifacts' % ver,
'--auto-props', '--config-option',
'config:auto-props:*.asc=svn:eol-style=native;svn:mime-type=text/plain',
target, dist_dev_url]
@@ -565,18 +794,19 @@ def post_candidates(args):
#----------------------------------------------------------------------
# Create tag
+# Bump versions on branch
-def create_tag(args):
+def create_tag_only(args):
'Create tag in the repository'
+ target = get_target(args)
+
logging.info('Creating tag for %s' % str(args.version))
- if args.branch:
- branch = secure_repos + '/' + args.branch
- else:
- branch = secure_repos + '/branches/%d.%d.x' % (args.version.major,
- args.version.minor)
- target = get_target(args)
+ if not args.branch:
+ args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor)
+
+ branch = secure_repos + '/' + args.branch.rstrip('/')
tag = secure_repos + '/tags/' + str(args.version)
@@ -590,95 +820,132 @@ def create_tag(args):
tag + '/subversion/include/svn_version.h']
# don't redirect stdout/stderr since svnmucc might ask for a password
- subprocess.check_call(svnmucc_cmd)
+ try:
+ subprocess.check_call(svnmucc_cmd)
+ except subprocess.CalledProcessError:
+ if args.version.is_prerelease():
+ logging.error("Do you need to pass --branch=trunk?")
+ raise
+
+def bump_versions_on_branch(args):
+ 'Bump version numbers on branch'
+
+ logging.info('Bumping version numbers on the branch')
+
+ if not args.branch:
+ args.branch = 'branches/%d.%d.x' % (args.version.major, args.version.minor)
+
+ branch = secure_repos + '/' + args.branch.rstrip('/')
+
+ def replace_in_place(fd, startofline, flat, spare):
+ """In file object FD, replace FLAT with SPARE in the first line
+ starting with regex STARTOFLINE."""
+
+ pattern = r'^(%s)%s' % (startofline, re.escape(flat))
+ repl = r'\g<1>%s' % (spare,)
+ fd.seek(0, os.SEEK_SET)
+ lines = fd.readlines()
+ for i, line in enumerate(lines):
+ replacement = re.sub(pattern, repl, line)
+ if replacement != line:
+ lines[i] = replacement
+ break
+ else:
+ raise RuntimeError("Could not replace r'%s' with r'%s' in '%s'"
+ % (pattern, repl, fd.url))
- if not args.version.is_prerelease():
- logging.info('Bumping revisions on the branch')
- def replace_in_place(fd, startofline, flat, spare):
- """In file object FD, replace FLAT with SPARE in the first line
- starting with STARTOFLINE."""
-
- fd.seek(0, os.SEEK_SET)
- lines = fd.readlines()
- for i, line in enumerate(lines):
- if line.startswith(startofline):
- lines[i] = line.replace(flat, spare)
- break
- else:
- raise RuntimeError('Definition of %r not found' % startofline)
+ fd.seek(0, os.SEEK_SET)
+ fd.writelines(lines)
+ fd.truncate() # for current callers, new value is never shorter.
+
+ new_version = Version('%d.%d.%d' %
+ (args.version.major, args.version.minor,
+ args.version.patch + 1))
+
+ HEAD = subprocess.check_output(['svn', 'info', '--show-item=revision',
+ '--', branch]).strip()
+ HEAD = int(HEAD)
+ def file_object_for(relpath):
+ fd = tempfile.NamedTemporaryFile()
+ url = branch + '/' + relpath
+ fd.url = url
+ subprocess.check_call(['svn', 'cat', '%s@%d' % (url, HEAD)],
+ stdout=fd)
+ return fd
+
+ svn_version_h = file_object_for('subversion/include/svn_version.h')
+ replace_in_place(svn_version_h, '#define SVN_VER_PATCH *',
+ str(args.version.patch), str(new_version.patch))
+
+ STATUS = file_object_for('STATUS')
+ replace_in_place(STATUS, 'Status of ',
+ str(args.version), str(new_version))
+
+ svn_version_h.seek(0, os.SEEK_SET)
+ STATUS.seek(0, os.SEEK_SET)
+ subprocess.check_call(['svnmucc', '-r', str(HEAD),
+ '-m', 'Post-release housekeeping: '
+ 'bump the %s branch to %s.'
+ % (branch.split('/')[-1], str(new_version)),
+ 'put', svn_version_h.name, svn_version_h.url,
+ 'put', STATUS.name, STATUS.url,
+ ])
+ del svn_version_h
+ del STATUS
+
+def create_tag_and_bump_versions(args):
+ '''Create tag in the repository and, if not a prerelease version,
+ bump version numbers on the branch'''
+
+ create_tag_only(args)
- fd.seek(0, os.SEEK_SET)
- fd.writelines(lines)
- fd.truncate() # for current callers, new value is never shorter.
-
- new_version = Version('%d.%d.%d' %
- (args.version.major, args.version.minor,
- args.version.patch + 1))
-
- def file_object_for(relpath):
- fd = tempfile.NamedTemporaryFile()
- url = branch + '/' + relpath
- fd.url = url
- subprocess.check_call(['svn', 'cat', '%s@%d' % (url, args.revnum)],
- stdout=fd)
- return fd
-
- svn_version_h = file_object_for('subversion/include/svn_version.h')
- replace_in_place(svn_version_h, '#define SVN_VER_PATCH ',
- str(args.version.patch), str(new_version.patch))
-
- STATUS = file_object_for('STATUS')
- replace_in_place(STATUS, 'Status of ',
- str(args.version), str(new_version))
-
- svn_version_h.seek(0, os.SEEK_SET)
- STATUS.seek(0, os.SEEK_SET)
- subprocess.check_call(['svnmucc', '-r', str(args.revnum),
- '-m', 'Post-release housekeeping: '
- 'bump the %s branch to %s.'
- % (branch.split('/')[-1], str(new_version)),
- 'put', svn_version_h.name, svn_version_h.url,
- 'put', STATUS.name, STATUS.url,
- ])
- del svn_version_h
- del STATUS
+ if not args.version.is_prerelease():
+ bump_versions_on_branch(args)
#----------------------------------------------------------------------
# Clean dist
def clean_dist(args):
- 'Clean the distribution directory of all but the most recent artifacts.'
+ '''Clean the distribution directory of release artifacts of
+ no-longer-supported minor lines.'''
stdout = subprocess.check_output(['svn', 'list', dist_release_url])
+ def minor(version):
+ """Return the minor release line of the parameter, which must be
+ a Version object."""
+ return (version.major, version.minor)
+
filenames = stdout.split('\n')
- tar_gz_archives = []
- for entry in filenames:
- if fnmatch.fnmatch(entry, 'subversion-*.tar.gz'):
- tar_gz_archives.append(entry)
-
- versions = []
- for archive in tar_gz_archives:
- versions.append(Version(archive))
+ filenames = filter(lambda x: x.startswith('subversion-'), filenames)
+ versions = set(map(Version, filenames))
+ to_keep = set()
+ # TODO: When we release 1.A.0 GA we'll have to manually remove 1.(A-2).* artifacts.
+ for line_to_keep in [minor(Version(x + ".0")) for x in supported_release_lines]:
+ candidates = list(
+ x for x in versions
+ if minor(x) == line_to_keep
+ )
+ if candidates:
+ to_keep.add(max(candidates))
+ for i in sorted(to_keep):
+ logging.info("Saving release '%s'", i)
svnmucc_cmd = ['svnmucc', '-m', 'Remove old Subversion releases.\n' +
'They are still available at ' +
- 'http://archive.apache.org/dist/subversion/']
+ 'https://archive.apache.org/dist/subversion/']
if (args.username):
svnmucc_cmd += ['--username', args.username]
- for k, g in itertools.groupby(sorted(versions),
- lambda x: (x.major, x.minor)):
- releases = list(g)
- logging.info("Saving release '%s'", releases[-1])
-
- for r in releases[:-1]:
- for filename in filenames:
- if fnmatch.fnmatch(filename, 'subversion-%s.*' % r):
- logging.info("Removing '%s'" % filename)
- svnmucc_cmd += ['rm', dist_release_url + '/' + filename]
+ for filename in filenames:
+ if Version(filename) not in to_keep:
+ logging.info("Removing %r", filename)
+ svnmucc_cmd += ['rm', dist_release_url + '/' + filename]
# don't redirect stdout/stderr since svnmucc might ask for a password
- subprocess.check_call(svnmucc_cmd)
+ if 'rm' in svnmucc_cmd:
+ subprocess.check_call(svnmucc_cmd)
+ else:
+ logging.info("Nothing to remove")
#----------------------------------------------------------------------
# Move to dist
@@ -717,6 +984,8 @@ def write_news(args):
'version' : str(args.version),
'version_base' : args.version.base,
'anchor': args.version.get_download_anchor(),
+ 'is_recommended': ezt_bool(args.version.is_recommended()),
+ 'announcement_url': args.announcement_url,
}
if args.version.is_prerelease():
@@ -726,43 +995,53 @@ def write_news(args):
template = ezt.Template()
template.parse(get_tmplfile(template_filename).read())
- template.generate(sys.stdout, data)
+
+ # Insert the output into an existing file if requested, else print it
+ if args.edit_html_file:
+ tmp_name = args.edit_html_file + '.tmp'
+ with open(args.edit_html_file, 'r') as f, open(tmp_name, 'w') as g:
+ inserted = False
+ for line in f:
+ if not inserted and line.startswith('<div class="h3" id="news-'):
+ template.generate(g, data)
+ g.write('\n')
+ inserted = True
+ g.write(line)
+ os.remove(args.edit_html_file)
+ os.rename(tmp_name, args.edit_html_file)
+ else:
+ template.generate(sys.stdout, data)
-def get_sha1info(args, replace=False):
- 'Return a list of sha1 info for the release'
+def get_fileinfo(args):
+ 'Return a list of file info (filenames) for the release tarballs'
target = get_target(args)
- sha1s = glob.glob(os.path.join(target, 'subversion*-%s*.sha1' % args.version))
+ files = glob.glob(os.path.join(target, 'subversion*-%s*.asc' % args.version))
+ files.sort()
class info(object):
pass
- sha1info = []
- for s in sha1s:
+ fileinfo = []
+ for f in files:
i = info()
- # strip ".sha1"
- fname = os.path.basename(s)[:-5]
- if replace:
- # replace the version number with the [version] reference
- i.filename = Version.regex.sub('[version]', fname)
- else:
- i.filename = fname
- i.sha1 = open(s, 'r').read()
- sha1info.append(i)
+ # strip ".asc"
+ i.filename = os.path.basename(f)[:-4]
+ fileinfo.append(i)
- return sha1info
+ return fileinfo
def write_announcement(args):
'Write the release announcement.'
- sha1info = get_sha1info(args)
- siginfo = "\n".join(get_siginfo(args, True)) + "\n"
+ siginfo = get_siginfo(args, True)
+ if not siginfo:
+ raise RuntimeError("No signatures found for %s at %s" % (args.version, args.target))
data = { 'version' : str(args.version),
- 'sha1info' : sha1info,
- 'siginfo' : siginfo,
+ 'siginfo' : "\n".join(siginfo) + "\n",
'major-minor' : args.version.branch,
'major-minor-patch' : args.version.base,
'anchor' : args.version.get_download_anchor(),
@@ -771,8 +1050,19 @@ def write_announcement(args):
if args.version.is_prerelease():
template_filename = 'rc-release-ann.ezt'
else:
+ data['dot-zero'] = ezt_bool(args.version.patch == 0)
+ # TODO: instead of requiring the RM to remember to pass --security,
+ # read the private repository where CVE announcements are staged,
+ # parse the json file that identifies which versions are affected,
+ # and accordingly automagically set data['security'].
+ data['security'] = ezt_bool(args.security)
template_filename = 'stable-release-ann.ezt'
+ # The template text assumes these two are mutually exclusive.
+ # If you ever find a reason to make a x.y.0 release with a security
+ # bug, just comment this out and update the template before sending.
+ assert not (data['dot-zero'] and data['security'])
+
template = ezt.Template(compress_whitespace = False)
template.parse(get_tmplfile(template_filename).read())
template.generate(sys.stdout, data)
@@ -780,10 +1070,10 @@ def write_announcement(args):
def write_downloads(args):
'Output the download section of the website.'
- sha1info = get_sha1info(args, replace=True)
+ fileinfo = get_fileinfo(args)
data = { 'version' : str(args.version),
- 'fileinfo' : sha1info,
+ 'fileinfo' : fileinfo,
}
template = ezt.Template(compress_whitespace = False)
@@ -795,7 +1085,20 @@ def write_downloads(args):
# Validate the signatures for a release
key_start = '-----BEGIN PGP SIGNATURE-----'
-fp_pattern = re.compile(r'^pub\s+(\w+\/\w+)[^\n]*\n\s+Key\sfingerprint\s=((\s+[0-9A-F]{4}){10})\nuid\s+([^<\(]+)\s')
+key_end = '-----END PGP SIGNATURE-----'
+
+PUBLIC_KEY_ALGORITHMS = {
+ # These values are taken from the RFC's registry at:
+ # https://www.iana.org/assignments/pgp-parameters/pgp-parameters.xhtml#pgp-parameters-12
+ #
+ # The values are callables that produce gpg1-like key length and type
+ # indications, e.g., "4096R" for a 4096-bit RSA key.
+ 1: (lambda keylen: str(keylen) + 'R'), # RSA
+}
+
+def _make_human_readable_fingerprint(fingerprint):
+ return re.compile(r'(....)' * 10).sub(r'\1 \2 \3 \4 \5 \6 \7 \8 \9 \10',
+ fingerprint)
def get_siginfo(args, quiet=False):
'Returns a list of signatures for the release.'
@@ -817,9 +1120,27 @@ def get_siginfo(args, quiet=False):
text = open(filename).read()
keys = text.split(key_start)
+ # Check the keys file syntax. We've been bitten in the past
+ # with syntax errors in the key delimiters that GPG didn't
+ # catch for us, but the ASF key checker tool did.
+ if keys[0]:
+ sys.stderr.write("SYNTAX ERROR: %s does not start with '%s'\n"
+ % (filename, key_start))
+ sys.exit(1)
+ keys = keys[1:]
+
if not quiet:
- logging.info("Checking %d sig(s) in %s" % (len(keys[1:]), filename))
- for key in keys[1:]:
+ logging.info("Checking %d sig(s) in %s" % (len(keys), filename))
+
+ n = 0
+ for key in keys:
+ n += 1
+ if not key.rstrip().endswith(key_end):
+ sys.stderr.write("SYNTAX ERROR: Key %d in %s"
+ " does not end with '%s'\n"
+ % (n, filename, key_end))
+ sys.exit(1)
+
fd, fn = tempfile.mkstemp()
os.write(fd, key_start + key)
os.close(fd)
@@ -827,33 +1148,76 @@ def get_siginfo(args, quiet=False):
os.unlink(fn)
if verified.valid:
- good_sigs[verified.key_id[-8:]] = True
+ good_sigs[verified.fingerprint] = True
else:
- sys.stderr.write("BAD SIGNATURE for %s\n" % filename)
+ sys.stderr.write("BAD SIGNATURE: Key %d in %s\n"
+ % (n, filename))
if verified.key_id:
sys.stderr.write(" key id: %s\n" % verified.key_id)
sys.exit(1)
for id in good_sigs.keys():
- gpg = subprocess.Popen(['gpg', '--fingerprint', id],
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- rc = gpg.wait()
- gpg_output = gpg.stdout.read()
- if rc:
- print(gpg_output)
- sys.stderr.write("UNABLE TO GET FINGERPRINT FOR %s" % id)
- sys.exit(1)
-
- gpg_output = "\n".join([ l for l in gpg_output.splitlines()
- if l[0:7] != 'Warning' ])
-
- fp = fp_pattern.match(gpg_output).groups()
- fingerprints["%s [%s] %s" % (fp[3], fp[0], fp[1])] = fp
+ # Most potential signers have public short keyid (32-bit) collisions in
+ # the https://evil32.com/ set, which has been uploaded to the
+ # keyservers, so generate the long keyid (see use of LONG_KEY_ID below).
+ #
+ # TODO: in the future it'd be nice to use the 'gnupg' module here.
+ gpg_output = subprocess.check_output(
+ ['gpg', '--fixed-list-mode', '--with-colons', '--fingerprint', id],
+ stderr=subprocess.STDOUT,
+ )
+ gpg_output = gpg_output.splitlines()
+
+ # This code was added in r934990, but there was no comment (nor log
+ # message text) explaining its purpose. I've commented it out since
+ # ignoring arbitrary warnings in a verification codepath is Bad. If
+ # you run into warnings on your machine, feel free to uncomment it,
+ # but when you do so please make it match specific warnings only.
+ #
+ #gpg_output = "\n".join([ l for l in gpg_output.splitlines()
+ # if l[0:7] != 'Warning' ])
+
+ # Parse gpg's output. This happens to work for both gpg1 and gpg2,
+ # even though their outputs are slightly different.
+ #
+ # See http://git.gnupg.org/cgi-bin/gitweb.cgi?p=gnupg.git;a=blob_plain;f=doc/DETAILS
+ for line in gpg_output:
+ parts = line.split(':')
+ if parts[0] == 'pub':
+ keylen = int(parts[2])
+ keytype = int(parts[3])
+ formatter = PUBLIC_KEY_ALGORITHMS[keytype]
+ long_key_id = parts[4]
+ length_and_type = formatter(keylen) + '/' + long_key_id
+ del keylen, keytype, formatter, long_key_id
+ break
+ else:
+ raise RuntimeError("Failed to determine LONG_KEY_ID")
+ for line in gpg_output:
+ parts = line.split(':')
+ if parts[0] == 'fpr':
+ fingerprint = parts[9]
+ break
+ else:
+ raise RuntimeError("Failed to determine FINGERPRINT")
+ for line in gpg_output:
+ parts = line.split(':')
+ if parts[0] == 'uid':
+ name = parts[9].split(' <')[0]
+ break
+ else:
+ raise RuntimeError("Failed to determine NAME")
- for entry in sorted(fingerprints.keys()):
- fp = fingerprints[entry]
- output.append(" %s [%s] with fingerprint:" % (fp[3], fp[0]))
- output.append(" %s" % fp[1])
+ format_expandos = dict(
+ name=name,
+ length_and_type=length_and_type,
+ fingerprint=_make_human_readable_fingerprint(fingerprint),
+ )
+ del name, length_and_type, fingerprint
+ line = " {name} [{length_and_type}] with fingerprint:"
+ output.append( line.format(**format_expandos) )
+ line = " {fingerprint}"
+ output.append( line.format(**format_expandos) )
return output
@@ -873,6 +1237,206 @@ def get_keys(args):
fd.seek(0)
subprocess.check_call(['gpg', '--import'], stdin=fd)
+def add_to_changes_dict(changes_dict, audience, section, change, revision):
+ # Normalize arguments
+ if audience:
+ audience = audience.upper()
+ if section:
+ section = section.lower()
+ change = change.strip()
+
+ if not audience in changes_dict:
+ changes_dict[audience] = dict()
+ if not section in changes_dict[audience]:
+ changes_dict[audience][section] = dict()
+
+ changes = changes_dict[audience][section]
+ if change in changes:
+ changes[change].add(revision)
+ else:
+ changes[change] = set([revision])
+
+def print_section(changes_dict, audience, section, title, mandatory=False):
+ if audience in changes_dict:
+ audience_changes = changes_dict[audience]
+ if mandatory or (section in audience_changes):
+ if title:
+ print(' - %s:' % title)
+ if section in audience_changes:
+ print_changes(audience_changes[section])
+ elif mandatory:
+ print(' (none)')
+
+def print_changes(changes):
+ # Print in alphabetical order, so entries with the same prefix are together
+ for change in sorted(changes):
+ revs = changes[change]
+ rev_string = 'r' + str(min(revs)) + (' et al' if len(revs) > 1 else '')
+ print(' * %s (%s)' % (change, rev_string))
+
+def write_changelog(args):
+ 'Write changelog, parsed from commit messages'
+ # Changelog lines are lines with the following format:
+ # '['[audience[:section]]']' <message>
+ # or:
+ # <message> '['[audience[:section]]']'
+ # where audience = U (User-visible) or D (Developer-visible)
+ # section = general|major|minor|client|server|clientserver|other|api|bindings
+ # (section is optional and is treated case-insensitively)
+ # message = the actual text for CHANGES
+ #
+ # This means the "changes label" can be used as prefix or suffix, and it
+ # can also be left empty (which results in an uncategorized changes entry),
+ # if the committer isn't sure where the changelog entry belongs.
+ #
+ # Putting [skip], [ignore], [c:skip] or [c:ignore] somewhere in the
+ # log message means this commit must be ignored for Changelog processing
+ # (ignored even with the --include-unlabeled-summaries option).
+ #
+ # If there is no changes label anywhere in the commit message, and the
+ # --include-unlabeled-summaries option is used, we'll consider the summary
+ # line of the commit message (= first line except if it starts with a *)
+ # as an uncategorized changes entry, except if it contains "status",
+ # "changes", "post-release housekeeping" or "follow-up".
+ #
+ # Examples:
+ # [U:major] Better interactive conflict resolution for tree conflicts
+ # ra_serf: Adjustments for serf versions with HTTP/2 support [U:minor]
+ # [U] Fix 'svn diff URL@REV WC' wrongly looks up URL@HEAD (issue #4597)
+ # Fix bug with canonicalizing Window-specific drive-relative URL []
+ # New svn_ra_list() API function [D:api]
+ # [D:bindings] JavaHL: Allow access to constructors of a couple JavaHL classes
+
+ branch = secure_repos + '/' + args.branch
+ previous = secure_repos + '/' + args.previous
+ include_unlabeled = args.include_unlabeled
+ separator_line = ('-' * 72) + '\n'
+
+ mergeinfo = subprocess.check_output(['svn', 'mergeinfo', '--show-revs',
+ 'eligible', '--log', branch, previous])
+ log_messages_dict = {
+ # This is a dictionary mapping revision numbers to their respective
+ # log messages. The expression in the "key:" part of the dict
+ # comprehension extracts the revision number, as integer, from the
+ # 'svn log' output.
+ int(log_message.splitlines()[0].split()[0][1:]): log_message
+ # The [1:-1] ignores the empty first and last element of the split().
+ for log_message in mergeinfo.split(separator_line)[1:-1]
+ }
+ mergeinfo = mergeinfo.splitlines()
+
+ separator_pattern = re.compile('^-{72}$')
+ revline_pattern = re.compile('^r(\d+) \| [^\|]+ \| [^\|]+ \| \d+ lines?$')
+ changes_prefix_pattern = re.compile(r'^\[(U|D)?:?([^\]]+)?\](.+)$')
+ changes_suffix_pattern = re.compile(r'^(.+)\[(U|D)?:?([^\]]+)?\]$')
+ # TODO: push this into backport.status as a library function
+ auto_merge_pattern = \
+ re.compile(r'^Merge (r\d+,? |the r\d+ group |the \S+ branch:)')
+
+ changes_dict = dict() # audience -> (section -> (change -> set(revision)))
+ revision = -1
+ got_firstline = False
+ unlabeled_summary = None
+ changes_ignore = False
+ audience = None
+ section = None
+ message = None
+
+ for line in mergeinfo:
+ if separator_pattern.match(line):
+ # New revision section. Reset variables.
+ # If there's an unlabeled summary from a previous section, and
+ # include_unlabeled is True, put it into uncategorized_changes.
+ if include_unlabeled and unlabeled_summary and not changes_ignore:
+ if auto_merge_pattern.match(unlabeled_summary):
+ # 1. Parse revision numbers from the first line
+ merged_revisions = [
+ int(x) for x in
+ re.compile(r'(?<=\br)\d+\b').findall(unlabeled_summary)
+ ]
+ # TODO pass each revnum in MERGED_REVISIONS through this
+ # logic, in order to extract CHANGES_PREFIX_PATTERN
+ # and CHANGES_SUFFIX_PATTERN lines from the trunk log
+ # message.
+
+ # 2. Parse the STATUS entry
+ this_log_message = log_messages_dict[revision]
+ status_paragraph = this_log_message.split('\n\n')[2]
+ logsummary = \
+ backport.status.StatusEntry(status_paragraph).logsummary
+ add_to_changes_dict(changes_dict, None, None,
+ ' '.join(logsummary), revision)
+ else:
+ add_to_changes_dict(changes_dict, None, None,
+ unlabeled_summary, revision)
+ revision = -1
+ got_firstline = False
+ unlabeled_summary = None
+ changes_ignore = False
+ audience = None
+ section = None
+ message = None
+ continue
+
+ revmatch = revline_pattern.match(line)
+ if revmatch and (revision == -1):
+ # A revision line: get the revision number
+ revision = int(revmatch.group(1))
+ logging.debug('Changelog processing revision r%d' % revision)
+ continue
+
+ if line.strip() == '':
+ # Skip empty / whitespace lines
+ continue
+
+ if not got_firstline:
+ got_firstline = True
+ if (not re.search(r'status|changes|post-release housekeeping|follow-up|^\*',
+ line, re.IGNORECASE)
+ and not changes_prefix_pattern.match(line)
+ and not changes_suffix_pattern.match(line)):
+ unlabeled_summary = line
+
+ if re.search(r'\[(c:)?(skip|ignore)\]', line, re.IGNORECASE):
+ changes_ignore = True
+
+ prefix_match = changes_prefix_pattern.match(line)
+ if prefix_match:
+ audience = prefix_match.group(1)
+ section = prefix_match.group(2)
+ message = prefix_match.group(3)
+ add_to_changes_dict(changes_dict, audience, section, message, revision)
+
+ suffix_match = changes_suffix_pattern.match(line)
+ if suffix_match:
+ message = suffix_match.group(1)
+ audience = suffix_match.group(2)
+ section = suffix_match.group(3)
+ add_to_changes_dict(changes_dict, audience, section, message, revision)
+
+ # Output the sorted changelog entries
+ # 1) Uncategorized changes
+ print_section(changes_dict, None, None, None)
+ print
+ # 2) User-visible changes
+ print(' User-visible changes:')
+ print_section(changes_dict, 'U', None, None)
+ print_section(changes_dict, 'U', 'general', 'General')
+ print_section(changes_dict, 'U', 'major', 'Major new features')
+ print_section(changes_dict, 'U', 'minor', 'Minor new features and improvements')
+ print_section(changes_dict, 'U', 'client', 'Client-side bugfixes', mandatory=True)
+ print_section(changes_dict, 'U', 'server', 'Server-side bugfixes', mandatory=True)
+ print_section(changes_dict, 'U', 'clientserver', 'Client-side and server-side bugfixes')
+ print_section(changes_dict, 'U', 'other', 'Other tool improvements and bugfixes')
+ print_section(changes_dict, 'U', 'bindings', 'Bindings bugfixes', mandatory=True)
+ print
+ # 3) Developer-visible changes
+ print(' Developer-visible changes:')
+ print_section(changes_dict, 'D', None, None)
+ print_section(changes_dict, 'D', 'general', 'General', mandatory=True)
+ print_section(changes_dict, 'D', 'api', 'API changes', mandatory=True)
+ print_section(changes_dict, 'D', 'bindings', 'Bindings')
+
#----------------------------------------------------------------------
# Main entry point for argument parsing and handling
@@ -913,10 +1477,13 @@ def main():
subparser.set_defaults(func=roll_tarballs)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
- subparser.add_argument('revnum', type=int,
+ subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')),
help='''The revision number to base the release on.''')
subparser.add_argument('--branch',
- help='''The branch to base the release on.''')
+ help='''The branch to base the release on,
+ relative to ^/subversion/.''')
+ subparser.add_argument('--patches',
+ help='''The path to the directory containing patches.''')
# Setup the parser for the sign-candidates subcommand
subparser = subparsers.add_parser('sign-candidates',
@@ -927,6 +1494,10 @@ def main():
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
+ subparser.add_argument('--userid',
+ help='''The (optional) USER-ID specifying the key to be
+ used for signing, such as '110B1C95' (Key-ID). If
+ omitted, uses the default key.''')
# Setup the parser for the post-candidates subcommand
subparser = subparsers.add_parser('post-candidates',
@@ -943,14 +1514,33 @@ def main():
# Setup the parser for the create-tag subcommand
subparser = subparsers.add_parser('create-tag',
- help='''Create the release tag.''')
- subparser.set_defaults(func=create_tag)
+ help='''Create the release tag and, if not a prerelease
+ version, bump version numbers on the branch.''')
+ subparser.set_defaults(func=create_tag_and_bump_versions)
+ subparser.add_argument('version', type=Version,
+ help='''The release label, such as '1.7.0-alpha1'.''')
+ subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')),
+ help='''The revision number to base the release on.''')
+ subparser.add_argument('--branch',
+ help='''The branch to base the release on,
+ relative to ^/subversion/.''')
+ subparser.add_argument('--username',
+ help='''Username for ''' + secure_repos + '''.''')
+ subparser.add_argument('--target',
+ help='''The full path to the directory containing
+ release artifacts.''')
+
+ # Setup the parser for the bump-versions-on-branch subcommand
+ subparser = subparsers.add_parser('bump-versions-on-branch',
+ help='''Bump version numbers on branch.''')
+ subparser.set_defaults(func=bump_versions_on_branch)
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
- subparser.add_argument('revnum', type=int,
+ subparser.add_argument('revnum', type=lambda arg: int(arg.lstrip('r')),
help='''The revision number to base the release on.''')
subparser.add_argument('--branch',
- help='''The branch to base the release on.''')
+ help='''The branch to base the release on,
+ relative to ^/subversion/.''')
subparser.add_argument('--username',
help='''Username for ''' + secure_repos + '''.''')
subparser.add_argument('--target',
@@ -959,8 +1549,7 @@ def main():
# The clean-dist subcommand
subparser = subparsers.add_parser('clean-dist',
- help='''Clean the distribution directory (and mirrors) of
- all but the most recent MAJOR.MINOR release.''')
+ help=clean_dist.__doc__.split('\n\n')[0])
subparser.set_defaults(func=clean_dist)
subparser.add_argument('--dist-dir',
help='''The directory to clean.''')
@@ -983,6 +1572,11 @@ def main():
help='''Output to stdout template text for use in the news
section of the Subversion website.''')
subparser.set_defaults(func=write_news)
+ subparser.add_argument('--announcement-url',
+ help='''The URL to the archived announcement email.''')
+ subparser.add_argument('--edit-html-file',
+ help='''Insert the text into this file
+ news.html, index.html).''')
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
@@ -991,6 +1585,9 @@ def main():
help='''Output to stdout template text for the emailed
release announcement.''')
subparser.set_defaults(func=write_announcement)
+ subparser.add_argument('--security', action='store_true', default=False,
+ help='''The release being announced includes security
+ fixes.''')
subparser.add_argument('--target',
help='''The full path to the directory containing
release artifacts.''')
@@ -1002,6 +1599,9 @@ def main():
help='''Output to stdout template text for the download
table for subversion.apache.org''')
subparser.set_defaults(func=write_downloads)
+ subparser.add_argument('--target',
+ help='''The full path to the directory containing
+ release artifacts.''')
subparser.add_argument('version', type=Version,
help='''The release label, such as '1.7.0-alpha1'.''')
@@ -1027,6 +1627,29 @@ def main():
separate subcommand.''')
subparser.set_defaults(func=cleanup)
+ # write-changelog
+ subparser = subparsers.add_parser('write-changelog',
+ help='''Output to stdout changelog entries parsed from
+ commit messages, optionally labeled with a category
+ like [U:client], [D:api], [U], ...''')
+ subparser.set_defaults(func=write_changelog)
+ subparser.add_argument('branch',
+ help='''The branch (or tag or trunk), relative to
+ ^/subversion/, of which to generate the
+ changelog, when compared to "previous".''')
+ subparser.add_argument('previous',
+ help='''The "previous" branch or tag, relative to
+ ^/subversion/, to compare "branch" against.''')
+ subparser.add_argument('--include-unlabeled-summaries',
+ dest='include_unlabeled',
+ action='store_true', default=False,
+ help='''Include summary lines that do not have a changes
+ label, unless an explicit [c:skip] or [c:ignore]
+ is part of the commit message (except if the
+ summary line contains 'STATUS', 'CHANGES',
+ 'Post-release housekeeping', 'Follow-up' or starts
+ with '*').''')
+
# Parse the arguments
args = parser.parse_args()
@@ -1045,6 +1668,9 @@ def main():
os.environ['PATH'] = os.path.join(get_prefix(args.base_dir), 'bin') + ':' \
+ os.environ['PATH']
+ # Make timestamps in tarballs independent of local timezone
+ os.environ['TZ'] = 'UTC'
+
# finally, run the subcommand, and give it the parsed arguments
args.func(args)