You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by ph...@apache.org on 2017/08/09 17:36:11 UTC
svn commit: r1804590 - /subversion/trunk/tools/dist/release.py
Author: philip
Date: Wed Aug 9 17:36:11 2017
New Revision: 1804590
URL: http://svn.apache.org/viewvc?rev=1804590&view=rev
Log:
Move the dist.sh processing into release.py so that the python script
now invokes the underlying commands directly. Change the processing
to checkout a working copy and then export the tarball trees from that
working copy rather than exporting directly from the repository. This
makes it simpler to add a patching step to apply security patches when
rolling tarballs.
* tools/dist/release.py
(Version.get_ver_tags): New.
(get_workdir, get_exportdir): New.
(run_script): Add option to hide stderr.
(replace_lines): New.
(roll_tarballs): Stop invoking dist.sh, checkout working copy, add
optional step applying patches, export from working copy, invoke
configure and roll tarballs.
(main): Add --patches option to roll subcommand.
Modified:
subversion/trunk/tools/dist/release.py
Modified: subversion/trunk/tools/dist/release.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/release.py?rev=1804590&r1=1804589&r2=1804590&view=diff
==============================================================================
--- subversion/trunk/tools/dist/release.py (original)
+++ subversion/trunk/tools/dist/release.py Wed Aug 9 17:36:11 2017
@@ -166,6 +166,27 @@ class Version(object):
else:
return 'supported-releases'
+ def get_ver_tags(self, revnum):
+ # These get substituted into svn_version.h
+ ver_tag = ''
+ ver_numtag = ''
+ if self.pre == 'alpha':
+ ver_tag = '" (Alpha %d)"' % self.pre_num
+ ver_numtag = '"-alpha%d"' % self.pre_num
+ elif self.pre == 'beta':
+ ver_tag = '" (Beta %d)"' % args.version.pre_num
+ ver_numtag = '"-beta%d"' % self.pre_num
+ elif self.pre == 'rc':
+ ver_tag = '" (Release Candidate %d)"' % self.pre_num
+ ver_numtag = '"-rc%d"' % self.pre_num
+ elif self.pre == 'nightly':
+ ver_tag = '" (Nightly Build r%d)"' % revnum
+ ver_numtag = '"-nightly-r%d"' % revnum
+ else:
+ ver_tag = '" (r%d)"' % revnum
+ ver_numtag = '""'
+ return (ver_tag, ver_numtag)
+
def __serialize(self):
return (self.major, self.minor, self.patch, self.pre, self.pre_num)
@@ -220,6 +241,18 @@ def get_prefix(base_dir):
def get_tempdir(base_dir):
return os.path.join(base_dir, 'tempdir')
+def get_workdir(base_dir):
+ return os.path.join(get_tempdir(base_dir), 'working')
+
+# The name of this directory is also used to name the tarball and for
+# the root of paths within the tarball, e.g. subversion-1.9.5 or
+# subversion-nightly-r1800000
+def get_exportdir(base_dir, version, revnum):
+ if version.pre != 'nightly':
+ return os.path.join(get_tempdir(base_dir), 'subversion-'+str(version))
+ return os.path.join(get_tempdir(base_dir),
+ 'subversion-%s-r%d' % (version, revnum))
+
def get_deploydir(base_dir):
return os.path.join(base_dir, 'deploy')
@@ -243,14 +276,17 @@ def get_tmplfile(filename):
def get_nullfile():
return open(os.path.devnull, 'w')
-def run_script(verbose, script):
+def run_script(verbose, script, hide_stderr=False):
+ stderr = None
if verbose:
stdout = None
else:
stdout = get_nullfile()
+ if hide_stderr:
+ stderr = get_nullfile()
for l in script.split('\n'):
- subprocess.check_call(l.split(), stdout=stdout)
+ subprocess.check_call(l.split(), stdout=stdout, stderr=stderr)
def download_file(url, target, checksum):
response = urllib2.urlopen(url)
@@ -482,6 +518,16 @@ def check_copyright_year(repos, branch,
check_file('NOTICE')
check_file('subversion/libsvn_subr/version.c')
+def replace_lines(path, actions):
+ with open(path, 'r') as old_content:
+ lines = old_content.readlines()
+ with open(path, 'w') as new_content:
+ for line in lines:
+ for start, pattern, repl in actions:
+ if line.startswith(start):
+ line = re.sub(pattern, repl, line)
+ new_content.write(line)
+
def roll_tarballs(args):
'Create the release artifacts.'
@@ -522,43 +568,158 @@ def roll_tarballs(args):
os.mkdir(get_deploydir(args.base_dir))
- # For now, just delegate to dist.sh to create the actual artifacts
- extra_args = ''
- if args.version.is_prerelease():
- if args.version.pre == 'nightly':
- extra_args = '-nightly'
+ logging.info('Preparing working copy source')
+ shutil.rmtree(get_workdir(args.base_dir), True)
+ run_script(args.verbose, 'svn checkout %s %s'
+ % (repos + '/' + branch + '@' + str(args.revnum),
+ get_workdir(args.base_dir)))
+
+ # Exclude stuff we don't want in the tarball, it will not be present
+ # in the exported tree.
+ exclude = ['contrib', 'notes']
+ if branch != 'trunk':
+ exclude += ['STATUS']
+ if args.version.minor < 7:
+ exclude += ['packages', 'www']
+ cwd = os.getcwd()
+ os.chdir(get_workdir(args.base_dir))
+ run_script(args.verbose,
+ 'svn update --set-depth exclude %s' % " ".join(exclude))
+ os.chdir(cwd)
+
+ if args.patches:
+ # Assume patches are independent and can be applied in any
+ # order, no need to sort.
+ majmin = '%d.%d' % (args.version.major, args.version.minor)
+ for name in os.listdir(args.patches):
+ if name.find(majmin) != -1 and name.endswith('patch'):
+ logging.info('Applying patch %s' % name)
+ run_script(args.verbose,
+ '''svn patch %s %s'''
+ % (os.path.join(args.patches, name),
+ get_workdir(args.base_dir)))
+
+ # Massage the new version number into svn_version.h.
+ ver_tag, ver_numtag = args.version.get_ver_tags(args.revnum)
+ replacements = [('#define SVN_VER_TAG',
+ '".*"', ver_tag),
+ ('#define SVN_VER_NUMTAG',
+ '".*"', ver_numtag),
+ ('#define SVN_VER_REVISION',
+ '[0-9][0-9]*', str(args.revnum))]
+ if args.version.pre != 'nightly':
+ # dist.sh does this but when would the numbers ever change?
+ replacements += [('#define SVN_VER_MAJOR',
+ '[0-9][0-9]*', str(args.version.major)),
+ ('#define SVN_VER_MINOR',
+ '[0-9][0-9]*', str(args.version.minor)),
+ ('#define SVN_VER_PATCH',
+ '[0-9][0-9]*', str(args.version.patch))]
+ replace_lines(os.path.join(get_workdir(args.base_dir),
+ 'subversion', 'include', 'svn_version.h'),
+ replacements)
+
+ # Basename for export and tarballs, e.g. subversion-1.9.5 or
+ # subversion-nightly-r1800000
+ exportdir = get_exportdir(args.base_dir, args.version, args.revnum)
+ basename = os.path.basename(exportdir)
+
+ def export(windows):
+ shutil.rmtree(exportdir, True)
+ if windows:
+ eol_style = "--native-eol CRLF"
else:
- extra_args = '-%s %d' % (args.version.pre, args.version.pre_num)
- # Build Unix last to leave Unix-style svn_version.h for tagging
+ eol_style = "--native-eol LF"
+ run_script(args.verbose, "svn export %s %s %s"
+ % (eol_style, get_workdir(args.base_dir), exportdir))
+
+ def transform_sql():
+ for root, dirs, files in os.walk(exportdir):
+ for fname in files:
+ if fname.endswith('.sql'):
+ run_script(args.verbose,
+ 'python build/transform_sql.py %s/%s %s/%s'
+ % (root, fname, root, fname[:-4] + '.h'))
+
+ def clean_autom4te():
+ for root, dirs, files in os.walk(get_workdir(args.base_dir)):
+ for dname in dirs:
+ if dname.startswith('autom4te') and dname.endswith('.cache'):
+ shutil.rmtree(os.path.join(root, dname))
+
logging.info('Building Windows tarballs')
- run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d -zip %s'
- % (sys.path[0], args.version.base, branch, args.revnum,
- extra_args) )
- logging.info('Building UNIX tarballs')
- run_script(args.verbose, '%s/dist.sh -v %s -pr %s -r %d %s'
- % (sys.path[0], args.version.base, branch, args.revnum,
- extra_args) )
+ export(windows=True)
+ os.chdir(exportdir)
+ transform_sql()
+ # Can't use the po-update.sh in the Windows export since it has CRLF
+ # line endings and won't run, so use the one in the working copy.
+ run_script(args.verbose,
+ '%s/tools/po/po-update.sh pot' % get_workdir(args.base_dir))
+ os.chdir(cwd)
+ clean_autom4te() # dist.sh does it but pointless on Windows?
+ os.chdir(get_tempdir(args.base_dir))
+ run_script(args.verbose,
+ 'zip -q -r %s %s' % (basename + '.zip', basename))
+ os.chdir(cwd)
+
+ logging.info('Building Unix tarballs')
+ export(windows=False)
+ os.chdir(exportdir)
+ transform_sql()
+ run_script(args.verbose,
+ '''tools/po/po-update.sh pot
+ ./autogen.sh --release''',
+ hide_stderr=True) # SWIG is noisy
+ os.chdir(cwd)
+ clean_autom4te() # dist.sh does it but probably pointless
+
+ # Do not use tar, it's probably GNU tar which produces tar files
+ # that are not compliant with POSIX.1 when including filenames
+ # longer than 100 chars. Platforms without a tar that understands
+ # the GNU tar extension will not be able to extract the resulting
+ # tar file. Use pax to produce POSIX.1 tar files.
+ #
+ # Use the gzip -n flag - this prevents it from storing the
+ # original name of the .tar file, and far more importantly, the
+ # mtime of the .tar file, in the produced .tar.gz file. This is
+ # important, because it makes the gzip encoding reproducable by
+ # anyone else who has an similar version of gzip, and also uses
+ # "gzip -9n". This means that committers who want to GPG-sign both
+ # the .tar.gz and the .tar.bz2 can download the .tar.bz2 (which is
+ # smaller), and locally generate an exact duplicate of the
+ # official .tar.gz file. This metadata is data on the temporary
+ # uncompressed tarball itself, not any of its contents, so there
+ # will be no effect on end-users.
+ os.chdir(get_tempdir(args.base_dir))
+ run_script(args.verbose,
+ '''pax -x ustar -w -f %s %s
+ bzip2 -9fk %s
+ gzip -9nf %s'''
+ % (basename + '.tar', basename,
+ basename + '.tar',
+ basename + '.tar'))
+ os.chdir(cwd)
# Move the results to the deploy directory
logging.info('Moving artifacts and calculating checksums')
for e in extns:
- if args.version.pre == 'nightly':
- filename = 'subversion-nightly.%s' % e
- else:
- filename = 'subversion-%s.%s' % (args.version, e)
-
- shutil.move(filename, get_deploydir(args.base_dir))
- filename = os.path.join(get_deploydir(args.base_dir), filename)
+ filename = basename + '.' + e
+ filepath = os.path.join(get_tempdir(args.base_dir), filename)
+ shutil.move(filepath, get_deploydir(args.base_dir))
+ filepath = os.path.join(get_deploydir(args.base_dir), filename)
m = hashlib.sha1()
- m.update(open(filename, 'r').read())
- open(filename + '.sha1', 'w').write(m.hexdigest())
+ m.update(open(filepath, 'r').read())
+ open(filepath + '.sha1', 'w').write(m.hexdigest())
m = hashlib.sha512()
- m.update(open(filename, 'r').read())
- open(filename + '.sha512', 'w').write(m.hexdigest())
+ m.update(open(filepath, 'r').read())
+ open(filepath + '.sha512', 'w').write(m.hexdigest())
- shutil.move('svn_version.h.dist',
- get_deploydir(args.base_dir) + '/' + 'svn_version.h.dist'
- + '-' + str(args.version))
+ # Nightlies do not get tagged so do not need the header
+ if args.version.pre != 'nightly':
+ shutil.copy(os.path.join(get_workdir(args.base_dir),
+ 'subversion', 'include', 'svn_version.h'),
+ os.path.join(get_deploydir(args.base_dir),
+ 'svn_version.h.dist-%s' % str(args.version)))
# And we're done!
@@ -1031,6 +1192,8 @@ def main():
subparser.add_argument('--branch',
help='''The branch to base the release on,
relative to ^/subversion/.''')
+ subparser.add_argument('--patches',
+ help='''The path to the directory containing patches.''')
# Setup the parser for the sign-candidates subcommand
subparser = subparsers.add_parser('sign-candidates',