You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by da...@apache.org on 2015/03/29 09:08:45 UTC
svn commit: r1669859 - in /subversion/trunk/tools/dist: backport/
backport/__init__.py backport/merger.py backport/status.py
detect-conflicting-backports.py merge-approved-backports.py
Author: danielsh
Date: Sun Mar 29 07:08:45 2015
New Revision: 1669859
URL: http://svn.apache.org/r1669859
Log:
backport.py: New set of scripts.
Reimplement backport.pl in Python. For now, only the two interactive modes
— the nightly mergebot and the hourly conflicts bot — are implemented. The
other two modes — the interactive review and nomination modes — have not yet
been ported.
* tools/dist/backport/__init__.py: New module marker.
* tools/dist/backport/merger.py,
* tools/dist/backport/status_file.py.
New submodules. A reimplementation of backport.pl.
* tools/dist/merge-approved-backports.py:
New script, implements backport.pl's nightly mergebot mode.
* tools/dist/detect-conflicting-backports.py
New script, implements backport.pl's hourly conflicts bot mode.
Added:
subversion/trunk/tools/dist/backport/
subversion/trunk/tools/dist/backport/__init__.py
subversion/trunk/tools/dist/backport/merger.py
subversion/trunk/tools/dist/backport/status.py
subversion/trunk/tools/dist/detect-conflicting-backports.py
subversion/trunk/tools/dist/merge-approved-backports.py
Added: subversion/trunk/tools/dist/backport/__init__.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/backport/__init__.py?rev=1669859&view=auto
==============================================================================
(empty)
Added: subversion/trunk/tools/dist/backport/merger.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/backport/merger.py?rev=1669859&view=auto
==============================================================================
--- subversion/trunk/tools/dist/backport/merger.py (added)
+++ subversion/trunk/tools/dist/backport/merger.py Sun Mar 29 07:08:45 2015
@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+backport.merger - library for running STATUS merges
+"""
+
+import backport.status
+
+import functools
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+import unittest
+
+logger = logging.getLogger(__name__)
+
+# The 'svn' binary
+SVN = os.getenv('SVN', 'svn')
+# TODO: maybe run 'svn info' to check if it works / fail early?
+
+# ### Hardcode these here.
+TRUNK = '^/subversion/trunk'
+BRANCHES = '^/subversion/branches'
+
+
+class UnableToMergeException(Exception):
+ pass
+
+
+def invoke_svn(argv, extra_env={}):
+ "Run svn with ARGV as argv[1:]. Return (exit_code, stdout, stderr)."
+ # TODO(interactive mode): disable --non-interactive
+ child_env = os.environ.copy()
+ child_env.update(extra_env)
+ child = subprocess.Popen([SVN, '--non-interactive'] + argv,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=child_env)
+ stdout, stderr = child.communicate()
+ return child.returncode, stdout.decode('UTF-8'), stderr.decode('UTF-8')
+
+def run_svn(argv, expected_stderr=None, extra_env={'LC_ALL': 'C'}):
+ """Run svn with ARGV as argv[1:]. If EXPECTED_STDERR is None, raise if the
+ exit code is non-zero or stderr is non-empty. Else, treat EXPECTED_STDERR as
+ a regexp, and ignore an errorful exit or stderr messages if the latter match
+ the regexp. Return exit_code, stdout, stderr."""
+
+ exit_code, stdout, stderr = invoke_svn(argv, extra_env)
+ if exit_code == 0 and not stderr:
+ return exit_code, stdout, stderr
+ elif expected_stderr and re.compile(expected_stderr).search(stderr):
+ return exit_code, stdout, stderr
+ else:
+ logger.warning("Unexpected stderr: %r", stderr)
+ # TODO: pass stdout/stderr to caller?
+ raise subprocess.CalledProcessError(returncode=exit_code,
+ cmd=[SVN] + argv)
+
+def run_svn_quiet(argv, *args, **kwargs):
+ "Wrapper for run_svn(-q)."
+ return run_svn(['-q'] + argv, *args, **kwargs)
+
+class Test_invoking_cmdline_client(unittest.TestCase):
+ def test_run_svn(self):
+ _, stdout, _ = run_svn(['--version', '-q'])
+ self.assertRegex(stdout, r'^1\.[0-9]+\.[0-9]+')
+
+ run_svn(['--version', '--no-such-option'], "invalid option")
+
+ with self.assertRaises(subprocess.CalledProcessError):
+ with self.assertLogs() as cm:
+ run_svn(['--version', '--no-such-option'])
+ self.assertRegex(cm.output[0], "Unexpected stderr.*")
+
+ def test_svn_version(self):
+ self.assertGreaterEqual(svn_version(), (1, 0))
+
+
+@functools.lru_cache(maxsize=1)
+def svn_version():
+ "Return the version number of the 'svn' binary as a (major, minor) tuple."
+ _, stdout, _ = run_svn(['--version', '-q'])
+ match = re.compile(r'(\d+)\.(\d+)').match(stdout)
+ assert match
+ return tuple(map(int, match.groups()))
+
+def run_revert():
+ return run_svn(['revert', '-q', '-R', './'])
+
+def last_changed_revision(path_or_url):
+ "Return the 'Last Changed Rev:' of PATH_OR_URL."
+
+ if svn_version() >= (1, 9):
+ return int(run_svn(['info', '--show-item=last-changed-revision', '--',
+ path_or_url])[1])
+ else:
+ _, lines, _ = run_svn(['info', '--', path_or_url]).splitlines()
+ for line in lines:
+ if line.startswith('Last Changed Rev:'):
+ return int(line.split(':', 1)[1])
+ else:
+ raise Exception("'svn info' did not print last changed revision")
+
+
+def merge(entry, expected_stderr=None, *, commit=False, sf=None):
+ """Merges ENTRY into the working copy at cwd.
+
+ Do not commit the result, unless COMMIT is true. When committing,
+ use parameter SF, a StatusFile instance, to remove ENTRY from the STATUS file
+ prior to committing.
+
+ EXPECTED_STDERR will be passed to run_svn() for the actual 'merge' command."""
+
+ assert (commit == False) or isinstance(sf, backport.status.StatusFile)
+ assert isinstance(entry, backport.status.StatusEntry)
+ assert entry.valid()
+
+ # TODO(interactive mode): catch the exception
+ validate_branch_contains_named_revisions(entry)
+
+ # Prepare mergeargs and logmsg.
+ logmsg = ""
+ if entry.branch:
+ branch_url = "%s/%s" % (BRANCHES, entry.branch)
+ if svn_version() >= (1, 8):
+ mergeargs = ['--', branch_url]
+ logmsg = "Merge {}:\n".format(entry.noun())
+ reintegrated_word = "merged"
+ else:
+ mergeargs = ['--reintegrate', '--', branch_url]
+ logmsg = "Reintegrate {}:\n".format(entry.noun())
+ reintegrated_word = "reintegrated"
+ logmsg += "\n"
+ elif entry.revisions:
+ mergeargs = []
+ if entry.accept:
+ mergeargs.append('--accept=%s' % (entry.accept,))
+ logmsg += "Merge {} from trunk, with --accept={}:\n".\
+ format(entry.noun(), entry.accept)
+ else:
+ logmsg += "Merge {} from trunk:\n".format(entry.noun())
+ logmsg += "\n"
+ mergeargs.extend('-c' + str(revision) for revision in entry.revisions)
+ mergeargs.extend(['--', TRUNK])
+ logmsg += entry.raw
+
+ # TODO(interactive mode): exclude STATUS from reverts
+ # TODO(interactive mode): save local mods to disk, as backport.pl does
+ run_revert()
+
+ run_svn_quiet(['update'])
+ # TODO: use select() to restore interweaving of stdout/stderr
+ _, stdout, stderr = run_svn_quiet(['merge'] + mergeargs, expected_stderr)
+ sys.stdout.write(stdout)
+ sys.stderr.write(stderr)
+ run_svn(['status', '-q'])
+
+ if commit:
+ sf.remove(entry)
+ sf.unparse(open('./STATUS', 'w'))
+ run_svn_quiet(['commit', '-m', logmsg])
+
+ # TODO: add the 'only mergeinfo changes' check (and regression test it)
+ # TODO(interactive mode): add the 'svn status' display
+
+ if entry.branch:
+ revnum = last_changed_revision('./STATUS')
+
+ if commit:
+ # TODO: disable this for test runs
+ # Sleep to avoid out-of-order commit notifications
+ time.sleep(15)
+ second_logmsg = "Remove the {!r} branch, {} in r{}."\
+ .format(entry.branch, reintegrated_word, revnum)
+ run_svn(['rm', '-m', second_logmsg, '--', branch_url])
+ time.sleep(1)
+
+def validate_branch_contains_named_revisions(entry):
+ """Validate that every revision explicitly named in ENTRY has either been
+ merged to its backport branch from trunk, or has been committed directly to
+ its backport branch. Entries that declare no backport branches are
+ considered valid. Return on success, raise on failure."""
+ if not entry.branch:
+ return # valid
+
+ if svn_version() < (1,5): # doesn't have 'svn mergeinfo' subcommand
+ return # skip check
+
+ branch_url = "%s/%s" % (BRANCHES, entry.branch)
+ present_str = \
+ run_svn(['mergeinfo', '--show-revs=merged', '--', TRUNK, branch_url])[1] + \
+ run_svn(['mergeinfo', '--show-revs=eligible', '--', branch_url])[1]
+
+ present = map(int, re.compile(r'(\d+)').findall(present_str))
+
+ absent = set(entry.revisions) - set(present)
+
+ if absent:
+ raise UnableToMergeException("Revisions '{}' nominated but not included "
+ "in branch".format(
+ ', '.join('r%d' % revno
+ for revno in absent)))
+
+
+
+def setUpModule():
+ "Set-up function, invoked by 'python -m unittest'."
+ # Suppress warnings generated by the test data.
+ # TODO: some test functions assume .assertLogs is available, they fail with
+ # AttributeError if it's absent (e.g., on python < 3.4).
+ try:
+ unittest.TestCase.assertLogs
+ except AttributeError:
+ logger.setLevel(logging.ERROR)
+
+if __name__ == '__main__':
+ unittest.main()
Added: subversion/trunk/tools/dist/backport/status.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/backport/status.py?rev=1669859&view=auto
==============================================================================
--- subversion/trunk/tools/dist/backport/status.py (added)
+++ subversion/trunk/tools/dist/backport/status.py Sun Mar 29 07:08:45 2015
@@ -0,0 +1,672 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+backport.status - library for parsing and unparsing STATUS files
+"""
+
+# Recipe for interactive testing:
+# % python3
+# >>> import backport.status
+# >>> sf = backport.status.StatusFile(open('STATUS'))
+# >>> entries = [p.entry() for p in sf.entries_paras()]
+# >>> entries[0]
+# <backport.status.StatusEntry object at 0x1b88f90>
+# >>>
+
+import collections
+import hashlib
+import io
+import logging
+import re
+import unittest
+
+logger = logging.getLogger(__name__)
+
+
+class ParseException(Exception):
+ pass
+
+
+class _ParagraphsIterator:
+ "A paragraph-based iterator for file-like objects."
+
+ def __init__(self, stream):
+ # KISS implementation, since STATUS files are small.
+ self.stream = stream
+ self.paragraphs = re.compile(r'\n\s*?\n+').split(stream.read())
+
+ def __iter__(self):
+ # Ensure there is exactly one trailing newline.
+ return iter(para.rstrip('\n') + "\n" for para in self.paragraphs)
+
+class Test_ParagraphsIterator(unittest.TestCase):
+ "Unit test for _ParagraphsIterator."
+ def test_basic(self):
+ stream = io.StringIO('foo\nfoo2\n\n\nbar\n')
+ paragraphs = _ParagraphsIterator(stream)
+ self.assertEqual(list(paragraphs), ['foo\nfoo2\n', 'bar\n'])
+
+
+class Kind:
+ "The kind of a single physical paragraph of STATUS. See 'Paragraph'."
+
+ preamble = object()
+ section_header = object()
+ nomination = object()
+ unknown = object()
+
+ # TODO: can avoid the repetition by using the 'enum' module of Python 3.4
+ # That will also make repr() useful.
+ @classmethod
+ def exists(cls, kind):
+ return kind in (cls.preamble, cls.section_header,
+ cls.nomination, cls.unknown)
+
+class Paragraph:
+ """A single physical paragraph of STATUS, which may be either a nomination
+ or something else."""
+
+ def __init__(self, kind, text, entry, containing_section):
+ """Constructor.
+
+ KIND is one of the Kind.* enumerators.
+
+ TEXT is the physical text in the file, used by unparsing.
+
+ ENTRY is the StatusEntry object, if Kind.nomination, else None.
+
+ CONTAINING_SECTION is the text of the section header this paragraph appears
+ within. (If this paragraph is a section header, this refers to itself.)
+ """
+
+ assert Kind.exists(kind)
+ assert (entry is not None) == (kind is Kind.nomination)
+ self.kind = kind
+ self.text = text
+ self._entry = entry
+ self._containing_section = containing_section
+
+ # Private for _paragraph_is_header()
+ _re_equals_line = re.compile(r'^=+$')
+
+ @classmethod
+ def is_header(cls, para_text):
+ """PARA_TEXT is a single physical paragraph, as a bare multiline string.
+
+ If PARA_TEXT is a section header, return the header text; else, return
+ False."""
+ lines = para_text.split('\n', 2)
+ valid = (len(lines) == 3
+ and lines[0].endswith(':')
+ and cls._re_equals_line.match(lines[1])
+ and lines[2] == '')
+ if valid:
+ header = lines[0].rstrip(':')
+ if header:
+ return header
+ return False
+
+ def entry(self):
+ "Validating accessor for ENTRY."
+ assert self.kind is Kind.nomination
+ return self._entry
+
+ def section(self):
+ "Validating accessor for CONTAINING_SECTION."
+ assert self.kind is not Kind.preamble
+ return self._containing_section
+
+ def approved(self):
+ "TRUE if this paragraph is in the approved section, false otherwise."
+ assert self.kind
+ # ### backport.pl used to check just .startswith() here.
+ return self.section() == "Approved changes"
+
+ def unparse(self, stream):
+ "Write this paragraph to STREAM, an open file-like object."
+ if self.kind in (Kind.preamble, Kind.section_header, Kind.unknown):
+ stream.write(self.text + "\n")
+ elif self.kind is Kind.nomination:
+ self.entry().unparse(stream)
+ else:
+ assert False, "Unknown paragraph kind"
+
+ def __repr__(self):
+ return "<Paragraph({!r}, {!r}, {!r}, {!r})>".format(
+ self.kind, self.text, self._entry, self._containing_section
+ )
+
+
+class StatusFile:
+ "Encapsulates the STATUS file."
+
+ def __init__(self, status_file):
+ "Constructor. STATUS_FILE is an open file-like object to parse."
+ self._parse(status_file)
+ self.validate_unique_entry_ids() # Use-case for making this optional?
+
+ def _parse(self, status_file):
+ "Parse self.status_file into self.paragraphs."
+
+ self.paragraphs = []
+ last_header = None
+ for para_text in _ParagraphsIterator(status_file):
+ kind = None
+ entry = None
+ header = Paragraph.is_header(para_text)
+ if para_text.isspace():
+ continue
+ elif header:
+ kind = Kind.section_header
+ last_header = header
+ elif last_header is not None:
+ try:
+ entry = StatusEntry(para_text)
+ kind = Kind.nomination
+ except ParseException:
+ kind = Kind.unknown
+ logger.warning("Failed to parse entry {!r} in {!r}".format(
+ para_text, status_file))
+ else:
+ kind = Kind.preamble
+
+ self.paragraphs.append(Paragraph(kind, para_text, entry, last_header))
+
+ def entries_paras(self):
+ "Return an iterator over entries"
+ return filter(lambda para: para.kind is Kind.nomination,
+ self.paragraphs)
+
+ def validate_unique_entry_ids(self):
+ # TODO: what about [r42, r43] and [r41, r43] entry pairs?
+ """Check if two entries have the same id. If so, mark them both
+ inoperative."""
+
+ # Build an auxiliary data structure.
+ id2entry = collections.defaultdict(list)
+ for para in self.entries_paras():
+ entry = para.entry()
+ id2entry[entry.id()].append(para)
+
+ # Examine it for problems.
+ for entry_id, entry_paras in id2entry.items():
+ if len(entry_paras) != 1:
+ # Found a problem.
+ #
+ # Warn about it, and ignore all involved entries.
+ logger.warning("There is more than one {} entry; ignoring them in "
+ "further processing".format(entry_id))
+ for para in entry_paras:
+ para.kind = Kind.unknown
+
+ def remove(self, entry):
+ "Remove ENTRY from SELF."
+ for para in self.entries_paras():
+ if para.entry() is entry:
+ self.paragraphs.remove(para)
+ return
+ else:
+ assert False, "Attempted to remove non-existent entry"
+
+ def unparse(self, stream):
+ "Write the STATUS file to STREAM, an open file-like object."
+ for para in self.paragraphs:
+ para.unparse(stream)
+
+
+class Test_StatusFile(unittest.TestCase):
+ def test__paragraph_is_header(self):
+ self.assertTrue(Paragraph.is_header("Nominations:\n========\n"))
+ self.assertFalse(Paragraph.is_header("Status of 1.9.12:\n"))
+
+ def test_parse_unparse(self):
+ s = (
+ "*** This release stream is used for testing. ***\n"
+ "\n"
+ "Candidate changes:\n"
+ "==================\n"
+ "\n"
+ " * r42\n"
+ " Bump version number to 1.0.\n"
+ " Votes:\n"
+ " +1: jrandom\n"
+ "\n"
+ "Approved changes:\n"
+ "=================\n"
+ "\n"
+ "This paragraph will trigger an exception.\n"
+ "\n"
+ " * r43\n"
+ " Bump version number to 1.0.\n"
+ " Votes:\n"
+ " +1: jrandom\n"
+ "\n"
+ )
+ test_file = io.StringIO(s)
+ with test_file:
+ with self.assertLogs() as cm:
+ sf = StatusFile(test_file)
+ self.assertRegex(cm.output[0], "Failed to parse.*'.*will trigger.*'")
+
+ self.assertSequenceEqual(
+ tuple(para.kind for para in sf.paragraphs),
+ (Kind.preamble,
+ Kind.section_header, Kind.nomination,
+ Kind.section_header, Kind.unknown, Kind.nomination)
+ )
+ self.assertFalse(sf.paragraphs[1].approved()) # header
+ self.assertFalse(sf.paragraphs[2].approved()) # nomination
+ self.assertTrue(sf.paragraphs[3].approved()) # header
+ self.assertTrue(sf.paragraphs[4].approved()) # unknown
+
+ output_file = io.StringIO()
+ sf.unparse(output_file)
+ self.assertEqual(s, output_file.getvalue())
+
+ def test_double_nomination(self):
+ "Test two nominations of the same group"
+
+ test_file = io.StringIO(
+ "Approved changes:\n"
+ "=================\n"
+ "\n"
+ " * r42\n"
+ " First time.\n"
+ "\n"
+ " * r42\n"
+ " Second time.\n"
+ "\n"
+ )
+
+ with test_file:
+ with self.assertLogs() as cm:
+ sf = StatusFile(test_file)
+ self.assertRegex(cm.output[0], "There is more than one r42 entry")
+ self.assertIs(sf.paragraphs[1].kind, Kind.unknown)
+ self.assertIs(sf.paragraphs[2].kind, Kind.unknown)
+
+
+class StatusEntry:
+ """Encapsulates a single nomination.
+
+ An Entry has the following attributes:
+
+ branch - the backport branch's basename, or None.
+ revisions - the revisions to nominated, as iterable of int.
+ logsummary - the text before the justification, as an array of lines.
+ depends - true if a "Depends:" entry was found, False otherwise.
+ accept - the value to pass to 'svn merge --accept=%s', or None.
+ votes_str - everything after the "Votes:" subheader. An unparsed string.
+ """
+
+ def __init__(self, para_text):
+ """Parse an entry from PARA_TEXT, and add it to SELF. PARA_TEXT must
+ contain exactly one entry, as a single multiline string."""
+ self.branch = None
+ self.revisions = []
+ self.logsummary = []
+ self.depends = False
+ self.accept = None
+ self.votes_str = None
+
+ self.raw = para_text
+
+ _re_entry_indentation = re.compile(r'^( *\* )')
+ _re_revisions_line = re.compile(r'^(?:r?\d+[,; ]*)+$')
+
+ lines = para_text.rstrip().split('\n')
+
+ # Strip indentation and trailing whitespace.
+ match = _re_entry_indentation.match(lines[0])
+ if not match:
+ raise ParseException("Entry found with no ' * ' line")
+ indentation = len(match.group(1))
+ lines = (line[indentation:] for line in lines)
+ lines = (line.rstrip() for line in lines)
+
+ # Consume the generator.
+ lines = list(lines)
+
+ # Parse the revisions lines.
+ match = re.compile(r'(\S*) branch|branches/(\S*)').search(lines[0])
+ if match:
+ # Parse whichever group matched.
+ self.branch = self.parse_branch(match.group(1) or match.group(2))
+ else:
+ while _re_revisions_line.match(lines[0]):
+ self.revisions.extend(map(int, re.compile(r'(\d+)').findall(lines[0])))
+ lines = lines[1:]
+
+ # Validate it now, since later exceptions rely on it.
+ if not(self.branch or self.revisions):
+ raise ParseException("Entry found with neither branch nor revisions")
+
+ # Parse the logsummary.
+ while lines and not self._is_subheader(lines[0]):
+ self.logsummary.append(lines[0])
+ lines = lines[1:]
+
+ # Parse votes.
+ if "Votes:" in lines:
+ index = lines.index("Votes:")
+ self.votes_str = '\n'.join(lines[index+1:]) + '\n'
+ lines = lines[:index]
+ del index
+ else:
+ self.votes_str = None
+
+ # depends, branch, notes
+ while lines:
+
+ if lines[0].strip().startswith('Depends:'):
+ self.depends = True
+ lines = lines[1:]
+ continue
+
+ if lines[0].strip().startswith('Branch:'):
+ maybe_value = lines[0].strip().split(':', 1)[1]
+ if maybe_value.strip():
+ # Value on same line as header
+ self.branch = self.parse_branch(maybe_value)
+ lines = lines[1:]
+ continue
+ else:
+ # Value should be on next line
+ if len(lines) == 1:
+ raise ParseException("'Branch:' header found without value")
+ self.branch = self.parse_branch(lines[1])
+ lines = lines[2:]
+ continue
+
+ if lines[0].strip().startswith('Notes:'):
+ notes = lines[0].strip().split(':', 1)[1] + "\n"
+ lines = lines[1:]
+
+ # Consume the indented body of the "Notes" field.
+ while lines and not lines[0][0].isalnum():
+ notes += lines[0] + "\n"
+ lines = lines[1:]
+
+ # Look for possible --accept directives.
+ matches = re.compile(r'--accept[ =]([a-z-]+)').findall(notes)
+ if len(matches) > 1:
+ raise ParseException("Too many --accept values at %s" % (self,))
+ elif len(matches) == 1:
+ self.accept = matches[0]
+
+ continue
+
+ # else
+ lines = lines[1:]
+ continue
+
+ # Some sanity checks.
+ if self.branch and self.accept:
+ raise ParseException("Entry %s has both --accept and branch" % (self,))
+
+ if not self.logsummary:
+ raise ParseException("No logsummary at %s" % (self,))
+
+ def digest(self):
+ """Return a unique digest of this entry, with the following property: any
+ change to the entry will cause the digest value to change."""
+
+ # Digest the raw text, canonicalizing the number of trailing newlines.
+ # There is no particular reason to use md5 over anything else, except for
+ # compatibility with existing .backports1 files in people's working copies.
+ return hashlib.md5(self.raw.rstrip('\n').encode('UTF-8')
+ + b"\n\n").hexdigest()
+
+ @staticmethod
+ def parse_branch(string):
+ "Extract a branch name from STRING."
+ return string.strip().rstrip('/').split('/')[-1]
+
+ def valid(self):
+ "Test the invariants."
+ return all([
+ self.branch or self.revisions,
+ self.logsummary,
+ not(self.branch and self.accept),
+ ])
+
+ def id(self):
+ "Return the first revision or branch's name."
+ # Assert a minimal invariant, since this is used by error paths.
+ assert self.branch or self.revisions
+ if self.branch is not None:
+ return self.branch
+ else:
+ return "r{:d}".format(self.revisions[0])
+
+ def noun(self, start_of_sentence=False):
+ """Return a noun phrase describing this entry.
+ START_OF_SENTENCE is used to correctly capitalize the result."""
+ # Assert a minimal invariant, since this is used by error paths.
+ assert self.branch or self.revisions
+ if start_of_sentence:
+ the = "The"
+ else:
+ the = "the"
+ if self.branch is not None:
+ return "{} {} branch".format(the, self.branch)
+ elif len(self.revisions) == 1:
+ return "r{:d}".format(self.revisions[0])
+ else:
+ return "{} r{:d} group".format(the, self.revisions[0])
+
+ def logsummarysummary(self):
+ "Return a one-line summary of the changeset."
+ assert self.valid()
+ suffix = "" if len(self.logsummary) == 1 else " [...]"
+ return self.logsummary[0] + suffix
+
+ # Private for is_vetoed()
+ _re_vetoed = re.compile(r'^\s*(-1:|-1\s*[()])', re.MULTILINE)
+ def is_vetoed(self):
+ "Return TRUE iff a -1 vote has been cast."
+ return self._re_vetoed.search(self.votes_str)
+
+ @staticmethod
+ def _is_subheader(string):
+ """Given a single line from an entry, is that line a subheader (such as
+ "Justification:" or "Notes:")?"""
+ # TODO: maybe change the 'subheader' heuristic? Perhaps "line starts with
+ # an uppercase letter and ends with a colon".
+ #
+ # This is currently only used for finding the end of logsummary, and all
+ # explicitly special-cased headers (e.g., "Depends:") match this, though.
+ return re.compile(r'^\s*\w+:').match(string)
+
+ def unparse(self, stream):
+ "Write this entry to STREAM, an open file-like object."
+ # For now, this is simple.. until we add interactive editing.
+ stream.write(self.raw + "\n")
+
+class Test_StatusEntry(unittest.TestCase):
+ def test___init__(self):
+ "Test the entry parser"
+
+ # All these entries actually have a "four spaces" line as their last line,
+ # but the parser doesn't care.
+
+ s = """\
+ * r42, r43,
+ r44
+ This is the logsummary.
+ Branch:
+ 1.8.x-rfourty-two
+ Votes:
+ +1: jrandom
+ """
+ entry = StatusEntry(s)
+ self.assertEqual(entry.branch, "1.8.x-rfourty-two")
+ self.assertEqual(entry.revisions, [42, 43, 44])
+ self.assertEqual(entry.logsummary, ["This is the logsummary."])
+ self.assertEqual(entry.logsummarysummary(), "This is the logsummary.")
+ self.assertFalse(entry.depends)
+ self.assertIsNone(entry.accept)
+ self.assertIn("+1: jrandom", entry.votes_str)
+ self.assertFalse(entry.is_vetoed())
+ self.assertEqual(entry.id(), "1.8.x-rfourty-two")
+ self.assertEqual(entry.noun(True), "The 1.8.x-rfourty-two branch")
+ self.assertEqual(entry.noun(), "the 1.8.x-rfourty-two branch")
+
+ s = """\
+ * r42
+ This is the logsummary.
+ It has multiple lines.
+ Depends: must be merged before the r43 entry"
+ Notes:
+ Merge with --accept=theirs-conflict.
+ Votes:
+ +1: jrandom
+ -1: jconstant
+ """
+ entry = StatusEntry(s)
+ self.assertIsNone(entry.branch)
+ self.assertEqual(entry.revisions, [42])
+ self.assertEqual(entry.logsummary,
+ ["This is the logsummary.",
+ "It has multiple lines."])
+ self.assertEqual(entry.logsummarysummary(),
+ "This is the logsummary. [...]")
+ self.assertTrue(entry.depends)
+ self.assertEqual(entry.accept, "theirs-conflict")
+ self.assertRegex(entry.votes_str, "(?s)jrandom.*jconstant") # re.DOTALL
+ self.assertTrue(entry.is_vetoed())
+ self.assertEqual(entry.id(), "r42")
+ self.assertEqual(entry.noun(), "r42")
+
+ s = """\
+ * ^/subversion/branches/1.8.x-fixes
+ This is the logsummary.
+ Votes:
+ +1: jrandom
+ -1 (see <message-id>): jconstant
+ """
+ entry = StatusEntry(s)
+ self.assertEqual(entry.branch, "1.8.x-fixes")
+ self.assertEqual(entry.revisions, [])
+ self.assertTrue(entry.is_vetoed())
+
+ s = """\
+ * r42
+ This is the logsummary.
+ Branch: ^/subversion/branches/on-the-same-line
+ Votes:
+ +1: jrandom
+ """
+ entry = StatusEntry(s)
+ self.assertEqual(entry.branch, "on-the-same-line")
+ self.assertEqual(entry.revisions, [42])
+
+ self.assertTrue(str(entry)) # tests __str__
+ self.assertEqual(entry.raw, s)
+
+ s = """\
+ * The 1.8.x-fixes branch
+ This is the logsummary.
+ Votes:
+ +1: jrandom
+ """
+ entry = StatusEntry(s)
+ self.assertEqual(entry.branch, "1.8.x-fixes")
+
+ s = """\
+ * The 1.8.x-fixes branch
+ This is the logsummary.
+ Notes: merge with --accept=tc.
+ Votes:
+ +1: jrandom
+ """
+ with self.assertRaisesRegex(ParseException, "both.*accept.*branch"):
+ entry = StatusEntry(s)
+
+ s = """\
+ * r42
+ Votes:
+ +1: jrandom
+ """
+ with self.assertRaisesRegex(ParseException, "No logsummary"):
+ entry = StatusEntry(s)
+
+ s = """\
+ * r42
+ This is the logsummary.
+ Notes: merge with --accept=mc.
+ This tests multi-line notes.
+ Merge with --accept=tc.
+ Votes:
+ +1: jrandom
+ """
+ with self.assertRaisesRegex(ParseException, "Too many.*--accept"):
+ entry = StatusEntry(s)
+
+ def test_digest(self):
+ s = """\
+ * r42
+ Fix a bug.
+ Votes:
+ +1: jrandom\n"""
+ digest = '92812e1f36a33f7d51670f89134ad2ee'
+ entry = StatusEntry(s)
+ self.assertEqual(entry.digest(), digest)
+
+ entry = StatusEntry(s + "\n\n\n")
+ self.assertEqual(entry.digest(), digest)
+
+ entry = StatusEntry(s.replace('Fix', 'Introduce'))
+ self.assertNotEqual(entry.digest(), digest)
+
+ def test_parse_branch(self):
+ inputs = (
+ "1.8.x-r42",
+ "branches/1.8.x-r42",
+ "branches/1.8.x-r42/",
+ "subversion/branches/1.8.x-r42",
+ "subversion/branches/1.8.x-r42/",
+ "^/subversion/branches/1.8.x-r42",
+ "^/subversion/branches/1.8.x-r42/",
+ )
+
+ for string in inputs:
+ self.assertEqual(StatusEntry.parse_branch(string), "1.8.x-r42")
+
+ def test__is_subheader(self):
+ "Test that all explicitly-special-cased headers are detected as subheaders."
+ subheaders = "Justification: Notes: Depends: Branch: Votes:".split()
+ for subheader in subheaders:
+ self.assertTrue(StatusEntry._is_subheader(subheader))
+ self.assertTrue(StatusEntry._is_subheader(subheader + " with value"))
+
+
+def setUpModule():
+ "Set-up function, invoked by 'python -m unittest'."
+ # Suppress warnings generated by the test data.
+ # TODO: some test functions assume .assertLogs is available, they fail with
+ # AttributeError if it's absent (e.g., on python < 3.4).
+ try:
+ unittest.TestCase.assertLogs
+ except AttributeError:
+ logger.setLevel(logging.ERROR)
+
+if __name__ == '__main__':
+ unittest.main()
Added: subversion/trunk/tools/dist/detect-conflicting-backports.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/detect-conflicting-backports.py?rev=1669859&view=auto
==============================================================================
--- subversion/trunk/tools/dist/detect-conflicting-backports.py (added)
+++ subversion/trunk/tools/dist/detect-conflicting-backports.py Sun Mar 29 07:08:45 2015
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""\
+Conflicts detector script.
+
+This script is used by buildbot.
+
+Run this script from the root of a stable branch's working copy (e.g.,
+a working copy of /branches/1.9.x). This script will iterate the STATUS file,
+attempt to merge each entry therein (nothing will be committed), and exit
+non-zero if any merge produced a conflict.
+
+
+Conflicts caused by entry interdependencies
+-------------------------------------------
+
+Occasionally, a nomination is added to STATUS that is expected to conflict (for
+example, because it textually depends on another revision that is also
+nominated). To prevent false positive failures in such cases, the dependent
+entry may be annotated by a "Depends:" header, to signal to this script that
+the conflict is expected. Expected conflicts never cause a non-zero exit code.
+
+A "Depends:" header looks as follows:
+
+ * r42
+ Make some change.
+ Depends:
+ Requires the r40 group to be merged first.
+ Votes:
+ +1: jrandom
+
+The value of the header is not parsed; the script only cares about its presence
+of absence.
+"""
+
+import sys
+assert sys.version_info[0] == 3, "This script targets Python 3"
+
+import backport.status
+import backport.merger
+
+import collections
+import logging
+import re
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+if sys.argv[1:]:
+ # Usage.
+ print(__doc__)
+ sys.exit(0)
+
+sf = backport.status.StatusFile(open('./STATUS'))
+
+ERRORS = collections.defaultdict(list)
+
+# Main loop.
+for entry_para in sf.entries_paras():
+ if entry_para.approved():
+ entry = entry_para.entry()
+ # SVN_ERR_WC_FOUND_CONFLICT = 155015
+ backport.merger.merge(entry, 'svn: E155015' if entry.depends else None)
+
+ _, output, _ = backport.merger.run_svn(['status'])
+
+ # Pre-1.6 svn's don't have the 7th column, so fake it.
+ if backport.merger.svn_version() < (1,6):
+ output = re.compile('^(......)', re.MULTILINE).sub(r'\1 ', output)
+
+ pattern = re.compile(r'(?:C......|.C.....|......C)\s(.*)', re.MULTILINE)
+ conflicts = pattern.findall(output)
+ if conflicts and not entry.depends:
+ if len(conflicts) == 1:
+ victims = conflicts[0]
+ else:
+ victims = '[{}]'.format(', '.join(conflicts))
+ ERRORS[entry].append("Conflicts on {}".format(victims))
+ sys.stderr.write(
+ "Conflicts merging {}!\n"
+ "\n"
+ "{}\n"
+ .format(entry.noun(), output)
+ )
+ subprocess.check_call([backport.merger.SVN, 'diff', '--'] + conflicts)
+ elif entry.depends and not conflicts:
+ # Not a warning since svn-role may commit the dependency without
+ # also committing the dependent in the same pass.
+ print("No conflicts merging {}, but conflicts were "
+ "expected ('Depends:' header set)".format(entry.noun()))
+ elif conflicts:
+ print("Conflicts found merging {}, as expected.".format(entry.noun()))
+ backport.merger.run_revert()
+
+# Summarize errors before exiting.
+if ERRORS:
+ warn = sys.stderr.write
+ warn("Warning summary\n")
+ warn("===============\n");
+ warn("\n");
+ for entry, warnings in ERRORS.items():
+ for warning in warnings:
+ title = entry.logsummarysummary()
+ warn('{} ({}): {}\n'.format(entry.id(), title, warning))
+ sys.exit(1)
Added: subversion/trunk/tools/dist/merge-approved-backports.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dist/merge-approved-backports.py?rev=1669859&view=auto
==============================================================================
--- subversion/trunk/tools/dist/merge-approved-backports.py (added)
+++ subversion/trunk/tools/dist/merge-approved-backports.py Sun Mar 29 07:08:45 2015
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""\
+Automatic backport merging script.
+
+This script is run from cron. It may also be run interactively, however, it
+has no interactive features.
+
+Run this script from the root of a stable branch's working copy (e.g.,
+a working copy of /branches/1.9.x). This script will iterate the STATUS file
+and commit every nomination in the section "Approved changes".
+"""
+
+import sys
+assert sys.version_info[0] == 3, "This script targets Python 3"
+
+import backport.status
+import backport.merger
+
+if sys.argv[1:]:
+ # Usage.
+ print(__doc__)
+ sys.exit(0)
+
+sf = backport.status.StatusFile(open('./STATUS'))
+
+# Duplicate sf.paragraphs, since merge() will be removing elements from it.
+entries_paras = list(sf.entries_paras())
+for entry_para in entries_paras:
+ if entry_para.approved():
+ entry = entry_para.entry()
+ backport.merger.merge(entry, commit=True, sf=sf)