You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by br...@apache.org on 2014/02/25 03:53:03 UTC
[20/26] git commit: SENTRY-113: Fix Sentry Precommit tests (Brock
Noland via Gregory Chanan)
SENTRY-113: Fix Sentry Precommit tests (Brock Noland via Gregory Chanan)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/7e1ce212
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/7e1ce212
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/7e1ce212
Branch: refs/heads/db_policy_store
Commit: 7e1ce212fb851b8233f0f5af9d23c9a82257b6cb
Parents: 55ca6c7
Author: Gregory Chanan <gr...@gmail.com>
Authored: Fri Feb 7 16:06:19 2014 -0800
Committer: Gregory Chanan <gr...@gmail.com>
Committed: Fri Feb 7 16:06:19 2014 -0800
----------------------------------------------------------------------
.gitignore | 1 +
dev-support/smart-apply-patch.sh | 99 +++++++++++++++++++++++++++++++++++
dev-support/test-patch.py | 73 +++++++++++++-------------
pom.xml | 2 +
4 files changed, 137 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/7e1ce212/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 920de9a..06f9ff4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ target/
*.war
*.ear
test-output/
+maven-repo/
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/7e1ce212/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
new file mode 100644
index 0000000..fce2735
--- /dev/null
+++ b/dev-support/smart-apply-patch.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+PATCH_FILE=$1
+DRY_RUN=$2
+if [ -z "$PATCH_FILE" ]; then
+ echo usage: $0 patch-file
+ exit 1
+fi
+
+PATCH=${PATCH:-patch} # allow overriding patch binary
+
+# Cleanup handler for temporary files
+TOCLEAN=""
+cleanup() {
+ rm $TOCLEAN
+ exit $1
+}
+trap "cleanup 1" HUP INT QUIT TERM
+
+# Allow passing "-" for stdin patches
+if [ "$PATCH_FILE" == "-" ]; then
+ PATCH_FILE=/tmp/tmp.in.$$
+ cat /dev/fd/0 > $PATCH_FILE
+ TOCLEAN="$TOCLEAN $PATCH_FILE"
+fi
+
+# Come up with a list of changed files into $TMP
+TMP=/tmp/tmp.paths.$$
+TOCLEAN="$TOCLEAN $TMP"
+
+if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
+ PLEVEL=0
+ #if the patch applied at P0 there is the possability that all we are doing
+ # is adding new files and they would apply anywhere. So try to guess the
+ # correct place to put those files.
+
+ TMP2=/tmp/tmp.paths.2.$$
+ TOCLEAN="$TOCLEAN $TMP2"
+
+ egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
+
+ if [ ! -s $TMP2 ]; then
+ echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
+ cleanup 1
+ fi
+
+ #first off check that all of the files do not exist
+ FOUND_ANY=0
+ for CHECK_FILE in $(cat $TMP2)
+ do
+ if [[ -f $CHECK_FILE ]]; then
+ FOUND_ANY=1
+ fi
+ done
+
+ if [[ "$FOUND_ANY" = "0" ]]; then
+ #all of the files are new files so we have to guess where the correct place to put it is.
+
+ # if all of the lines start with a/ or b/, then this is a git patch that
+ # was generated without --no-prefix
+ if ! grep -qv '^a/\|^b/' $TMP2 ; then
+ echo Looks like this is a git patch. Stripping a/ and b/ prefixes
+ echo and incrementing PLEVEL
+ PLEVEL=$[$PLEVEL + 1]
+ sed -i -e 's,^[ab]/,,' $TMP2
+ fi
+
+ fi
+elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+ PLEVEL=1
+elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+ PLEVEL=2
+else
+ echo "The patch does not appear to apply with p0, p1, or p2";
+ cleanup 1;
+fi
+
+# If this is a dry run then exit instead of applying the patch
+if [[ -n $DRY_RUN ]]; then
+ cleanup 0;
+fi
+
+echo Going to apply patch with: $PATCH -p$PLEVEL
+$PATCH -p$PLEVEL -E < $PATCH_FILE
+
+cleanup $?
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/7e1ce212/dev-support/test-patch.py
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.py b/dev-support/test-patch.py
index 8fd158d..449fe0e 100644
--- a/dev-support/test-patch.py
+++ b/dev-support/test-patch.py
@@ -88,7 +88,7 @@ def jira_post_comment(result, defect, branch, username, password):
# hack (from hadoop) but REST api doesn't list attachments?
def jira_get_attachment(result, defect, username, password):
html = jira_get_defect_html(result, defect, username, password)
- pattern = "(/secure/attachment/[0-9]+/%s[0-9\-]*\.(patch|txt|patch\.txt))" % (re.escape(defect))
+ pattern = "(/secure/attachment/[0-9]+/%s[0-9\.\-]*\.(patch|txt|patch\.txt))" % (re.escape(defect))
matches = []
for match in re.findall(pattern, html, re.IGNORECASE):
matches += [ match[0] ]
@@ -118,26 +118,25 @@ def git_checkout(result, branch):
if execute("git merge --ff-only origin/%s" % (branch)):
result.fatal("git merge failed")
-def git_apply(result, cmd, patch_file, strip, output_dir):
+def git_apply(result, cmd, patch_file, output_dir):
output_file = "%s/apply.txt" % (output_dir)
- rc = execute("%s -p%s < %s 1>%s 2>&1" % (cmd, strip, patch_file, output_file))
+ rc = execute("%s %s 1>%s 2>&1" % (cmd, patch_file, output_file))
output = ""
if os.path.exists(output_file):
with open(output_file) as fh:
output = fh.read()
- if rc == 0:
- if output:
- print output
- else:
+ if output:
+ print output
+ if rc != 0:
result.fatal("failed to apply patch (exit code %d):\n%s\n" % (rc, output))
-def mvn_clean(result, workspace, output_dir):
- rc = execute("mvn clean -Dmaven.repo.local=%s 1>%s/clean.txt 2>&1" % (workspace, output_dir))
+def mvn_clean(result, mvn_repo, output_dir):
+ rc = execute("mvn clean -Dmaven.repo.local=%s 1>%s/clean.txt 2>&1" % (mvn_repo, output_dir))
if rc != 0:
result.fatal("failed to clean project (exit code %d)" % (rc))
-def mvn_install(result, workspace, output_dir):
- rc = execute("mvn install -DskipTests -Dmaven.repo.local=%s 1>%s/install.txt 2>&1" % (workspace, output_dir))
+def mvn_install(result, mvn_repo, output_dir):
+ rc = execute("mvn install -DskipTests -Dmaven.repo.local=%s 1>%s/install.txt 2>&1" % (mvn_repo, output_dir))
if rc != 0:
result.fatal("failed to build with patch (exit code %d)" % (rc))
@@ -146,8 +145,8 @@ def find_all_files(top):
for f in files:
yield os.path.join(root, f)
-def mvn_test(result, workspace, output_dir):
- rc = execute("mvn verify -Dmaven.repo.local=%s 1>%s/test.txt 2>&1" % (workspace, output_dir))
+def mvn_test(result, mvn_repo, output_dir):
+ rc = execute("mvn verify -Dmaven.repo.local=%s 1>%s/test.txt 2>&1" % (mvn_repo, output_dir))
if rc == 0:
result.success("all tests passed")
else:
@@ -189,9 +188,10 @@ class Result(object):
if self._fatal or self._error:
if tmp_dir:
print "INFO: output is located %s" % (tmp_dir)
+ sys.exit(1)
elif tmp_dir:
shutil.rmtree(tmp_dir)
- sys.exit(0)
+ sys.exit(0)
usage = "usage: %prog [options]"
parser = OptionParser(usage)
@@ -205,16 +205,10 @@ parser.add_option("--run-tests", dest="run_tests",
help="Run Tests", action="store_true")
parser.add_option("--username", dest="username",
help="JIRA Username", metavar="USERNAME", default="hiveqa")
-parser.add_option("--output", dest="output_dir",
- help="Directory to write output", metavar="DIRECTORY")
parser.add_option("--post-results", dest="post_results",
help="Post results to JIRA (only works in defect mode)", action="store_true")
parser.add_option("--password", dest="password",
help="JIRA Password", metavar="PASSWORD")
-parser.add_option("--patch-command", dest="patch_cmd", default="git apply",
- help="Patch command such as `git apply' or `patch'", metavar="COMMAND")
-parser.add_option("-p", "--strip", dest="strip", default="1",
- help="Remove <n> leading slashes from diff paths", metavar="N")
parser.add_option("--workspace", dest="workspace",
help="Jenkins workspace directory", metavar="DIR")
@@ -227,10 +221,6 @@ if options.defect and options.filename:
print "FATAL: Both --defect and --file cannot be specified."
sys.exit(1)
-if options.output_dir and not os.path.isdir(options.output_dir):
- print "FATAL: Output directory %s does not exist" % (options.output_dir)
- sys.exit(1)
-
if options.post_results and not options.password:
print "FATAL: --post-results requires --password"
sys.exit(1)
@@ -239,15 +229,13 @@ if not options.workspace:
print "FATAL: --workspace is required"
sys.exit(1)
+patch_cmd = "bash ./dev-support/smart-apply-patch.sh"
branch = options.branch
-output_dir = options.output_dir
defect = options.defect
username = options.username
password = options.password
run_tests = options.run_tests
post_results = options.post_results
-strip = options.strip
-patch_cmd = options.patch_cmd
workspace = options.workspace
result = Result()
@@ -270,12 +258,19 @@ if post_results:
result.exit()
result.exit_handler = post_jira_comment_and_exit
-if not output_dir:
- tmp_dir = tempfile.mkdtemp()
- output_dir = tmp_dir
-
-if output_dir.endswith("/"):
- output_dir = output_dir[:-1]
+if workspace.endswith("/"):
+ workspace = workspace[:-1]
+mvn_repo = workspace + "/maven-repo"
+output_dir = workspace + "/test-output"
+if os.path.exists(mvn_repo):
+ if not os.path.isdir(mvn_repo):
+ shutil.rmtree(mvn_repo)
+ os.mkdir(mvn_repo)
+else:
+ os.mkdir(mvn_repo)
+if os.path.exists(output_dir):
+ shutil.rmtree(output_dir)
+os.mkdir(output_dir)
if defect:
jira_json = jira_get_defect(result, defect, username, password)
@@ -294,14 +289,16 @@ if defect:
elif options.filename:
patch_file = options.filename
else:
- raise Exception("Not reachable")
+ print "ERROR: Reached unreachable code. Please report."
+ sys.exit(1)
+
-mvn_clean(result, workspace, output_dir)
+mvn_clean(result, mvn_repo, output_dir)
git_checkout(result, branch)
-git_apply(result, patch_cmd, patch_file, strip, output_dir)
-mvn_install(result, workspace, output_dir)
+git_apply(result, patch_cmd, patch_file, output_dir)
+mvn_install(result, mvn_repo, output_dir)
if run_tests:
- mvn_test(result, workspace, output_dir)
+ mvn_test(result, mvn_repo, output_dir)
else:
result.info("patch applied and built but tests did not execute")
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/7e1ce212/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7db0088..7fdfb72 100644
--- a/pom.xml
+++ b/pom.xml
@@ -345,6 +345,8 @@ limitations under the License.
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>.idea/</exclude>
+ <exclude>maven-repo/</exclude>
+ <exclude>test-output/</exclude>
<exclude>**/*.iml</exclude>
<exclude>*.log</exclude>
<exclude>**/nb-configuration.xml</exclude>