You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/11/17 18:44:43 UTC
hive git commit: HIVE-15226: Add a different masking comment to
qtests blobstore output (Sergio Pena,
reviewed by Mohit Sabharwal and Sahil Takiar)
Repository: hive
Updated Branches:
refs/heads/master 2de8b3192 -> c5d147b68
HIVE-15226: Add a different masking comment to qtests blobstore output (Sergio Pena, reviewed by Mohit Sabharwal and Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5d147b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5d147b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5d147b6
Branch: refs/heads/master
Commit: c5d147b68a922a69aef163a2287bc3761b3a7186
Parents: 2de8b31
Author: Sergio Pena <se...@cloudera.com>
Authored: Thu Nov 17 12:43:26 2016 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Thu Nov 17 12:44:33 2016 -0600
----------------------------------------------------------------------
.../results/clientpositive/insert_into.q.out | 4 ++--
.../control/AbstractCoreBlobstoreCliDriver.java | 2 ++
.../org/apache/hadoop/hive/ql/QTestUtil.java | 24 ++++++++++++++++++--
3 files changed, 26 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c5d147b6/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
index 919f3e7..93f545d 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
@@ -4,12 +4,12 @@ POSTHOOK: query: DROP TABLE qtest
POSTHOOK: type: DROPTABLE
#### A masked pattern was here ####
PREHOOK: type: CREATETABLE
-PREHOOK: Input: #### A masked pattern was here ####
+PREHOOK: Input: ### test.blobstore.path ###/qtest
PREHOOK: Output: database:default
PREHOOK: Output: default@qtest
#### A masked pattern was here ####
POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: #### A masked pattern was here ####
+POSTHOOK: Input: ### test.blobstore.path ###/qtest
POSTHOOK: Output: database:default
POSTHOOK: Output: default@qtest
PREHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000)
http://git-wip-us.apache.org/repos/asf/hive/blob/c5d147b6/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index e797eb5..2f57683 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -163,5 +163,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime())
+ "-" + String.format("%03d", (int)(Math.random() * 999));
testBlobstorePathUnique = testBlobstorePath + uid;
+
+ qt.addPatternWithMaskComment(testBlobstorePathUnique, String.format("### %s ###", HCONF_TEST_BLOBSTORE_PATH));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c5d147b6/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index ab21589..d08ecc7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -67,6 +67,8 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
@@ -1643,6 +1645,18 @@ public class QTestUtil {
}
if (!partialMaskWasMatched) {
+ for (Pair<Pattern, String> pair : patternsWithMaskComments) {
+ Pattern pattern = pair.getLeft();
+ String maskComment = pair.getRight();
+
+ matcher = pattern.matcher(line);
+ if (matcher.find()) {
+ line = matcher.replaceAll(maskComment);
+ partialMaskWasMatched = true;
+ break;
+ }
+ }
+
for (Pattern pattern : patterns) {
line = pattern.matcher(line).replaceAll(maskPattern);
}
@@ -1707,8 +1721,7 @@ public class QTestUtil {
".*at com\\.sun\\.proxy.*",
".*at com\\.jolbox.*",
".*at com\\.zaxxer.*",
- "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
- "(s3.?|swift|wasb.?):\\/\\/[\\w\\.\\/-]*"
+ "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*"
});
private final Pattern[] partialReservedPlanMask = toPattern(new String[] {
@@ -1716,6 +1729,13 @@ public class QTestUtil {
//TODO: add more expected test result here
});
+ /* This list may be modified by specific cli drivers to mask strings that change on every test */
+ private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<>();
+
+ public void addPatternWithMaskComment(String patternStr, String maskComment) {
+ patternsWithMaskComments.add(ImmutablePair.of(Pattern.compile(patternStr), maskComment));
+ }
+
public int checkCliDriverResults(String tname) throws Exception {
assert(qMap.containsKey(tname));