You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by yc...@apache.org on 2017/05/24 14:30:28 UTC
hive git commit: HIVE-16643: BeeLine tests output should keep the
PREHOOK/POSTHOOK Input/Output orderdering (Peter Vary,
reviewed by Yongzhi Chen)
Repository: hive
Updated Branches:
refs/heads/master 1bdc5413d -> dec96ca6f
HIVE-16643: BeeLine tests output should keep the PREHOOK/POSTHOOK Input/Output orderdering (Peter Vary, reviewed by Yongzhi Chen)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dec96ca6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dec96ca6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dec96ca6
Branch: refs/heads/master
Commit: dec96ca6f6fb8de04b2847be2285792275ea9423
Parents: 1bdc541
Author: Yongzhi Chen <yc...@apache.org>
Authored: Wed May 24 10:30:13 2017 -0400
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Wed May 24 10:30:13 2017 -0400
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 3 +-
.../java/org/apache/hive/beeline/QFile.java | 32 ++++-
.../beeline/create_merge_compressed.q.out | 138 +++++++++++++++++++
.../beeline/escape_comments.q.out | 2 -
4 files changed, 168 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e23ef63..8e99b85 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -756,7 +756,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_auto_purge_tables.q \
encryption_drop_table_in_encrypted_db.q
-beeline.positive.include=drop_with_concurrency.q,\
+beeline.positive.include=create_merge_compressed.q,\
+ drop_with_concurrency.q,\
escape_comments.q,\
mapjoin2.q,\
materialized_view_create_rewrite.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
index af464b9..0d00a9b 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
@@ -31,8 +31,11 @@ import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@@ -59,6 +62,8 @@ public final class QFile {
private static final Pattern USE_PATTERN =
Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE);
+ private static final Pattern ENTITYLIST_PATTERN =
+ Pattern.compile("(((PREHOOK|POSTHOOK): (Input|Output): \\S+\n)+)", Pattern.MULTILINE);
private static final String MASK_PATTERN = "#### A masked pattern was here ####\n";
@@ -174,13 +179,29 @@ public final class QFile {
return source;
}
+ /**
+ * The PREHOOK/POSTHOOK Input/Output lists should be sorted again after reverting the database
+ * name in those strings to match the original Cli output.
+ * @param source The original query output
+ * @return The query output where the input/output list are alphabetically ordered
+ */
+ private String sortInputOutput(String source) {
+ Matcher matcher = ENTITYLIST_PATTERN.matcher(source);
+ while(matcher.find()) {
+ List<String> lines = Arrays.asList(matcher.group(1).split("\n"));
+ Collections.sort(lines);
+ source = source.replaceAll(matcher.group(1), String.join("\n", lines) + "\n");
+ }
+ return source;
+ }
+
public void filterOutput() throws IOException {
- String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8");
+ String output = FileUtils.readFileToString(rawOutputFile, "UTF-8");
+ output = staticFilterSet.filter(specificFilterSet.filter(output));
if (rewriteSourceTables) {
- rawOutput = revertReplaceTableNames(rawOutput);
+ output = sortInputOutput(revertReplaceTableNames(output));
}
- String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput));
- FileUtils.writeStringToFile(outputFile, filteredOutput);
+ FileUtils.writeStringToFile(outputFile, output);
}
public QTestProcessExecResult compareResults() throws IOException, InterruptedException {
@@ -285,6 +306,9 @@ public final class QFile {
.addFilter(".*file\\..*\n", MASK_PATTERN)
.addFilter(".*CreateTime.*\n", MASK_PATTERN)
.addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
+ .addFilter(".*lastUpdateTime.*\n", MASK_PATTERN)
+ .addFilter(".*lastAccessTime.*\n", MASK_PATTERN)
+ .addFilter(".*[Oo]wner.*\n", MASK_PATTERN)
.addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out b/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out
new file mode 100644
index 0000000..58f517c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/beeline/create_merge_compressed.q.out
@@ -0,0 +1,138 @@
+PREHOOK: query: create table src_rc_merge_test(key int, value string) stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_rc_merge_test
+POSTHOOK: query: create table src_rc_merge_test(key int, value string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_rc_merge_test
+PREHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' into table src_rc_merge_test
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_rc_merge_test
+POSTHOOK: query: load data local inpath '../../data/files/smbbucket_1.rc' into table src_rc_merge_test
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_rc_merge_test
+PREHOOK: query: create table tgt_rc_merge_test(key int, value string) stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: create table tgt_rc_merge_test(key int, value string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tgt_rc_merge_test
+PREHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: Lineage: tgt_rc_merge_test.key SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: tgt_rc_merge_test.value SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: insert into table tgt_rc_merge_test select * from src_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: Lineage: tgt_rc_merge_test.key SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: tgt_rc_merge_test.value SIMPLE [(src_rc_merge_test)src_rc_merge_test.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: show table extended like `tgt_rc_merge_test`
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like `tgt_rc_merge_test`
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:tgt_rc_merge_test
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+columns:struct columns { i32 key, string value}
+partitioned:false
+partitionColumns:
+totalNumberFiles:2
+totalFileSize:342
+maxFileSize:171
+minFileSize:171
+#### A masked pattern was here ####
+
+PREHOOK: query: select count(1) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+10
+PREHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+46 -751895388
+PREHOOK: query: alter table tgt_rc_merge_test concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@tgt_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: alter table tgt_rc_merge_test concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@tgt_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
+PREHOOK: query: show table extended like `tgt_rc_merge_test`
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like `tgt_rc_merge_test`
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:tgt_rc_merge_test
+#### A masked pattern was here ####
+inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+columns:struct columns { i32 key, string value}
+partitioned:false
+partitionColumns:
+totalNumberFiles:1
+totalFileSize:243
+maxFileSize:243
+minFileSize:243
+#### A masked pattern was here ####
+
+PREHOOK: query: select count(1) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+10
+PREHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from tgt_rc_merge_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tgt_rc_merge_test
+#### A masked pattern was here ####
+46 -751895388
+PREHOOK: query: drop table src_rc_merge_test
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_rc_merge_test
+PREHOOK: Output: default@src_rc_merge_test
+POSTHOOK: query: drop table src_rc_merge_test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_rc_merge_test
+POSTHOOK: Output: default@src_rc_merge_test
+PREHOOK: query: drop table tgt_rc_merge_test
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tgt_rc_merge_test
+PREHOOK: Output: default@tgt_rc_merge_test
+POSTHOOK: query: drop table tgt_rc_merge_test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tgt_rc_merge_test
+POSTHOOK: Output: default@tgt_rc_merge_test
http://git-wip-us.apache.org/repos/asf/hive/blob/dec96ca6/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/escape_comments.q.out b/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
index 07fef57..aa26c57 100644
--- a/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
+++ b/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
@@ -95,7 +95,6 @@ p1 string a\nb
NULL NULL
# Detailed Table Information NULL NULL
Database: escape_comments_db NULL
-Owner: user NULL
#### A masked pattern was here ####
LastAccessTime: UNKNOWN NULL
Retention: 0 NULL
@@ -171,7 +170,6 @@ col1 string a\nb
NULL NULL
# Detailed Table Information NULL NULL
Database: escape_comments_db NULL
-Owner: user NULL
#### A masked pattern was here ####
LastAccessTime: UNKNOWN NULL
Retention: 0 NULL