You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/11/22 02:46:37 UTC
[28/35] hive git commit: HIVE-15246: Add a making comment to
blobstore staging paths on qtest output (Sergio Pena,
reviewed by Mohit Sabharwal and Sahil Takiar)
HIVE-15246: Add a making comment to blobstore staging paths on qtest output (Sergio Pena, reviewed by Mohit Sabharwal and Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d2cb3277
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d2cb3277
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d2cb3277
Branch: refs/heads/hive-14535
Commit: d2cb3277f2fbb71c9ce27034e2538459a31a6e9a
Parents: e2a6273
Author: Sergio Pena <se...@cloudera.com>
Authored: Sun Nov 20 18:03:12 2016 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Sun Nov 20 18:03:12 2016 -0600
----------------------------------------------------------------------
.../test/queries/clientpositive/insert_into.q | 3 +
.../results/clientpositive/insert_into.q.out | 295 +++++++++++++++++++
.../org/apache/hadoop/hive/ql/QTestUtil.java | 10 +-
3 files changed, 306 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d2cb3277/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q
index e36ef1d..919ff7d 100644
--- a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q
+++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q
@@ -1,4 +1,7 @@
+set hive.blobstore.use.blobstore.as.scratchdir=true;
+
DROP TABLE qtest;
CREATE TABLE qtest (value int) LOCATION '${hiveconf:test.blobstore.path.unique}/qtest/';
INSERT INTO qtest VALUES (1), (10), (100), (1000);
+EXPLAIN EXTENDED INSERT INTO qtest VALUES (1), (10), (100), (1000);
SELECT * FROM qtest;
http://git-wip-us.apache.org/repos/asf/hive/blob/d2cb3277/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
index 93f545d..c25d0c4 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out
@@ -21,6 +21,301 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@values__tmp__table__1
POSTHOOK: Output: default@qtest
POSTHOOK: Lineage: qtest.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: EXPLAIN EXTENDED INSERT INTO qtest VALUES (1), (10), (100), (1000)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED INSERT INTO qtest VALUES (1), (10), (100), (1000)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-2 depends on stages: Stage-0
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: values__tmp__table__2
+ Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: UDFToInteger(tmp_values_col1) (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ directory: ### BLOBSTORE_STAGING_PATH ###
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
+ Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.qtest
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: Values__Tmp__Table__2
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns tmp_values_col1
+ columns.comments
+ columns.types string
+#### A masked pattern was here ####
+ name default.values__tmp__table__2
+ serialization.ddl struct values__tmp__table__2 { string tmp_values_col1}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns tmp_values_col1
+ columns.comments
+ columns.types string
+#### A masked pattern was here ####
+ name default.values__tmp__table__2
+ serialization.ddl struct values__tmp__table__2 { string tmp_values_col1}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.values__tmp__table__2
+ name: default.values__tmp__table__2
+ Truncated Path -> Alias:
+#### A masked pattern was here ####
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+ source: ### BLOBSTORE_STAGING_PATH ###
+ destination: ### BLOBSTORE_STAGING_PATH ###
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ source: ### BLOBSTORE_STAGING_PATH ###
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.qtest
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ GatherStats: false
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ directory: ### BLOBSTORE_STAGING_PATH ###
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.qtest
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+ ### BLOBSTORE_STAGING_PATH ###
+ Path -> Partition:
+ ### BLOBSTORE_STAGING_PATH ###
+ Partition
+ base file name: -ext-10002
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.qtest
+ name: default.qtest
+ Truncated Path -> Alias:
+ ### BLOBSTORE_STAGING_PATH ###
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ GatherStats: false
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ directory: ### BLOBSTORE_STAGING_PATH ###
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.qtest
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+ ### BLOBSTORE_STAGING_PATH ###
+ Path -> Partition:
+ ### BLOBSTORE_STAGING_PATH ###
+ Partition
+ base file name: -ext-10002
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns value
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ location ### test.blobstore.path ###/qtest
+ name default.qtest
+ numFiles 1
+ serialization.ddl struct qtest { i32 value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 14
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.qtest
+ name: default.qtest
+ Truncated Path -> Alias:
+ ### BLOBSTORE_STAGING_PATH ###
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+ source: ### BLOBSTORE_STAGING_PATH ###
+ destination: ### BLOBSTORE_STAGING_PATH ###
+
PREHOOK: query: SELECT * FROM qtest
PREHOOK: type: QUERY
PREHOOK: Input: default@qtest
http://git-wip-us.apache.org/repos/asf/hive/blob/d2cb3277/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index d08ecc7..5bf23e7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1730,10 +1730,16 @@ public class QTestUtil {
});
/* This list may be modified by specific cli drivers to mask strings that change on every test */
- private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<>();
+ private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
+ add(toPatternPair("(s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
+ }};
+
+ private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) {
+ return ImmutablePair.of(Pattern.compile(patternStr), maskComment);
+ }
public void addPatternWithMaskComment(String patternStr, String maskComment) {
- patternsWithMaskComments.add(ImmutablePair.of(Pattern.compile(patternStr), maskComment));
+ patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
}
public int checkCliDriverResults(String tname) throws Exception {