You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/05/16 18:55:06 UTC
[1/2] hive git commit: HIVE-19572: Add option to mask stats and data
size in q files (Jesus Camacho Rodriguez, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master ce0f7b034 -> 3d834677a
HIVE-19572: Add option to mask stats and data size in q files (Jesus Camacho Rodriguez, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6325c1ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6325c1ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6325c1ab
Branch: refs/heads/master
Commit: 6325c1abe24fca335168779e10db404e408d3004
Parents: ce0f7b0
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed May 16 11:53:29 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 16 11:53:29 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/QOutProcessor.java | 31 ++-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 19 +-
.../hadoop/hive/ql/TestQOutProcessor.java | 2 +-
.../queries/clientpositive/default_constraint.q | 5 +-
.../llap/default_constraint.q.out | 234 +++++++++----------
5 files changed, 165 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6325c1ab/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index 52d9668..392d376 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -43,7 +43,7 @@ public class QOutProcessor {
public static final String PATH_HDFS_WITH_DATE_USER_GROUP_REGEX =
"([a-z]+) ([a-z]+)([ ]+)([0-9]+) ([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}) "
+ PATH_HDFS_REGEX;
-
+
public static final String HDFS_MASK = "### HDFS PATH ###";
public static final String HDFS_DATE_MASK = "### HDFS DATE ###";
public static final String HDFS_USER_MASK = "### USER ###";
@@ -52,6 +52,13 @@ public class QOutProcessor {
public static final String MASK_PATTERN = "#### A masked pattern was here ####";
public static final String PARTIAL_MASK_PATTERN = "#### A PARTIAL masked pattern was here ####";
+ private static final PatternReplacementPair MASK_STATS = new PatternReplacementPair(
+ Pattern.compile(" Num rows: [1-9][0-9]* Data size: [1-9][0-9]*"),
+ " Num rows: ###Masked### Data size: ###Masked###");
+ private static final PatternReplacementPair MASK_DATA_SIZE = new PatternReplacementPair(
+ Pattern.compile(" Data size: [1-9][0-9]*"),
+ " Data size: ###Masked###");
+
private FsType fsType = FsType.local;
public static class LineProcessingResult {
@@ -130,7 +137,7 @@ public class QOutProcessor {
return patterns;
}
- public void maskPatterns(String fname) throws Exception {
+ public void maskPatterns(String fname, boolean maskStats, boolean maskDataSize) throws Exception {
String line;
BufferedReader in;
BufferedWriter out;
@@ -145,7 +152,7 @@ public class QOutProcessor {
boolean lastWasMasked = false;
while (null != (line = in.readLine())) {
- LineProcessingResult result = processLine(line);
+ LineProcessingResult result = processLine(line, maskStats, maskDataSize);
if (result.line.equals(MASK_PATTERN)) {
// We're folding multiple masked lines into one.
@@ -167,7 +174,7 @@ public class QOutProcessor {
out.close();
}
- public LineProcessingResult processLine(String line) {
+ public LineProcessingResult processLine(String line, boolean maskStats, boolean maskDataSize) {
LineProcessingResult result = new LineProcessingResult(line);
Matcher matcher = null;
@@ -205,6 +212,22 @@ public class QOutProcessor {
}
}
+ if (!result.partialMaskWasMatched && maskStats) {
+ matcher = MASK_STATS.pattern.matcher(result.line);
+ if (matcher.find()) {
+ result.line = result.line.replaceAll(MASK_STATS.pattern.pattern(), MASK_STATS.replacement);
+ result.partialMaskWasMatched = true;
+ }
+ }
+
+ if (!result.partialMaskWasMatched && maskDataSize) {
+ matcher = MASK_DATA_SIZE.pattern.matcher(result.line);
+ if (matcher.find()) {
+ result.line = result.line.replaceAll(MASK_DATA_SIZE.pattern.pattern(), MASK_DATA_SIZE.replacement);
+ result.partialMaskWasMatched = true;
+ }
+ }
+
for (Pattern pattern : planMask) {
result.line = pattern.matcher(result.line).replaceAll(MASK_PATTERN);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6325c1ab/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 16571b3..894832a 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -164,6 +164,8 @@ public class QTestUtil {
private final Set<String> qHashQuerySet;
private final Set<String> qSortNHashQuerySet;
private final Set<String> qNoSessionReuseQuerySet;
+ private final Set<String> qMaskStatsQuerySet;
+ private final Set<String> qMaskDataSizeQuerySet;
private final Set<String> qJavaVersionSpecificOutput;
private static final String SORT_SUFFIX = ".sorted";
private static Set<String> srcTables;
@@ -590,6 +592,8 @@ public class QTestUtil {
qHashQuerySet = new HashSet<String>();
qSortNHashQuerySet = new HashSet<String>();
qNoSessionReuseQuerySet = new HashSet<String>();
+ qMaskStatsQuerySet = new HashSet<String>();
+ qMaskDataSizeQuerySet = new HashSet<String>();
qJavaVersionSpecificOutput = new HashSet<String>();
this.clusterType = clusterType;
@@ -824,6 +828,13 @@ public class QTestUtil {
if (matches(NO_SESSION_REUSE, query)) {
qNoSessionReuseQuerySet.add(qf.getName());
}
+
+ if (matches(MASK_STATS, query)) {
+ qMaskStatsQuerySet.add(qf.getName());
+ }
+ if (matches(MASK_DATA_SIZE, query)) {
+ qMaskDataSizeQuerySet.add(qf.getName());
+ }
}
private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
@@ -831,6 +842,8 @@ public class QTestUtil {
private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
+ private static final Pattern MASK_STATS = Pattern.compile("-- MASK_STATS");
+ private static final Pattern MASK_DATA_SIZE = Pattern.compile("-- MASK_DATA_SIZE");
private boolean matches(Pattern pattern, String query) {
Matcher matcher = pattern.matcher(query);
@@ -1725,7 +1738,7 @@ public class QTestUtil {
File f = new File(logDir, tname + outFileExtension);
- qOutProcessor.maskPatterns(f.getPath());
+ qOutProcessor.maskPatterns(f.getPath(), qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname));
QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
outFileName, false,
qSortSet.contains(tname));
@@ -1742,9 +1755,9 @@ public class QTestUtil {
public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
throws Exception {
assert outputs.size() > 1;
- qOutProcessor.maskPatterns(outputs.get(0));
+ qOutProcessor.maskPatterns(outputs.get(0), qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname));
for (int i = 1; i < outputs.size(); ++i) {
- qOutProcessor.maskPatterns(outputs.get(i));
+ qOutProcessor.maskPatterns(outputs.get(i), qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname));
QTestProcessExecResult result = executeDiffCommand(
outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
if (result.getReturnCode() != 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6325c1ab/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
----------------------------------------------------------------------
diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java b/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
index aa363ef..c373f8c 100644
--- a/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
+++ b/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
@@ -68,6 +68,6 @@ public class TestQOutProcessor {
}
private String processLine(String line) {
- return qOutProcessor.processLine(line).get();
+ return qOutProcessor.processLine(line, false, false).get();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/6325c1ab/ql/src/test/queries/clientpositive/default_constraint.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/default_constraint.q b/ql/src/test/queries/clientpositive/default_constraint.q
index c16dda3..a86622b 100644
--- a/ql/src/test/queries/clientpositive/default_constraint.q
+++ b/ql/src/test/queries/clientpositive/default_constraint.q
@@ -1,5 +1,8 @@
-- create table
- -- numeric type
+-- numeric type
+
+-- MASK_DATA_SIZE
+
set hive.stats.autogather=false;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
http://git-wip-us.apache.org/repos/asf/hive/blob/6325c1ab/ql/src/test/results/clientpositive/llap/default_constraint.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out
index d987ef2..80ac628 100644
--- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out
+++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out
@@ -91,22 +91,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(3)) (type: array<struct<col1:int>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: col1 (type: int), 32767S (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: UDFToByte(_col0) (type: tinyint)
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2))
Execution mode: llap
LLAP IO: no inputs
@@ -116,10 +116,10 @@ STAGE PLANS:
Select Operator
expressions: UDFToByte(VALUE._col0) (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -186,22 +186,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(4.5,678.4)) (type: array<struct<col1:decimal(2,1),col2:decimal(4,1)>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: 127Y (type: tinyint), 32767S (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), col1 (type: decimal(2,1)), col2 (type: decimal(4,1))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: decimal(2,1)), _col5 (type: decimal(4,1))
Execution mode: llap
LLAP IO: no inputs
@@ -211,10 +211,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), UDFToDouble(VALUE._col4) (type: double), CAST( VALUE._col5 AS decimal(9,2)) (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -354,21 +354,21 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('1985-12-31 12:45:07')) (type: array<struct<col1:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: DATE'2018-02-14' (type: date), CAST( col1 AS TIMESTAMP) (type: timestamp), TIMESTAMPLOCALTZ'2016-01-03 12:26:34.0 US/Pacific' (type: timestamp with local time zone), CURRENT_DATE() (type: date), null (type: timestamp)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -431,21 +431,21 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('1985-12-31','2018-02-27 17:32:14.259')) (type: array<struct<col1:string,col2:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: CAST( col1 AS DATE) (type: date), TIMESTAMP'2016-02-22 12:45:07.0' (type: timestamp), TIMESTAMPLOCALTZ'2016-01-03 12:26:34.0 US/Pacific' (type: timestamp with local time zone), CURRENT_DATE() (type: date), CAST( col2 AS TIMESTAMP) (type: timestamp)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -589,22 +589,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('default')) (type: array<struct<col1:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: col1 (type: string), CURRENT_USER() (type: string), 'Current_User()' (type: string), CAST( 'varchar_default_value' AS varchar(350)) (type: varchar(350)), CAST( 'char_value' AS CHAR(20)) (type: char(20))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 405 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 405 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: varchar(350)), _col4 (type: char(20))
Execution mode: llap
LLAP IO: no inputs
@@ -614,10 +614,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: varchar(350)), VALUE._col4 (type: char(20))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 405 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 405 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -683,22 +683,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('varchar_default2','char')) (type: array<struct<col1:string,col2:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: 'current_database()' (type: string), CURRENT_USER() (type: string), 'Current_User()' (type: string), col1 (type: string), col2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 298 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 298 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -708,10 +708,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), CAST( VALUE._col3 AS varchar(350)) (type: varchar(350)), CAST( VALUE._col4 AS CHAR(20)) (type: char(20))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 836 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 836 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -839,22 +839,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(false)) (type: array<struct<col1:boolean>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: col1 (type: boolean), CAST( 'bin' AS BINARY) (type: binary)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col0 (type: boolean)
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: boolean), _col1 (type: binary)
Execution mode: llap
LLAP IO: no inputs
@@ -864,10 +864,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: boolean), VALUE._col1 (type: binary)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -930,22 +930,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('011')) (type: array<struct<col1:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: true (type: boolean), col1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col0 (type: boolean)
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: boolean), _col1 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -955,10 +955,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: boolean), CAST( VALUE._col1 AS BINARY) (type: binary)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 148 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 148 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1142,21 +1142,21 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('vi')) (type: array<struct<col1:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: UDFToInteger(UDFToDouble(4)) (type: int), UDFToBoolean('true') (type: boolean), UDFToInteger(5.67) (type: int), UDFToByte(45) (type: tinyint), UDFToFloat(45.4) (type: float), UDFToLong(567) (type: bigint), UDFToShort(88) (type: smallint), CAST( CURRENT_TIMESTAMP() AS varchar(50)) (type: varchar(50)), UDFToString(CAST( CURRENT_USER() AS varchar(50))) (type: string), CAST( '2016-01-03 12:26:34 America/Los_Angeles' AS timestamp with local time zone) (type: timestamp with local time zone), CAST( '2016-01-01 12:01:01' AS TIMESTAMP) (type: timestamp), CAST( 4.5 AS decimal(8,2)) (type: decimal(8,2)), UDFToDouble(5) (type: double), CAST( col1 AS CHAR(2)) (type: char(2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 1 Data size: 523 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 523 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1228,21 +1228,21 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(true,'ga')) (type: array<struct<col1:boolean,col2:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: UDFToInteger(UDFToDouble(4)) (type: int), col1 (type: boolean), UDFToInteger(5.67) (type: int), UDFToByte(45) (type: tinyint), UDFToFloat(45.4) (type: float), UDFToLong(567) (type: bigint), UDFToShort(88) (type: smallint), CAST( CURRENT_TIMESTAMP() AS varchar(50)) (type: varchar(50)), UDFToString(CAST( CURRENT_USER() AS varchar(50))) (type: string), CAST( '2016-01-03 12:26:34 America/Los_Angeles' AS timestamp with local time zone) (type: timestamp with local time zone), CAST( '2016-01-01 12:01:01' AS TIMESTAMP) (type: timestamp), CAST( 4.5 AS decimal(8,2)) (type: decimal(8,2)), UDFToDouble(5) (type: double), CAST( col2 AS CHAR(2)) (type: char(2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 1 Data size: 519 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 519 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1398,22 +1398,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(456)) (type: array<struct<col1:int>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: UDFToShort(_col1) (type: smallint)
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2))
Execution mode: llap
LLAP IO: no inputs
@@ -1423,10 +1423,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1562,22 +1562,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(56)) (type: array<struct<col1:int>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: UDFToShort(_col1) (type: smallint)
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2))
Execution mode: llap
LLAP IO: no inputs
@@ -1587,10 +1587,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1821,25 +1821,25 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(847.45)) (type: array<struct<col1:decimal(5,2)>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: 127Y (type: tinyint), 32767S (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), col1 (type: decimal(5,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: enforce_constraint(127Y is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col1 (type: smallint)
- Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(5,2))
Execution mode: llap
LLAP IO: no inputs
@@ -1849,10 +1849,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), CAST( VALUE._col5 AS decimal(9,2)) (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2157,25 +2157,25 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(847.45)) (type: array<struct<col1:decimal(5,2)>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: 108Y (type: tinyint), 32767S (type: smallint), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), col1 (type: decimal(5,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: enforce_constraint(108Y is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: _col1 (type: smallint)
- Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(5,2))
Execution mode: llap
LLAP IO: no inputs
@@ -2185,10 +2185,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), CAST( VALUE._col5 AS decimal(9,2)) (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2311,21 +2311,21 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(3)) (type: array<struct<col1:int>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: col1 (type: int), '1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123' (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 337 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 337 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2405,24 +2405,24 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct('not','null','constraint')) (type: array<struct<col1:string,col2:string,col3:string>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: col1 (type: string), col2 (type: string), col3 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: enforce_constraint((_col0 is not null and _col2 is not null)) (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2560,22 +2560,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(456)) (type: array<struct<col1:int>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: UDFToShort(_col1) (type: smallint)
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2))
Execution mode: llap
LLAP IO: no inputs
@@ -2585,10 +2585,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -2724,22 +2724,22 @@ STAGE PLANS:
TableScan
alias: _dummy_table
Row Limit Per Split: 1
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: array(const struct(56)) (type: array<struct<col1:int>>)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
function name: inline
Select Operator
expressions: null (type: tinyint), col1 (type: int), 2147483647 (type: int), 9223372036854775807L (type: bigint), 3.4E38D (type: double), 1234567.89 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Map-reduce partition columns: UDFToShort(_col1) (type: smallint)
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: double), _col5 (type: decimal(9,2))
Execution mode: llap
LLAP IO: no inputs
@@ -2749,10 +2749,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: tinyint), UDFToShort(VALUE._col1) (type: smallint), VALUE._col2 (type: int), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(9,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
[2/2] hive git commit: HIVE-19575: TestAutoPurgeTables seems flaky
(Prasanth Jayachandran, reviewed by Jesus Camacho Rodriguez)
Posted by jc...@apache.org.
HIVE-19575: TestAutoPurgeTables seems flaky (Prasanth Jayachandran, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3d834677
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3d834677
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3d834677
Branch: refs/heads/master
Commit: 3d834677a46380ddf618f21a0a1c3b59061d5229
Parents: 6325c1a
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Wed May 16 11:54:44 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 16 11:54:44 2018 -0700
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3d834677/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
index 7fc9432..c45c959 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAutoPurgeTables.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.common.util.RetryTestRunner;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -39,9 +40,11 @@ import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+@RunWith(RetryTestRunner.class)
public class TestAutoPurgeTables {
private static final String driverName = "org.apache.hive.jdbc.HiveDriver";
private static final String testDbName = "auto_purge_test_db";