You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/06/09 03:57:09 UTC
[01/50] [abbrv] hive git commit: HIVE-10527: NPE in
SparkUtilities::isDedicatedCluster [Spark Branch] (Rui reviewed by Jimmy)
Repository: hive
Updated Branches:
refs/heads/spark 52e008612 -> 8cf95355f
HIVE-10527: NPE in SparkUtilities::isDedicatedCluster [Spark Branch] (Rui reviewed by Jimmy)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/81e8e665
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/81e8e665
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/81e8e665
Branch: refs/heads/spark
Commit: 81e8e665247f8bff927cb206b5bb178455047d73
Parents: cee1183
Author: Rui Li <ru...@intel.com>
Authored: Thu Apr 30 10:15:34 2015 +0800
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:03:30 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/81e8e665/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index 058d91d..21398d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -112,6 +112,7 @@ public class HiveSparkClientFactory {
String sparkMaster = hiveConf.get("spark.master");
if (sparkMaster == null) {
sparkMaster = sparkConf.get("spark.master");
+ hiveConf.set("spark.master", sparkMaster);
}
if (sparkMaster.equals("yarn-cluster")) {
sparkConf.put("spark.yarn.maxAppAttempts", "1");
[39/50] [abbrv] hive git commit: HIVE-10941: Provide option to
disable spark tests outside itests (Hari Sankar Sivarama Subramaniyan,
reviewed by Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10941: Provide option to disable spark tests outside itests (Hari Sankar Sivarama Subramaniyan, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/29af41c7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/29af41c7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/29af41c7
Branch: refs/heads/spark
Commit: 29af41c7822789b178416a6b416fb87e75deda78
Parents: 8a10730
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Fri Jun 5 21:48:56 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Fri Jun 5 21:48:56 2015 -0700
----------------------------------------------------------------------
pom.xml | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/29af41c7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9b40909..51b26ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,7 @@
<!-- To add additional exclude patterns set this property -->
<test.excludes.additional></test.excludes.additional>
+ <skip.spark.files></skip.spark.files>
<!-- Plugin and Plugin Dependency Versions -->
<ant.contrib.version>1.0b3</ant.contrib.version>
@@ -825,6 +826,7 @@
<exclude>**/TestHiveServer2Concurrency.java</exclude>
<exclude>**/TestHiveMetaStore.java</exclude>
<exclude>${test.excludes.additional}</exclude>
+ <exclude>%regex[${skip.spark.files}]</exclude>
</excludes>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<reuseForks>false</reuseForks>
@@ -1207,5 +1209,16 @@
<test.log4j.scheme>file:/</test.log4j.scheme>
</properties>
</profile>
+ <profile>
+ <id>spark-test</id>
+ <activation>
+ <property>
+ <name>!skipSparkTests</name>
+ </property>
+ </activation>
+ <properties>
+ <skip.spark.files>.*[TestSparkSessionManagerImpl|TestMultiSessionsHS2WithLocalClusterSpark|TestJdbcWithLocalClusterSpark].class</skip.spark.files>
+ </properties>
+ </profile>
</profiles>
</project>
[02/50] [abbrv] hive git commit: HIVE-10626 Spark paln need to be
updated [Spark Branch] (Chinna via Jimmy)
Posted by xu...@apache.org.
HIVE-10626 Spark paln need to be updated [Spark Branch] (Chinna via Jimmy)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2df14f9b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2df14f9b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2df14f9b
Branch: refs/heads/spark
Commit: 2df14f9b04dec968751f7b336bde7a5451af9d93
Parents: 81e8e66
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Fri May 8 13:14:24 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:03:42 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/spark/GroupByShuffler.java | 4 +
.../hadoop/hive/ql/exec/spark/MapInput.java | 8 +-
.../hadoop/hive/ql/exec/spark/MapTran.java | 9 +-
.../hadoop/hive/ql/exec/spark/ReduceTran.java | 9 +-
.../hadoop/hive/ql/exec/spark/ShuffleTran.java | 16 +-
.../hive/ql/exec/spark/SortByShuffler.java | 5 +
.../hadoop/hive/ql/exec/spark/SparkPlan.java | 165 ++++++++++++++-----
.../hive/ql/exec/spark/SparkShuffler.java | 2 +
.../hadoop/hive/ql/exec/spark/SparkTran.java | 2 +
9 files changed, 176 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
index b8e36cb..e128dd2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
@@ -33,4 +33,8 @@ public class GroupByShuffler implements SparkShuffler {
return input.groupByKey();
}
+ @Override
+ public String getName() {
+ return "GroupBy";
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
index 157e4d8..26cfebd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapInput.java
@@ -36,6 +36,7 @@ public class MapInput implements SparkTran<WritableComparable, Writable,
private JavaPairRDD<WritableComparable, Writable> hadoopRDD;
private boolean toCache;
private final SparkPlan sparkPlan;
+ private String name = "MapInput";
public MapInput(SparkPlan sparkPlan, JavaPairRDD<WritableComparable, Writable> hadoopRDD) {
this(sparkPlan, hadoopRDD, false);
@@ -88,11 +89,16 @@ public class MapInput implements SparkTran<WritableComparable, Writable,
@Override
public String getName() {
- return "MapInput";
+ return name;
}
@Override
public Boolean isCacheEnable() {
return new Boolean(toCache);
}
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
index f6a4d77..2170243 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
@@ -24,6 +24,7 @@ import org.apache.spark.api.java.JavaPairRDD;
public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey, BytesWritable> {
private HiveMapFunction mapFunc;
+ private String name = "MapTran";
@Override
public JavaPairRDD<HiveKey, BytesWritable> transform(
@@ -37,12 +38,16 @@ public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey,
@Override
public String getName() {
- return "MapTran";
+ return name;
}
@Override
public Boolean isCacheEnable() {
- // TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
index fd6b31c..e60dfac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
@@ -24,6 +24,7 @@ import org.apache.spark.api.java.JavaPairRDD;
public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, HiveKey, BytesWritable> {
private HiveReduceFunction reduceFunc;
+ private String name = "Reduce";
@Override
public JavaPairRDD<HiveKey, BytesWritable> transform(
@@ -37,12 +38,16 @@ public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, H
@Override
public String getName() {
- return "Reduce";
+ return name;
}
@Override
public Boolean isCacheEnable() {
- // TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
index 6cdab20..a774395 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ShuffleTran.java
@@ -28,6 +28,7 @@ public class ShuffleTran implements SparkTran<HiveKey, BytesWritable, HiveKey, I
private final int numOfPartitions;
private final boolean toCache;
private final SparkPlan sparkPlan;
+ private String name = "Shuffle";
public ShuffleTran(SparkPlan sparkPlan, SparkShuffler sf, int n) {
this(sparkPlan, sf, n, false);
@@ -50,13 +51,26 @@ public class ShuffleTran implements SparkTran<HiveKey, BytesWritable, HiveKey, I
return result;
}
+ public int getNoOfPartitions() {
+ return numOfPartitions;
+ }
+
@Override
public String getName() {
- return "Shuffle";
+ return name;
}
@Override
public Boolean isCacheEnable() {
return new Boolean(toCache);
}
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public SparkShuffler getShuffler() {
+ return shuffler;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
index 2545a9d..766813c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SortByShuffler.java
@@ -56,6 +56,11 @@ public class SortByShuffler implements SparkShuffler {
return rdd.mapPartitionsToPair(new ShuffleFunction());
}
+ @Override
+ public String getName() {
+ return "SortBy";
+ }
+
private static class ShuffleFunction implements
PairFlatMapFunction<Iterator<Tuple2<HiveKey, BytesWritable>>,
HiveKey, Iterable<BytesWritable>> {
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
index 81b7e85..ee5c78a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
@@ -91,57 +91,146 @@ public class SparkPlan {
return finalRDD;
}
+ private void addNumberToTrans() {
+ int i = 1;
+ String name = null;
+
+ // Traverse leafTran & transGraph add numbers to trans
+ for (SparkTran leaf : leafTrans) {
+ name = leaf.getName() + " " + i++;
+ leaf.setName(name);
+ }
+ Set<SparkTran> sparkTrans = transGraph.keySet();
+ for (SparkTran tran : sparkTrans) {
+ name = tran.getName() + " " + i++;
+ tran.setName(name);
+ }
+ }
+
private void logSparkPlan() {
- LOG.info("------------------------------ Spark Plan -----------------------------");
- Set<SparkTran> keySet = invertedTransGraph.keySet();
- for (SparkTran sparkTran : keySet) {
- if (sparkTran instanceof ReduceTran) {
- String sparkPlan = " " + sparkTran.getName();
- sparkPlan = getSparkPlan(sparkTran, sparkPlan);
- LOG.info(sparkPlan);
+ addNumberToTrans();
+ ArrayList<SparkTran> leafTran = new ArrayList<SparkTran>();
+ leafTran.addAll(leafTrans);
+
+ for (SparkTran leaf : leafTrans) {
+ collectLeafTrans(leaf, leafTran);
+ }
+
+ // Start Traverse from the leafTrans and get parents of each leafTrans till
+ // the end
+ StringBuilder sparkPlan = new StringBuilder(
+ "\n\t!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Spark Plan !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! \n\n");
+ for (SparkTran leaf : leafTran) {
+ sparkPlan.append(leaf.getName());
+ getSparkPlan(leaf, sparkPlan);
+ sparkPlan.append("\n");
+ }
+ sparkPlan
+ .append(" \n\t!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Spark Plan !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ");
+ LOG.info(sparkPlan);
+ }
+
+ private void collectLeafTrans(SparkTran leaf, List<SparkTran> reduceTrans) {
+ List<SparkTran> parents = getParents(leaf);
+ if (parents.size() > 0) {
+ SparkTran nextLeaf = null;
+ for (SparkTran leafTran : parents) {
+ if (leafTran instanceof ReduceTran) {
+ reduceTrans.add(leafTran);
+ } else {
+ if (getParents(leafTran).size() > 0)
+ nextLeaf = leafTran;
+ }
}
+ if (nextLeaf != null)
+ collectLeafTrans(nextLeaf, reduceTrans);
}
- LOG.info("------------------------------ Spark Plan -----------------------------");
- }
-
- private String getSparkPlan(SparkTran leaf, String sparkPlanMsg) {
- if (leaf != null) {
- List<SparkTran> parents = getParents(leaf);
- if (parents.size() > 0) {
- sparkPlanMsg = sparkPlanMsg + " <-- ";
- boolean isFirst = true;
- SparkTran parent = null;
- for (SparkTran sparkTran : parents) {
- if (isFirst) {
- sparkPlanMsg = sparkPlanMsg + "( " + sparkTran.getName();
- sparkPlanMsg = logCacheStatus(sparkPlanMsg, sparkTran);
- isFirst = false;
- } else {
- sparkPlanMsg = sparkPlanMsg + "," + sparkTran.getName();
- sparkPlanMsg = logCacheStatus(sparkPlanMsg, sparkTran);
- }
- if (getParents(sparkTran).size() > 0 && !(sparkTran instanceof ReduceTran)) {
- parent = sparkTran;
- }
- }
- sparkPlanMsg = sparkPlanMsg + " ) ";
- return getSparkPlan(parent, sparkPlanMsg);
+ }
+
+ private void getSparkPlan(SparkTran tran, StringBuilder sparkPlan) {
+ List<SparkTran> parents = getParents(tran);
+ List<SparkTran> nextLeaf = new ArrayList<SparkTran>();
+ if (parents.size() > 0) {
+ sparkPlan.append(" <-- ");
+ boolean isFirst = true;
+ for (SparkTran leaf : parents) {
+ if (isFirst) {
+ sparkPlan.append("( " + leaf.getName());
+ if (leaf instanceof ShuffleTran) {
+ logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
+ } else {
+ logCacheStatus(leaf, sparkPlan);
+ }
+ isFirst = false;
+ } else {
+ sparkPlan.append("," + leaf.getName());
+ if (leaf instanceof ShuffleTran) {
+ logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
+ } else {
+ logCacheStatus(leaf, sparkPlan);
+ }
+ }
+ // Leave reduceTran it will be expanded in the next line
+ if (getParents(leaf).size() > 0 && !(leaf instanceof ReduceTran)) {
+ nextLeaf.add(leaf);
+ }
+ }
+ sparkPlan.append(" ) ");
+ if (nextLeaf.size() > 1) {
+ logLeafTran(nextLeaf, sparkPlan);
} else {
- return sparkPlanMsg;
+ if (nextLeaf.size() != 0)
+ getSparkPlan(nextLeaf.get(0), sparkPlan);
}
}
- return sparkPlanMsg;
}
- private String logCacheStatus(String sparkPlanMsg, SparkTran sparkTran) {
+ private void logLeafTran(List<SparkTran> parent, StringBuilder sparkPlan) {
+ sparkPlan.append(" <-- ");
+ boolean isFirst = true;
+ for (SparkTran sparkTran : parent) {
+ List<SparkTran> parents = getParents(sparkTran);
+ SparkTran leaf = parents.get(0);
+ if (isFirst) {
+ sparkPlan.append("( " + leaf.getName());
+ if (leaf instanceof ShuffleTran) {
+ logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
+ } else {
+ logCacheStatus(leaf, sparkPlan);
+ }
+ isFirst = false;
+ } else {
+ sparkPlan.append("," + leaf.getName());
+ if (leaf instanceof ShuffleTran) {
+ logShuffleTranStatus((ShuffleTran) leaf, sparkPlan);
+ } else {
+ logCacheStatus(leaf, sparkPlan);
+ }
+ }
+ }
+ sparkPlan.append(" ) ");
+ }
+
+ private void logShuffleTranStatus(ShuffleTran leaf, StringBuilder sparkPlan) {
+ int noOfPartitions = leaf.getNoOfPartitions();
+ sparkPlan.append(" ( Partitions " + noOfPartitions);
+ SparkShuffler shuffler = leaf.getShuffler();
+ sparkPlan.append(", " + shuffler.getName());
+ if (leaf.isCacheEnable()) {
+ sparkPlan.append(", Cache on");
+ } else {
+ sparkPlan.append(", Cache off");
+ }
+ }
+
+ private void logCacheStatus(SparkTran sparkTran, StringBuilder sparkPlan) {
if (sparkTran.isCacheEnable() != null) {
if (sparkTran.isCacheEnable().booleanValue()) {
- sparkPlanMsg = sparkPlanMsg + " (cache on) ";
+ sparkPlan.append(" (cache on) ");
} else {
- sparkPlanMsg = sparkPlanMsg + " (cache off) ";
+ sparkPlan.append(" (cache off) ");
}
}
- return sparkPlanMsg;
}
public void addTran(SparkTran tran) {
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
index 53845a0..40e251f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkShuffler.java
@@ -27,4 +27,6 @@ public interface SparkShuffler {
JavaPairRDD<HiveKey, Iterable<BytesWritable>> shuffle(
JavaPairRDD<HiveKey, BytesWritable> input, int numPartitions);
+ public String getName();
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2df14f9b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
index c3c48a0..671c983 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTran.java
@@ -28,5 +28,7 @@ public interface SparkTran<KI extends WritableComparable, VI, KO extends Writabl
public String getName();
+ public void setName(String name);
+
public Boolean isCacheEnable();
}
[48/50] [abbrv] hive git commit: HIVE-10929: In Tez mode,
dynamic partitioning query with union all fails at moveTask,
Invalid partition key & values (Vikram Dixit K reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/vector_leftsemi_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_leftsemi_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/vector_leftsemi_mapjoin.q.out
index 9f40f9e..6e4971d 100644
--- a/ql/src/test/results/clientpositive/tez/vector_leftsemi_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_leftsemi_mapjoin.q.out
@@ -435,42 +435,42 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
alias: b
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < 15) (type: boolean)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: key (type: int), key (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -616,24 +616,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t3
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key > 5) (type: boolean)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Map 2
Map Operator Tree:
TableScan
@@ -651,24 +651,24 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 1
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -902,10 +902,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -915,11 +915,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -947,10 +947,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1121,15 +1121,15 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: int), _col3 (type: string)
Map 3
Map Operator Tree:
@@ -1149,33 +1149,33 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: key
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: key (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1233,10 +1233,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key is not null and value is not null) (type: boolean)
- Statistics: Num rows: 2 Data size: 238 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -1246,11 +1246,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
input vertices:
1 Map 3
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -1278,10 +1278,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1337,10 +1337,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -1353,11 +1353,11 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -1407,10 +1407,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1468,7 +1468,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -1481,11 +1481,11 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -1520,10 +1520,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1603,12 +1603,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 5
Map Operator Tree:
TableScan
@@ -1639,20 +1639,20 @@ STAGE PLANS:
1 key (type: int)
2 _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1725,12 +1725,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
@@ -1771,20 +1771,20 @@ STAGE PLANS:
1 _col0 (type: int)
2 key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1857,12 +1857,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
@@ -1903,20 +1903,20 @@ STAGE PLANS:
1 _col0 (type: int)
2 key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1991,12 +1991,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
@@ -2037,20 +2037,20 @@ STAGE PLANS:
1 _col0 (type: int)
2 key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2136,10 +2136,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 3 Data size: 358 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -2149,7 +2149,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
input vertices:
1 Map 3
- Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -2159,11 +2159,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 4
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -2201,10 +2201,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2282,14 +2282,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((key > 100) and value is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 119 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 119 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -2666,43 +2666,43 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
alias: b
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < 15) (type: boolean)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: key (type: int), key (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2849,24 +2849,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t3
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key > 5) (type: boolean)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Map 2
Map Operator Tree:
TableScan
@@ -2884,25 +2884,25 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 1
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3138,10 +3138,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -3151,12 +3151,12 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -3184,10 +3184,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3359,16 +3359,16 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: int), _col3 (type: string)
Map 3
Map Operator Tree:
@@ -3388,33 +3388,33 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: key
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: key (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3472,10 +3472,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key is not null and value is not null) (type: boolean)
- Statistics: Num rows: 2 Data size: 238 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -3485,12 +3485,12 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
input vertices:
1 Map 3
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -3518,10 +3518,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3577,10 +3577,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -3593,12 +3593,12 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -3648,10 +3648,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3709,7 +3709,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -3722,12 +3722,12 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -3762,10 +3762,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3845,12 +3845,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 5
Map Operator Tree:
TableScan
@@ -3881,20 +3881,20 @@ STAGE PLANS:
1 key (type: int)
2 _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3967,12 +3967,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
@@ -4013,20 +4013,20 @@ STAGE PLANS:
1 _col0 (type: int)
2 key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -4099,12 +4099,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
@@ -4145,20 +4145,20 @@ STAGE PLANS:
1 _col0 (type: int)
2 key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -4233,12 +4233,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: int)
sort order: +
Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
@@ -4279,20 +4279,20 @@ STAGE PLANS:
1 _col0 (type: int)
2 key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -4378,10 +4378,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 3 Data size: 358 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -4391,7 +4391,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
input vertices:
1 Map 3
- Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -4402,12 +4402,12 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 4
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -4445,10 +4445,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1237 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -4526,14 +4526,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((key > 100) and value is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 119 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 119 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -4917,44 +4917,44 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 3
Map Operator Tree:
TableScan
alias: b
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < 15) (type: boolean)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: key (type: int), key (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5104,24 +5104,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t3
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key > 5) (type: boolean)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 59 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 651 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 2
Map Operator Tree:
@@ -5140,25 +5140,25 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 1
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 64 Data size: 259 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7 Data size: 716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5399,10 +5399,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -5412,11 +5412,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 3
Map Operator Tree:
@@ -5446,10 +5446,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 99 Data size: 396 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 1125 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5624,15 +5624,15 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: int), _col3 (type: string)
Execution mode: vectorized
Map 3
@@ -5654,34 +5654,34 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: key
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: key (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: int), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5740,10 +5740,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 6 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key is not null and value is not null) (type: boolean)
- Statistics: Num rows: 2 Data size: 238 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 558 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -5753,11 +5753,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
input vertices:
1 Map 3
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 3
Map Operator Tree:
@@ -5787,10 +5787,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 613 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5847,10 +5847,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 90 Data size: 360 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1023 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Semi Join 0 to 1
@@ -5863,11 +5863,11 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 3
Map Operator Tree:
@@ -5920,10 +5920,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 198 Data size: 792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 2250 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5982,7 +5982,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 179 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 2046 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -5995,11 +5995,11 @@ STAGE PLANS:
input vertices:
1 Map 3
2 Map 4
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 48 Data size: 4501 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 3
Map Operator Tree:
@@ -6037,10 +6037,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 393 Data size: 1575 Basic stats: COMPLETE Column stats: NONE
+
<TRUNCATED>
[22/50] [abbrv] hive git commit: HIVE-10705 Update tests for
HIVE-9302 after removing binaries(Ferdinand Xu,
reviewed by Hari Sankar Sivarama Subramaniyan)
Posted by xu...@apache.org.
HIVE-10705 Update tests for HIVE-9302 after removing binaries(Ferdinand Xu, reviewed by Hari Sankar Sivarama Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aec98e9f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aec98e9f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aec98e9f
Branch: refs/heads/spark
Commit: aec98e9f1b9d3bd0926b0b84c6ed49bbe494667a
Parents: 4140da5
Author: Ferdinand Xu <ch...@intel.com>
Authored: Wed Jun 3 14:02:13 2015 -0400
Committer: Ferdinand Xu <ch...@intel.com>
Committed: Wed Jun 3 14:02:20 2015 -0400
----------------------------------------------------------------------
beeline/pom.xml | 6 ++
.../hive/beeline/TestBeelineArgParsing.java | 32 +++++-----
.../test/resources/DummyDriver-1.0-SNAPSHOT.jar | Bin 2812 -> 0 bytes
beeline/src/test/resources/DummyDriver.txt | 59 +++++++++++++++++++
.../src/test/resources/postgresql-9.3.jdbc3.jar | Bin 539003 -> 0 bytes
.../apache/hive/common/util/HiveTestUtils.java | 46 +++++++++++++++
.../hive/ql/session/TestSessionState.java | 40 ++-----------
7 files changed, 133 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/pom.xml
----------------------------------------------------------------------
diff --git a/beeline/pom.xml b/beeline/pom.xml
index 352f561..4ead3d3 100644
--- a/beeline/pom.xml
+++ b/beeline/pom.xml
@@ -111,6 +111,12 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>9.1-901.jdbc4</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
index 7a354f3..702805f 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
@@ -18,6 +18,7 @@
package org.apache.hive.beeline;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -30,6 +31,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hive.common.util.HiveTestUtils;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -41,6 +43,9 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestBeelineArgParsing {
private static final Log LOG = LogFactory.getLog(TestBeelineArgParsing.class.getName());
+
+ private static final String dummyDriverClazzName = "DummyDriver";
+
private String connectionString;
private String driverClazzName;
private String driverJarFileName;
@@ -85,15 +90,17 @@ public class TestBeelineArgParsing {
}
}
- @Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(
- new Object[][] {
- {"jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver", "postgresql-9.3.jdbc3.jar",
- true},
- {"jdbc:dummy://host:5432/testdb", "org.apache.dummy.DummyDriver",
- "DummyDriver-1.0-SNAPSHOT.jar", false}
- });
+ @Parameters public static Collection<Object[]> data() throws IOException, InterruptedException {
+ // generate the dummy driver by using txt file
+ String u = HiveTestUtils.getFileFromClasspath("DummyDriver.txt");
+ File jarFile = HiveTestUtils.genLocalJarForTest(u, dummyDriverClazzName);
+ String pathToDummyDriver = jarFile.getAbsolutePath();
+ return Arrays.asList(new Object[][] {
+ { "jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver",
+ System.getProperty("maven.local.repository") + File.separator + "postgresql"
+ + File.separator + "postgresql" + File.separator + "9.1-901.jdbc4" + File.separator
+ + "postgresql-9.1-901.jdbc4.jar", true },
+ { "jdbc:dummy://host:5432/testdb", dummyDriverClazzName, pathToDummyDriver, false } });
}
@Test
@@ -217,9 +224,8 @@ public class TestBeelineArgParsing {
Assert.assertNull(bl.findLocalDriver(connectionString));
LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName);
- String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName);
- bl.addLocalJar(mysqlDriverPath);
+ bl.addLocalJar(driverJarFileName);
bl.addlocaldrivername(driverClazzName);
Assert.assertEquals(bl.findLocalDriver(connectionString).getClass().getName(), driverClazzName);
}
@@ -229,9 +235,7 @@ public class TestBeelineArgParsing {
TestBeeline bl = new TestBeeline();
LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName);
- String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName);
-
- bl.addLocalJar(mysqlDriverPath);
+ bl.addLocalJar(driverJarFileName);
if (!defaultSupported) {
Assert.assertNull(bl.findLocalDriver(connectionString));
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar b/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar
deleted file mode 100644
index 3dadc9e..0000000
Binary files a/beeline/src/test/resources/DummyDriver-1.0-SNAPSHOT.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/resources/DummyDriver.txt
----------------------------------------------------------------------
diff --git a/beeline/src/test/resources/DummyDriver.txt b/beeline/src/test/resources/DummyDriver.txt
new file mode 100644
index 0000000..05f8b12
--- /dev/null
+++ b/beeline/src/test/resources/DummyDriver.txt
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.DriverPropertyInfo;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.util.Properties;
+import java.util.logging.Logger;
+
+public class DummyDriver implements Driver {
+ @Override public Connection connect(String url, Properties info) throws SQLException {
+ return null;
+ }
+
+ @Override public boolean acceptsURL(String url) throws SQLException {
+ if (url == null) {
+ return false;
+ } else {
+ return url.startsWith("jdbc:dummy://");
+ }
+ }
+
+ @Override public DriverPropertyInfo[] getPropertyInfo(String url, Properties info)
+ throws SQLException {
+ return new DriverPropertyInfo[0];
+ }
+
+ @Override public int getMajorVersion() {
+ return 0;
+ }
+
+ @Override public int getMinorVersion() {
+ return 0;
+ }
+
+ @Override public boolean jdbcCompliant() {
+ return false;
+ }
+
+ @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/beeline/src/test/resources/postgresql-9.3.jdbc3.jar
----------------------------------------------------------------------
diff --git a/beeline/src/test/resources/postgresql-9.3.jdbc3.jar b/beeline/src/test/resources/postgresql-9.3.jdbc3.jar
deleted file mode 100644
index f537b98..0000000
Binary files a/beeline/src/test/resources/postgresql-9.3.jdbc3.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/HiveTestUtils.java b/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
index db34494..06caa53 100644
--- a/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
+++ b/common/src/java/org/apache/hive/common/util/HiveTestUtils.java
@@ -18,14 +18,27 @@
package org.apache.hive.common.util;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URL;
+import com.google.common.io.Files;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class HiveTestUtils {
+ public static final Log LOG = LogFactory.getLog(HiveTestUtils.class);
+
+ public final static String JAVA_FILE_EXT = ".java";
+ public final static String CLAZZ_FILE_EXT = ".class";
+ public final static String JAR_FILE_EXT = ".jar";
+ public final static String TXT_FILE_EXT = ".txt";
public static String getFileFromClasspath(String name) {
URL url = ClassLoader.getSystemResource(name);
@@ -34,4 +47,37 @@ public class HiveTestUtils {
}
return url.getPath();
}
+
+ private static void executeCmd(String[] cmdArr, File dir) throws IOException, InterruptedException {
+ final Process p1 = Runtime.getRuntime().exec(cmdArr, null, dir);
+ new Thread(new Runnable() {
+ public void run() {
+ BufferedReader input = new BufferedReader(new InputStreamReader(p1.getErrorStream()));
+ String line;
+ try {
+ while ((line = input.readLine()) != null) {
+ System.out.println(line);
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to execute the command due the exception " + e);
+ }
+ }
+ }).start();
+ p1.waitFor();
+ }
+
+ public static File genLocalJarForTest(String pathToClazzFile, String clazzName)
+ throws IOException, InterruptedException {
+ String u = pathToClazzFile;
+ File dir = new File(u);
+ File parentDir = dir.getParentFile();
+ File f = new File(parentDir, clazzName + JAVA_FILE_EXT);
+ Files.copy(dir, f);
+ executeCmd(new String[] { "javac", clazzName + JAVA_FILE_EXT }, parentDir);
+ executeCmd(new String[] { "jar", "cf", clazzName + JAR_FILE_EXT, clazzName + CLAZZ_FILE_EXT },
+ parentDir);
+ f.delete();
+ new File(parentDir, clazzName + CLAZZ_FILE_EXT).delete();
+ return new File(parentDir, clazzName + JAR_FILE_EXT);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/aec98e9f/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
index 45ad22a..9e16c0c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
@@ -57,15 +57,8 @@ public class TestSessionState {
private final static String reloadClazzFileName = "RefreshedJarClass.jar";
private final static String versionMethodName = "version";
private final static String RELOADED_CLAZZ_PREFIX_NAME = "RefreshedJarClass";
- private final static String JAVA_FILE_EXT = ".java";
- private final static String CLAZZ_FILE_EXT = ".class";
- private final static String JAR_FILE_EXT = ".jar";
- private final static String TXT_FILE_EXT = ".txt";
private final static String V1 = "V1";
private final static String V2 = "V2";
- private final String clazzFile = RELOADED_CLAZZ_PREFIX_NAME + CLAZZ_FILE_EXT;
- private final String jarFile = RELOADED_CLAZZ_PREFIX_NAME + JAR_FILE_EXT;
- private final String javaFile = RELOADED_CLAZZ_PREFIX_NAME + JAVA_FILE_EXT;
private static String hiveReloadPath;
private File reloadFolder;
public static final Log LOG = LogFactory.getLog(TestSessionState.class);
@@ -191,35 +184,10 @@ public class TestSessionState {
}
private void generateRefreshJarFiles(String version) throws IOException, InterruptedException {
- String u = HiveTestUtils.getFileFromClasspath(
- RELOADED_CLAZZ_PREFIX_NAME + version + TXT_FILE_EXT);
- File dir = new File(u);
- File parentDir = dir.getParentFile();
- File f = new File(parentDir, javaFile);
- Files.copy(dir, f);
- executeCmd(new String[]{"javac", javaFile}, parentDir);
- executeCmd(new String[]{"jar", "cf", jarFile, clazzFile}, parentDir);
- Files.move(new File(parentDir, jarFile), new File(parentDir, jarFile + "." + version));
- f.delete();
- new File(parentDir, clazzFile).delete();
- }
-
- private void executeCmd(String[] cmdArr, File dir) throws IOException, InterruptedException {
- final Process p1 = Runtime.getRuntime().exec(cmdArr, null, dir);
- new Thread(new Runnable() {
- public void run() {
- BufferedReader input = new BufferedReader(new InputStreamReader(p1.getErrorStream()));
- String line;
- try {
- while ((line = input.readLine()) != null) {
- System.out.println(line);
- }
- } catch (IOException e) {
- LOG.error("Failed to execute the command due the exception " + e);
- }
- }
- }).start();
- p1.waitFor();
+ String u = HiveTestUtils
+ .getFileFromClasspath(RELOADED_CLAZZ_PREFIX_NAME + version + HiveTestUtils.TXT_FILE_EXT);
+ File jarFile = HiveTestUtils.genLocalJarForTest(u, RELOADED_CLAZZ_PREFIX_NAME);
+ Files.move(jarFile, new File(jarFile.getAbsolutePath() + "." + version));
}
@Test
[38/50] [abbrv] hive git commit: HIVE-10932 : Unit test
udf_nondeterministic failure due to HIVE-10728 (Aihua Xu via Ashutosh
Chauhan)
Posted by xu...@apache.org.
HIVE-10932 : Unit test udf_nondeterministic failure due to HIVE-10728 (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8a107300
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8a107300
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8a107300
Branch: refs/heads/spark
Commit: 8a107300d43ea11a5951194482fcdfc3a362ca96
Parents: c9b7d25
Author: Aihua Xu <ai...@gmail.com>
Authored: Thu Jun 4 06:13:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Jun 4 16:23:39 2015 -0700
----------------------------------------------------------------------
.../clientpositive/udf_folder_constants.q | 11 ++
.../clientpositive/udf_nondeterministic.q | 11 --
.../clientpositive/udf_folder_constants.q.out | 120 +++++++++++++++++++
.../clientpositive/udf_nondeterministic.q.out | 120 -------------------
4 files changed, 131 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8a107300/ql/src/test/queries/clientpositive/udf_folder_constants.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_folder_constants.q b/ql/src/test/queries/clientpositive/udf_folder_constants.q
new file mode 100644
index 0000000..fcb5162
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_folder_constants.q
@@ -0,0 +1,11 @@
+drop table if exists udf_tb1;
+drop table if exists udf_tb2;
+
+create table udf_tb1 (year int, month int);
+create table udf_tb2(month int);
+insert into udf_tb1 values(2001, 11);
+insert into udf_tb2 values(11);
+
+explain
+select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month);
+select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month);
http://git-wip-us.apache.org/repos/asf/hive/blob/8a107300/ql/src/test/queries/clientpositive/udf_nondeterministic.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_nondeterministic.q b/ql/src/test/queries/clientpositive/udf_nondeterministic.q
deleted file mode 100644
index fcb5162..0000000
--- a/ql/src/test/queries/clientpositive/udf_nondeterministic.q
+++ /dev/null
@@ -1,11 +0,0 @@
-drop table if exists udf_tb1;
-drop table if exists udf_tb2;
-
-create table udf_tb1 (year int, month int);
-create table udf_tb2(month int);
-insert into udf_tb1 values(2001, 11);
-insert into udf_tb2 values(11);
-
-explain
-select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month);
-select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month);
http://git-wip-us.apache.org/repos/asf/hive/blob/8a107300/ql/src/test/results/clientpositive/udf_folder_constants.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_folder_constants.q.out b/ql/src/test/results/clientpositive/udf_folder_constants.q.out
new file mode 100644
index 0000000..65ee693
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_folder_constants.q.out
@@ -0,0 +1,120 @@
+PREHOOK: query: drop table if exists udf_tb1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists udf_tb1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists udf_tb2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists udf_tb2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table udf_tb1 (year int, month int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@udf_tb1
+POSTHOOK: query: create table udf_tb1 (year int, month int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@udf_tb1
+PREHOOK: query: create table udf_tb2(month int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@udf_tb2
+POSTHOOK: query: create table udf_tb2(month int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@udf_tb2
+PREHOOK: query: insert into udf_tb1 values(2001, 11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@udf_tb1
+POSTHOOK: query: insert into udf_tb1 values(2001, 11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@udf_tb1
+POSTHOOK: Lineage: udf_tb1.month EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: udf_tb1.year EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: insert into udf_tb2 values(11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@udf_tb2
+POSTHOOK: query: insert into udf_tb2 values(11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@udf_tb2
+POSTHOOK: Lineage: udf_tb2.month EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: explain
+select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: udf_tb1
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((year = 2001) and month is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: month (type: int)
+ outputColumnNames: _col1
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ alias: b
+ Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: month is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: month (type: int)
+ sort order: +
+ Map-reduce partition columns: month (type: int)
+ Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 month (type: int)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 978336000 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@udf_tb1
+PREHOOK: Input: default@udf_tb2
+#### A masked pattern was here ####
+POSTHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@udf_tb1
+POSTHOOK: Input: default@udf_tb2
+#### A masked pattern was here ####
+978336000
http://git-wip-us.apache.org/repos/asf/hive/blob/8a107300/ql/src/test/results/clientpositive/udf_nondeterministic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_nondeterministic.q.out b/ql/src/test/results/clientpositive/udf_nondeterministic.q.out
deleted file mode 100644
index eef5555..0000000
--- a/ql/src/test/results/clientpositive/udf_nondeterministic.q.out
+++ /dev/null
@@ -1,120 +0,0 @@
-PREHOOK: query: drop table if exists udf_tb1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists udf_tb1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table if exists udf_tb2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists udf_tb2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table udf_tb1 (year int, month int)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@udf_tb1
-POSTHOOK: query: create table udf_tb1 (year int, month int)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@udf_tb1
-PREHOOK: query: create table udf_tb2(month int)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@udf_tb2
-POSTHOOK: query: create table udf_tb2(month int)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@udf_tb2
-PREHOOK: query: insert into udf_tb1 values(2001, 11)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@values__tmp__table__1
-PREHOOK: Output: default@udf_tb1
-POSTHOOK: query: insert into udf_tb1 values(2001, 11)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@values__tmp__table__1
-POSTHOOK: Output: default@udf_tb1
-POSTHOOK: Lineage: udf_tb1.month EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
-POSTHOOK: Lineage: udf_tb1.year EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: insert into udf_tb2 values(11)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@values__tmp__table__2
-PREHOOK: Output: default@udf_tb2
-POSTHOOK: query: insert into udf_tb2 values(11)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@values__tmp__table__2
-POSTHOOK: Output: default@udf_tb2
-POSTHOOK: Lineage: udf_tb2.month EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: explain
-select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain
-select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: udf_tb1
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((year = 2001) and month is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: month (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- TableScan
- alias: b
- Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: month is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: month (type: int)
- sort order: +
- Map-reduce partition columns: month (type: int)
- Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: int)
- 1 month (type: int)
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: unix_timestamp('2001-01-01 00:00:00') (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@udf_tb1
-PREHOOK: Input: default@udf_tb2
-#### A masked pattern was here ####
-POSTHOOK: query: select unix_timestamp(concat(a.year, '-01-01 00:00:00')) from (select * from udf_tb1 where year=2001) a join udf_tb2 b on (a.month=b.month)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@udf_tb1
-POSTHOOK: Input: default@udf_tb2
-#### A masked pattern was here ####
-978336000
[34/50] [abbrv] hive git commit: HIVE-10427 - collect_list() and
collect_set() should accept struct types as argument (Chao Sun,
reviewed by Alexander Pivovarov)
Posted by xu...@apache.org.
HIVE-10427 - collect_list() and collect_set() should accept struct types as argument (Chao Sun, reviewed by Alexander Pivovarov)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c45f925
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c45f925
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c45f925
Branch: refs/heads/spark
Commit: 9c45f925ebac0c16b1af71aa774cbfce2e65c95a
Parents: 244386a
Author: Chao Sun <su...@apache.org>
Authored: Thu Jun 4 13:54:59 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Thu Jun 4 13:54:59 2015 -0700
----------------------------------------------------------------------
data/files/customers.txt | 3 +
data/files/nested_orders.txt | 8 +
data/files/orders.txt | 8 +
.../ql/udf/generic/GenericUDAFCollectList.java | 15 +-
.../ql/udf/generic/GenericUDAFCollectSet.java | 14 +-
.../GenericUDAFMkCollectionEvaluator.java | 21 +-
.../udaf_collect_set_unsupported.q | 3 +
.../queries/clientpositive/udaf_collect_set_2.q | 222 ++++++
.../udaf_collect_set_unsupported.q.out | 1 +
.../clientpositive/udaf_collect_set_2.q.out | 742 +++++++++++++++++++
10 files changed, 1017 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/data/files/customers.txt
----------------------------------------------------------------------
diff --git a/data/files/customers.txt b/data/files/customers.txt
new file mode 100644
index 0000000..fb85554
--- /dev/null
+++ b/data/files/customers.txt
@@ -0,0 +1,3 @@
+1,Chris,25
+2,John,20
+3,Martin,30
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/data/files/nested_orders.txt
----------------------------------------------------------------------
diff --git a/data/files/nested_orders.txt b/data/files/nested_orders.txt
new file mode 100644
index 0000000..b0581dc
--- /dev/null
+++ b/data/files/nested_orders.txt
@@ -0,0 +1,8 @@
+1,3,2014-05-11,"apple":30.50$"orange":41.35
+2,1,2013-06-21,"juice":21.45$"bread":15.20
+3,2,2013-08-10,"yogurt":126.57$"beef":210.57
+4,1,2014-10-11,"rice":29.36$"grape":1200.50
+5,3,2014-12-12,"icecream":210.03$"banana":100.56$"coffee:500.00
+6,2,2015-01-15,"milk":27.45
+7,2,2014-06-25,"chocolate":3.65$"water":420.36
+8,1,2013-06-21,"juice":21.45$"bread":15.20
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/data/files/orders.txt
----------------------------------------------------------------------
diff --git a/data/files/orders.txt b/data/files/orders.txt
new file mode 100644
index 0000000..06b422b
--- /dev/null
+++ b/data/files/orders.txt
@@ -0,0 +1,8 @@
+1,3,2014-05-11,30.50
+2,1,2013-06-21,21.45
+3,2,2013-08-10,126.57
+4,1,2014-10-11,29.36
+5,3,2014-12-12,210.03
+6,2,2015-01-15,27.45
+7,2,2014-06-25,3.65
+8,1,2013-06-21,21.45
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
index 536c4a7..b10c4ab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
@@ -42,10 +42,17 @@ public class GenericUDAFCollectList extends AbstractGenericUDAFResolver {
throw new UDFArgumentTypeException(parameters.length - 1,
"Exactly one argument is expected.");
}
- if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(0,
- "Only primitive type arguments are accepted but "
- + parameters[0].getTypeName() + " was passed as parameter 1.");
+
+ switch (parameters[0].getCategory()) {
+ case PRIMITIVE:
+ case STRUCT:
+ case MAP:
+ case LIST:
+ break;
+ default:
+ throw new UDFArgumentTypeException(0,
+ "Only primitive, struct, list or map type arguments are accepted but "
+ + parameters[0].getTypeName() + " was passed as parameter 1.");
}
return new GenericUDAFMkCollectionEvaluator(BufferType.LIST);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
index 6dc424a..312a698 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectSet.java
@@ -44,10 +44,16 @@ public class GenericUDAFCollectSet extends AbstractGenericUDAFResolver {
throw new UDFArgumentTypeException(parameters.length - 1,
"Exactly one argument is expected.");
}
- if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(0,
- "Only primitive type arguments are accepted but "
- + parameters[0].getTypeName() + " was passed as parameter 1.");
+ switch (parameters[0].getCategory()) {
+ case PRIMITIVE:
+ case STRUCT:
+ case MAP:
+ case LIST:
+ break;
+ default:
+ throw new UDFArgumentTypeException(0,
+ "Only primitive, struct, list or map type arguments are accepted but "
+ + parameters[0].getTypeName() + " was passed as parameter 1.");
}
return new GenericUDAFMkCollectionEvaluator(BufferType.SET);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
index efcc8f5..2b5e6dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
@@ -40,7 +39,7 @@ public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
enum BufferType { SET, LIST }
// For PARTIAL1 and COMPLETE: ObjectInspectors for original data
- private transient PrimitiveObjectInspector inputOI;
+ private transient ObjectInspector inputOI;
// For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations (list
// of objs)
private transient StandardListObjectInspector loi;
@@ -64,21 +63,19 @@ public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
// init output object inspectors
// The output of a partial aggregation is a list
if (m == Mode.PARTIAL1) {
- inputOI = (PrimitiveObjectInspector) parameters[0];
- return ObjectInspectorFactory
- .getStandardListObjectInspector((PrimitiveObjectInspector) ObjectInspectorUtils
- .getStandardObjectInspector(inputOI));
+ inputOI = parameters[0];
+ return ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorUtils.getStandardObjectInspector(inputOI));
} else {
if (!(parameters[0] instanceof ListObjectInspector)) {
//no map aggregation.
- inputOI = (PrimitiveObjectInspector) ObjectInspectorUtils
- .getStandardObjectInspector(parameters[0]);
- return (StandardListObjectInspector) ObjectInspectorFactory
- .getStandardListObjectInspector(inputOI);
+ inputOI = ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
+ return ObjectInspectorFactory.getStandardListObjectInspector(inputOI);
} else {
internalMergeOI = (ListObjectInspector) parameters[0];
- inputOI = (PrimitiveObjectInspector) internalMergeOI.getListElementObjectInspector();
- loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
+ inputOI = internalMergeOI.getListElementObjectInspector();
+ loi = (StandardListObjectInspector)
+ ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
return loi;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q b/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q
new file mode 100644
index 0000000..ed03620
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/udaf_collect_set_unsupported.q
@@ -0,0 +1,3 @@
+SELECT key, collect_set(create_union(value))
+FROM src
+GROUP BY key ORDER BY key limit 20;
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/queries/clientpositive/udaf_collect_set_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udaf_collect_set_2.q b/ql/src/test/queries/clientpositive/udaf_collect_set_2.q
new file mode 100644
index 0000000..d18509b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udaf_collect_set_2.q
@@ -0,0 +1,222 @@
+set hive.support.sql11.reserved.keywords=false;
+
+DESCRIBE FUNCTION collect_set;
+DESCRIBE FUNCTION EXTENDED collect_set;
+
+DESCRIBE FUNCTION collect_list;
+DESCRIBE FUNCTION EXTENDED collect_list;
+
+
+-- initialize tables
+
+CREATE TABLE customers (id int, name varchar(10), age int)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ',';
+
+LOAD DATA LOCAL INPATH "../../data/files/customers.txt" INTO TABLE customers;
+
+CREATE TABLE orders (id int, cid int, date date, amount double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ',';
+
+LOAD DATA LOCAL INPATH "../../data/files/orders.txt" INTO TABLE orders;
+
+CREATE TABLE nested_orders (id int, cid int, date date, sub map<string,double>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+COLLECTION ITEMS TERMINATED BY '$'
+MAP KEYS TERMINATED BY ':';
+
+LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_orders;
+
+-- 1. test struct
+
+-- 1.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- cast decimal
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+
+SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+
+-- 1.2 when field is map
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+
+-- 1.3 when field is list
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_set(struct(c.name, o.date, map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(struct(c.name, o.date, map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+
+-- 2. test array
+
+-- 2.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(array(o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(array(o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- cast decimal
+
+SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- 2.2 when field is struct
+
+SELECT c.id, sort_array(collect_set(array(o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(array(o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- 2.3 when field is list
+
+SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+
+-- 3. test map
+
+-- 3.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- cast decimal
+
+SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- 3.2 when field is struct
+
+SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+-- 3.3 when field is list
+
+SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id;
+
+
+-- clean up
+
+DROP TABLE customer;
+DROP TABLE orders;
+DROP TABLE nested_orders
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out b/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out
new file mode 100644
index 0000000..4298215
--- /dev/null
+++ b/ql/src/test/results/clientnegative/udaf_collect_set_unsupported.q.out
@@ -0,0 +1 @@
+FAILED: UDFArgumentTypeException Only primitive, struct, list or map type arguments are accepted but uniontype<> was passed as parameter 1.
http://git-wip-us.apache.org/repos/asf/hive/blob/9c45f925/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out b/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out
new file mode 100644
index 0000000..536234f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udaf_collect_set_2.q.out
@@ -0,0 +1,742 @@
+PREHOOK: query: DESCRIBE FUNCTION collect_set
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION collect_set
+POSTHOOK: type: DESCFUNCTION
+collect_set(x) - Returns a set of objects with duplicate elements eliminated
+PREHOOK: query: DESCRIBE FUNCTION EXTENDED collect_set
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION EXTENDED collect_set
+POSTHOOK: type: DESCFUNCTION
+collect_set(x) - Returns a set of objects with duplicate elements eliminated
+PREHOOK: query: DESCRIBE FUNCTION collect_list
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION collect_list
+POSTHOOK: type: DESCFUNCTION
+collect_list(x) - Returns a list of objects with duplicates
+PREHOOK: query: DESCRIBE FUNCTION EXTENDED collect_list
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION EXTENDED collect_list
+POSTHOOK: type: DESCFUNCTION
+collect_list(x) - Returns a list of objects with duplicates
+PREHOOK: query: -- initialize tables
+
+CREATE TABLE customers (id int, name varchar(10), age int)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@customers
+POSTHOOK: query: -- initialize tables
+
+CREATE TABLE customers (id int, name varchar(10), age int)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@customers
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/customers.txt" INTO TABLE customers
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@customers
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/customers.txt" INTO TABLE customers
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@customers
+PREHOOK: query: CREATE TABLE orders (id int, cid int, date date, amount double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orders
+POSTHOOK: query: CREATE TABLE orders (id int, cid int, date date, amount double)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orders
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/orders.txt" INTO TABLE orders
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orders
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/orders.txt" INTO TABLE orders
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orders
+PREHOOK: query: CREATE TABLE nested_orders (id int, cid int, date date, sub map<string,double>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+COLLECTION ITEMS TERMINATED BY '$'
+MAP KEYS TERMINATED BY ':'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@nested_orders
+POSTHOOK: query: CREATE TABLE nested_orders (id int, cid int, date date, sub map<string,double>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ','
+COLLECTION ITEMS TERMINATED BY '$'
+MAP KEYS TERMINATED BY ':'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@nested_orders
+PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_orders
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@nested_orders
+POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_orders
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@nested_orders
+PREHOOK: query: -- 1. test struct
+
+-- 1.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1. test struct
+
+-- 1.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","amount":21.45},{"name":"Chris","date":"2014-10-11","amount":29.36}]
+2 [{"name":"John","date":"2013-08-10","amount":126.57},{"name":"John","date":"2014-06-25","amount":3.65},{"name":"John","date":"2015-01-15","amount":27.45}]
+3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210.03}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","amount":21.45},{"name":"Chris","date":"2013-06-21","amount":21.45},{"name":"Chris","date":"2014-10-11","amount":29.36}]
+2 [{"name":"John","date":"2013-08-10","amount":126.57},{"name":"John","date":"2014-06-25","amount":3.65},{"name":"John","date":"2015-01-15","amount":27.45}]
+3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210.03}]
+PREHOOK: query: -- cast decimal
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- cast decimal
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","amount":21.5},{"name":"Chris","date":"2014-10-11","amount":29.4}]
+2 [{"name":"John","date":"2013-08-10","amount":126.6},{"name":"John","date":"2014-06-25","amount":3.7},{"name":"John","date":"2015-01-15","amount":27.5}]
+3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","amount":21.5},{"name":"Chris","date":"2013-06-21","amount":21.5},{"name":"Chris","date":"2014-10-11","amount":29.4}]
+2 [{"name":"John","date":"2013-08-10","amount":126.6},{"name":"John","date":"2014-06-25","amount":3.7},{"name":"John","date":"2015-01-15","amount":27.5}]
+3 [{"name":"Martin","date":"2014-05-11","amount":30.5},{"name":"Martin","date":"2014-12-12","amount":210}]
+PREHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"col1":"Chris","col2":"2013-06-21","col3":21.45},{"col1":"Chris","col2":"2014-10-11","col3":29.36}]
+2 [{"col1":"John","col2":"2013-08-10","col3":126.57},{"col1":"John","col2":"2014-06-25","col3":3.65},{"col1":"John","col2":"2015-01-15","col3":27.45}]
+3 [{"col1":"Martin","col2":"2014-05-11","col3":30.5},{"col1":"Martin","col2":"2014-12-12","col3":210.03}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"col1":"Chris","col2":"2013-06-21","col3":21.45},{"col1":"Chris","col2":"2013-06-21","col3":21.45},{"col1":"Chris","col2":"2014-10-11","col3":29.36}]
+2 [{"col1":"John","col2":"2013-08-10","col3":126.57},{"col1":"John","col2":"2014-06-25","col3":3.65},{"col1":"John","col2":"2015-01-15","col3":27.45}]
+3 [{"col1":"Martin","col2":"2014-05-11","col3":30.5},{"col1":"Martin","col2":"2014-12-12","col3":210.03}]
+PREHOOK: query: -- 1.2 when field is map
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1.2 when field is map
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"name":"Chris","date":"2014-10-11","sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
+2 [{"name":"John","date":"2013-08-10","sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"name":"John","date":"2014-06-25","sub":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"name":"John","date":"2015-01-15","sub":{"\"milk\"":27.45}}]
+3 [{"name":"Martin","date":"2014-05-11","sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"name":"Martin","date":"2014-12-12","sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"name":"Chris","date":"2013-06-21","sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"name":"Chris","date":"2014-10-11","sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
+2 [{"name":"John","date":"2013-08-10","sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"name":"John","date":"2014-06-25","sub":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"name":"John","date":"2015-01-15","sub":{"\"milk\"":27.45}}]
+3 [{"name":"Martin","date":"2014-05-11","sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"name":"Martin","date":"2014-12-12","sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
+PREHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"col1":"Chris","col2":"2013-06-21","col3":{"\"bread\"":15.2,"\"juice\"":21.45}},{"col1":"Chris","col2":"2014-10-11","col3":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
+2 [{"col1":"John","col2":"2013-08-10","col3":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"col1":"John","col2":"2014-06-25","col3":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"col1":"John","col2":"2015-01-15","col3":{"\"milk\"":27.45}}]
+3 [{"col1":"Martin","col2":"2014-05-11","col3":{"\"orange\"":41.35,"\"apple\"":30.5}},{"col1":"Martin","col2":"2014-12-12","col3":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"col1":"Chris","col2":"2013-06-21","col3":{"\"bread\"":15.2,"\"juice\"":21.45}},{"col1":"Chris","col2":"2013-06-21","col3":{"\"bread\"":15.2,"\"juice\"":21.45}},{"col1":"Chris","col2":"2014-10-11","col3":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
+2 [{"col1":"John","col2":"2013-08-10","col3":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"col1":"John","col2":"2014-06-25","col3":{"\"chocolate\"":3.65,"\"water\"":420.36}},{"col1":"John","col2":"2015-01-15","col3":{"\"milk\"":27.45}}]
+3 [{"col1":"Martin","col2":"2014-05-11","col3":{"\"orange\"":41.35,"\"apple\"":30.5}},{"col1":"Martin","col2":"2014-12-12","col3":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
+PREHOOK: query: -- 1.3 when field is list
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1.3 when field is list
+
+SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","sub":[15.2,21.45]},{"name":"Chris","date":"2014-10-11","sub":[1200.5,29.36]}]
+2 [{"name":"John","date":"2013-08-10","sub":[210.57,126.57]},{"name":"John","date":"2014-06-25","sub":[3.65,420.36]},{"name":"John","date":"2015-01-15","sub":[27.45]}]
+3 [{"name":"Martin","date":"2014-05-11","sub":[41.35,30.5]},{"name":"Martin","date":"2014-12-12","sub":[210.03,500.0,100.56]}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.date, "sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"name":"Chris","date":"2013-06-21","sub":[15.2,21.45]},{"name":"Chris","date":"2013-06-21","sub":[15.2,21.45]},{"name":"Chris","date":"2014-10-11","sub":[1200.5,29.36]}]
+2 [{"name":"John","date":"2013-08-10","sub":[210.57,126.57]},{"name":"John","date":"2014-06-25","sub":[3.65,420.36]},{"name":"John","date":"2015-01-15","sub":[27.45]}]
+3 [{"name":"Martin","date":"2014-05-11","sub":[41.35,30.5]},{"name":"Martin","date":"2014-12-12","sub":[210.03,500.0,100.56]}]
+PREHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_set(struct(c.name, o.date, map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"col1":"Chris","col2":"2013-06-21","col3":[15.2,21.45]},{"col1":"Chris","col2":"2014-10-11","col3":[1200.5,29.36]}]
+2 [{"col1":"John","col2":"2013-08-10","col3":[210.57,126.57]},{"col1":"John","col2":"2014-06-25","col3":[3.65,420.36]},{"col1":"John","col2":"2015-01-15","col3":[27.45]}]
+3 [{"col1":"Martin","col2":"2014-05-11","col3":[41.35,30.5]},{"col1":"Martin","col2":"2014-12-12","col3":[210.03,500.0,100.56]}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(struct(c.name, o.date, map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"col1":"Chris","col2":"2013-06-21","col3":[15.2,21.45]},{"col1":"Chris","col2":"2013-06-21","col3":[15.2,21.45]},{"col1":"Chris","col2":"2014-10-11","col3":[1200.5,29.36]}]
+2 [{"col1":"John","col2":"2013-08-10","col3":[210.57,126.57]},{"col1":"John","col2":"2014-06-25","col3":[3.65,420.36]},{"col1":"John","col2":"2015-01-15","col3":[27.45]}]
+3 [{"col1":"Martin","col2":"2014-05-11","col3":[41.35,30.5]},{"col1":"Martin","col2":"2014-12-12","col3":[210.03,500.0,100.56]}]
+PREHOOK: query: -- 2. test array
+
+-- 2.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(array(o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2. test array
+
+-- 2.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(array(o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [[21.45],[29.36]]
+2 [[3.65],[27.45],[126.57]]
+3 [[30.5],[210.03]]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(array(o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [[21.45],[21.45],[29.36]]
+2 [[3.65],[27.45],[126.57]]
+3 [[30.5],[210.03]]
+PREHOOK: query: -- cast decimal
+
+SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- cast decimal
+
+SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [[21.5],[29.4]]
+2 [[3.7],[27.5],[126.6]]
+3 [[30.5],[210]]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [[21.5],[21.5],[29.4]]
+2 [[3.7],[27.5],[126.6]]
+3 [[30.5],[210]]
+PREHOOK: query: -- 2.2 when field is struct
+
+SELECT c.id, sort_array(collect_set(array(o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2.2 when field is struct
+
+SELECT c.id, sort_array(collect_set(array(o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [[{"\"bread\"":15.2,"\"juice\"":21.45}],[{"\"grape\"":1200.5,"\"rice\"":29.36}]]
+2 [[{"\"milk\"":27.45}],[{"\"beef\"":210.57,"\"yogurt\"":126.57}],[{"\"chocolate\"":3.65,"\"water\"":420.36}]]
+3 [[{"\"orange\"":41.35,"\"apple\"":30.5}],[{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}]]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(array(o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [[{"\"bread\"":15.2,"\"juice\"":21.45}],[{"\"bread\"":15.2,"\"juice\"":21.45}],[{"\"grape\"":1200.5,"\"rice\"":29.36}]]
+2 [[{"\"milk\"":27.45}],[{"\"beef\"":210.57,"\"yogurt\"":126.57}],[{"\"chocolate\"":3.65,"\"water\"":420.36}]]
+3 [[{"\"orange\"":41.35,"\"apple\"":30.5}],[{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}]]
+PREHOOK: query: -- 2.3 when field is list
+
+SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2.3 when field is list
+
+SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [[[15.2,21.45]],[[1200.5,29.36]]]
+2 [[[3.65,420.36]],[[27.45]],[[210.57,126.57]]]
+3 [[[41.35,30.5]],[[210.03,500.0,100.56]]]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [[[15.2,21.45]],[[15.2,21.45]],[[1200.5,29.36]]]
+2 [[[3.65,420.36]],[[27.45]],[[210.57,126.57]]]
+3 [[[41.35,30.5]],[[210.03,500.0,100.56]]]
+PREHOOK: query: -- 3. test map
+
+-- 3.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 3. test map
+
+-- 3.1 when field is primitive
+
+SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"amount":21.45},{"amount":29.36}]
+2 [{"amount":3.65},{"amount":27.45},{"amount":126.57}]
+3 [{"amount":30.5},{"amount":210.03}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"amount":21.45},{"amount":21.45},{"amount":29.36}]
+2 [{"amount":3.65},{"amount":27.45},{"amount":126.57}]
+3 [{"amount":30.5},{"amount":210.03}]
+PREHOOK: query: -- cast decimal
+
+SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- cast decimal
+
+SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"amount":21.5},{"amount":29.4}]
+2 [{"amount":3.7},{"amount":27.5},{"amount":126.6}]
+3 [{"amount":30.5},{"amount":210}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
+FROM customers c
+INNER JOIN orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@orders
+#### A masked pattern was here ####
+1 [{"amount":21.5},{"amount":21.5},{"amount":29.4}]
+2 [{"amount":3.7},{"amount":27.5},{"amount":126.6}]
+3 [{"amount":30.5},{"amount":210}]
+PREHOOK: query: -- 3.2 when field is struct
+
+SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 3.2 when field is struct
+
+SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
+2 [{"sub":{"\"milk\"":27.45}},{"sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"sub":{"\"chocolate\"":3.65,"\"water\"":420.36}}]
+3 [{"sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"sub":{"\"bread\"":15.2,"\"juice\"":21.45}},{"sub":{"\"grape\"":1200.5,"\"rice\"":29.36}}]
+2 [{"sub":{"\"milk\"":27.45}},{"sub":{"\"beef\"":210.57,"\"yogurt\"":126.57}},{"sub":{"\"chocolate\"":3.65,"\"water\"":420.36}}]
+3 [{"sub":{"\"orange\"":41.35,"\"apple\"":30.5}},{"sub":{"\"icecream\"":210.03,"\"coffee":500.0,"\"banana\"":100.56}}]
+PREHOOK: query: -- 3.3 when field is list
+
+SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: -- 3.3 when field is list
+
+SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"sub":[15.2,21.45]},{"sub":[1200.5,29.36]}]
+2 [{"sub":[3.65,420.36]},{"sub":[27.45]},{"sub":[210.57,126.57]}]
+3 [{"sub":[41.35,30.5]},{"sub":[210.03,500.0,100.56]}]
+PREHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@customers
+PREHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
+FROM customers c
+INNER JOIN nested_orders o
+ON (c.id = o.cid) GROUP BY c.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@customers
+POSTHOOK: Input: default@nested_orders
+#### A masked pattern was here ####
+1 [{"sub":[15.2,21.45]},{"sub":[15.2,21.45]},{"sub":[1200.5,29.36]}]
+2 [{"sub":[3.65,420.36]},{"sub":[27.45]},{"sub":[210.57,126.57]}]
+3 [{"sub":[41.35,30.5]},{"sub":[210.03,500.0,100.56]}]
+PREHOOK: query: -- clean up
+
+DROP TABLE customer
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- clean up
+
+DROP TABLE customer
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orders
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orders
+PREHOOK: Output: default@orders
+POSTHOOK: query: DROP TABLE orders
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orders
+POSTHOOK: Output: default@orders
+PREHOOK: query: DROP TABLE nested_orders
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@nested_orders
+PREHOOK: Output: default@nested_orders
+POSTHOOK: query: DROP TABLE nested_orders
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@nested_orders
+POSTHOOK: Output: default@nested_orders
[32/50] [abbrv] hive git commit: HIVE-9069: Simplify filter
predicates for CBO (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Posted by xu...@apache.org.
HIVE-9069: Simplify filter predicates for CBO (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/991bb454
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/991bb454
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/991bb454
Branch: refs/heads/spark
Commit: 991bb454b6f08e989425cc97a7d2e17becf9e879
Parents: 0e5d907
Author: jpullokk <jp...@apache.org>
Authored: Thu Jun 4 01:46:07 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Thu Jun 4 01:46:07 2015 -0700
----------------------------------------------------------------------
.../calcite/rules/HivePreFilteringRule.java | 218 +++++++
.../hadoop/hive/ql/parse/CalcitePlanner.java | 16 +-
.../clientpositive/filter_cond_pushdown.q | 19 +
.../results/clientpositive/auto_join16.q.out | 4 +-
.../results/clientpositive/auto_join8.q.out | 14 +-
.../clientpositive/cbo_rp_auto_join1.q.out | 4 +-
.../clientpositive/explain_logical.q.out | 22 +-
.../clientpositive/filter_cond_pushdown.q.out | 382 ++++++++++++
ql/src/test/results/clientpositive/join8.q.out | 16 +-
.../list_bucket_query_oneskew_2.q.out | 2 +-
.../test/results/clientpositive/ppd_gby.q.out | 41 +-
.../test/results/clientpositive/ppd_gby2.q.out | 49 +-
.../results/clientpositive/ppd_gby_join.q.out | 42 +-
.../test/results/clientpositive/ppd_join.q.out | 44 +-
.../test/results/clientpositive/ppd_join2.q.out | 81 +--
.../test/results/clientpositive/ppd_join3.q.out | 91 +--
.../results/clientpositive/ppd_udf_col.q.out | 178 +++---
.../test/results/clientpositive/ppd_union.q.out | 42 +-
ql/src/test/results/clientpositive/ppd_vc.q.out | 8 +-
.../clientpositive/spark/ppd_join2.q.out | 81 +--
.../clientpositive/spark/ppd_join3.q.out | 91 +--
.../results/clientpositive/subquery_notin.q.out | 16 +-
.../subquery_notin_having.q.java1.7.out | 16 +-
.../subquery_unqualcolumnrefs.q.out | 2 +-
.../results/clientpositive/subquery_views.q.out | 8 +-
.../clientpositive/tez/explainuser_1.q.out | 592 +++++++++----------
26 files changed, 1411 insertions(+), 668 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
new file mode 100644
index 0000000..dde6288
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.calcite.plan.RelOptPredicateList;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.RelFactories.FilterFactory;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+
+
+public class HivePreFilteringRule extends RelOptRule {
+
+ protected static final Log LOG = LogFactory
+ .getLog(HivePreFilteringRule.class.getName());
+
+
+ public static final HivePreFilteringRule INSTANCE =
+ new HivePreFilteringRule();
+
+ private final FilterFactory filterFactory;
+
+
+ private static final Set<String> COMPARISON_UDFS = Sets.newHashSet(
+ GenericUDFOPEqual.class.getAnnotation(Description.class).name(),
+ GenericUDFOPEqualNS.class.getAnnotation(Description.class).name(),
+ GenericUDFOPEqualOrGreaterThan.class.getAnnotation(Description.class).name(),
+ GenericUDFOPEqualOrLessThan.class.getAnnotation(Description.class).name(),
+ GenericUDFOPGreaterThan.class.getAnnotation(Description.class).name(),
+ GenericUDFOPLessThan.class.getAnnotation(Description.class).name(),
+ GenericUDFOPNotEqual.class.getAnnotation(Description.class).name());
+ private static final String IN_UDF =
+ GenericUDFIn.class.getAnnotation(Description.class).name();
+ private static final String BETWEEN_UDF =
+ GenericUDFBetween.class.getAnnotation(Description.class).name();
+
+
+ private HivePreFilteringRule() {
+ super(operand(Filter.class,
+ operand(RelNode.class, any())));
+ this.filterFactory = HiveFilter.DEFAULT_FILTER_FACTORY;
+ }
+
+ public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+ final RelNode filterChild = call.rel(1);
+
+ // 0. If the filter is already on top of a TableScan,
+ // we can bail out
+ if (filterChild instanceof TableScan) {
+ return;
+ }
+
+ final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
+
+ final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
+
+ // 1. We extract possible candidates to be pushed down
+ List<RexNode> commonOperands = new ArrayList<>();
+ switch (condition.getKind()) {
+ case AND:
+ ImmutableList<RexNode> operands = RexUtil.flattenAnd(((RexCall) condition).getOperands());
+ for (RexNode operand: operands) {
+ if (operand.getKind() == SqlKind.OR) {
+ commonOperands.addAll(extractCommonOperands(rexBuilder,operand));
+ }
+ }
+ break;
+ case OR:
+ commonOperands = extractCommonOperands(rexBuilder,condition);
+ break;
+ default:
+ return;
+ }
+
+ // 2. If we did not generate anything for the new predicate, we bail out
+ if (commonOperands.isEmpty()) {
+ return;
+ }
+
+ // 3. If the new conjuncts are already present in the plan, we bail out
+ final RelOptPredicateList predicates = RelMetadataQuery.getPulledUpPredicates(filter);
+ final List<RexNode> newConjuncts = new ArrayList<>();
+ for (RexNode commonOperand : commonOperands) {
+ boolean found = false;
+ for (RexNode conjunct : predicates.pulledUpPredicates) {
+ if (commonOperand.toString().equals(conjunct.toString())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ newConjuncts.add(commonOperand);
+ }
+ }
+ if (newConjuncts.isEmpty()) {
+ return;
+ }
+
+ // 4. Otherwise, we create a new condition
+ final RexNode newCondition = RexUtil.pullFactors(rexBuilder,
+ RexUtil.composeConjunction(rexBuilder, newConjuncts, false));
+
+ // 5. We create the new filter that might be pushed down
+ RelNode newFilter = filterFactory.createFilter(filterChild, newCondition);
+ RelNode newTopFilter = filterFactory.createFilter(newFilter, condition);
+
+ call.transformTo(newTopFilter);
+
+ }
+
+ private static List<RexNode> extractCommonOperands(RexBuilder rexBuilder, RexNode condition) {
+ assert condition.getKind() == SqlKind.OR;
+ Multimap<String,RexNode> reductionCondition = LinkedHashMultimap.create();
+
+ // 1. We extract the information necessary to create the predicate for the new
+ // filter; currently we support comparison functions, in and between
+ ImmutableList<RexNode> operands = RexUtil.flattenOr(((RexCall) condition).getOperands());
+ for (RexNode operand: operands) {
+ final RexNode operandCNF = RexUtil.toCnf(rexBuilder, operand);
+ final List<RexNode> conjunctions = RelOptUtil.conjunctions(operandCNF);
+ boolean addedToReductionCondition = false; // Flag to control whether we have added a new factor
+ // to the reduction predicate
+ for (RexNode conjunction: conjunctions) {
+ if (!(conjunction instanceof RexCall)) {
+ continue;
+ }
+ RexCall conjCall = (RexCall) conjunction;
+ if(COMPARISON_UDFS.contains(conjCall.getOperator().getName())) {
+ if (conjCall.operands.get(0) instanceof RexInputRef &&
+ conjCall.operands.get(1) instanceof RexLiteral) {
+ reductionCondition.put(conjCall.operands.get(0).toString(),
+ conjCall);
+ addedToReductionCondition = true;
+ } else if (conjCall.operands.get(1) instanceof RexInputRef &&
+ conjCall.operands.get(0) instanceof RexLiteral) {
+ reductionCondition.put(conjCall.operands.get(1).toString(),
+ conjCall);
+ addedToReductionCondition = true;
+ }
+ } else if(conjCall.getOperator().getName().equals(IN_UDF)) {
+ reductionCondition.put(conjCall.operands.get(0).toString(),
+ conjCall);
+ addedToReductionCondition = true;
+ } else if(conjCall.getOperator().getName().equals(BETWEEN_UDF)) {
+ reductionCondition.put(conjCall.operands.get(1).toString(),
+ conjCall);
+ addedToReductionCondition = true;
+ }
+ }
+
+ // If we did not add any factor, we can bail out
+ if (!addedToReductionCondition) {
+ return new ArrayList<>();
+ }
+ }
+
+ // 2. We gather the common factors and return them
+ List<RexNode> commonOperands = new ArrayList<>();
+ for (Entry<String,Collection<RexNode>> pair : reductionCondition.asMap().entrySet()) {
+ if (pair.getValue().size() == operands.size()) {
+ commonOperands.add(RexUtil.composeDisjunction(rexBuilder, pair.getValue(), false));
+ }
+ }
+ return commonOperands;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 02fc796..48e3cc7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -142,6 +142,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveInsertExchange4Join
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinAddNotNullRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinToMultiJoinRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter;
@@ -933,17 +934,18 @@ public class CalcitePlanner extends SemanticAnalyzer {
basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE);
}
- // 3. PPD
+ // 3. Constant propagation, common filter extraction, and PPD
basePlan = hepPlan(basePlan, true, mdProvider,
ReduceExpressionsRule.PROJECT_INSTANCE,
ReduceExpressionsRule.FILTER_INSTANCE,
ReduceExpressionsRule.JOIN_INSTANCE,
- new HiveFilterProjectTransposeRule(
- Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.class,
- HiveProject.DEFAULT_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule(
- HiveFilter.DEFAULT_FILTER_FACTORY),
- new FilterMergeRule(HiveFilter.DEFAULT_FILTER_FACTORY), HiveFilterJoinRule.JOIN,
- HiveFilterJoinRule.FILTER_ON_JOIN, new FilterAggregateTransposeRule(Filter.class,
+ HivePreFilteringRule.INSTANCE,
+ new HiveFilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY,
+ HiveProject.class, HiveProject.DEFAULT_PROJECT_FACTORY),
+ new HiveFilterSetOpTransposeRule(HiveFilter.DEFAULT_FILTER_FACTORY),
+ HiveFilterJoinRule.JOIN,
+ HiveFilterJoinRule.FILTER_ON_JOIN,
+ new FilterAggregateTransposeRule(Filter.class,
HiveFilter.DEFAULT_FILTER_FACTORY, Aggregate.class));
// 4. Transitive inference & Partition Pruning
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/queries/clientpositive/filter_cond_pushdown.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/filter_cond_pushdown.q b/ql/src/test/queries/clientpositive/filter_cond_pushdown.q
new file mode 100644
index 0000000..5e23b71
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/filter_cond_pushdown.q
@@ -0,0 +1,19 @@
+EXPLAIN
+SELECT f.key, g.value
+FROM src f JOIN src m JOIN src g ON(g.value = m.value AND m.value is not null AND m.value !='')
+WHERE (f.key = m.key AND f.value='2008-04-08' AND m.value='2008-04-08') OR (f.key = m.key AND f.value='2008-04-09');
+
+EXPLAIN
+SELECT f.key, g.value
+FROM src f JOIN src m JOIN src g ON(g.value = m.value AND m.value is not null AND m.value !='')
+WHERE (f.key = m.key AND f.value IN ('2008-04-08','2008-04-10') AND m.value='2008-04-08') OR (f.key = m.key AND f.value='2008-04-09');
+
+EXPLAIN
+SELECT t1.key
+FROM cbo_t1 t1
+JOIN (
+ SELECT t2.key
+ FROM cbo_t2 t2
+ JOIN (SELECT * FROM cbo_t3 t3 WHERE c_int=1) t3 ON t2.key=t3.c_int
+ WHERE ((t2.key=t3.key) AND (t2.c_float + t3.c_float > 2)) OR
+ ((t2.key=t3.key) AND (t2.c_int + t3.c_int > 2))) t4 ON t1.key=t4.key;
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/auto_join16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join16.q.out b/ql/src/test/results/clientpositive/auto_join16.q.out
index b07eaf6..1bad0f9 100644
--- a/ql/src/test/results/clientpositive/auto_join16.q.out
+++ b/ql/src/test/results/clientpositive/auto_join16.q.out
@@ -23,11 +23,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:a
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:a
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:a
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:a
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/auto_join8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join8.q.out b/ql/src/test/results/clientpositive/auto_join8.q.out
index 485622a..5b02597 100644
--- a/ql/src/test/results/clientpositive/auto_join8.q.out
+++ b/ql/src/test/results/clientpositive/auto_join8.q.out
@@ -73,12 +73,12 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -86,17 +86,17 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col2 is null (type: boolean)
- Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(null) (type: int), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out b/ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out
index 2e23197..f4b25ae 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out
@@ -610,7 +610,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean)
@@ -1277,7 +1277,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((key < 8) and (key < 6)) and key is not null) (type: boolean)
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/explain_logical.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/explain_logical.q.out b/ql/src/test/results/clientpositive/explain_logical.q.out
index 831535e..545034a 100644
--- a/ql/src/test/results/clientpositive/explain_logical.q.out
+++ b/ql/src/test/results/clientpositive/explain_logical.q.out
@@ -503,23 +503,23 @@ TOK_QUERY
LOGICAL PLAN:
-$hdt$_0:srcpart
+$hdt$_0:$hdt$_0:srcpart
TableScan (TS_0)
alias: srcpart
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- Filter Operator (FIL_12)
+ Filter Operator (FIL_14)
predicate: key is not null (type: boolean)
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Select Operator (SEL_2)
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator (RS_6)
+ Reduce Output Operator (RS_8)
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Join Operator (JOIN_9)
+ Join Operator (JOIN_11)
condition map:
Inner Join 0 to 1
keys:
@@ -527,11 +527,11 @@ $hdt$_0:srcpart
1 _col0 (type: string)
outputColumnNames: _col0, _col2
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
- Select Operator (SEL_10)
+ Select Operator (SEL_12)
expressions: _col0 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
- File Output Operator (FS_11)
+ File Output Operator (FS_13)
compressed: false
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
table:
@@ -539,23 +539,23 @@ $hdt$_0:srcpart
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
$hdt$_1:src2
- TableScan (TS_3)
+ TableScan (TS_5)
alias: src2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator (FIL_13)
+ Filter Operator (FIL_15)
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator (SEL_4)
+ Select Operator (SEL_6)
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator (RS_8)
+ Reduce Output Operator (RS_10)
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Join Operator (JOIN_9)
+ Join Operator (JOIN_11)
condition map:
Inner Join 0 to 1
keys:
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
new file mode 100644
index 0000000..e09057a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
@@ -0,0 +1,382 @@
+PREHOOK: query: EXPLAIN
+SELECT f.key, g.value
+FROM src f JOIN src m JOIN src g ON(g.value = m.value AND m.value is not null AND m.value !='')
+WHERE (f.key = m.key AND f.value='2008-04-08' AND m.value='2008-04-08') OR (f.key = m.key AND f.value='2008-04-09')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT f.key, g.value
+FROM src f JOIN src m JOIN src g ON(g.value = m.value AND m.value is not null AND m.value !='')
+WHERE (f.key = m.key AND f.value='2008-04-08' AND m.value='2008-04-08') OR (f.key = m.key AND f.value='2008-04-09')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: f
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((value = '2008-04-08') or (value = '2008-04-09')) and key is not null) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: f
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((value <> '') and key is not null) and value is not null) (type: boolean)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((_col1 = '2008-04-08') and (_col3 = '2008-04-08')) or (_col1 = '2008-04-09')) (type: boolean)
+ Statistics: Num rows: 205 Data size: 2177 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col3 (type: string)
+ outputColumnNames: _col0, _col3
+ Statistics: Num rows: 205 Data size: 2177 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col3 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: string)
+ Statistics: Num rows: 205 Data size: 2177 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: f
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value <> '') (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT f.key, g.value
+FROM src f JOIN src m JOIN src g ON(g.value = m.value AND m.value is not null AND m.value !='')
+WHERE (f.key = m.key AND f.value IN ('2008-04-08','2008-04-10') AND m.value='2008-04-08') OR (f.key = m.key AND f.value='2008-04-09')
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT f.key, g.value
+FROM src f JOIN src m JOIN src g ON(g.value = m.value AND m.value is not null AND m.value !='')
+WHERE (f.key = m.key AND f.value IN ('2008-04-08','2008-04-10') AND m.value='2008-04-08') OR (f.key = m.key AND f.value='2008-04-09')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: f
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((value) IN ('2008-04-08', '2008-04-10') or (value = '2008-04-09')) and key is not null) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: f
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((value <> '') and key is not null) and value is not null) (type: boolean)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((_col1) IN ('2008-04-08', '2008-04-10') and (_col3 = '2008-04-08')) or (_col1 = '2008-04-09')) (type: boolean)
+ Statistics: Num rows: 205 Data size: 2177 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col3 (type: string)
+ outputColumnNames: _col0, _col3
+ Statistics: Num rows: 205 Data size: 2177 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col3 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: string)
+ Statistics: Num rows: 205 Data size: 2177 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: f
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value <> '') (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT t1.key
+FROM cbo_t1 t1
+JOIN (
+ SELECT t2.key
+ FROM cbo_t2 t2
+ JOIN (SELECT * FROM cbo_t3 t3 WHERE c_int=1) t3 ON t2.key=t3.c_int
+ WHERE ((t2.key=t3.key) AND (t2.c_float + t3.c_float > 2)) OR
+ ((t2.key=t3.key) AND (t2.c_int + t3.c_int > 2))) t4 ON t1.key=t4.key
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT t1.key
+FROM cbo_t1 t1
+JOIN (
+ SELECT t2.key
+ FROM cbo_t2 t2
+ JOIN (SELECT * FROM cbo_t3 t3 WHERE c_int=1) t3 ON t2.key=t3.c_int
+ WHERE ((t2.key=t3.key) AND (t2.c_float + t3.c_float > 2)) OR
+ ((t2.key=t3.key) AND (t2.c_int + t3.c_int > 2))) t4 ON t1.key=t4.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t2
+ Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (UDFToDouble(key) = 1.0) (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: c_int (type: int), c_float (type: float)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: '1.0' (type: string)
+ sort order: +
+ Map-reduce partition columns: '1.0' (type: string)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: float)
+ TableScan
+ alias: t3
+ Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((c_int = 1) and (UDFToDouble(key) = 1.0)) (type: boolean)
+ Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: c_float (type: float)
+ outputColumnNames: _col2
+ Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: '1.0' (type: string)
+ sort order: +
+ Map-reduce partition columns: '1.0' (type: string)
+ Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: float)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2, _col5
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((_col2 + _col5) > 2.0) or ((_col1 + 1) > 2)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (UDFToDouble(key) = 1.0) (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: '1.0' (type: string)
+ sort order: +
+ Map-reduce partition columns: '1.0' (type: string)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: '1.0' (type: string)
+ sort order: +
+ Map-reduce partition columns: '1.0' (type: string)
+ Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: '1.0' (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/join8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join8.q.out b/ql/src/test/results/clientpositive/join8.q.out
index daf4b17..71792c1 100644
--- a/ql/src/test/results/clientpositive/join8.q.out
+++ b/ql/src/test/results/clientpositive/join8.q.out
@@ -53,17 +53,17 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
TableScan
alias: src1
@@ -89,17 +89,17 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col2 is null (type: boolean)
- Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(null) (type: int), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out
index aab5506..177da44 100644
--- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out
+++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out
@@ -758,7 +758,7 @@ STAGE PLANS:
name: default.fact_daily
name: default.fact_daily
Truncated Path -> Alias:
- /fact_tz/ds=1/x=484 [$hdt$_0:fact_daily]
+ /fact_tz/ds=1/x=484 [$hdt$_0:$hdt$_0:fact_daily]
Needs Tagging: false
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_gby.q.out b/ql/src/test/results/clientpositive/ppd_gby.q.out
index 523842e..87cb907 100644
--- a/ql/src/test/results/clientpositive/ppd_gby.q.out
+++ b/ql/src/test/results/clientpositive/ppd_gby.q.out
@@ -25,38 +25,45 @@ STAGE PLANS:
predicate: ((value > 'val_10') and (value > 'val_200')) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string), key (type: string)
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(_col1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Filter Operator
+ predicate: (_col1 > 'val_200') (type: boolean)
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col1 > 30) or (_col0 < 'val_400')) (type: boolean)
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_gby2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_gby2.q.out b/ql/src/test/results/clientpositive/ppd_gby2.q.out
index 1ee4b49..bc00149 100644
--- a/ql/src/test/results/clientpositive/ppd_gby2.q.out
+++ b/ql/src/test/results/clientpositive/ppd_gby2.q.out
@@ -28,41 +28,48 @@ STAGE PLANS:
predicate: ((value > 'val_10') and (value > 'val_200')) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string), key (type: string)
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(_col1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Filter Operator
+ predicate: (_col1 > 'val_200') (type: boolean)
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col1 > 30) or (_col0 < 'val_400')) (type: boolean)
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: bigint), _col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(_col1)
keys: _col0 (type: bigint)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -78,7 +85,7 @@ STAGE PLANS:
key expressions: _col0 (type: bigint)
sort order: +
Map-reduce partition columns: _col0 (type: bigint)
- Statistics: Num rows: 18 Data size: 190 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reduce Operator Tree:
Group By Operator
@@ -86,14 +93,14 @@ STAGE PLANS:
keys: KEY._col0 (type: bigint)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_gby_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_gby_join.q.out b/ql/src/test/results/clientpositive/ppd_gby_join.q.out
index 60ae8f9..579c827 100644
--- a/ql/src/test/results/clientpositive/ppd_gby_join.q.out
+++ b/ql/src/test/results/clientpositive/ppd_gby_join.q.out
@@ -42,16 +42,19 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > '1') and ((_col0 > '20') and (_col0 < '400'))) (type: boolean)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '4') (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
+ predicate: ((_col0 > '1') and ((_col0 > '20') and (_col0 < '400'))) (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -59,20 +62,27 @@ STAGE PLANS:
predicate: (((((((key > '1') and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key < '400')) and (key > '2')) and (key <> '4')) and key is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > '2') and (_col0 <> '4')) (type: boolean)
+ predicate: ((_col0 > '20') and (((_col1 < 'val_50') or (_col0 > '2')) and (_col0 < '400'))) (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Filter Operator
+ predicate: ((_col0 > '2') and (_col0 <> '4')) (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_join.q.out b/ql/src/test/results/clientpositive/ppd_join.q.out
index 8fde2d7..ae5fb27 100644
--- a/ql/src/test/results/clientpositive/ppd_join.q.out
+++ b/ql/src/test/results/clientpositive/ppd_join.q.out
@@ -39,17 +39,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > '1') and ((_col0 > '20') and (_col0 < '400'))) (type: boolean)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '4') (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
+ predicate: ((_col0 > '1') and ((_col0 > '20') and (_col0 < '400'))) (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -57,20 +60,27 @@ STAGE PLANS:
predicate: (((((((key > '1') and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key < '400')) and (key > '2')) and (key <> '4')) and key is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > '2') and (_col0 <> '4')) (type: boolean)
+ predicate: ((_col0 > '20') and (((_col1 < 'val_50') or (_col0 > '2')) and (_col0 < '400'))) (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Filter Operator
+ predicate: ((_col0 > '2') and (_col0 <> '4')) (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_join2.q.out b/ql/src/test/results/clientpositive/ppd_join2.q.out
index b821d91..88624ea 100644
--- a/ql/src/test/results/clientpositive/ppd_join2.q.out
+++ b/ql/src/test/results/clientpositive/ppd_join2.q.out
@@ -46,17 +46,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '302') and ((_col0 <> '311') and (_col0 < '400'))) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '14') (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ predicate: ((_col0 <> '302') and ((_col0 <> '311') and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -68,17 +71,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '305') and (_col0 <> '14')) (type: boolean)
- Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col0 <> '311') and (((_col1 <> 'val_50') or (_col0 > '1')) and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ predicate: ((_col0 <> '305') and (_col0 <> '14')) (type: boolean)
+ Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reduce Operator Tree:
Join Operator
condition map:
@@ -87,21 +93,21 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: string), _col3 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col2 <> '10') or (_col0 <> '10')) (type: boolean)
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col1 is not null (type: boolean)
- Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -119,23 +125,30 @@ STAGE PLANS:
predicate: (((key <> '306') and (sqrt(key) <> 13.0)) and value is not null) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ predicate: (sqrt(_col0) <> 13.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
TableScan
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col3 (type: string)
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_join3.q.out b/ql/src/test/results/clientpositive/ppd_join3.q.out
index 49da9e0..6c5c0da 100644
--- a/ql/src/test/results/clientpositive/ppd_join3.q.out
+++ b/ql/src/test/results/clientpositive/ppd_join3.q.out
@@ -46,17 +46,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '11') and ((_col0 > '0') and (_col0 < '400'))) (type: boolean)
- Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '4') (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ predicate: ((_col0 <> '11') and ((_col0 > '0') and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -64,20 +67,27 @@ STAGE PLANS:
predicate: (((((((((key <> '11') and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key < '400')) and (key <> '12')) and (key <> '4')) and key is not null) and (key <> '13')) and (key <> '1')) (type: boolean)
Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '12') and (_col0 <> '4')) (type: boolean)
- Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 19 Data size: 201 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 19 Data size: 201 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col0 > '0') and (((_col1 <> 'val_500') or (_col0 > '1')) and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 <> '12') and (_col0 <> '4')) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
@@ -86,24 +96,24 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col1 > '10') or (_col0 <> '10')) (type: boolean)
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col0 <> '13') and (_col0 <> '1')) (type: boolean)
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -125,22 +135,25 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '11') and ((_col0 > '0') and ((_col0 < '400') and ((_col0 <> '12') and (_col0 <> '4'))))) (type: boolean)
- Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '1') (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ predicate: ((_col0 <> '11') and ((_col0 > '0') and ((_col0 < '400') and ((_col0 <> '12') and (_col0 <> '4'))))) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
TableScan
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: string)
Reduce Operator Tree:
Join Operator
@@ -150,14 +163,14 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col1, _col3
- Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
[33/50] [abbrv] hive git commit: HIVE-10909 : Make TestFilterHooks
robust (Ashutosh Chauhan via Thejas Nair)
Posted by xu...@apache.org.
HIVE-10909 : Make TestFilterHooks robust (Ashutosh Chauhan via Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/244386a4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/244386a4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/244386a4
Branch: refs/heads/spark
Commit: 244386a4d25f75bd15c64410ec1cb1d914daf3c0
Parents: 991bb45
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Thu Jun 4 11:18:29 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Jun 4 11:18:29 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/TestFilterHooks.java | 29 +++++++++++---------
1 file changed, 16 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/244386a4/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
index 4ceee8c..b8cf94f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hive.UtilsForTest;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -37,8 +38,8 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -157,23 +158,25 @@ public class TestFilterHooks {
private static final String TAB1 = "tab1";
private static final String TAB2 = "tab2";
private static final String INDEX1 = "idx1";
- private HiveConf hiveConf;
- private HiveMetaStoreClient msc;
- private Driver driver;
+ private static HiveConf hiveConf;
+ private static HiveMetaStoreClient msc;
+ private static Driver driver;
- @Before
- public void setUp() throws Exception {
+ @BeforeClass
+ public static void setUp() throws Exception {
DummyMetaStoreFilterHookImpl.blockResults = false;
- int port = MetaStoreUtils.findFreePort();
- MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
- hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+ hiveConf = new HiveConf(TestFilterHooks.class);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName());
+ UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName());
+ int port = MetaStoreUtils.findFreePort();
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
+
SessionState.start(new CliSessionState(hiveConf));
msc = new HiveMetaStoreClient(hiveConf, null);
driver = new Driver(hiveConf);
@@ -190,8 +193,8 @@ public class TestFilterHooks {
driver.run("CREATE INDEX " + INDEX1 + " on table " + TAB1 + "(id) AS 'COMPACT' WITH DEFERRED REBUILD");
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
DummyMetaStoreFilterHookImpl.blockResults = false;
driver.run("drop database if exists " + DBNAME1 + " cascade");
driver.run("drop database if exists " + DBNAME2 + " cascade");
[50/50] [abbrv] hive git commit: HIVE-10962: Merge master to Spark
branch 6/7/2015 [Spark Branch]
Posted by xu...@apache.org.
HIVE-10962: Merge master to Spark branch 6/7/2015 [Spark Branch]
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8cf95355
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8cf95355
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8cf95355
Branch: refs/heads/spark
Commit: 8cf95355f2758ff4f78112e9e758c3c196c60179
Parents: 52e0086 4d59230
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Mon Jun 8 18:56:33 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 8 18:56:33 2015 -0700
----------------------------------------------------------------------
RELEASE_NOTES.txt | 439 +++-
.../positive/accumulo_predicate_pushdown.q.out | 76 +-
.../results/positive/accumulo_queries.q.out | 70 +-
beeline/pom.xml | 6 +
.../hive/beeline/TestBeelineArgParsing.java | 32 +-
.../test/resources/DummyDriver-1.0-SNAPSHOT.jar | Bin 2812 -> 0 bytes
beeline/src/test/resources/DummyDriver.txt | 59 +
.../src/test/resources/postgresql-9.3.jdbc3.jar | Bin 539003 -> 0 bytes
bin/hive.cmd | 2 +-
common/pom.xml | 20 +
.../hadoop/hive/common/JvmPauseMonitor.java | 225 ++
.../common/jsonexplain/tez/TezJsonParser.java | 2 +-
.../hive/common/metrics/LegacyMetrics.java | 262 +++
.../hadoop/hive/common/metrics/Metrics.java | 253 ---
.../hive/common/metrics/common/Metrics.java | 68 +
.../common/metrics/common/MetricsFactory.java | 48 +
.../metrics/metrics2/CodahaleMetrics.java | 366 ++++
.../metrics/metrics2/MetricsReporting.java | 27 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 44 +-
.../apache/hive/common/util/HiveTestUtils.java | 46 +
.../hive/common/metrics/TestLegacyMetrics.java | 295 +++
.../hadoop/hive/common/metrics/TestMetrics.java | 286 ---
.../metrics/metrics2/TestCodahaleMetrics.java | 138 ++
.../hive/common/type/TestHiveBaseChar.java | 2 +-
.../hive/common/type/TestHiveVarchar.java | 22 +-
.../test/results/clientpositive/url_hook.q.out | 4 +
data/files/customers.txt | 3 +
data/files/emp2.txt | 16 +
data/files/nested_orders.txt | 8 +
data/files/orders.txt | 8 +
data/files/vectortab_a_1k | 1000 +++++++++
data/files/vectortab_b_1k | 1000 +++++++++
data/scripts/q_test_cleanup.sql | 2 +-
.../hive/hcatalog/common/HCatException.java | 2 +-
.../hive/hcatalog/data/TestJsonSerDe.java | 2 +-
.../apache/hive/hcatalog/pig/HCatLoader.java | 8 -
.../hive/hcatalog/pig/TestHCatLoader.java | 44 -
.../hcatalog/pig/TestHCatLoaderEncryption.java | 9 +-
.../streaming/DelimitedInputWriter.java | 2 +-
.../hive/hcatalog/streaming/HiveEndPoint.java | 2 +-
.../hive/hcatalog/streaming/TestStreaming.java | 2 +-
.../hive/hcatalog/api/TestHCatClient.java | 2 +-
.../templeton/tool/JobSubmissionConstants.java | 1 +
.../hcatalog/templeton/tool/LaunchMapper.java | 27 +-
.../hive/minikdc/TestHs2HooksWithMiniKdc.java | 2 +-
.../org/apache/hadoop/hive/UtilsForTest.java | 41 +
.../apache/hadoop/hive/hooks/TestHs2Hooks.java | 10 +-
.../hadoop/hive/metastore/TestFilterHooks.java | 29 +-
.../hive/metastore/TestMetaStoreMetrics.java | 94 +
.../TestHiveAuthorizerCheckInvocation.java | 34 +-
.../plugin/TestHiveAuthorizerShowFilters.java | 6 +-
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 123 ++
.../authorization/TestJdbcMetadataApiAuth.java | 260 +++
.../TestJdbcWithSQLAuthorization.java | 18 +
.../test/resources/testconfiguration.properties | 4 +
.../org/apache/hive/jdbc/HiveConnection.java | 45 +-
.../apache/hive/jdbc/HivePreparedStatement.java | 2 +-
.../apache/hive/jdbc/HiveQueryResultSet.java | 25 +-
.../org/apache/hive/jdbc/HiveStatement.java | 24 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 10 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 136 +-
.../hive/metastore/HiveMetaStoreFsImpl.java | 1 -
.../apache/hadoop/hive/metastore/Warehouse.java | 14 +-
.../metastore/txn/CompactionTxnHandler.java | 9 +-
pom.xml | 18 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 37 +-
.../hive/ql/exec/DefaultFetchFormatter.java | 1 -
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 4 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../hadoop/hive/ql/exec/MapJoinOperator.java | 2 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 9 +-
.../persistence/BytesBytesMultiHashMap.java | 48 +-
.../persistence/MapJoinBytesTableContainer.java | 4 +-
.../ql/exec/spark/HiveSparkClientFactory.java | 3 -
.../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 12 +-
.../hive/ql/exec/tez/TezSessionPoolManager.java | 21 +-
.../hive/ql/exec/tez/TezSessionState.java | 10 -
.../hive/ql/exec/vector/ColumnVector.java | 10 +
.../ql/exec/vector/VectorColumnSetInfo.java | 42 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 35 +-
.../ql/exec/vector/VectorFilterOperator.java | 5 +-
.../ql/exec/vector/VectorizationContext.java | 46 +
.../ql/exec/vector/VectorizedBatchUtil.java | 2 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 50 +-
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 27 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 1 -
.../hive/ql/io/sarg/SearchArgumentImpl.java | 6 +
.../formatting/JsonMetaDataFormatter.java | 2 +-
.../metadata/formatting/MetaDataFormatter.java | 2 +-
.../formatting/TextMetaDataFormatter.java | 76 +-
.../optimizer/ConstantPropagateProcFactory.java | 57 +-
.../hive/ql/optimizer/GenMRTableScan1.java | 2 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 5 +-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 2 +-
.../ql/optimizer/calcite/HiveCalciteUtil.java | 2 +-
.../ql/optimizer/calcite/HiveRelOptUtil.java | 414 ----
.../ql/optimizer/calcite/RelOptHiveTable.java | 2 +-
.../calcite/reloperators/HiveAggregate.java | 58 +
.../calcite/reloperators/HiveProject.java | 5 +-
.../calcite/reloperators/HiveSemiJoin.java | 79 +
.../rules/HiveInsertExchange4JoinRule.java | 2 +-
.../calcite/rules/HivePreFilteringRule.java | 218 ++
.../calcite/translator/HiveOpConverter.java | 6 +-
.../spark/SparkReduceSinkMapJoinProc.java | 2 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 26 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 3 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 8 +
.../apache/hadoop/hive/ql/parse/HiveParser.g | 8 +-
.../hadoop/hive/ql/parse/PTFTranslator.java | 1 +
.../hive/ql/parse/ProcessAnalyzeTable.java | 4 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 68 +-
.../hadoop/hive/ql/parse/WindowingSpec.java | 38 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 2 +-
.../apache/hadoop/hive/ql/plan/FilterDesc.java | 4 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 25 +-
.../hadoop/hive/ql/plan/TableScanDesc.java | 2 +-
.../AuthorizationPreEventListener.java | 8 +-
.../AuthorizationMetaStoreFilterHook.java | 9 +-
.../authorization/plugin/HiveOperationType.java | 19 +-
.../plugin/HivePrivilegeObjectUtils.java | 51 +
.../plugin/sqlstd/Operation2Privilege.java | 17 +-
.../SQLStdHiveAuthorizationValidator.java | 5 +
.../hadoop/hive/ql/session/SessionState.java | 4 +-
.../hive/ql/txn/compactor/CompactorMR.java | 9 +-
.../hadoop/hive/ql/txn/compactor/Initiator.java | 2 +-
.../hadoop/hive/ql/txn/compactor/Worker.java | 3 +-
.../ql/udf/generic/GenericUDAFCollectList.java | 15 +-
.../ql/udf/generic/GenericUDAFCollectSet.java | 14 +-
.../ql/udf/generic/GenericUDAFFirstValue.java | 41 +-
.../ql/udf/generic/GenericUDAFLastValue.java | 33 +-
.../GenericUDAFMkCollectionEvaluator.java | 21 +-
.../generic/GenericUDAFStreamingEvaluator.java | 22 -
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 23 -
.../ql/udf/generic/GenericUDFSortArray.java | 17 +-
.../udf/generic/GenericUDFSubstringIndex.java | 159 ++
.../udf/generic/GenericUDFToUnixTimeStamp.java | 6 +-
.../ql/udf/generic/GenericUDFUnixTimeStamp.java | 36 +-
.../hive/ql/udf/ptf/WindowingTableFunction.java | 31 +
.../hadoop/hive/ql/io/orc/TestFileDump.java | 7 +-
.../hive/ql/io/sarg/TestSearchArgumentImpl.java | 27 +
.../hadoop/hive/ql/plan/TestViewEntity.java | 53 +-
.../hive/ql/session/TestSessionState.java | 40 +-
.../ql/udf/generic/TestGenericUDFSortArray.java | 158 ++
.../generic/TestGenericUDFSubstringIndex.java | 97 +
.../table_nonprintable_negative.q | 11 +
.../udaf_collect_set_unsupported.q | 3 +
.../clientnegative/udf_sort_array_wrong3.q | 2 +-
.../queries/clientpositive/alter_merge_2_orc.q | 6 +-
.../clientpositive/alter_table_partition_drop.q | 11 +
.../clientpositive/authorization_view_sqlstd.q | 4 +
ql/src/test/queries/clientpositive/auto_join0.q | 11 +-
ql/src/test/queries/clientpositive/auto_join1.q | 2 +
.../clientpositive/auto_sortmerge_join_5.q | 3 +
.../queries/clientpositive/autogen_colalias.q | 4 +-
ql/src/test/queries/clientpositive/cbo_gby.q | 3 +
.../test/queries/clientpositive/cbo_udf_udaf.q | 2 +
.../test/queries/clientpositive/cbo_windowing.q | 2 +
.../clientpositive/cross_product_check_1.q | 9 +-
.../clientpositive/cross_product_check_2.q | 6 +-
ql/src/test/queries/clientpositive/ctas.q | 1 +
ql/src/test/queries/clientpositive/cteViews.q | 41 +
.../clientpositive/dynpart_sort_optimization2.q | 7 +-
.../test/queries/clientpositive/explainuser_2.q | 7 +-
.../clientpositive/filter_cond_pushdown.q | 19 +
.../clientpositive/filter_join_breaktask2.q | 2 +
.../test/queries/clientpositive/insert_into1.q | 15 +-
.../test/queries/clientpositive/insert_into2.q | 14 +-
.../clientpositive/insert_nonacid_from_acid.q | 2 +
.../join_merge_multi_expressions.q | 1 +
.../test/queries/clientpositive/join_nullsafe.q | 2 +
.../queries/clientpositive/mapjoin_decimal.q | 6 +-
ql/src/test/queries/clientpositive/mrr.q | 2 +
ql/src/test/queries/clientpositive/orc_merge1.q | 2 +
.../queries/clientpositive/partition_date2.q | 2 +
.../queries/clientpositive/script_env_var1.q | 4 +-
.../queries/clientpositive/selectDistinctStar.q | 2 +
ql/src/test/queries/clientpositive/skewjoin.q | 11 +-
.../queries/clientpositive/subquery_exists.q | 1 +
.../queries/clientpositive/table_nonprintable.q | 30 +
ql/src/test/queries/clientpositive/temp_table.q | 6 +-
ql/src/test/queries/clientpositive/tez_dml.q | 2 +
.../test/queries/clientpositive/tez_join_hash.q | 2 +
.../queries/clientpositive/tez_joins_explain.q | 1 -
.../queries/clientpositive/tez_multi_union.q | 1 +
.../clientpositive/tez_schema_evolution.q | 12 +-
.../tez_union_dynamic_partition.q | 21 +
.../queries/clientpositive/udaf_collect_set_2.q | 222 ++
.../clientpositive/udf_folder_constants.q | 11 +
.../queries/clientpositive/udf_sort_array.q | 10 +
.../clientpositive/udf_substring_index.q | 32 +
.../queries/clientpositive/udf_unix_timestamp.q | 8 +-
.../queries/clientpositive/unionDistinct_2.q | 2 +
.../update_after_multiple_inserts.q | 2 +
.../clientpositive/update_all_partitioned.q | 4 +-
.../clientpositive/update_where_partitioned.q | 4 +-
.../queries/clientpositive/vector_data_types.q | 12 +-
.../clientpositive/vector_interval_mapjoin.q | 87 +
.../clientpositive/vector_null_projection.q | 2 +
.../queries/clientpositive/vector_outer_join1.q | 8 +-
.../queries/clientpositive/vector_outer_join2.q | 8 +-
.../queries/clientpositive/vector_outer_join3.q | 8 +-
.../clientpositive/vectorization_part_project.q | 4 +-
.../clientpositive/vectorized_distinct_gby.q | 2 +-
.../queries/clientpositive/vectorized_mapjoin.q | 2 +
.../clientpositive/vectorized_nested_mapjoin.q | 2 +
.../clientpositive/vectorized_rcfile_columnar.q | 2 +-
.../clientpositive/vectorized_shufflejoin.q | 10 +-
.../clientpositive/vectorized_timestamp_funcs.q | 5 +-
.../clientpositive/windowing_windowspec2.q | 18 +
.../clientpositive/windowing_windowspec3.q | 31 +
.../clientnegative/authorization_ctas2.q.out | 2 +
.../table_nonprintable_negative.q.out | 19 +
.../udaf_collect_set_unsupported.q.out | 1 +
.../clientnegative/udf_sort_array_wrong1.q.out | 2 +-
.../clientnegative/udf_sort_array_wrong2.q.out | 2 +-
.../clientnegative/udf_sort_array_wrong3.q.out | 2 +-
.../results/clientpositive/add_part_exist.q.out | 2 +
ql/src/test/results/clientpositive/alter1.q.out | 4 +
ql/src/test/results/clientpositive/alter2.q.out | 4 +
ql/src/test/results/clientpositive/alter3.q.out | 4 +
ql/src/test/results/clientpositive/alter4.q.out | 4 +
ql/src/test/results/clientpositive/alter5.q.out | 4 +
.../alter_change_db_location.q.out | 2 +
.../results/clientpositive/alter_db_owner.q.out | 6 +
.../results/clientpositive/alter_index.q.out | 2 +
.../clientpositive/alter_merge_2_orc.q.out | 16 +-
.../clientpositive/alter_rename_partition.q.out | 6 +
.../alter_table_partition_drop.q.out | 74 +
.../authorization_owner_actions_db.q.out | 2 +
.../authorization_view_sqlstd.q.out | 14 +
.../results/clientpositive/auto_join0.q.out | 235 +-
.../results/clientpositive/auto_join1.q.out | 8 +-
.../results/clientpositive/auto_join16.q.out | 4 +-
.../results/clientpositive/auto_join8.q.out | 14 +-
.../clientpositive/auto_sortmerge_join_5.q.out | 6 +
.../clientpositive/autogen_colalias.q.out | 12 +-
.../test/results/clientpositive/cbo_gby.q.out | 18 +-
.../clientpositive/cbo_rp_auto_join1.q.out | 4 +-
.../results/clientpositive/cbo_udf_udaf.q.out | 18 +-
.../results/clientpositive/cbo_windowing.q.out | 66 +-
.../results/clientpositive/create_view.q.out | 2 +
.../clientpositive/cross_product_check_1.q.out | 20 +-
.../clientpositive/cross_product_check_2.q.out | 72 +-
ql/src/test/results/clientpositive/ctas.q.out | 2 +
.../test/results/clientpositive/cteViews.q.out | 242 +++
.../test/results/clientpositive/database.q.out | 12 +
.../clientpositive/database_location.q.out | 8 +
.../clientpositive/database_properties.q.out | 6 +
.../clientpositive/dbtxnmgr_query5.q.out | 2 +
.../clientpositive/describe_database.q.out | 4 +
.../clientpositive/describe_database_json.q.out | 12 +
.../clientpositive/describe_table_json.q.out | 4 +
.../dynpart_sort_optimization2.q.out | 166 +-
.../encrypted/encryption_drop_table.q.out | 6 +
.../encryption_join_unencrypted_tbl.q.out | 8 +-
.../encrypted/encryption_move_tbl.q.out | 4 +
.../clientpositive/explain_logical.q.out | 22 +-
.../clientpositive/filter_cond_pushdown.q.out | 382 ++++
.../clientpositive/filter_join_breaktask2.q.out | 8 +-
.../test/results/clientpositive/fold_case.q.out | 8 +-
.../results/clientpositive/index_creation.q.out | 2 +
ql/src/test/results/clientpositive/input2.q.out | 4 +
ql/src/test/results/clientpositive/input3.q.out | 4 +
.../results/clientpositive/insert_into1.q.out | 86 +-
.../results/clientpositive/insert_into2.q.out | 82 +-
.../insert_nonacid_from_acid.q.out | 20 +-
ql/src/test/results/clientpositive/join8.q.out | 16 +-
.../join_merge_multi_expressions.q.out | 6 +-
.../results/clientpositive/join_nullsafe.q.out | 6 +-
.../list_bucket_query_oneskew_2.q.out | 2 +-
.../results/clientpositive/orc_merge1.q.out | 8 +-
.../clientpositive/partition_date2.q.out | 8 +-
.../test/results/clientpositive/ppd_gby.q.out | 41 +-
.../test/results/clientpositive/ppd_gby2.q.out | 49 +-
.../results/clientpositive/ppd_gby_join.q.out | 42 +-
.../test/results/clientpositive/ppd_join.q.out | 44 +-
.../test/results/clientpositive/ppd_join2.q.out | 81 +-
.../test/results/clientpositive/ppd_join3.q.out | 91 +-
.../results/clientpositive/ppd_udf_col.q.out | 178 +-
.../test/results/clientpositive/ppd_union.q.out | 42 +-
ql/src/test/results/clientpositive/ppd_vc.q.out | 8 +-
.../results/clientpositive/rename_column.q.out | 4 +
.../clientpositive/script_env_var1.q.out | 8 +-
.../clientpositive/selectDistinctStar.q.out | 92 +-
.../results/clientpositive/show_functions.q.out | 1 +
.../results/clientpositive/show_tables.q.out | 18 +
.../test/results/clientpositive/skewjoin.q.out | 8 +-
.../clientpositive/spark/auto_join0.q.out | 97 +-
.../clientpositive/spark/auto_join1.q.out | 8 +-
.../spark/auto_sortmerge_join_5.q.out | 6 +
.../results/clientpositive/spark/cbo_gby.q.out | 24 +-
.../clientpositive/spark/cbo_udf_udaf.q.out | 18 +-
.../spark/cross_product_check_1.q.out | 20 +-
.../spark/cross_product_check_2.q.out | 68 +-
.../results/clientpositive/spark/ctas.q.out | 2 +
.../spark/filter_join_breaktask2.q.out | 8 +-
.../clientpositive/spark/insert_into1.q.out | 92 +-
.../clientpositive/spark/insert_into2.q.out | 88 +-
.../spark/join_merge_multi_expressions.q.out | 6 +-
.../clientpositive/spark/join_nullsafe.q.out | 6 +-
.../clientpositive/spark/mapjoin_decimal.q.out | 41 +-
.../clientpositive/spark/ppd_join2.q.out | 81 +-
.../clientpositive/spark/ppd_join3.q.out | 91 +-
.../clientpositive/spark/script_env_var1.q.out | 8 +-
.../results/clientpositive/spark/skewjoin.q.out | 8 +-
.../clientpositive/spark/stats_noscan_2.q.out | 8 +-
.../clientpositive/spark/subquery_exists.q.out | 2 +
.../clientpositive/spark/temp_table.q.out | 90 +-
.../spark/tez_joins_explain.q.out | 2 -
.../clientpositive/spark/udf_in_file.q.out | 2 +-
.../spark/vector_data_types.q.out | 116 +-
.../spark/vectorization_part_project.q.out | 40 +-
.../spark/vectorized_mapjoin.q.out | 8 +-
.../spark/vectorized_nested_mapjoin.q.out | 8 +-
.../spark/vectorized_rcfile_columnar.q.out | 24 +-
.../spark/vectorized_shufflejoin.q.out | 37 +-
.../spark/vectorized_timestamp_funcs.q.out | 292 +--
.../results/clientpositive/stats_noscan_2.q.out | 8 +-
.../clientpositive/subquery_exists.q.out | 2 +
.../results/clientpositive/subquery_notin.q.out | 16 +-
.../subquery_notin_having.q.java1.7.out | 16 +-
.../subquery_unqualcolumnrefs.q.out | 2 +-
.../results/clientpositive/subquery_views.q.out | 8 +-
.../clientpositive/table_nonprintable.q.out | 72 +
.../results/clientpositive/temp_table.q.out | 84 +-
.../clientpositive/temp_table_names.q.out | 8 +
.../clientpositive/tez/alter_merge_2_orc.q.out | 16 +-
.../results/clientpositive/tez/auto_join0.q.out | 91 +-
.../results/clientpositive/tez/auto_join1.q.out | 8 +-
.../tez/auto_sortmerge_join_5.q.out | 6 +
.../results/clientpositive/tez/cbo_gby.q.out | 18 +-
.../clientpositive/tez/cbo_udf_udaf.q.out | 18 +-
.../clientpositive/tez/cbo_windowing.q.out | 66 +-
.../tez/cross_product_check_1.q.out | 20 +-
.../tez/cross_product_check_2.q.out | 88 +-
.../test/results/clientpositive/tez/ctas.q.out | 2 +
.../tez/dynpart_sort_optimization2.q.out | 166 +-
.../clientpositive/tez/explainuser_1.q.out | 592 +++--
.../clientpositive/tez/explainuser_2.q.out | 71 +-
.../tez/filter_join_breaktask2.q.out | 8 +-
.../clientpositive/tez/insert_into1.q.out | 86 +-
.../clientpositive/tez/insert_into2.q.out | 82 +-
.../clientpositive/tez/join_nullsafe.q.out | 6 +-
.../clientpositive/tez/mapjoin_decimal.q.out | 42 +-
.../test/results/clientpositive/tez/mrr.q.out | 2052 +++++++++---------
.../results/clientpositive/tez/orc_merge1.q.out | 8 +-
.../clientpositive/tez/script_env_var1.q.out | 8 +-
.../clientpositive/tez/selectDistinctStar.q.out | 92 +-
.../results/clientpositive/tez/skewjoin.q.out | 8 +-
.../clientpositive/tez/subquery_exists.q.out | 2 +
.../results/clientpositive/tez/temp_table.q.out | 87 +-
.../results/clientpositive/tez/tez_dml.q.out | 2008 ++++++++---------
.../clientpositive/tez/tez_join_hash.q.out | 286 +--
.../clientpositive/tez/tez_joins_explain.q.out | 2 -
.../clientpositive/tez/tez_multi_union.q.out | 1002 ++++-----
.../tez/tez_schema_evolution.q.out | 96 +-
.../tez/tez_union_dynamic_partition.q.out | 148 ++
.../results/clientpositive/tez/union4.q.out | 4 +
.../results/clientpositive/tez/union6.q.out | 4 +
.../clientpositive/tez/unionDistinct_2.q.out | 38 +-
.../tez/update_after_multiple_inserts.q.out | 12 +-
.../tez/update_all_partitioned.q.out | 14 +-
.../tez/update_where_partitioned.q.out | 20 +-
.../clientpositive/tez/vector_data_types.q.out | 116 +-
.../tez/vector_interval_mapjoin.q.out | 281 +++
.../tez/vector_leftsemi_mapjoin.q.out | 1032 ++++-----
.../tez/vector_multi_insert.q.out | 20 +-
.../tez/vector_null_projection.q.out | 8 +-
.../clientpositive/tez/vector_outer_join1.q.out | 236 +-
.../clientpositive/tez/vector_outer_join2.q.out | 40 +-
.../clientpositive/tez/vector_outer_join3.q.out | 82 +-
.../clientpositive/tez/vector_outer_join4.q.out | 48 +-
.../tez/vectorization_part_project.q.out | 40 +-
.../tez/vectorized_distinct_gby.q.out | 4 +-
.../clientpositive/tez/vectorized_mapjoin.q.out | 8 +-
.../tez/vectorized_nested_mapjoin.q.out | 8 +-
.../tez/vectorized_rcfile_columnar.q.out | 24 +-
.../tez/vectorized_shufflejoin.q.out | 37 +-
.../tez/vectorized_timestamp_funcs.q.out | 292 +--
.../clientpositive/udaf_collect_set_2.q.out | 742 +++++++
ql/src/test/results/clientpositive/udf5.q.out | 30 +-
.../clientpositive/udf_folder_constants.q.out | 120 +
.../results/clientpositive/udf_in_file.q.out | 2 +-
.../results/clientpositive/udf_sort_array.q.out | 33 +
.../clientpositive/udf_substring_index.q.out | 107 +
.../clientpositive/udf_unix_timestamp.q.out | 32 +-
.../clientpositive/unionDistinct_2.q.out | 42 +-
.../update_after_multiple_inserts.q.out | 12 +-
.../clientpositive/update_all_partitioned.q.out | 14 +-
.../update_where_partitioned.q.out | 20 +-
.../clientpositive/vector_data_types.q.out | 116 +-
.../vector_interval_mapjoin.q.out | 281 +++
.../clientpositive/vector_null_projection.q.out | 8 +-
.../clientpositive/vector_outer_join1.q.out | 228 +-
.../clientpositive/vector_outer_join2.q.out | 36 +-
.../clientpositive/vector_outer_join3.q.out | 70 +-
.../vectorization_part_project.q.out | 40 +-
.../vectorized_distinct_gby.q.out | 4 +-
.../clientpositive/vectorized_mapjoin.q.out | 8 +-
.../vectorized_nested_mapjoin.q.out | 8 +-
.../vectorized_rcfile_columnar.q.out | 24 +-
.../clientpositive/vectorized_shufflejoin.q.out | 44 +-
.../vectorized_timestamp_funcs.q.out | 292 +--
.../clientpositive/windowing_windowspec2.q.out | 1302 +++++++++++
.../clientpositive/windowing_windowspec3.q.out | 110 +
.../hadoop/hive/serde2/NullStructSerDe.java | 2 +-
.../objectinspector/ObjectInspectorUtils.java | 2 +-
.../hive/serde2/lazy/TestLazySimpleSerDe.java | 9 +-
.../apache/hive/service/auth/HttpAuthUtils.java | 2 +-
.../java/org/apache/hive/service/cli/Type.java | 2 +-
.../cli/operation/GetCatalogsOperation.java | 12 +-
.../cli/operation/GetColumnsOperation.java | 34 +
.../cli/operation/GetFunctionsOperation.java | 25 +
.../cli/operation/GetSchemasOperation.java | 13 +
.../cli/operation/GetTableTypesOperation.java | 4 +
.../cli/operation/GetTablesOperation.java | 10 +
.../cli/operation/GetTypeInfoOperation.java | 4 +
.../cli/operation/MetadataOperation.java | 35 +
.../service/cli/thrift/ThriftHttpServlet.java | 2 +-
.../apache/hive/service/server/HiveServer2.java | 25 +-
.../hadoop/hive/shims/Hadoop20SShims.java | 5 -
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 13 -
.../apache/hadoop/hive/shims/HadoopShims.java | 2 -
424 files changed, 18314 insertions(+), 8234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8cf95355/pom.xml
----------------------------------------------------------------------
[04/50] [abbrv] hive git commit: HIVE-10721 SparkSessionManagerImpl
leaks SparkSessions [Spark Branch] (Jimmy reviewed by Xuefu, Chengxiang)
Posted by xu...@apache.org.
HIVE-10721 SparkSessionManagerImpl leaks SparkSessions [Spark Branch] (Jimmy reviewed by Xuefu, Chengxiang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b3214f99
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b3214f99
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b3214f99
Branch: refs/heads/spark
Commit: b3214f996c9889035110ba548ea8eddf8390cdce
Parents: 3f27c6e
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Fri May 15 08:34:38 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:04:09 2015 -0700
----------------------------------------------------------------------
.../ql/exec/spark/LocalHiveSparkClient.java | 8 ++-
.../ql/exec/spark/RemoteHiveSparkClient.java | 6 +-
.../ql/exec/spark/session/SparkSessionImpl.java | 2 +-
.../spark/session/SparkSessionManagerImpl.java | 63 ++++++++++----------
.../hive/spark/client/SparkClientImpl.java | 2 +-
5 files changed, 43 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
index 7e33a3f..19d3fee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
@@ -199,7 +199,11 @@ public class LocalHiveSparkClient implements HiveSparkClient {
@Override
public void close() {
- sc.stop();
- client = null;
+ synchronized (LocalHiveSparkClient.class) {
+ client = null;
+ }
+ if (sc != null) {
+ sc.stop();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index bae30f3..8b15099 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -199,15 +199,19 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
@Override
public void close() {
- remoteClient.stop();
+ if (remoteClient != null) {
+ remoteClient.stop();
+ }
}
private static class JobStatusJob implements Job<Serializable> {
+ private static final long serialVersionUID = 1L;
private final byte[] jobConfBytes;
private final byte[] scratchDirBytes;
private final byte[] sparkWorkBytes;
+ @SuppressWarnings("unused")
private JobStatusJob() {
// For deserialization.
this(null, null, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
index 603f1ca..49e5f6c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
@@ -53,7 +53,7 @@ public class SparkSessionImpl implements SparkSession {
isOpen = true;
try {
hiveSparkClient = HiveSparkClientFactory.createHiveSparkClient(conf);
- } catch (Exception e) {
+ } catch (Throwable e) {
throw new HiveException("Failed to create spark client.", e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
index ad012b6..616807c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
@@ -17,23 +17,19 @@
*/
package org.apache.hadoop.hive.ql.exec.spark.session;
-import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.spark.client.SparkClientFactory;
-
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClientFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hive.spark.client.SparkClientFactory;
/**
* Simple implementation of <i>SparkSessionManager</i>
@@ -44,8 +40,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class SparkSessionManagerImpl implements SparkSessionManager {
private static final Log LOG = LogFactory.getLog(SparkSessionManagerImpl.class);
- private Set<SparkSession> createdSessions;
- private AtomicBoolean inited = new AtomicBoolean(false);
+ private Set<SparkSession> createdSessions = Collections.synchronizedSet(new HashSet<SparkSession>());
+ private volatile boolean inited = false;
private static SparkSessionManagerImpl instance;
@@ -78,14 +74,18 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
@Override
public void setup(HiveConf hiveConf) throws HiveException {
- if (inited.compareAndSet(false, true)) {
- LOG.info("Setting up the session manager.");
- createdSessions = Collections.synchronizedSet(new HashSet<SparkSession>());
- Map<String, String> conf = HiveSparkClientFactory.initiateSparkConf(hiveConf);
- try {
- SparkClientFactory.initialize(conf);
- } catch (IOException e) {
- throw new HiveException("Error initializing SparkClientFactory", e);
+ if (!inited) {
+ synchronized (this) {
+ if (!inited) {
+ LOG.info("Setting up the session manager.");
+ Map<String, String> conf = HiveSparkClientFactory.initiateSparkConf(hiveConf);
+ try {
+ SparkClientFactory.initialize(conf);
+ inited = true;
+ } catch (IOException e) {
+ throw new HiveException("Error initializing SparkClientFactory", e);
+ }
+ }
}
}
}
@@ -104,14 +104,12 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
if (existingSession != null) {
// Open the session if it is closed.
if (!existingSession.isOpen() && doOpen) {
- existingSession.open(conf);
+ existingSession.open(conf);
}
return existingSession;
}
SparkSession sparkSession = new SparkSessionImpl();
- createdSessions.add(sparkSession);
-
if (doOpen) {
sparkSession.open(conf);
}
@@ -119,6 +117,7 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("New session (%s) is created.", sparkSession.getSessionId()));
}
+ createdSessions.add(sparkSession);
return sparkSession;
}
@@ -144,17 +143,15 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
@Override
public void shutdown() {
LOG.info("Closing the session manager.");
- if (createdSessions != null) {
- synchronized (createdSessions) {
- Iterator<SparkSession> it = createdSessions.iterator();
- while (it.hasNext()) {
- SparkSession session = it.next();
- session.close();
- }
- createdSessions.clear();
+ synchronized (createdSessions) {
+ Iterator<SparkSession> it = createdSessions.iterator();
+ while (it.hasNext()) {
+ SparkSession session = it.next();
+ session.close();
}
+ createdSessions.clear();
}
- inited.set(false);
+ inited = false;
SparkClientFactory.stop();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b3214f99/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
index 1bcd221..9e34a49 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
@@ -97,7 +97,7 @@ class SparkClientImpl implements SparkClient {
try {
// The RPC server will take care of timeouts here.
this.driverRpc = rpcServer.registerClient(clientId, secret, protocol).get();
- } catch (Exception e) {
+ } catch (Throwable e) {
LOG.warn("Error while waiting for client to connect.", e);
driverThread.interrupt();
try {
[19/50] [abbrv] hive git commit: HIVE-10855: with vectorization
enabled join operation involving interval_day_time fails (Matt McCline
reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/data/files/vectortab_b_1k
----------------------------------------------------------------------
diff --git a/data/files/vectortab_b_1k b/data/files/vectortab_b_1k
new file mode 100644
index 0000000..812d98a
--- /dev/null
+++ b/data/files/vectortab_b_1k
@@ -0,0 +1,1000 @@
+38|28413|-469870330|7061809776248545280|5038.36|4899225.93||true|american history|wendy van buren|2052-03-21 12:31:39|2041-01-06 14:03:47.179206937|2074-03-12
+108|-5135|-2042831105|6969599299897163776|-40691.70|-1564446.85|-4298929701839.924316|false|philosophy||2033-10-13 05:36:52|2014-06-18 16:42:39.89307305|2079-04-19
+-34|11168|1846184880|-7535857766791577600|-12097.91|296884.75|189835086132.674805|false|wind surfing|zach ellison|2022-01-21 10:58:22|2014-08-14 21:48:41.258440751|2058-03-24
+42|-7323|1625751062|3724|19175.45|-4645302.81||true|yard duty|jessica nixon|2046-12-23 10:11:15|2040-05-12 13:49:04.312695207|2067-12-19
+-101|20084|343362793|2786|-5759.71|-3710545.39|-747309143932.177734|false|xylophone band|nick johnson|2055-11-22 06:32:07|2032-12-27 00:39:24.505679054|2013-11-15
+33|-18263|735732067|1948|38213.05|2668590.35|915569735455.541016|true|quiet hour|rachel ovid|2018-03-02 05:25:36|2063-07-27 10:41:50.110054963|2046-11-16
+-54|-21795|-1017629298|-7328087811698909184|-20010.24|-1103528.09|171257816833.722656|true|geology|wendy brown|2022-12-25 05:14:20|2069-06-10 23:30:18.814929160|2078-03-20
+16|-14888|1190302173|7753882935005880320|-13338.98|-3830938.65|-16966857031.200195|false|kindergarten|priscilla ichabod|2079-09-20 10:22:11|2066-05-11 11:44:44.967158232|2058-05-02
+-104|22213|-1818456584|-8384695077413412864|43493.35|-828620.69|2303273645231.165039|true|nap time|fred garcia||2058-03-20 03:18:32.441094681|2005-09-08
+76|-26087|-295186284|-9075302542655684608|-16572.92|2456302.57|1695145412960.172852|true|mathematics|rachel thompson|2067-11-29 10:49:14|2023-02-24 06:11:31.127903962|1970-10-04
+-105|-4953|315973457|-8844949406948671488|-9860.94|998897.88|-4775905812492.879883|false|education|fred davidson|2025-09-30 18:11:16|2025-06-11 16:43:58.941547150|1999-12-25
+-127|16110|904604938|8337549596011102208|2848.64|-3418610.14|-2009642318242.228516|true|debate|mike steinbeck|2023-04-26 20:45:39|2048-06-21 02:28:33.302451076|2068-01-05
+-109|31986|-1218581850|-7624057992767782912|-9642.07||4524736841293.623047|true|chemistry|ethan ovid|2046-01-30 17:53:58|2064-06-05 07:51:39.21714322|2097-09-03
+116|-32119|-2144241640|9073672806863790080|-7691.49|-4810810.06|-3529351191949.139648|true|zync studies|quinn ichabod|2025-09-14 05:57:47|2052-08-21 07:01:03.126893909|2101-08-31
+-69|-28864|1766517223|7948803266578161664|-33985.35|3550700.26|4086092318223.570312|false|debate||2071-01-12 01:01:39|2059-09-08 02:39:24.970547582|1989-11-05
+-13|12814|881673558|7566273236152721408|-4594.45|-3286472.17|423483325042.503906|false|linguistics|nick underhill|2023-06-25 22:14:52||2064-02-13
+4|-10513|536876888|7062382339142156288|13262.02|4083053.68||false||wendy young|2073-07-01 02:10:21|2068-11-13 07:56:42.534738020|2026-07-12
+-109|-20188|-1289665817|7909645665163804672|42529.03|2938682.72|4492997493634.832031|false|values clariffication||2036-02-03 11:35:01|2016-09-22 08:56:11.169028039|2065-08-11
+-33|20120|-340462064|-8086577583338061824|30435.53|4916056.68|2409115470740.985352|true|zync studies|calvin quirinius|2052-08-12 00:26:09|2019-08-25 23:11:02.671444276|1991-04-05
+-82|-23630|1425456189|-8122639684164501504|13589.93|-4405035.26|3998360085714.693359|true|geology||2055-05-29 17:37:08||2006-11-30
+-18|-19786|-1642207005|-8117838333114212352|17411.05||1866266907288.434570|false|wind surfing|sarah garcia|2028-01-29 13:44:26|2039-09-01 15:14:46.549821420|1975-12-21
+-83|23910|1376818328|336|-27132.88|1204353.22|2806463824394.356445|true|undecided|sarah ellison|2080-10-30 17:04:15|2065-12-02 08:54:56.954538295|2014-02-02
+111|32231|-1919939921|2320|-6568.58|1215239.64|-1869700480167.893066|true|kindergarten|xavier van buren||2016-07-10 10:53:59.626066496|2021-05-21
+-86|12915|-1434562279|-8731068123910987776|-8835.22|-1335137.34|-1538575730095.495117|false|american history||2061-12-03 00:29:54|2072-08-27 13:08:21.852423838|2016-07-12
+-107|-13|1070989126|1906|-12761.32|3064168.48|-3462773458406.651367|true||fred carson|2018-12-15 02:28:19|2071-03-20 11:13:17.416475043|2018-06-15
+95|-20789|1028204648|-7838598833900584960|1716.08|-2554550.12|4714480698801.324219|true|geology|bob falkner|2045-07-31 00:13:55|2059-03-29 14:57:28.222857121|2099-08-04
+-80|-8852|477857533|1165|37201.95||-1261362806924.886230|true|forestry|jessica ovid|2062-04-27 20:14:10||2064-08-16
+|-14783|-224865887|-7456869587112255488|5122.61|-2102976.63|1531516008788.393555|false|values clariffication|fred nixon|2048-10-12 23:56:55|2026-06-07 14:49:02.306537960|2089-05-15
+-15|17581|1142098316|2013|6003.12|-1151629.14|4348345225726.326172|false|history|yuri van buren|2024-12-02 01:39:53|2073-03-07 20:51:20.326376818|2050-01-09
+-8|-24885|-332125121|7333512171174223872|26535.92|2399386.54|-2960944140681.114258|false|opthamology|ethan white|2079-02-15 19:05:52||2087-01-27
+-89|-10935|1240875512|-7571293705217687552|21259.00|-629766.26|-1636595770961.833008|false|american history|david quirinius|2033-01-07 17:02:27|2063-10-15 04:11:02.388323396|2096-08-29
+-19|6306|-590374062|-8523434203900674048|28590.02|1265094.91|3833331173378.199219|true|undecided|victor thompson|2056-03-02 15:07:50|2047-02-04 15:17:32.647082767|1990-11-23
+-39|-16680|492968645|8099215208813903872|11064.77|750795.33|-3997116358934.107422|false|zync studies|sarah underhill|2032-12-13 09:50:46|2029-10-11 21:31:15.496921510|2001-01-25
+-48|-30157|-1635301453|9040958359122640896|42146.27|-1984020.04|-3919914678779.932129|false|yard duty|luke ichabod|2050-05-21 14:47:18|2072-06-27 06:27:52.850736616|2012-07-08
+118|-21389|1319589591|-7356685674003021824|48287.81|4413852.26|4754891500688.814453|false|philosophy|||2031-07-02 15:06:48.138419705|2092-03-31
+-118|13138|-1652600376|2072|31787.11|4413364.49||false|opthamology|victor zipper||2058-07-13 04:38:16.443396189|1996-08-31
+32|29023|-856843296|2073|48554.24|-2764932.99|4532387406205.238281|false|mathematics|jessica polk|2066-09-16 03:57:39|2065-06-11 16:20:38.164287829|2011-01-20
+-31|-9496|915505006|871|37794.69|4782774.03|3414437080001.526367|false|geology|luke hernandez|2061-05-20 02:26:59|2068-01-21 09:27:30.854734737|2023-12-26
+22|-6460|1179528290|-7551394356730339328|-23041.73|-4296206.08|4962408770099.849609|true|xylophone band|luke falkner|2058-10-05 19:19:16||2021-12-19
+83|-29675|-113253627|-8172827216441573376|-1794.84|-317220.62|688329351482.579102|true|biology|david robinson|2039-03-30 18:42:28|2020-02-03 05:20:18.744684653|2050-09-20
+88|6761|-1878838836|-8082793390939193344|36116.48|1298897.65|1744469246864.212891|false|chemistry|tom ellison|2030-01-25 11:28:31|2016-05-22 14:39:37.64576689|2044-06-20
+-49|-12588|2065408093|8854495099223375872|25810.56||-3371566685461.750977|false|opthamology|priscilla polk|2029-08-07 14:29:01|2048-03-29 02:01:01.562257980|2000-08-18
+-14|28008|-122391516|-8358130693961195520|-33737.35|4303885.61|207192707064.258789|true|joggying|alice garcia|2025-11-21 05:44:38|2029-03-08 07:36:34.632835852|2105-01-06
+-52|27527|-1240048334|9050032047355125760|-41056.39|-2126298.83|3424231918266.365234|false|education|sarah ellison|2055-05-29 10:01:01|2053-06-19 04:48:57.10910076|2004-06-03
+84||1813010930|-7162299524557471744|-4970.18|910733.08|-4747435156455.921875|false|history|quinn ovid|2056-11-24 23:44:36|2054-10-30 00:49:45.287291218|2070-02-03
+28|-21506|-682333536|809|-13948.95|2291401.51|33266358053.083008|false|geology|ulysses steinbeck|2080-06-19 15:17:59||1984-12-28
+4||-759911896|-8946656952763777024|-4183.99|-4038048.77|-3167988150233.824707|false|opthamology|katie white|2061-09-28 17:08:58|2015-09-01 23:29:55.301806098|2032-01-18
+-100|13491|-1369302744|1053|-10809.39|4440019.40|-1117844143988.975098||values clariffication|quinn thompson|2023-03-09 21:06:33|2027-12-13 08:22:59.973086123|2015-09-29
+4||765084282|482|-24177.64|-3926632.94|-128454528929.609375||biology|ulysses johnson|2039-12-01 00:28:17|2047-02-12 11:40:45.586393217|2090-06-30
+126|-16307|470993066|-6968892545529896960|2416.88|-767525.41|-4599746514511.089844|true|industrial engineering|luke garcia||2068-09-30 15:34:57.912726771|2043-08-23
+-3|24819|2070969353|203|49013.15|3648039.90|-2530166692170.396973|false|religion|zach davidson|2028-03-11 20:18:02|2023-01-31 06:26:26.819065415|2026-12-24
+-79|-19571|-296195507|1614|1339.61|472805.29|-4628165921333.362305|false|geology|victor thompson|2025-05-07 23:33:22|2076-12-14 19:01:33.673146196|2040-05-30
+88|7057|6266567|-8593419958317056000|20993.37|3470920.76|2907965923466.875000|false|opthamology|jessica van buren|2026-04-10 03:58:42|2027-05-24 20:24:57.709957719|1985-01-18
+42|-562|604460005|8190967051000659968|10778.57|1505668.86|-4298212148464.983887|false|american history|priscilla king|2071-01-07 17:48:53|2050-08-28 10:51:32.79079993|2036-07-23
+-5|4536|-1836166334|808|-26348.93|1609583.36|-2450581027868.068359|false|biology|zach ellison|2064-03-18 16:36:24|2040-12-19 14:19:07.29019956|2094-02-23
+31|2855|458910170||4194.56|-2703308.04|-3920065100013.161621|false|geology|holly ellison||2064-10-06 03:04:54.202018334|2058-01-16
+-36|813|2144454927|412|-17894.49|3767875.02|-4076956189315.090820|true|zync studies|irene young|2051-03-22 13:42:47||2069-11-02
+-19|-16002||8656571350884048896|44168.85|4141077.81|2689794991277.487305|true|yard duty|rachel steinbeck|2038-02-19 06:53:09|2052-02-15 19:04:26.991933690|2018-01-25
+-123|25732|2100377172|8769199243315814400|14038.74|191532.01|-3373496595819.993652|false|quiet hour|quinn allen|2080-05-30 20:51:34|2038-06-11 05:22:17.584841494|1997-09-20
+-28|-9278|-1236536142|-8546758906409312256|-18615.91|-385172.08|3145518148645.542969|false|values clariffication|quinn ellison||2050-08-26 21:46:55.776228072|2006-05-04
+90|-27844||8829545979081744384|-7890.36|-4687872.68|-189051209331.019531|true|forestry|holly ellison|2078-01-22 08:05:44|2051-12-30 01:07:06.843556116|2019-10-31
+-33|6756|-1380678829|7545689659010949120|3119.59|1219503.88|1383475996151.766602|true|chemistry|gabriella nixon|2013-11-15 10:53:04|2074-01-13 06:13:44.309707032|2086-07-20
+-27|32063|-412333994|618|18352.29|-2682908.49|-2196296807639.192871|false|xylophone band|mike ichabod|2049-05-17 01:36:05|2016-01-06 04:27:12.937792140|1988-07-21
+115|14089|1583280136|8573305425181941760|-12596.11|-4306107.96|3399085059351.882812|true|forestry|bob carson|2077-10-22 01:28:01|2070-11-22 05:00:18.148513417|2060-02-01
+87|-5837||94|31117.44|4356549.19|3648188962203.846680|false|joggying|quinn nixon|2013-11-03 06:51:21|2060-06-22 12:16:37.962088375|2101-04-20
+42|6036|960187615|-7266719102957125632|-47893.89|-295854.59|2917455244518.621094|false|topology|calvin zipper||2064-05-28 07:51:38.18135161|2091-10-08
+-57|-16290|1835749815|2772|-2499.18|-2208789.27|-1579859560936.931641|true|joggying|wendy carson|2068-08-26 13:27:51|2021-11-05 06:16:08.956633171|2012-03-06
+34|-2518|1625699061|379|-24309.03|3375470.42|-1718131278688.766113|false|education|ethan johnson|2046-02-23 07:23:16|2040-11-20 23:57:22.271546670|1978-07-29
+69|28358|-1524081566|8302473563519950848|-297.67|-3373283.08|-2009241907914.223145|true|joggying|ethan van buren|2047-08-15 06:48:02|2043-04-13 19:11:51.590383118|
+-11|31334|215759857|-7802538500225777664|33944.93|-2034061.98|-3034548052198.289062|false|industrial engineering|sarah steinbeck|2055-02-11 13:24:46|2053-11-27 08:12:32.905325041|2039-02-11
+76|-25155|2009890220|-7273694358642851840|18007.61|-1882990.85|-1586767512900.379883|false|religion|nick underhill|2043-09-05 07:30:29|2039-05-08 08:23:28.744105983|2012-06-13
+-42|-27015|-1024500955|8987827141270880256|6071.06|||false|history|rachel thompson|2080-11-04 06:42:28|2022-05-07 19:23:26.462479493|1990-08-27
+-91|-7300|-1257859205|914|-5866.74|-1809849.07|-1670602067707.562988|true|xylophone band|mike garcia|2057-06-04 13:48:28|2056-12-01 04:07:20.17327137|2090-07-13
+|-13972|1616782308|723|15887.15|-2469559.45|-380705728848.234375|false|american history|jessica quirinius|2057-08-11 06:45:24||2003-06-16
+-69|-10569|-1928197479|-7104310188119834624|26076.44|2535185.38|2312022922718.455078|false|linguistics|calvin falkner|2033-03-03 06:25:26|2015-06-15 18:44:04.445242368|2020-04-30
+-101|12583|998793176|154|-39236.48|2570101.79|-2715354441339.101562|false|history|xavier falkner|2062-06-17 13:37:19|2065-12-13 13:46:25.594101554|1983-02-08
+28|25518|631207613|-8494118409594650624|33125.44|-315524.15|4237441963242.361328|false|study skills|tom garcia|2048-09-29 05:52:03|2066-12-04 01:20:15.241688388|1988-03-29
+-40|-31164|-161884324|3781|-49606.91|1038717.47|155199366725.398438|false|joggying|oscar nixon|2023-10-07 10:39:24|2071-02-16 02:10:47.198374359|2055-05-12
+-124|-5267|574069547|1466|4445.84|3318110.58|3262707195109.015625|true|industrial engineering|tom falkner|2049-01-22 07:20:24|2013-04-19 01:40:57.733559293|2064-01-22
+-28|-20663|-616724730|724|43976.01|-254306.16|-2564468363230.716309|true|industrial engineering|mike brown|2040-06-27 02:12:07|2027-09-24 00:10:50.868223193|2043-01-07
+|2542|-1984079412|-7270034223527993344|42132.29|1872487.12|4239681407336.525391|true|undecided|holly ichabod|2036-02-28 11:43:29|2058-09-07 02:46:35.228498480|2052-03-13
+-62|-25077|1845797092|913|18763.32|622185.07|2205467425785.875977|false|religion|alice zipper|2072-02-01 03:42:53|2054-05-21 06:31:30.994303844|2024-02-23
+64|32309|-605370177|1704|46536.51|-2429187.29|-3869444972623.164062|false|linguistics|priscilla davidson|2047-07-10 07:46:00|2047-03-10 19:56:43.16082472|2077-03-07
+-21|2590|-915104901||-19995.58|2053186.77|957236234850.747070|false|wind surfing|yuri carson|2015-06-30 01:25:11|2070-03-17 03:33:19.30902666|2036-02-16
+-75|-12782|1136548971|-7883252982752665600|-814.76|2576885.92|2555233457696.850586|false|geology||2024-05-25 02:43:22|2040-11-17 01:26:44.81314350|2000-03-04
+127|-27312|-505879576|412|39913.52|2317989.11|4849612269862.681641|false|opthamology|holly steinbeck|2048-01-19 12:38:07|2054-11-08 01:30:13.939944047|2000-05-28
+-28|-25412|-202035134|4024|-4465.40|-1742902.49|4667839539207.853516|true|undecided|katie underhill|2039-09-02 21:29:19|2071-08-10 05:34:49.654645949|2098-09-08
+-26||-935723237|7226360892091416576|-31695.38|2802498.83|2390739186802.050781|true|kindergarten||2051-11-30 06:15:36|2050-03-18 13:45:32.683739800|2052-09-26
+-115|-28939|688547276|-8244657976255889408|36162.80|-4989122.80|-2552765883743.006836|true|xylophone band||2034-11-26 00:25:11||2025-05-13
+-86|-12071|1956887369|2862|1570.24|-4626990.97|3666109195511.175781|false||bob king|2051-01-24 01:38:45|2055-03-11 20:34:57.197479653|2034-02-08
+-67|-1443|-993029335|1521|-46155.68|4524183.68||false|debate|oscar laertes|2058-04-21 07:57:00|2078-08-31 09:56:45.671467607|1982-04-01
+7|7530|-549167249||20406.25|1525395.61|-1198863806969.718750|false|history|holly nixon|2043-01-28 08:16:12|2045-05-21 07:53:06.829103969|2030-09-22
+3|7603|2038381675|7386087924003676160|-21580.07|933815.29|-3828528440938.576660|false|linguistics|quinn ichabod|2026-12-02 16:54:36|2044-04-29 02:35:27.537827490|2086-05-11
+-69|-13474|-373038706|3907|-18880.83|-939028.09|117666480427.907227|false|undecided|priscilla ovid|2067-02-17 21:38:22||2075-08-06
+92|2833|-2119724898|7818464507324121088|30292.00|884488.54|871664464870.037109|false|joggying|holly young|2065-11-24 15:01:57|2071-09-02 12:05:40.306146425|2041-05-01
+21|30075|-1464514590|-8293833565967810560|-36263.23|-1474035.29|-4690640135324.102539|false|quiet hour|nick garcia|2033-09-22 16:37:28|2072-01-29 18:50:14.473666434|1994-05-25
+119|-12571|-779743333|-7892780594910871552|4572.65|995018.67|-425783093743.490234|true|joggying|ethan white|2061-07-26 18:52:34|2071-07-05 00:55:22.981130585|2029-09-01
+-83||1920662116|1509|33314.97|-2730529.51|84511300621.094727||wind surfing|tom hernandez|2050-03-09 05:14:35|2078-12-03 09:09:20.319905333|2029-09-01
+85|-13713||7592440105065308160|42570.35|4483692.57|-2934238933405.062500|false|xylophone band|jessica quirinius|2030-03-23 21:15:46|2045-01-19 02:06:48.899240157|2103-08-27
+53|17436|-722294882|-7600138468036386816|36893.19|-4542207.94|-1743688633337.790039|false|education|quinn quirinius|2017-05-28 14:36:27|2066-02-21 06:53:29.204553365|2009-09-12
+3|10727|-1849091666|9064847977742032896|17555.77|-4634541.04|2527955410135.795898|true|nap time|oscar carson|2048-03-24 17:24:21|1970-01-01 00:00:00.846186492|2033-08-07
+9|20766|-181122344|-8379964450833367040|-35831.96|2510460.11|3065070467778.455078|true|geology|irene davidson|2050-02-22 15:05:52|2027-10-08 09:45:46.728679053|2050-06-14
+113|-3165|-90029636|7217123582035116032|12022.56|3658068.63|-2382124601444.530762|true|linguistics|wendy laertes|2033-11-09 23:52:55||2070-09-29
+93|24677|-472303419|-7879864376629567488||-4715822.32|3836587003422.593750|false|linguistics|rachel garcia|2020-12-08 01:26:41|2018-07-13 07:36:49.359511410|2067-01-14
+9|-3885|-906545548|2878|-38407.73|-1928000.84|-2800032052132.195312|false|zync studies|xavier falkner|2056-02-23 09:06:30|2047-05-31 06:42:08.683378784|2016-11-13
+-37|23750|-1749841790|2412|46750.89|-2458475.76|4439101915043.935547|false|zync studies|katie thompson|2073-10-11 11:47:19|2026-01-28 22:23:58.98308516|2092-06-27
+-91|-29218|-1326025787|524|42679.72|-662744.93|310684347775.804688|true|chemistry|calvin garcia|2068-11-17 02:45:29|2059-01-25 08:18:48.682643621|1971-12-01
+75|21407|44595790|784|-39388.69|-4752379.35|-4679521339883.549805|false|debate|xavier brown|2059-07-30 11:32:09|2029-01-31 01:58:51.459910049|2007-05-08
+35|4397|-117723745|-7046180371529351168|-22128.92|-1882093.86|604464981663.425781|true|quiet hour|oscar white|2014-11-01 18:19:21|2080-04-14 06:26:44.785796402|2054-06-03
+-26|-30730|-207899360|471|22281.44|-2302502.73|-1952325542067.043945|true|debate|tom ellison|2041-01-14 15:00:44|2080-03-13 16:08:00.809220690|2063-04-29
+-81|-7564|1131663263|612|19632.85|3876730.76|-3157294779103.096680|true|study skills|david laertes|2062-12-12 19:12:58|2035-06-20 23:05:28.501460278|
+-98|21941|-1665164127|8368012468775608320|528.05|3412429.91|-950235709930.815430|true|industrial engineering|victor polk|2043-04-29 14:31:44||2060-04-22
+75|-716|1784291853|-7547245548870025216|33682.36|-2277813.00|-2651511947479.434570|true|history|yuri falkner|2060-09-06 05:02:04|2015-12-10 05:12:43.991653609|1970-04-26
+1|8466|-901079162|3841|26403.70|-2787173.70|-4078808993300.313477|false|zync studies|fred ovid|2042-01-27 06:10:24|2065-09-04 01:31:40.42530548|
+-97|913|-1502924486|8752150411997356032|31964.76||4205989218534.185547|true|opthamology|rachel nixon|2027-03-17 19:37:19|2028-03-14 08:43:52.398149158|2064-03-06
+-34|15015|1637295757|-8623965248051789824|-13910.96|-1899040.07||false|linguistics||2074-06-10 07:25:52|2064-06-11 04:29:32.55418236|2094-06-14
+-33|20036|1880017800|7637152193832886272|-13101.15|-4590457.77|-1226073411038.548340||quiet hour|bob nixon|2055-07-25 23:02:17|2072-05-17 10:12:38.513326012|2012-07-07
+22|-16940|-595769210|9191943992860327936|42065.10|353607.21|-3757504944023.821777|false||bob polk|2049-10-28 12:03:35|2028-12-24 08:45:04.462025798|2099-03-24
+-81|25454|-123529324|2700|-49851.20|-199346.24|3039471429849.208008|true|american history|luke underhill|2041-05-03 12:44:35|2062-05-13 13:04:05.633293665|2016-04-11
+111|-6349|1950882901|9180098147855769600|8936.06|4586140.88|-2562228399414.543457||undecided|xavier allen|2047-04-04 14:41:47|2029-10-19 11:17:14.241599813|2076-10-10
+32|-6564|1928365430|1775|16609.42|3708243.51|-1030172091977.021973|false|chemistry|alice thompson|2057-06-03 18:50:23|2016-01-12 02:37:54.734293795|2010-10-15
+87|5550|996831203|797|-32213.23|-348044.95|2004890357846.349609|false||quinn quirinius|2025-05-25 18:20:23|2065-11-02 11:48:50.631850931|
+48|-9943|270090617|-7773957003968675840|39329.34|4135595.75|-243786022935.737305||wind surfing|holly laertes|2030-12-19 19:08:33|2023-04-14 22:34:26.707421905|2043-08-05
+-115|28301|1317690178|-8660149447361404928|-37823.29|-2814333.34|-3512730030774.058594|false|geology|holly miller|2029-03-26 00:28:37|2039-02-23 12:54:55.42365758|2076-09-26
+-46|18350|-1531040609|8641221723991433216|23823.69|-427642.08|2819404917896.939453|true|yard duty|rachel davidson|2024-07-29 07:41:06|2025-01-31 13:56:58.481985366|2067-01-10
+51|-13904|1664736741|392|-33079.41|4087589.11|4145813588397.863281|true|chemistry|jessica king|2035-02-25 00:23:44|2052-01-19 21:04:20.398663046|2079-08-01
+19|-26613|-971615370||33112.52|4554924.96|-4303832304648.800293|true|chemistry|calvin allen|2073-04-14 09:13:06|2039-12-03 11:31:50.851455297|2009-09-29
+-89|29333|-1124028213|8489735221193138176|27673.25|-4285255.26|4961373696863.767578|false|linguistics|katie ellison|2072-02-05 07:19:52|2067-07-12 01:46:05.391290184|2018-03-24
+0|-12203|372099650|7944741547145502720|-6724.11|-3694995.69|-630159357028.894531|true|undecided|luke robinson|2033-06-16 06:37:14|2014-10-03 05:32:19.137455217|2068-06-05
+111|18910|780859673|6933731240564056064|-44662.14|-1927984.24|-3491883806228.379883|true|biology|katie van buren|2038-08-20 15:28:40|2064-04-16 22:33:47.576062785|1970-06-03
+57|1280|-1359838019|9083704659251798016|20916.87|-1497681.60|2846930959096.063477|true|biology|quinn miller|2053-01-28 18:41:42|2060-07-25 07:46:20.708072689|2002-05-27
+122|9177|128430191|-9084940280061485056|-42499.27|3229184.24|-4017461708820.384766|true|values clariffication|ulysses ovid|2034-11-06 04:21:49|2034-07-30 16:22:19.644823674|2023-10-05
+-78|-10532|-318380015|8222714144797368320|-11740.33|-2082364.30|-2110282626871.757812|false|xylophone band|katie ichabod|2046-04-25 00:51:33||
+-82|-24320|1550375386|8817665768680906752||-953044.44|-2273400242763.017090|false|geology|sarah carson|2048-08-22 04:37:46||2086-10-16
+113|-27401|1012230484|1995|40659.62|-4935987.77|-1008868027448.919434|true|debate|sarah polk|2062-01-02 11:36:14||2080-04-14
+-111|30736||1561|-31482.56|-1580110.12|-1566816988300.167969|true|zync studies|quinn quirinius|2060-03-07 21:51:32|2061-11-09 12:07:37.596342844|2026-11-23
+-5|-4037|-379643543|2485|-9312.35|3850072.85|2057210782083.452148|false|industrial engineering|david quirinius||2061-11-11 05:38:01.34481671|2021-04-19
+27|-13623|505902480|1826|-42341.81||2681464429083.355469|true|chemistry|ulysses garcia|2074-12-19 04:00:03|2049-02-17 19:06:31.903289514|2013-01-23
+|14234|-1026746699|845|35740.11|4705168.38|-4722254262111.391602|false|quiet hour|rachel hernandez|2019-03-25 04:02:54|2080-10-07 11:37:55.957012102|2004-08-13
+-58|3325|1665724041|8376440110255243264|46586.97|1120362.22|3853942610392.876953|false|history|luke laertes|2029-12-12 07:11:57||2004-04-01
+-17|22289|712816880|9075404705968840704|-15320.85|-3930645.10|-4500316102425.934570|false|linguistics|alice king|2062-09-20 07:45:00|2021-06-10 09:55:52.478946811|1994-01-27
+-28|-11740|1566607834|-8379109122834997248|21449.12|-4905990.36|-1952272677926.849609|true|debate|david brown|2018-12-26 15:21:05|2055-07-04 00:03:47.54540038|2058-11-22
+-102|30420|980732494|-6938706403992854528||4290144.13|3973620252675.828125|false|nap time|holly polk|2013-06-14 16:30:17|2079-04-01 01:20:33.477066908|1973-06-12
+-27|10473|1805139501|961|540.66|1846904.66||false|study skills|katie thompson|2080-12-15 12:30:07|2018-10-01 01:35:46.827604858|2065-10-01
+93||-1402821064|1422|-32181.01|1471767.66|3888068884720.255859|false|forestry|luke brown|2055-04-11 22:16:15|2023-10-05 09:42:54.182435030|2001-11-08
+-72|-31033|-694520014|9149216169284091904|-21438.32|592620.51|-868405318561.400391|false||quinn steinbeck|2038-03-12 13:12:37|2019-08-02 08:13:35.567323306|2015-09-08
+-83|-21268|962091264|2752|-9102.58|-1920670.21||false|forestry|tom polk|2048-07-06 23:16:50||2103-08-02
+-91|5014|-1561738723|2255|11659.02|-1238767.70|2257301642704.199219|true|opthamology|fred underhill|2034-05-29 03:12:18|2056-10-14 08:06:18.341166711|1973-02-06
+-46|24913|906074599|-9080568167841226752|-1633.77|2280305.65|3418787511905.457031|||wendy nixon|2054-11-07 20:23:51|2069-03-08 13:59:21.935346573|2027-06-01
+-55||-990781312|1046|-17223.20|-2914331.70|-3061715672539.690918|true|american history|victor xylophone|2025-06-24 19:34:32|2029-05-31 09:16:06.592040235|2067-06-22
+85|14176|1582537271|7926898770090491904|-48480.35|-4206386.08|-3830188857046.366211|false|kindergarten|jessica garcia|2027-03-21 01:29:39|2050-06-01 01:40:27.460760962|2094-01-29
+5|26915|-158848747|7784489776013295616|25599.75|3264338.39|1398225707031.031250|false|geology|sarah ellison|2081-02-08 13:31:19|2016-12-22 00:48:27.635611363|2019-01-04
+48|20655|-1345085327|6991316084916879360|5659.91|2971588.36|944624924616.661133|true|philosophy|bob johnson|2042-07-11 18:36:57|2037-12-17 04:00:52.301645045|2015-05-25
+-60|-23750|747122546|1566|-32617.06|3930932.70|904359634710.800781|true|american history|david thompson|2049-08-15 13:53:19|2049-05-25 04:35:12.968980434|2054-07-15
+1|19276|1504919241|1671|-35828.00|-1990218.76|1742934975017.433594|true|undecided|calvin miller|2033-10-17 07:51:41|2045-07-16 13:46:26.28793094|2089-07-13
+76|10382|-607667405|-8543982423727128576|15112.99|-1144078.40|1538907131524.070312|true|nap time|ethan young|2039-03-18 13:48:12|2022-05-16 00:50:53.345008930|2072-03-19
+20|5947|-66010816|-8832750849949892608|43254.26|-4684380.49||false|debate|rachel davidson|2022-03-02 20:53:13|2029-05-15 20:54:25.943787316|2051-12-13
+|10083|-1340213051|6963217546192322560|-23734.81||42087741032.439453|false|wind surfing|rachel ovid|2026-01-17 13:10:48|2068-04-09 14:34:20.845374064|2046-04-19
+25|-11129|514833409|236|14759.04|-4445156.14|-2471323804264.180664|true|chemistry|katie nixon||2062-06-11 00:50:35.200240078|2012-03-25
+-117|21418|-1106685577|7086206629592252416|-20613.82|-4669723.14|1287664454410.020508||undecided|ulysses ovid|2014-05-27 23:10:23|2075-10-09 13:36:21.138709539|2055-09-11
+73|-32208|1075444504|9053187076403060736|-20607.86|-663557.15|3604091262736.055664|false|religion|luke robinson|2064-03-29 13:46:25|2028-09-13 12:25:32.422151087|2001-09-18
+68|-9365|214068706|-8067243114610532352|-40369.97|-372273.18|-4679196929122.006836|true|american history|bob thompson||2016-06-02 13:49:05.762346775|2031-11-23
+38|29288|-667383951|1751|-40969.80|352106.97|-3590646924755.134277|false|religion|alice white|2021-01-31 08:56:35|2074-02-24 20:30:17.797682689|2054-10-30
+73|1077|-336625622|2502|34593.52|-3380008.10|4861797358093.511719|true|linguistics|yuri underhill|2031-10-04 05:10:58|2020-09-16 21:07:49.9039871|2000-08-20
+86|-8927|-1817096156|294|18420.41|-718931.02|-812599933568.947266|false|topology|yuri miller|2030-03-08 13:49:25|2059-08-20 13:04:07.326107637|2026-09-01
+46|-26138|-371779520|7892281003266408448|8622.19|4294576.25|-2935717014226.568359|true|education|fred xylophone|2025-08-10 03:36:42|2058-08-31 20:49:43.13681707|2017-12-24
+125|7954||8577096957495025664|-40074.79|-3740147.62|1353404175657.040039|false|education|sarah garcia|2020-04-24 05:38:29|2059-09-29 14:12:16.157602035|
+|23725|-45460011|-8665764757143658496|-10058.15|3111392.39|4140724331387.878906|false|wind surfing|mike nixon|2022-12-19 01:07:38|2077-04-10 05:41:52.802703873|2039-07-14
+95|-25151|-1770250407|2855|-8224.46|-1232102.28|1061929942085.519531|true|education|victor davidson|2024-08-05 07:04:05|2076-04-22 18:20:42.949891304|2082-07-08
+-92|22075|-170643477|2811|-6333.87|-2996672.74|-4596379165591.137695|false|education|oscar johnson|2075-03-08 07:27:49|2031-01-22 04:35:56.338245755|2001-02-13
+-107|15530|1028092807|8785153741735616512|23063.22|4917736.71|-1387735872479.432129|false|study skills|ulysses quirinius|2051-06-21 00:48:20|2064-05-26 10:29:04.116652584|2001-03-24
+-35|-4509||1726|-36045.69|-957009.39|-1436845214707.389648|true|industrial engineering|katie ichabod|2061-12-03 22:11:19|2037-09-03 21:28:18.120030510|1976-03-25
+10|8243|1430614653|7186401810812059648|21430.58|-4117132.25|4273903933165.144531||values clariffication|david brown|2080-09-06 01:51:56|2067-02-25 00:06:03.464200151|2008-03-31
+-100|-18358|390124976|-7603569103205916672|9320.42|820509.76|-2378674120573.869141|true|philosophy|gabriella laertes|2054-02-15 22:48:03|2023-07-16 09:03:23.416142255|1987-01-17
+-34|-12896|-396852483|4018|14230.43|1948755.04|996493613976.418945|true|american history|jessica nixon|2035-11-08 16:07:43|2042-02-21 13:24:08.768185439|2024-08-27
+-79|-3883|-519978947|3566|-25444.08|-4338469.48|-2225952289175.303711|true|biology|bob van buren||2053-09-12 14:38:23.627646057|2061-08-19
+38|11067|257821327|2725||2123890.76|-4311579608982.931641|true|quiet hour|bob ellison|2074-07-08 16:12:25|2028-02-16 07:26:07.314316265|2046-10-04
+-46|9447|-1921909135|1234|43332.86|-1313189.64|3483740651742.017578||debate|ethan hernandez|2053-12-13 02:32:42|2024-05-02 15:41:49.244300741|2012-02-14
+|-31217|-1880877824|346||-4470321.21|-2089255269314.307617|true|zync studies|tom quirinius|2032-11-07 23:12:11|2057-06-05 07:40:56.143268958|1986-12-22
+||866084887|7961515985722605568|-13131.71||-2394101676301.012695|false|wind surfing|david polk|2052-08-30 08:01:09||2052-02-02
+21|32611|482977302|7274777328897802240|-6306.38|1252331.81|-533108289688.186523|false|study skills|jessica robinson|2061-05-19 17:30:01|2079-02-13 03:05:01.151556875|2064-10-20
+-80||581259902|-6933565857643814912|12765.75|-3696105.78||false|wind surfing|yuri ellison|2032-01-12 17:09:47|2026-12-04 01:38:04.606186772|2046-12-26
+62|-776|-409404534|-8330233444291084288|-13235.88|-4942351.80|1478986789735.259766|false|wind surfing|luke nixon||2015-09-27 04:55:14.418545051|2079-08-04
+-15|7988|1969650228|34|49433.36|-1454704.21|3304426642736.508789|false|debate|bob white|2032-05-25 11:24:01|2050-08-06 03:07:56.829882190|2011-02-25
+-100|-19374|-337073639|7080269176324218880|9636.84|4955437.13|-1605703810903.698242|true|joggying|priscilla nixon|2016-06-18 01:36:41|2076-06-24 17:32:46.629540610|2051-08-11
+31|11050|-1862095575|2941||4773173.81|-4916009895587.691406|false||tom ichabod|2026-02-05 06:06:27|2033-11-01 09:06:22.861384004|2064-04-06
+42|29954|-297664578|9117063974299148288|-36654.25|1809785.84|-2990108911038.724609|true|linguistics|quinn zipper|2063-01-04 13:11:49|2032-04-05 17:24:19.790913068|2082-02-07
+36|-1307|-603273425|-6917607783359897600|20495.55|-717022.00|401293377521.448242|false|xylophone band|ethan steinbeck|2043-08-16 10:35:12|2074-08-29 01:50:39.79906002|
+-47|20307|-1409508377|-8566940231897874432|-23112.40||-4906953690985.069336|false|american history|katie laertes|2018-04-24 10:05:45||2070-04-05
+-27|16664|-1817938378|-8710298418608619520|-19481.20|2004949.48|-2008363741307.021973|false|philosophy|sarah laertes|2038-09-29 23:53:07|1970-01-01 00:00:00.626944877|2041-07-12
+-31|20391||1520|39315.23|4922532.64|-1940382804363.878418|false|industrial engineering|priscilla brown|2045-10-19 14:00:02|2047-12-07 08:51:00.336787227|1988-07-12
+113|-21146|-800799595|3728|-6051.92||4288442203475.107422|false|history|luke ovid|2042-04-07 13:12:41|2017-11-07 00:47:02.498508838|1980-07-10
+127|-26526|-938112972|-8835408234247168000|-5042.40|-639514.18|2253781196684.378906|false|topology|rachel polk|2059-06-15 22:27:11|2078-04-15 00:06:23.865331718|2002-01-27
+47|3374|527598540|7705445437881278464|5750.75|265215.83|-3997464183110.714844|false|topology|luke ovid||2055-04-30 05:00:45.288293151|2038-08-04
+-57|7826|987734049|6926925215281774592|4092.06|-4043613.94|1825916166900.114258|false|wind surfing|alice hernandez|2066-02-26 20:09:56||1994-01-31
+30|-4159|-1054609414|835|-24688.28|-3625707.02|-1643333177209.917969|false|biology||||2097-09-12
+23|1326|-199587670||23788.91|-602696.25|3307871238283.475586|false||priscilla miller|2017-03-07 08:16:25|2052-04-04 23:29:00.786514521|2091-03-28
+59|3270|1434588588|3232|1480.61|743880.56|2593124571056.222656|true|wind surfing|nick allen|2056-12-31 22:55:26|2076-06-29 19:41:36.482632437|2051-05-15
+66|3245|-300717684|-7840338174858199040|40071.10|-905100.40|-4829089800174.344727|false|wind surfing|sarah ovid|2056-03-09 19:09:50|1970-01-01 00:00:00.59977853|2047-10-04
+85|6074|-540401598|7748799008146366464|20761.91|-4916297.37|2180818539557.698242||joggying|david brown|2024-11-21 14:50:04|2028-01-24 01:00:27.143579402|1970-07-16
+38|-7948|1987336880|7410096605330227200|18427.96|1858816.32|2290676735205.039062|false|topology|fred ichabod|2052-07-29 14:33:52||2090-05-17
+-127|11451|316438994|188|6220.87|2395035.62|1762390956829.769531|false|american history|sarah white|2035-12-04 05:59:54|2037-10-21 10:24:49.395474612|1976-11-08
+-49|19041|-1464762852||-9900.51||4912867697965.523438|false|biology|bob garcia|2080-03-12 06:31:47|2079-09-20 05:14:58.169640521|2008-01-13
+-104|5191|595836061|-7709958788604936192|-17146.91|-3981389.85|1420240216579.884766|true|nap time|priscilla johnson|2018-10-25 09:42:50|2027-12-11 09:57:50.980641227|2019-11-03
+115|-26832|-764412063|-6920172215209426944|17394.03|561210.22|997882374014.622070|false|linguistics|victor falkner|2018-12-21 14:56:23|2074-08-29 04:26:14.412387056|2100-03-23
+-84|-16327|407098216|-9109392978217484288|24409.31|4875980.88|2156284924808.034180|true|undecided|priscilla garcia|2049-08-03 07:20:14|2072-03-18 16:05:07.216021739|2020-01-11
+56|21814|773730574|3608|19617.72|2612031.03|472615987880.900391|true|values clariffication|holly polk|2079-07-31 02:12:53|2033-04-30 22:04:27.178988903|2065-03-04
+109|-5097||-8214462866994339840|14926.06|4194003.54|-3157298893601.385742|false||ulysses falkner|2051-01-12 23:18:22|2056-07-05 00:35:53.936899939|2083-09-24
+31|6900|-604362582|2306|25232.04|-301550.41|289949139351.364258|true|zync studies|gabriella ichabod|2041-03-14 14:44:09|2068-11-06 15:41:10.293161863|2098-08-25
+-122|30119|-2065287410|-7759238919361888256|-48628.97|4508399.10|-1946145505911.474121|true|topology|rachel quirinius|2042-12-15 05:57:04|2057-11-03 01:20:57.262442758|2077-07-27
+-81|32567|-536315467|-8922409715403112448|-49532.02|-2541992.13|-366989953592.735352|true|forestry|luke allen|2059-07-26 01:05:11||2075-01-17
+58|-10247|-186600427|3664|-20024.15|2499207.86|3164512989036.332031|false|history|sarah thompson|2036-09-10 12:17:11|2064-10-23 01:33:56.949763617|2085-10-20
+87|-22431|-1625800024|-9203942396257984512|10700.82|-4871285.36|1078033806441.389648|true|zync studies|fred falkner|2079-08-03 03:53:40|2077-02-22 11:17:37.324498592|2090-11-24
+89|24782|1914993018|8116738401948377088|28477.87|4428379.36|-3581726665433.642090|true|linguistics|yuri laertes|2042-05-04 19:59:28|2015-11-03 09:21:31.868284245|
+-19|-25973|-1900369503|1791|38658.65|3263702.68|-971581426672.542969|false|history|gabriella thompson|2019-12-18 15:34:18|2059-11-10 14:30:30.756815026|2094-09-03
+112|-5635|-4393552|-7419068456205385728|-16898.67|-3147119.24|-4071056405979.831543|false|history|rachel ichabod|2020-05-25 05:03:07||2041-02-27
+6|-5946|1366402722|8795069490394882048|-5903.91|-302873.33|-3374663867091.989258|false|quiet hour|gabriella robinson|2053-11-12 16:58:09||2071-08-09
+-115|325|692666133|3043|-22120.22|3867286.85|3593749673935.349609|true|yard duty|mike miller|2030-09-24 22:40:43|2059-05-19 21:39:40.717511206|2054-10-11
+46|-24248|-1871446009|3174|24920.44|-2650736.01|3646290606200.395508|true||david johnson|2021-12-02 02:13:49|2027-06-25 15:06:55.689498978|2092-05-03
+-92|28558|-1699044525|7625728883085025280|-26098.47|-201231.45|-879898512524.555176|false|philosophy|nick zipper|2039-09-05 02:14:13|2067-05-11 07:05:39.707943033|2082-09-07
+22|8786|-1196808950|-8585134536083660800|36606.56|-1258722.35|3594939606490.008789||philosophy|irene polk|2033-07-09 03:38:52|2070-09-24 03:28:58.997336852|2069-12-08
+6|-30638|-370901197|8558000156325707776|-9840.93|-4341927.96|1094936081745.956055|false|linguistics|katie laertes|2078-08-17 11:00:57|2047-01-10 07:06:46.173146425|2028-12-05
+48||1787826883|-8572949572756774912|-7447.88|-4657467.74|-1120139371327.120117|false|kindergarten|sarah allen|2045-11-09 08:09:15|2041-12-23 20:40:16.590539947|1996-09-10
+-2||1045719941|661|48481.57|-199335.41|1697710053431.365234|false|wind surfing|oscar allen|2077-10-09 07:08:17|2044-08-23 18:37:23.765446063|2039-05-20
+-18|-2636|1852725744|2393||-327510.29||true|opthamology|david garcia|2071-07-30 15:03:51|2049-10-01 11:47:21.417323976|2022-09-06
+108|-8578|1352739140|-7800879252150779904|27366.30||-180060584501.582031|false|history|luke miller|2067-12-30 02:21:40|2029-03-05 23:41:09.158607395|2027-02-24
+70|-30163|2044130430|7534549597202194432|-3684.66|3361124.51|-544906712728.980469|false|debate|luke brown|2015-07-17 03:26:40|2054-07-10 04:48:29.103251644|1981-06-25
+-72||583458404|-7642381493746483200|-41758.86|-432625.76|739045453673.981445|false|xylophone band|fred falkner|2075-02-09 15:25:34|2024-10-11 18:27:50.785894213|2038-05-15
+-31|14335|-2024003241|-7330413050756235264|-20201.82|672555.54|1469464763006.626953|false|topology|||2044-04-09 14:18:23.656514547|2038-02-20
+-44|31242|605946758|7596563216912211968|30099.30|2097061.15|3078551005937.946289|false|chemistry|calvin ellison|2040-08-01 15:36:49|2034-04-12 16:31:19.919073203|1981-01-10
+-115|-5240|-1128317466|3307|-17763.36|-3640432.08|-3634683485380.314453|true|chemistry|ethan young|2014-07-01 14:59:22|2065-10-29 07:42:38.763916292|2102-08-12
+-60|4725|-373541958|2971|-21091.82|3177096.44|2070825437177.284180|false|values clariffication|priscilla quirinius|2030-01-02 10:09:12|2017-02-05 15:11:27.916015499|2098-04-30
+-35|-3427|1533817551|2285|22842.27|434947.31|-233465238088.080078|true|biology|david underhill|2069-11-27 12:11:44|2038-10-25 03:53:58.936213000|2018-10-02
+23|-32383|1293876597|1880|15273.59|1943186.18|1318023503084.724609|true|quiet hour|mike brown|2036-06-15 05:33:48|2035-04-17 20:50:04.917479654|2086-09-08
+15|5972|947846543|4088|-24858.15|-1682574.63|2790323958356.459961|false|religion|quinn garcia|2039-02-08 09:45:49|2051-06-08 01:29:12.240654179|2020-09-22
+18|-10277|1004241194|743|37468.44|90556.54|-1721911019841.009277|false|study skills|calvin hernandez|2017-11-06 20:56:23|2066-11-30 20:25:38.323872312|2087-05-31
+96|-28730|-670925379|-8317591428117274624|45783.32|-4491871.75|1347547267070.333008|true|religion|ethan ovid|2039-04-05 21:26:27|2047-02-15 03:00:04.876322903|2016-12-02
+49|22511|1301997393|8854715632851345408|-7866.49|-4806269.72|4690235057502.966797|false|geology|victor van buren|2077-11-06 21:58:21|2026-07-12 18:24:40.599693692|2023-01-30
+116|21606||7768984605670604800|-37668.86|-4451507.54|1896615693548.895508|true|chemistry|victor steinbeck|2049-10-18 09:39:20|2017-04-25 06:11:37.174630171|2045-09-29
+102|-26461|2114363167|2900|42096.38|273171.55|-2395416839539.450195|false|industrial engineering|calvin falkner|2038-01-28 15:37:14|2021-10-28 12:53:49.438479282|2012-09-25
+-35||-1614194712|7062605127422894080|10143.00|3278700.42|-8877240475.761719|true|debate|alice laertes|2047-08-19 00:56:41|2041-07-30 06:11:21.601946450|2060-10-26
+30|-23672|-1983567458|7394967727502467072|42769.25|245529.05||true|yard duty|nick laertes|2052-08-02 11:29:42|2048-02-09 16:35:20.385399131|2092-01-30
+60|22378||1781|39850.06|-4048951.22|3015653383409.677734|true|yard duty|gabriella ovid|2054-01-29 18:44:50|2063-07-31 19:03:05.155926850|2006-12-12
+38|-12193|-1770229099|7238339720750948352|10757.18||2824970709025.791016|false|philosophy|alice miller||2058-07-11 09:32:37.491928305|2001-09-05
+|-2287|92777932|1638|-25700.79|-181051.84|4692362804086.371094|true|american history|ulysses davidson|2037-12-15 16:39:08|2075-09-17 23:16:13.795444412|2079-12-10
+93|15158|1475025489||8969.07|-4073690.87|909614012102.934570|true|geology|alice davidson|2074-01-01 16:32:52|2019-06-28 19:19:52.904542868|2048-06-04
+|6899|633813435|-8522878384019169280|-8564.75|4996750.69|755521930470.130859|false|forestry|oscar king|2078-04-19 11:14:16|2029-01-24 18:55:58.605673322|
+78|-9398|51376784|-8051587217208967168|42110.77|4237872.52|-480292543988.199219|true|religion|fred young|2043-08-31 12:19:13|2027-11-28 17:42:04.124425023|2043-05-27
+-98|22678|-765102534|-7425160895830573056|-36404.91|-232802.04|70171166399.717773|false|mathematics|oscar king|2061-03-29 17:11:56|2025-01-04 05:18:13.67002718|2070-03-30
+115|12263|-1669848306|7344029858387820544|45009.74|-4145959.26|-1287206623226.708984|false|study skills|holly thompson|2030-07-06 18:59:28||2068-03-31
+-98|-18295|-1391183008|-8013397854633648128|-12751.03|4087534.61|1251848312304.524414|true|values clariffication|ulysses brown|2042-11-01 14:59:40|2050-08-30 06:33:17.144659188|2100-07-17
+59|297|-1706867123|8808467247666241536|-23178.82|1057885.16|4299677631158.154297|false|industrial engineering|calvin allen|2017-05-25 20:58:54||2071-03-31
+-68|-26481|-726879427|-8768744394742235136|-46383.26|1261211.13|-2782916269764.896973|true|xylophone band|calvin young|2030-05-23 20:23:28|2051-07-05 23:08:45.51977303|2013-03-12
+-6|31316|-1011125931|9185458640237641728|-17508.25|-658586.36|1020511437056.228516|false|industrial engineering|zach ichabod||2061-07-05 08:12:13.181218526|2006-12-06
+-51|-20182|-892839693|-7686220526274502656|45425.18|1655396.08|3720881118135.359375|false|yard duty|wendy davidson|2065-08-25 22:34:57|2077-06-29 07:51:23.99926741|2003-09-22
+93|15815|-626484313|-8203075743525806080|1895.94||977607524337.074219|false|nap time|ethan underhill|2066-01-12 20:36:33|2016-07-24 00:19:40.599709326|2040-02-16
+122|-22390|511836073|3462|-13499.91|-2876022.64|1328013667669.125977|true|opthamology|calvin ellison|2038-09-26 01:22:16|2050-10-02 05:27:49.739848630|1997-07-24
+31|10544|2146312499|6964585306125008896|-1730.59|1232330.61|4495354594042.523438|true|nap time|calvin allen|2051-08-10 11:03:28||2099-02-09
+-125||-940504641|3418|-42213.02|1998604.86|-2596755613328.838379|true|chemistry|victor robinson|2069-06-09 00:13:59|2049-07-11 09:28:46.795925700|2066-06-17
+-55|21440|-606214770|3366|-7764.69|150515.54|3713473719274.773438|false|philosophy|holly king|2065-10-25 16:36:46|2058-03-17 19:37:18.893032333|1998-04-11
+|-11123|-1078579367|-7867219225874571264|3544.10|-4440011.92|-1597354768432.488281|true|education|katie nixon|2021-12-20 01:53:24|2016-04-25 20:51:22.134986617|2009-08-13
+14|-12517|-1269216718|8367680396909404160|41509.01|-1881292.71|-2614008444640.525391|true|topology|wendy nixon|2031-06-01 22:05:01|2019-05-30 05:53:53.829470930|1986-10-09
+-78|-7027|-664856187|7524958388842078208|-49058.42|2299272.82|1193868149375.028320|false|kindergarten|alice xylophone||2038-05-26 15:00:22.242283005|2072-09-03
+-69|4718|-47662800|2897|32798.27|3454639.78|1810203200294.040039|false|philosophy|zach polk|2069-08-29 08:29:09|2014-11-20 05:08:33.941609402|2082-06-01
+-72|-15957|-630900418|8391785334471589888|-49753.88||-3594602463335.583008|true|geology|oscar steinbeck|2034-06-08 17:12:48|2048-09-23 09:12:15.792761284|2039-09-20
+-57|15379|-674478103|-8581979259158929408|24804.72|516411.04|-2087053480555.418457|true|debate|ethan van buren|2039-03-15 22:10:39|2059-02-19 09:59:56.329119790|1980-04-11
+116|-4799|1485934602|587|-36250.77||-3139538476660.561523|false|zync studies|tom garcia|2070-11-21 14:24:06|2077-06-02 22:53:29.588878692|2021-02-13
+5|20183|-2081501748|130|16581.21|2049671.63|162654497074.683594||opthamology|priscilla falkner|2022-12-17 04:17:53|2050-08-15 19:53:57.340023498|1985-05-04
+25|-17429|-300429552|1030|48749.30|891980.69|-51603607081.567383|true|yard duty|calvin johnson|2017-11-20 04:16:35|2055-03-24 21:11:15.57401629|2053-12-19
+45|-31764|89366322|8362046808797306880|-28518.05|2976251.39|2342010012983.273438|false|philosophy|wendy underhill|2015-07-02 23:30:23|2076-07-01 17:36:04.583971136|1999-02-20
+124|3714|-664111469|3691|-30852.26|2707139.90|-4940400326044.844727|true|debate|katie king|2029-04-17 21:35:26||
+-97|4749|859140926|7454632396542074880|46493.77|4790547.07||false|values clariffication|jessica robinson|2072-02-26 01:04:57|2056-06-21 06:38:40.564403087|2042-05-03
+104|-7467|-2111312205|7125231541858205696|9390.22|-4975282.68|-2004428657765.586914|false|nap time|katie nixon|2051-03-30 14:45:15|2039-09-13 12:20:18.915778346|2105-04-10
+51|-31881|1933545427|2580|-38988.06|-4829221.83|3300559907076.631836|false|forestry|victor allen|2064-06-06 09:21:23|2070-06-29 22:13:10.207059753|2066-02-22
+63|17680|-1164833898|2512|18258.24|-1641179.75|752962351021.224609|false|study skills|rachel white|2056-09-13 22:53:52|2044-06-06 09:44:23.311696677|1978-12-09
+-50|29999|-290558484|7061498706968428544|1510.13|-4232212.06|-4574735317597.173828|true|debate|ulysses allen|2037-07-06 08:01:26|2050-06-04 17:30:11.711624931|2005-07-17
+45|-21005|2100839074|-7255686273677328384|34455.77|-1253884.38|4472794135666.830078|true|philosophy|calvin ovid|2070-07-26 14:31:09|2036-04-08 23:28:54.417802561|2044-10-24
+-15|14982|-1079231269|9048002942653710336|-22879.28|4867276.99|-3098102239046.767090|true|american history|victor carson|2059-03-26 12:10:56|2014-08-29 13:16:50.336506060|2012-02-05
+52|19003|1830870769|8868529429494071296|36316.44|2540170.52|76874893588.936523||kindergarten|ulysses robinson|2053-02-19 15:45:55|2062-12-03 14:50:03.474294016|1984-03-16
+103||-1701502632|8815398225009967104|16082.15|3286175.37|3474939567092.790039|false|religion|mike xylophone|2036-09-01 16:20:32|2049-09-25 10:34:17.329238191|2015-12-01
+54|29171|-283378057|7128222874437238784|5784.30|-2487846.11|1273026486246.777344|true|study skills|ulysses xylophone|2065-03-26 18:47:07|2046-10-10 08:53:02.163030900|2065-06-02
+-29|-22000|826143442|8371939471056470016|-9954.17|1165753.75|-3327839964882.955078|false||zach zipper|2025-04-26 13:39:51|2033-12-28 12:16:51.968462869|1997-05-08
+45|-6045|-1562552002|-8335810316927213568|18734.01|-1945504.67|-3344796345914.708984|true|undecided|katie brown|2057-08-05 00:41:39|2022-11-18 17:27:46.282162052|2040-01-17
+-37|-13426|-876122064|-7144791190333546496|-27443.20|3276942.49||false|zync studies|sarah ellison|2027-09-21 21:34:07|2028-07-26 18:55:38.467639955|2046-09-01
+84|-13038|1107502179||26203.53|1826616.72|52385263076.928711|||oscar brown|2062-03-24 05:55:44|2080-06-08 12:00:45.876270498|2068-05-03
+-37|16654|1652349607||23892.92|1116670.95|2593325712512.423828|true|chemistry|victor young|2079-05-05 06:48:25|2052-09-05 13:46:09.928597565|2077-10-05
+5|30730|-841268868|-7572962089372991488|-40151.14|-600165.15|4768177899583.748047|true|geology|ulysses ellison|2024-10-07 05:46:34|2076-04-07 07:58:46.169024390|1972-09-03
+21|-20657|1503176016|8850055384477401088|-36132.96|3738524.26|2378765399686.458008|false|geology|david nixon|2029-11-13 14:57:22|2018-08-17 10:16:14.518949916|1992-03-07
+107|-15779|1620529246|2626|10150.78|3475802.90|-2294284421075.900391|false|study skills|katie underhill|2037-09-11 08:16:17|2058-04-21 23:50:14.231746612|1999-03-02
+-27|-25112|2069258195|3599|3093.83|3254136.59|523934597183.667969|false|opthamology|nick allen|2080-02-16 07:15:57|2060-05-16 16:45:20.894463536|
+-118|-16813|-361944328|213|-40953.43||-4751644152447.696289|false|undecided|david white|2070-12-23 10:54:21|2048-10-22 12:20:49.521887540|1979-08-27
+17|28606|277582670|2232|45641.30|128509.04|4503808618217.277344|true|education|tom underhill|2080-09-02 15:00:49|2037-02-08 08:15:49.8202484|2080-09-01
+102|-18601|283618733|-8297230235506343936|-1052.55|-2765868.79|4944203659075.371094|true|geology|quinn ovid|2075-12-13 06:47:46|2023-03-25 01:12:26.114510579|2001-02-02
+-110|29515|-395499919|3430|-39432.90|3255575.14|-1079961002014.897461|true|chemistry|wendy zipper|2032-05-15 10:18:11|2062-08-26 11:43:13.962436079|1985-01-02
+|-9375|1107258026|391|39049.41|2111980.57|-1263586613411.327148|true|education|quinn steinbeck||2028-11-03 23:51:03.652949047|2089-06-26
+96|28011|830944953|-7395343938785738752|-17062.37|-4047697.78|2447642338433.482422|false|chemistry|irene davidson|2070-06-10 17:13:05|2049-03-17 04:12:33.703429425|1977-08-25
+74|-14836||9038087402564657152|46621.33|1011931.42|2571176224975.069336|false|american history|oscar nixon|2056-02-11 14:02:01|2061-02-10 07:18:47.459454314|2066-11-29
+-42|-15695|-1052493316|-9013952631912325120|24814.36|225913.19|1857357798507.415039|true|kindergarten|mike van buren|2032-07-31 17:27:25|2020-07-02 01:16:38.343352835|2081-07-07
+-80|-11081|440393309|3446|-27153.14|-3115694.15|3504105610850.736328|false|geology|sarah ovid|2080-11-21 06:41:56|2022-09-23 02:34:19.579830812|
+5|-22582|-397683105|-8703026916864802816|-21676.69|2972179.30|-3159033035088.328125|false|||2016-03-08 14:54:40|2039-10-18 00:49:44.471616221|2045-10-19
+92|-27715|589546540|-7833618000492109824|14121.48|-2112484.76|-2346132497060.402344|true|geology|david hernandez|2070-12-19 18:40:20|2030-10-05 20:00:42.51205588|2019-03-26
+60|-23853|-1430903652|1541|-22380.78|2905459.89|-4830217659481.838867|false|xylophone band|bob zipper|2044-11-07 17:03:34|2040-06-07 07:46:11.981745178|2022-05-14
+-2|-373|435426302|8759184090543857664|36669.53|-4226813.46|-2014561647846.981934|false||ulysses hernandez|2019-01-12 08:00:20|2038-11-27 08:44:14.384170737|2015-01-09
+121|22333|658636280|-7042183597114081280||1390895.25|3030971793285.637695|false|wind surfing|alice white|2069-01-21 19:40:51|2030-03-05 11:21:30.696290529|2045-07-03
+25|-10312|1759741857|-7147490721376591872|11100.55|-1549045.77|4731512443034.197266|true|joggying|nick steinbeck|2020-10-20 08:00:11|2061-08-30 15:56:42.855551352|1998-03-27
+68|25170|1911834442|3725|-46855.76|1255180.75|-1442847310094.883301|false|study skills|priscilla white|2031-05-04 10:11:19|2037-03-01 08:47:26.266061470|1976-03-08
+85|22852|-1138530007|7961909238130270208|-6093.71|3270007.69|-2674772136505.545410|false|education||2030-05-16 10:48:35|2068-01-21 12:11:12.864391307|2077-01-10
+116|7258|-966979668|-8930307926221807616|16695.39|-3896225.81|||education|alice falkner|2026-04-02 23:02:34|2061-07-08 03:11:42.545676274|2004-05-21
+-127|9863|1516149502|2719|-41428.66|4157540.50|638265920153.422852|true|yard duty|david ichabod|2017-04-04 18:50:25|2050-03-30 05:09:44.583713357|1976-02-24
+-68||-1230459100|-6988970700649168896|-32345.84|-4878754.52|2626801691463.143555|false|forestry|mike laertes||2015-12-05 16:38:41.173038216|2046-11-10
+-86|-22421|-345542922|-7155539549555105792|42860.19|-4587464.51|2546447982029.782227|true|biology|tom steinbeck|2076-09-06 07:07:12|2057-12-31 11:27:12.154351101|
+123|-1433|-1656822229|3625||1784691.26|-1490403618753.184570|true||priscilla ichabod|2023-04-14 13:50:19|2022-08-17 04:36:05.763580269|2019-11-15
+67|-17254|129675822|8113585123802529792||1292322.18|-2056228636604.906738|false|religion|alice ichabod|2075-02-23 01:44:22|2019-09-04 19:27:52.964797916|2101-04-17
+116|18354|2066707767|9207927479837319168|11869.69|782908.70|-3561486225626.175781|true|kindergarten|zach thompson|2036-01-14 22:33:10|2078-12-19 21:06:17.223542373|1981-03-03
+2|-6487|-2011708220|-8387347109404286976|3033.19|1933471.30|4046390706154.064453|true|biology|katie zipper|2024-01-09 10:21:56|2048-02-11 17:28:42.54177107|2069-08-17
+-53|-16052|-768305191||17946.80|303090.34|-364312091649.842773|false|linguistics|rachel johnson|2027-04-27 06:12:38|2052-06-15 15:08:20.185912632|2073-07-08
+-54|-31404|1540680149|8896237972875370496|2155.10|-4075392.96|4275511770663.023438|true|kindergarten|mike white|2053-02-17 10:06:35|2033-01-10 10:54:27.769712510|2019-02-12
+73|-29468|564349193|8372408423196270592|8882.78|2172977.04|335964103970.418945|true|debate|tom laertes|2070-07-27 23:21:50|2060-05-03 18:17:59.362453382|2082-10-06
+28|-16425|932774185|922|36886.25|4970929.84|-2211537110226.693359|true|yard duty|bob king|2042-10-20 10:28:56|2037-10-28 21:57:54.89341833|2079-05-16
+-108|-5118|1281159709|7255302164215013376|43517.90|-370366.94||false|chemistry|priscilla davidson|2074-11-03 03:30:08|2027-10-18 04:36:01.205338700|1994-01-16
+-31|-16978|318631333|-8585966098173870080|18570.09|898302.80||false|mathematics|ulysses davidson|2062-03-03 06:35:19||2095-12-10
+-111|-20112|1847210729|8424515140664360960|-13301.82|-3521114.58|-1467777131906.373535|true|quiet hour|sarah polk|2035-03-23 23:34:06|2068-12-06 16:35:56.147946401|2036-05-11
+-118|3486|-76654979|-6997233584896229376|-43892.40|1141130.14|3496895382259.453125|true|mathematics|zach van buren|2078-06-17 15:41:41|2017-02-03 01:26:03.765359003|2079-06-11
+-1|18900|-2137168636|8087737899452432384|24766.64|-1466134.14||false|industrial engineering|katie garcia|2072-04-07 08:27:00|2046-08-31 17:44:23.281747989|2038-04-03
+-60|-4501|557053197|1493|-28572.29|-3264979.44||true|topology||2069-10-07 09:02:18|2031-11-14 01:37:26.277625746|2093-10-03
+71|27960|-859535015|8779711700787298304|43414.93|-4284284.11|23723101115.607422||forestry|bob garcia|2065-07-01 07:28:35|2040-10-27 09:40:45.325661811|2078-10-25
+74|-17056|672266669|2533|14993.12|4181055.99|2330393617025.130859|false|values clariffication|jessica underhill|||2066-04-21
+105|8328|-44559184||34668.32|-3358597.04|2107086403699.476562|false|industrial engineering|priscilla garcia|2073-01-23 05:17:11|2041-01-24 08:27:17.399682734|1982-09-04
+35|13020|-1491722659|8017403886247927808|-27501.34|-3084795.60|1361019533518.934570|true|biology|ethan laertes|2049-12-25 02:26:54|2057-08-09 18:37:09.476222463|2091-03-14
+|-18151|-1140071443|1282|32571.05|-218555.28|2241361269159.725586||kindergarten|mike laertes|2038-07-20 14:27:31|2076-01-14 17:32:40.202975480|2076-03-16
+-55||-1960344717|2177|-44515.88|-75942.87|873619012141.020508|false|opthamology|holly steinbeck|2037-12-11 11:02:50|2068-04-24 14:56:03.866851484|1973-07-26
+-74|-12876||-8632237187473088512|27660.85|-3161619.93|3454616098419.087891|false|kindergarten|nick laertes|2054-07-06 18:44:09|2049-03-20 15:41:29.13229610|2043-01-06
+-121|-11110|2022944702|8109381965028548608|4900.65|-1214641.94|-782319272106.413086|true|biology||2077-04-19 06:19:04|2080-04-29 19:51:25.500331032|2029-03-19
+29|67|590719541|1157|36502.48|-4871524.01|-2077483805740.065918|false|wind surfing||2034-08-25 19:02:03|2037-08-20 21:02:49.427519908|1992-09-10
+-45|-24911|1870464222|7378993334503694336|-16461.48|2813386.84|-2450662360123.097168|false|quiet hour||2024-08-11 09:42:40||2059-08-24
+-72|2074|1013517056||-42455.06|-142043.75|3719431660678.935547|false||xavier ovid|2023-08-30 07:09:18|2018-09-20 23:47:19.977608647|2007-01-02
+-110||-1946023520|2560|-46654.69|407244.54|-622449463768.788086|false|industrial engineering|irene brown|2049-12-25 17:32:04|2044-10-31 04:37:40.524530302|2017-04-21
+-71|2335|1003667927|4037|-40079.96|374952.12|4337472356507.205078|false|topology|mike polk|2018-06-24 18:36:35|2064-11-10 09:43:52.16046799|2028-12-03
+-54|-4364|1775355987|-8562524688907485184|-10552.10|-494146.63|2527866513015.055664|true|nap time|david ovid|2036-03-05 02:43:58|2033-02-15 05:01:02.667963573|2061-02-21
+-12|-24847|1471913583|2325|26530.25|-11708.56|2137890018516.585938|false|zync studies|rachel robinson|2041-10-21 00:45:05|2031-10-12 16:59:24.57008285|2099-02-12
+48|-4923|2051470532|6962726713896484864|-31562.99|-3107233.90|1876626916355.661133|true|american history|rachel zipper|2047-01-05 01:41:41|2067-01-14 02:21:16.602793258|2104-10-13
+-50|-31709|-1379039356|8120593157178228736|20553.47|-2588951.95|3745237629497.139648|true|geology|yuri white|2072-09-13 15:30:55|2040-11-14 23:29:32.319041291|
+87|26324|-1709117770|6924820982050758656|10187.76|-787959.05|-455009287847.438477|true|opthamology|calvin miller|2037-08-01 12:37:21|2059-09-14 10:15:36.572649743|2083-11-10
+-32|20746|1592153312|-7366430883634929664|-48828.20|3542628.51|-4174317808621.062012|true|religion|ulysses young|2022-07-28 16:59:23|2055-09-14 11:55:11.844515490|2032-03-31
+|-23284|-2071851852|-7209060152494817280|34473.73|698236.93||false|debate|ethan ichabod||2039-04-13 07:12:39.959754595|2051-04-11
+-79||488559595|-8689606130068611072|46873.75|-4585927.20|1922555768632.854492|true|history|ethan hernandez|2016-07-20 02:40:21|2014-04-06 14:33:25.947936182|2006-02-12
+-124||297577612|3190|26589.02|2183845.19|-1193317944887.937012|true|geology|bob ovid|2014-05-18 11:54:56|2059-07-31 15:56:05.368353709|2063-06-28
+43|1762|-2119539915|3725|-45749.50|3070034.88|-3994813234986.907227|false|industrial engineering|holly carson|2039-11-03 00:15:58||2092-10-07
+-38|-28098|-890374552|-8581765103969312768||-3283040.77|2656201211529.278320|true|industrial engineering|gabriella polk|2029-01-04 19:28:12|2014-02-05 04:57:17.37210387|2038-02-26
+-67|-9676|-1889139541||-11999.87|-2509288.07|4834823806574.134766|false|linguistics|irene davidson|2039-06-08 03:04:50|2044-07-10 14:18:16.503028090|2094-11-08
+-59|-15053|459269456|3542|24044.42|3949871.39|3217181341598.660156|true|kindergarten|mike nixon|2054-05-19 16:38:12|2053-10-27 15:11:39.341374749|1984-09-17
+45|-9065|566646177|8553195689344991232|-8135.26|-1939444.49|1642913015191.496094|true|forestry|calvin ovid|2027-08-13 20:32:10|2034-04-21 19:49:54.457098155|2062-06-17
+12|-4618|-1098379914|1789|36168.36|2705339.34|-3690347823231.547852|true|biology|holly johnson|2042-06-30 11:41:46|2018-07-12 01:39:37.64101956|2056-03-04
+71|27905|-1755088362|8698055291501543424|-19752.00|33234.12|688042997736.531250|true|american history|oscar ovid||2053-09-30 17:29:18.665697992|2009-03-20
+-71|-18503|597657990|296|-11362.53|-4728535.24|1779931909010.094727|true|religion|david falkner|2072-03-01 20:11:45|2037-09-21 12:33:06.143755779|2000-03-05
+11|10940|1687784247|-9095689235523264512|4559.16|-3019879.00|-874527173245.560547|true|zync studies|bob king|2014-07-11 06:14:44|2077-11-30 06:12:58.616698981|2034-11-19
+-50|5591||7998687089080467456||285146.00|3230421493464.880859|false|quiet hour|priscilla ellison||2042-01-18 02:32:10.14671907|2094-02-22
+-90|19986|-1808960215|8160569434550403072|-36016.61|-2975819.86|4812496108620.009766|false|zync studies|ethan davidson|2040-08-07 19:44:11|2076-10-03 23:03:12.999374869|2062-08-07
+4|-21009|-928013434|489|37854.16||4399713240319.478516|false|values clariffication|ulysses hernandez|2047-03-25 04:38:53|2054-05-31 05:02:46.363507150|2069-11-23
+20|-23241|-1701492480|-9175038118837149696|-26613.47|1842617.05|-874710832768.563965|true|american history|jessica carson|2037-03-24 21:59:48|2075-10-04 10:31:47.885981681|2055-06-11
+119|-6384|1187495452|8571268359622172672|-8449.84|-4234142.32||false|american history|xavier ellison|2058-04-04 16:57:08|2031-10-24 10:16:43.56250862|1981-11-13
+26|23468|-2136052026|-7916510129632296960||-3375.58|-932636901856.333984|false|||2063-08-14 11:27:00|2029-05-07 11:16:44.211702565|2093-04-29
+-43|24298|-1524554771|8323460620425330688|38854.35|-4288521.90|816149960555.847656|true|joggying|luke johnson|2039-12-24 23:32:35|2074-12-19 05:35:31.748853096|1994-05-03
+|-11767|-938756287|346|-14372.50||2123388956108.485352|false|philosophy|fred steinbeck|2023-06-14 17:07:18|2013-08-25 12:57:38.526376049|2068-09-07
+82|-6334|-564495517|3980||-3367097.22|-974002789414.605469|true|american history|sarah steinbeck|2077-09-25 01:00:46|2031-02-06 14:50:01.97188046|1986-09-18
+-34|-13335|1568180994|-7707242953271500800|43312.27|-4190279.40|3157550021812.021484|true|education|alice underhill|2077-03-07 09:32:11|2061-12-11 17:47:50.656826729|2033-07-02
+-41|10278|-922200749|1811|-5675.84|2126493.90||true|linguistics|ulysses polk|2021-09-26 02:58:42|2037-02-07 22:52:41.845445061|
+-3|16134|923980398|2803|47958.63|-123069.85|3302684840602.587891|true|xylophone band|katie van buren||2017-11-28 19:30:26.747429591|
+-48|-27284|-215703544|7370078518278397952|-44853.80|2971392.13|-2865549427171.991699|true|values clariffication|yuri thompson|2040-10-13 04:09:32|2042-06-15 18:08:24.614905000|1987-10-30
+122|-21805|1543611951|7497276415392407552|-21005.13|2798310.44|-4898025473643.805664|false|american history|tom quirinius|2027-09-08 19:38:10|2073-02-25 17:53:54.931182325|1974-09-07
+29|-4772|-2028355450|2323|-14463.60|-2063292.11|3614336014381.258789|false|quiet hour|quinn thompson|2040-10-17 05:30:18||2073-04-07
+22|-23622|916057807|8467976965865799680|-45069.06|-3169540.55|-3121144891991.676270|false|american history|katie ellison|2039-06-07 19:13:41|2064-10-13 03:30:27.872003019|2064-06-14
+54|25636|-1156193121|691|14845.29|2712830.87||false|industrial engineering|yuri johnson|2075-04-20 17:39:07|2036-11-16 16:14:15.788195860|2053-03-05
+61||-1462331586|1914|-19390.54|-2888414.09|-1458422440761.921875|false|topology|gabriella xylophone|2043-05-11 03:31:25|2016-03-24 12:00:25.125315901|2067-01-30
+-68|-27049|-941433219|6982145326341423104|-17847.79|2416407.91|3913372951778.839844|true|quiet hour|oscar white|2016-12-13 22:46:01|2042-12-23 09:34:08.712953951|2092-12-18
+-14|-5357|-671853199|-9203804401302323200|-49229.73|3168248.28||false|linguistics|fred falkner|2045-09-30 02:09:20|2069-07-07 15:06:02.724854172|2093-12-04
+-125|-23546|344239980|7823874904139849728|41579.55|1950776.66|1201402518499.143555|false|joggying|david hernandez|2034-11-18 20:10:51|2054-11-24 08:43:56.953788150|2105-09-20
+54|26155|-532755480|7534145866886782976|6967.23|-3226525.02|-1855303775517.675781|true|debate|rachel van buren||2055-01-08 05:36:23.516296700|2012-04-21
+-1|-14551|76381404|9085434340468473856|-35645.13|2319559.14|-308755508523.432617|true|quiet hour|rachel garcia|2017-01-24 05:00:43|2013-12-09 10:54:08.794663094|2052-11-16
+-97|-3619|-1545388906|8579974641030365184|-21168.24|-2764375.21|-3347449615248.976562|false|values clariffication|bob ellison|2055-05-20 19:13:26|2033-04-21 06:38:32.498310577|2017-11-30
+100|-6024|1723691683|8536948829863198720|47627.04|-1569764.42|-2866718883273.335938|false|zync studies||2054-06-22 01:40:38|2058-02-06 15:15:47.940520185|2072-11-26
+126|5516|278643258|341|-11228.83|634413.34||true|history|xavier ichabod|2022-09-19 23:14:38|2079-08-22 14:14:01.57697301|2067-03-01
+78|-10439|742866312|-9102482277760983040|17001.89|3890002.99|2756529687561.446289|true|history|victor zipper|2017-08-01 15:57:45|2070-05-02 05:20:55.632052613|2030-10-04
+-103|32210|-1254129998|658|-29957.29|-2795444.37|75939180315.778320||mathematics|ulysses brown|2027-08-02 14:55:50||2081-08-17
+80|-5089|-588547970||40137.08|4196447.38|2249064278951.813477|false|values clariffication||2044-10-13 20:35:22|2063-07-28 15:07:41.30831543|1990-09-22
+-17|-14705|-887790938|2843|27083.98|-4094095.34|-4538477190343.535156|true|study skills|priscilla ichabod||2060-07-21 05:29:07.891389875|2020-08-08
+41|-22910|1410516523|7584007864107778048|-26850.55|1924125.21|2728934571209.719727|true|education|yuri ovid|2069-09-25 12:15:22|2081-01-21 23:09:27.167408818|2105-06-13
+53|-16247|-186764959|590|-43932.82|1488803.45|426645021297.744141|true|yard duty|xavier steinbeck|2052-08-30 02:52:56|2057-09-22 01:17:56.602173912|1977-06-25
+124|-1067|-2146432765|8899122608190930944|-8534.65||3325424877285.349609|true|religion|nick van buren|2058-01-20 15:06:15|2077-04-04 20:32:46.931020333|2026-01-11
+-36|27156|850625480|3588|42187.91|1811969.63|-3976647851697.982910|false|topology|zach allen|2014-07-18 20:02:05|2039-08-04 00:06:01.390839073|2058-05-02
+104|10457|-1380191654|3609||4276263.85|-265494965739.847656|true|chemistry|wendy steinbeck|2076-10-08 03:34:56|2043-11-26 06:58:33.589491132|2082-02-15
+111|-260|1372224352|3824|-10063.81|3381864.81|4651675165956.277344|false|education|tom underhill|2015-09-12 05:40:29|2024-06-13 07:25:09.53150568|1983-10-31
+-41|276|-2124994385|7690986322714066944|10570.59|1504218.24|4721071679740.945312|false|undecided|ethan young|2057-12-06 16:20:41|2044-12-03 15:12:53.223286147|
+-98|-4286|1074488452|7765456790394871808|34635.92|2380624.11|1160277570495.260742|false|industrial engineering|tom polk|2046-08-09 21:35:24|2063-02-09 19:10:28.968599555|1985-02-08
+5||-500921094|-8649711322250362880|2739.41|2974096.30|-3103906430076.932617|false|debate|irene van buren|2078-11-04 19:08:18|2030-03-03 09:27:14.362039938|2065-12-20
+-106|11895|-628446314|1948|-16324.00|-4498818.60|1427707877153.423828|false|debate|||1970-01-01 00:00:00.172894766|2080-10-14
+86|-16390|-1918847735|-9101953184875757568|-11869.39|2819946.57|597670762650.892578|true|geology|priscilla miller|2048-11-20 13:09:10|2027-04-22 12:21:15.759028066|2038-12-26
+52|-30492|2031604236|2463|38249.67||2340697225707.053711|false|nap time|quinn johnson|2027-10-15 08:46:59|2066-12-22 01:05:49.826136001|2088-10-12
+-69|-13410|-738157651|1813|20147.37|4298419.86|-471086287570.549805|false|kindergarten|david allen||2078-02-24 23:37:09.324361343|2023-05-09
+50|-12422|-1266138408|7054271419461812224|33076.17|4726793.74|-1825975796516.701172|false|study skills|gabriella steinbeck|2034-09-27 07:53:11|2067-05-04 08:00:04.76094355|2045-08-30
+119|6913|1836499981|7548958830580563968|-789.80|-4853521.51|-2520439759952.566406|false|education|yuri davidson|2037-12-20 12:22:00|2060-03-04 05:27:06.411478902|
+-68|22588|2084666529|-9206329156028112896|-46857.55|4518647.92|3514007061626.251953|false|industrial engineering|mike brown|2025-09-06 20:02:42|2017-06-08 12:00:40.56190634|1976-03-22
+30|-26180|1522208504|2637|24228.24|4916304.14|2690767997858.607422|false|study skills|fred hernandez|2018-08-31 02:11:20|2034-06-17 01:40:03.264737765|2005-11-23
+-106|-25689|-693249555|-7661250850555633664|-24825.24|1483670.11|-4990192589949.166992|true|geology|nick king|2019-01-30 15:29:01|2016-03-23 21:48:37.760268674|2081-06-09
+113|24019|899810881|664|44245.25|4097479.70|294259545397.430664|true|biology|quinn garcia|2017-11-03 16:12:54|2065-09-09 02:55:30.992457875|2044-04-01
+-73|-7873||2487|-21944.26|-1353294.67|3144741114101.847656|true|american history|sarah quirinius|2068-12-01 14:02:38|2025-09-28 09:32:04.373329529|1984-03-30
+-29|-4211|-1626062014|8221561626658881536|-17167.97|2376404.62|-2396766266715.104492|true|religion|jessica allen|2076-10-17 13:16:32|2041-07-29 04:57:59.28367872|2052-06-27
+76|-19545|1765874562|8169878743136043008|4737.74|2522751.77|4492331567972.839844|true|undecided|mike ovid|2025-02-11 02:25:59|2057-07-18 07:47:20.80373888|2035-05-28
+120|15578|1668094749|6927260280037097472|-12442.12|-3625208.20|-2505949310240.964844|true|values clariffication|luke carson|2060-08-28 06:39:49|2042-08-21 19:21:53.22689204|2036-04-04
+-121|-32403|-884796655|342|43804.75||3086918973716.556641|true|philosophy|oscar robinson|2042-10-22 23:52:01|2027-05-27 10:44:21.656527878|2057-06-19
+-2|-27807|1809795770|-7501803640821456896|-31728.95|2217700.27|1303564912300.209961|true|linguistics|david ichabod|2058-03-12 04:48:33|2026-10-07 09:37:11.609376049|2096-11-02
+39|-2410|1134416796|2745|32844.17|-1496658.25|-634697885935.402344|false|opthamology|quinn thompson||2043-04-08 21:14:44.886763154|1974-03-31
+57|-3449|-1038565721|677|-32939.72|3205392.52|-3723568349288.726074|true|yard duty|wendy laertes|2039-08-20 17:31:07|2043-12-22 07:02:29.625016869|2068-08-18
+-52|-19028|-2081809883|8435912708683087872|-49142.74|-1903864.82|3512222649259.086914|false|mathematics|gabriella brown|2040-06-21 05:21:16|2058-07-03 23:16:43.395150800|2059-05-01
+-123|12674|1817671655|7412924364686458880|22076.92|1732310.78|-198468750423.259766|true|values clariffication|yuri van buren|2062-02-20 04:56:42|2074-11-24 19:41:00.713104566|2003-07-31
+-76|-7533|1332181668|3563|16233.53||3796582343071.613281|false|yard duty|ulysses laertes|2029-10-24 07:37:56|2071-10-19 15:34:38.872812372|2071-04-18
+-119|-14053|8040290||9147.84|-4731232.91|1812411477716.757812|false|values clariffication|holly ichabod|2031-01-05 20:55:56|2050-12-24 02:26:09.954349620|2097-09-19
+110|-17426||7153922334283776000|10695.41|-4157898.82|-3205627224037.357422|true|yard duty|calvin ichabod|2022-06-25 07:08:22|2061-01-13 03:07:32.453720097|2066-09-03
+-28||718692886|8849475396952514560|-22341.30|-1839215.40|-2738602396779.903320|false|opthamology|victor white|2030-06-25 00:58:04|2061-03-21 18:52:13.283910292|2095-01-04
+90|-16129|-2007662579|2977|15304.02||-4049128118326.663574|false||wendy quirinius|2080-06-24 16:38:20|2017-06-18 02:28:45.172735440|2050-09-13
+96|1409|-1259611508|-7910019233726242816|-38530.51|-44517.83|-3803225081574.588379|true|industrial engineering|calvin brown|2017-03-12 03:01:53|2033-08-29 02:29:46.117205169|2010-04-10
+117|-32688|144428297|2835|-20190.35|3425255.76|1239390523307.939453|false|opthamology|rachel falkner|2056-11-05 14:04:47|2075-05-31 12:50:08.854908788|2084-04-26
+55|999|1281277970|2335|-2914.82|||false|opthamology|fred young|2072-08-24 05:04:35|2058-05-10 01:43:32.344081382|2029-07-31
+41|28165|1069486136||-37952.82|-276339.96|3109359968514.944336|false|opthamology|victor ichabod|2069-02-08 14:25:38|2076-05-15 06:44:30.554693842|2053-03-25
+-86|23850|-601946913|2515|10326.69|4649531.05|2268855975314.246094|false|topology|ulysses robinson|2013-04-25 03:23:27|2056-06-03 12:33:31.602556689|1984-05-06
+-101|718|386741352|-7617860842651017216|-42832.01|2064169.43|-4943806409743.167969|false|industrial engineering|sarah ichabod|2057-03-27 09:22:04|2064-01-08 17:40:10.787568720|2020-01-01
+-127|-10662|648935848|-7637755520917741568|-33941.20|1359686.42|1579030228803.083984|true|chemistry|luke laertes|2016-12-23 11:45:20|2036-05-11 12:47:07.733851638|2003-04-13
+80|-1223|92834720|2647|-36870.03|1719246.27|2540178498144.847656|true|nap time|bob falkner|2038-06-08 07:54:27|2042-11-05 13:52:50.588354549|
+-3|22320|1343581455|707|15471.72|-4962575.43|-1611227671309.833984|false|study skills|mike robinson|2078-04-16 04:19:35||2048-11-11
+||-4943292|8856674723376668672|2164.62|893829.85|-631328089657.294922|false|opthamology|oscar quirinius|2027-03-09 19:18:57|2054-01-23 04:24:15.292126981|2060-03-20
+-50|2616|977624089|7857878068300898304|-30457.66|-1477221.11|859058050811.647461|true|american history|ulysses ellison|2035-05-02 05:58:22|2050-12-28 10:26:48.409801540|
+3|-17916|-191704948|-8887058200926093312|35825.27|4647161.69|2477546519048.629883|true|mathematics|wendy carson|2027-09-25 05:34:03|2032-03-05 14:53:42.83179281|2024-12-19
+100|-5919|-835107230|108|-37662.39|-3446860.49|4914447452945.439453|false|kindergarten|sarah carson|2031-07-27 00:13:59|2066-12-30 00:14:45.43797107|2058-07-30
+37|-24696|314232856|2762|-9928.97|-2740875.06|3528496539480.343750|false|geology|fred thompson|2032-05-15 14:11:38|2057-07-08 21:29:58.104912394|2091-09-09
+27|-8398|-1583445177|3622|32876.52|285737.42|-4676053735656.750977|true|undecided|luke xylophone|2071-11-07 15:31:50|2021-08-10 08:22:04.527867145|2086-02-10
+|-14644|-2133145181|868|-5084.93|-77615.10|4647718670978.201172|true|chemistry|yuri thompson|2030-06-15 16:57:13|2063-01-20 09:04:49.362142228|
+36|-7046|843282593|138|-1437.75|1530832.04||true|biology|zach van buren|2049-05-13 12:51:19||2039-05-11
+31|-10258|-1009249550|1786|-45853.99|-2996009.61|-1270664267650.971680|true|zync studies|calvin laertes|2054-10-23 17:10:03|2054-10-22 11:44:42.793294685|2064-05-02
+||-236700442|9116137265342169088|13586.84|-3397114.60|1665543214302.829102|true|biology|holly carson|2076-12-15 12:30:43|2067-07-10 06:37:03.526969971|2038-06-13
+-8|-25988|1447438548|7955126053367119872|-37404.36|-4278525.05|-1539681145987.756836|true|forestry|david thompson|2056-07-24 05:41:13|2038-11-27 05:16:00.826863713|1976-09-08
+-93|4747|-201554470|491||-4736059.79|-4074148068973.322266|false|religion|luke brown|2052-04-12 11:55:51||2019-08-20
+18|-26064|-807242371|-7778829032042790912|-497.18|3018359.10|4165268868128.960938|false|history|quinn xylophone|2064-08-04 14:39:21|2049-07-11 03:38:36.188505486|1979-06-14
+-2|28921|-1599905147|-7753051494275432448|38746.13|1924853.86|2728705276812.348633|false|chemistry|mike miller|2063-11-22 05:40:00|2040-04-01 02:20:42.508563674|1999-07-08
+|-26946|-329695030|8962097525980225536|47503.36|-817384.58|955232900660.511719|true|zync studies|holly van buren|2020-01-09 00:26:09|2033-08-15 04:18:59.595230815|2066-06-06
+0|-2835|1968813171|8163948965373386752|8908.65|-3820148.18|-481479853467.270508|false||sarah miller||2068-01-02 01:54:57.346312106|2075-09-25
+114|20265|669484010|1145|13147.10|-1843756.18|554432518470.839844|false||yuri ovid|2039-05-04 13:47:10|2038-08-21 17:57:56.403669763|2081-07-12
+|28775|232405034|-8438554249514491904|46640.35|-4159360.82|-255361794196.424805||education|quinn robinson|2026-01-11 21:41:33|2040-12-26 20:16:11.824788390|2024-05-19
+111|22074|-853606287|522|35504.72|1790821.65|38856848277.219727|true|american history|alice van buren|2054-05-22 23:30:43|2046-11-03 15:13:41.781244421|2079-09-30
+55|4319|-524189419|1785|34553.22|-1096127.27|4436479409217.000000|false|biology|xavier van buren|2079-08-25 13:19:06|2080-03-03 09:31:40.884489078|1979-09-24
+51|-800|564366133|1545|-31100.82|-1936194.76|1798080391637.881836|false|undecided|alice xylophone|2065-08-30 18:15:20|2055-12-07 19:37:09.85869965|2078-10-08
+107|11159|-346607939|999|-24238.72|794247.13|835410305018.271484|true|forestry||2072-11-24 12:45:43|2027-01-26 04:52:45.940702976|2001-01-27
+-116|-31182|-54793232|1941|-33327.99|-4081209.43|-4789172370818.662109|true|wind surfing|jessica van buren|2062-10-28 14:12:34|2047-11-19 19:37:15.731253997|1983-02-23
+-10|-32371|1333214263||-32279.82|4486493.62|4879631964298.525391|true|undecided|quinn falkner|2045-05-25 12:09:59|2068-07-04 02:37:10.599103732|2089-02-09
+80|21377|-267554590|7454442625055145984|8701.65|-905741.94|1537464945669.313477|false|joggying|david van buren|2077-05-12 07:35:39||2061-02-10
+-104|-3190|197056787|3510|-22360.28|4523037.34|3240433039663.095703|false|xylophone band|xavier white|2051-07-06 22:05:57|2051-01-02 05:43:39.718899624|1996-03-18
+-64|6163|-1602792666|2373|16736.31|-3740028.62|3690526967552.294922|true|joggying|yuri brown|2040-02-10 20:14:48|2080-02-11 07:48:36.902436157|2000-11-17
+-30|-24670|1701817607|-8127494999848919040|-13803.97|3963022.39|1474152640583.561523|true|geology|tom hernandez|2079-07-30 04:14:31|2057-05-04 01:14:00.609761058|1974-09-16
+-74|-23109|1346627771|1643|-43168.85|3155689.66|-4448781974974.628906|true|forestry||2039-08-04 16:00:45|2028-08-01 19:19:13.322219484|
+118|-23389|-370093295|-7819437864839495680|-26609.73|-4237190.22||false|kindergarten|bob thompson|2019-09-22 15:35:21|2037-04-28 10:43:14.893096143|2021-09-12
+69|24253|60847311|-7822452149325094912|-40655.11|-3881485.82|-1314247053468.484863|false|debate|oscar young|2067-05-18 04:05:27|2039-10-06 08:12:08.884589651|1984-06-30
+-5|-32000|-177025818|7411793502161182720||2485530.29|1594683877449.583008|true|opthamology|yuri young|2035-11-20 23:20:18|2078-01-19 12:17:21.914800889|2015-02-06
+-35|7474|-1676261015|2274|38942.74|-37023.70|3451540509748.127930|true|opthamology|katie polk|2043-05-05 01:41:36|2044-09-13 08:37:07.823303166|1973-01-15
+-102||-714270951|8783241818558193664|-24442.85|50032.07|3379969757973.632812|false|topology|calvin xylophone|2049-08-05 08:30:47|2065-10-10 06:49:24.92342328|2078-07-28
+84|-18485|345556325|8316336224427483136||4778.81|-213703503553.759766|true|geology|irene laertes|2027-03-23 13:59:49|2068-08-29 13:45:16.22118413|2004-10-29
+57|-19535|-1484033125|-7669169138124275712|5132.56|-3812127.71|-2880380214433.984375|false|values clariffication|xavier ellison|2013-05-28 14:20:54||2032-11-10
+19|-16815|1440427914|2984|-33347.21|-2217545.87|-1622352479297.113281||xylophone band|katie garcia|2037-08-01 18:21:41|2074-09-05 16:07:02.893370421|2050-05-01
+-42||-1201785350|-7772064021830574080|-19385.56|-2417269.98|4254639589730.294922|true|wind surfing|quinn thompson|2056-10-03 11:35:20|2050-10-14 15:57:47.94057206|1994-04-07
+-26|-20787|-2066134281|3397|38532.91|-2052731.66|-2212037711168.371094|false|xylophone band|alice zipper|2081-02-22 01:21:53|2066-09-15 08:59:43.712981923|2068-06-30
+-27|-20093|-226635871||39730.55|2920764.76|3736425396480.705078||american history|alice davidson|2063-06-30 00:25:29|2039-10-04 08:34:32.485344607|
+-52|17720|2134433675|8523972434954510336|-41237.62|713755.68|-2484484479175.712402|true|education|irene ovid|2035-11-11 02:58:59|2036-10-18 01:59:34.57776882|2071-03-20
+-28|28089|260463232|-7127548949860818944|44331.03|-3844217.64|-1326537974723.827148||undecided|ethan davidson|2041-01-01 12:36:24|2023-11-30 11:12:45.269637203|2029-01-15
+3|6587|-916495008|8286706213485297664||-454411.58||true|xylophone band|priscilla polk|2066-01-13 23:59:59|2028-04-05 20:31:53.571987516|2031-08-14
+-96|-7065|1102069050|3147|-5252.56|2525210.28|-2140534616823.686035|false|wind surfing|oscar polk|2078-12-19 18:51:30|2024-01-19 18:54:53.308875328|2070-05-01
+-87|-134|-1289501869|-7536330682873937920|-32963.40|4026456.79|-4233812360710.160645|false|american history|irene king|2076-12-17 13:17:28|2026-03-07 02:44:03.954816308|2039-03-16
+-88|-29977||-7115054815375073280|-49346.30|-4958976.45|-2795424166054.196777|false|mathematics|zach steinbeck|2033-10-10 03:50:46|2072-01-30 01:12:19.980530173|1988-11-14
+-54|11077|-235238928|-7319315187617587200|-21327.60|3102506.04|3723093064701.678711|false|biology|quinn hernandez|2050-07-10 07:39:25|2067-01-01 12:42:39.897268043|1977-03-31
+-127|14408|1390704286|1099|-26406.94|4860414.94|4855359545894.408203|false|quiet hour|mike miller|2057-01-23 19:19:39|2064-10-13 08:42:40.250692511|2030-08-25
+-110||1310360849|-8989473881707921408|29776.70|4921339.21|-1711490983970.094238|false|american history|mike van buren|2017-06-03 10:24:21|2017-06-10 20:23:52.816880923|2042-06-26
+-108|6226|-912429611|2816|30162.78|682391.80|618292737301.046875|false|debate|sarah steinbeck|2059-03-16 19:54:08|2015-05-02 04:32:12.949465541|2008-06-28
+60|18845|-1369253050|-6986178228432322560|-35269.73|2961506.86|1006481369666.687500|false||katie white|2022-02-26 02:53:55|2031-08-28 00:04:15.166189356|2054-08-15
+46||-938762477|-7759425383684849664|46392.36|1976120.50|4269437008221.785156|false|mathematics|ethan steinbeck|2061-01-29 15:15:46|2064-07-10 10:33:03.635139975|1978-09-04
+-71||268888160|-7893577088764174336|-35474.68|-4088895.16|-617279427364.288086|false|nap time|bob hernandez||2040-02-11 11:11:34.256681234|2061-05-04
+91|22232||8091421389575282688|9502.48|-3837325.10|-1866384122906.517090|false|zync studies|holly young|2043-03-07 03:52:18|2033-07-28 01:11:14.500349492|2010-08-03
+77|-6884|-624029057|-7409653086454030336|39017.19|-2245679.39|-997439424357.757812|false|debate|irene robinson|2077-03-18 03:11:00|2040-04-20 17:23:11.66448920|2025-08-31
+-101|12411|-1940205653|7348598907182800896|43885.62|-4414072.29|-1555511953931.052246|false|american history|david xylophone|2029-10-21 01:48:10|2043-08-31 14:12:30.675070864|2085-10-24
+-57|11804|-496915240|-7362189611124563968|-8203.40|3975044.25|1092860346014.425781|false|chemistry|oscar allen|2042-05-21 21:47:45|2031-01-19 14:17:45.573707182|1978-09-30
+59|27945|474795096|||-1041376.93|1049208781647.408203|true|zync studies|mike young|2042-08-04 13:26:56|2076-09-18 22:30:21.635931282|2100-08-21
+|6490|-1819075185|2465|271.01|-3315662.78|178512873486.944336|true|philosophy|luke miller|2058-11-13 12:26:19|2055-04-29 01:32:52.338899070|2072-05-20
+59|-13144|330302407|350|4198.95|-3293708.78|-837074147937.393555|false||calvin davidson|2048-05-03 07:09:22|2016-02-14 09:35:08.980648052|2050-12-28
+-58|-8895|1895751360|2619|44281.94|-3137628.74||false|topology|ethan underhill|2067-05-09 03:59:27|2070-01-06 00:22:49.266686299|1988-03-03
+41|-5283|-1322736153|3722|33698.34|4876900.77|2424764462849.715820|true|wind surfing|jessica polk|2016-04-22 18:45:33|2078-05-27 02:08:56.801166778|1997-08-03
+56|-22608|338805871|898|4188.81|4744554.21|730519058324.112305||mathematics|priscilla falkner|2069-12-21 21:41:25||2070-01-08
+-56|10702|-1552053883|782|-38874.21|-1017367.57|4561020549763.021484|true|religion|calvin xylophone|2071-01-01 17:03:06|2016-11-14 05:12:35.708167457|2080-10-11
+17|-9585|-71433796|1780||509725.97|-1030876895734.564941|false|religion|mike johnson||2077-05-06 09:41:04.29662719|1978-11-15
+110|-24095|-1655396452|2186|-45152.50|3996472.52|888655102684.872070|false|quiet hour|mike hernandez|2065-12-16 23:03:21|2065-07-26 03:28:40.392848789|1991-03-29
+52|21673|-1066775085|-6921654334727036928|-9981.07|-1650150.10|-4371310034780.968750|false|history|katie zipper|2031-09-27 21:23:48|2080-08-25 23:49:27.604332416|2069-04-08
+-113|20052|-1447140800|4020|-22503.13||-4288708971525.778809||wind surfing|mike underhill|2079-05-03 00:37:57|2078-03-29 14:33:49.853013719|2043-01-06
+-61|28000|-178568841|8325227661920133120|-22983.25|1809174.68|-2297090489976.945312|false|history|nick quirinius|2069-09-27 08:08:25|2043-04-20 02:53:33.962471217|2044-06-19
+102|-31458|-1933192293|-7036607470351654912|-21380.15|1896367.69|1605659965343.057617|true||rachel white|2015-06-12 13:38:42|2068-03-09 14:21:05.750561437|2099-06-14
+-8|-25417|1961954939|7304839835188609024|-41942.73|191389.71|-4592978910553.853516|false|undecided|nick ovid|2035-11-09 10:25:30|2077-04-15 14:29:55.59700199|2010-11-12
+41|-10569|-745678338|8792059919353348096|-14404.47|-906177.24|326664391884.108398|false|philosophy|priscilla steinbeck|2034-04-20 09:29:51|2042-10-01 01:12:43.143080614|2048-10-17
+-41|4233|618321042|-8856821118526734336|-27396.30|4504910.53|180650253126.657227||debate|tom xylophone|2027-02-11 05:29:58|2066-07-09 11:19:47.657330816|1995-12-16
+-93|18820|825677248|8720504651219001344|9437.05|-4519097.40|-506526954412.187500|true|philosophy|alice miller||2077-05-05 12:20:54.799217277|2001-02-16
+33|-3103|371383749|1055|-43829.05|-3580973.49|3029406193241.257812|true|wind surfing|jessica quirinius|2026-05-11 10:04:20|2074-01-01 07:56:24.948489929|2096-02-29
+-95|26529|-223311809|1368|-19919.52|2770068.53|-2537836240149.431641|false|debate|gabriella miller|2040-03-14 14:51:41|2037-09-24 04:28:20.146749934|2102-01-22
+79|-28084|-1974972123|8736061027343859712|-28922.94|-2498365.96|-985441433336.760254|false||rachel laertes|2032-06-23 14:29:26|2023-01-25 12:10:28.866537625|2105-06-23
+70|6781|2125311222|7919597361814577152|47154.09|-1192322.16|-850435843660.583984|false|religion|alice polk||2065-10-31 11:02:12.417523639|
+-1|22463|867587289|7381659098423926784|-29302.26|-2211405.89|1156004227362.394531|false|study skills|alice white|2038-06-11 11:52:35|2026-02-06 12:13:27.232943870|2090-01-01
+15|7392|869288953|8731960288562044928|41942.98|3624760.68||false|zync studies|alice miller|2058-09-06 21:08:50|2060-02-22 03:34:09.543087589|2076-07-25
+-57|-24942|824743780|-7594824008626372608|35489.13|1006528.68|-4120539026250.966797|false|geology|priscilla falkner|2066-03-31 07:55:39|2053-11-01 10:52:13.944770605|2080-12-02
+-26|-14810|-1407817977|-9178166810751909888|37075.67|-4876175.79|2103363485344.971680||chemistry|tom quirinius|2026-05-24 15:05:33|2014-12-24 04:57:37.782711517|2000-06-22
+-70|6484|-385247581|3083||-2073947.26|890806455827.118164|true|values clariffication|ethan steinbeck|2018-06-04 10:12:46||2055-12-17
+79|-15946|-1078397698|-8948335470186373120|-22913.97|227203.97|4075094455625.994141|true|industrial engineering|david robinson|2033-07-28 05:43:26|2020-05-29 18:18:23.223813235|1982-05-04
+-75|-17873|-837503491|2569|6361.99|1161789.65|-1726978053243.932617|true|values clariffication|quinn robinson|2045-04-18 18:19:14|2021-01-27 02:30:40.164402829|2017-01-10
+96||1660088606|823|-28084.37|-3275847.50|2762686693256.810547|false|american history|ulysses nixon|2070-08-11 19:10:10|2016-09-07 01:20:11.409060890|
+-113|-30515|-922875124|259|4946.14|284363.03|-4858481689197.126953|false|kindergarten|fred nixon|2023-06-28 22:37:13|2049-06-26 22:24:15.711126823|2069-02-06
+49|3186|1636364987|8461498293348065280|-25371.38|-1184912.70|-993292644578.538574||education|alice white|2058-11-21 02:58:56|2065-03-02 04:53:08.176223587|1994-05-02
+63|150|1783034168|-8961059046745669632|||-1661232720992.700195|true|topology|quinn young|2038-09-04 07:44:08|2047-01-27 11:44:14.355445111|2043-02-21
+74|-21648|-1111937842|-8607195685207408640|-38757.30|4280929.21|-3715598460104.345703|true|joggying|irene white|2017-10-30 12:48:38|2013-03-25 15:32:17.128329365|2017-10-02
+79|3251|-125419186|-8754966081778565120|32512.77|1838279.26|-1177230386696.095215|true||zach laertes|2059-05-28 03:36:52|2049-09-18 13:52:36.954456644|2013-05-30
+96|-10126|-41864614|-8418913260807217152|-41581.70|3132782.56|-4989436752632.237305|true|opthamology|victor king|2025-08-12 00:24:07|2025-04-07 14:05:54.461436636|1987-03-18
+|-6474|706823078|-8877053610728161280|-32523.08|2431805.10|959431399353.079102|true|joggying|calvin ovid|2024-11-30 09:52:29|2026-10-20 13:44:56.808161722|2038-08-30
+-85|24858|-1062159435|-6935548339131138048|38973.86|-347553.95|3408594399945.599609|false|biology|oscar zipper|2026-05-18 20:07:36|2018-09-17 07:34:48.856373956|2044-03-13
+-44|30883|1452244326|-8219876839318716416|24860.92|-3405708.68|-3460325348420.880859|true|education|wendy quirinius|2038-08-09 15:24:29|2054-12-15 15:20:18.843218245|2061-01-18
+88|-9076|239078089|1132|28538.60|-735298.99|-4690879292129.750000|true|wind surfing|zach quirinius|2071-11-14 00:07:27|2047-06-15 18:39:40.150298534|2079-04-15
+48|23320|-1948257321|1337|-43790.15|3928891.37|3704624417356.896484|false|joggying|tom hernandez|2053-10-13 18:39:56|2026-03-31 20:18:01.770207236|2034-08-29
+-98|7197|492120544|1341|-2808.88|-686072.73|-803606583331.628418|true|education|tom xylophone|2015-05-08 21:19:10|2077-10-24 08:05:04.192595595|
+72|7058|-1563676282|976|-3004.90|1860479.52|-706916831724.306641|true|joggying|victor polk|||2058-01-01
+38|-14661|195281533|-7557017910095650816|-9383.79|-2348124.75|4636426753661.630859|true|mathematics|tom thompson|2067-05-15 15:21:33|2068-10-26 13:18:32.366613022|1975-07-07
+16|-28061|-292588406||23350.65|4918613.06|-3364353490553.036133|true|nap time|calvin steinbeck|2054-01-11 20:28:18|2037-02-15 00:30:35.900105726|1970-08-29
+-104|19636|-1344287228|-8683802826440105984|8738.40|-1978806.64|4981312863247.886719|false|biology|holly davidson|2017-11-02 22:33:43|2061-05-26 11:57:49.809511168|1993-04-14
+-59|-2113|-909024258|1845|41057.97|213462.82|-323293361486.882812|false|xylophone band|ethan carson|2049-05-23 19:46:12|2079-02-24 16:42:08.448187732|
+70|-13309|1173098061|1965|1703.69|2429715.56|-2779513442254.710938|false|joggying|calvin brown|2079-05-18 11:45:16|2018-04-22 08:38:53.938611457|2020-02-01
+19|-17269|-1091003492|-8104684579106914304|-1914.23|281948.98|2761939531515.835938|true|kindergarten|holly robinson|2044-05-20 18:29:37|2037-05-21 10:21:17.589154522|2003-03-13
+-96||1768399622|1835|5444.81||-137609076468.194336|false|xylophone band|ulysses king|2031-03-19 20:16:45|2034-05-25 03:38:47.816839184|2064-05-20
+-28|10916|849859032|7345991518378442752|-3117.17|1217133.63|-3849320444967.645508|true|linguistics|irene quirinius|2060-08-22 17:37:02|2013-09-04 14:35:51.471949124|2092-02-05
+10|19458|900992177|3212|36450.28|1766157.28|-1882794242083.676270|true|yard duty|fred king|2042-03-05 10:51:23|2075-06-04 16:36:19.586166561|2094-02-10
+52|-18283|-1969751342|-7081500255163727872|36032.40|4272798.19|-3071209073548.714844|false|opthamology|victor ichabod|2034-03-30 05:28:25|2026-11-18 10:38:29.612055317|2045-12-20
+125|16516|161210995|1074|-24350.82|2460973.22|921740273811.693359|true|linguistics|priscilla thompson|2024-07-31 16:19:23|2052-06-19 19:48:51.128000435|2031-11-09
+62|30936|1321678350|8372588378498777088|1059.15|-3300550.22|3839852559580.664062||study skills|ulysses ovid|2049-11-14 21:22:02|2034-12-30 19:24:13.874541216|2037-11-28
+-56|-23387|-1811563127|-7593363318079610880|-17859.26|-4870696.94|-1931143655238.862793|false|philosophy|jessica hernandez|2017-09-26 18:25:13|2023-12-17 02:12:59.770265368|2026-03-19
+97|10217|1338047392|-7451660755269853184||587440.58|3232315873104.952148|true|undecided|victor ovid|2041-06-18 17:20:25|2029-12-16 14:12:32.298550349|1987-03-13
+50|-12085|-1954890941|1983|-23284.45|129671.37|-2210750828357.674805|false|values clariffication|gabriella van buren|2055-07-03 23:20:38|2018-04-10 00:19:11.305802202|2103-07-27
+52|-13805|415234946|8514851182589771776|-39177.49|3119458.07|-4784612318346.788086|false|forestry|bob brown|2019-06-19 22:27:39|2065-11-12 11:35:37.144557823|2041-11-16
+104|||1864|13318.31|-3863136.50|-4419006380365.076172|false|joggying|luke steinbeck|2057-07-21 14:03:34|2026-03-06 18:45:53.361673899|2031-03-23
+-81|25986|-1643714866|8463868417649524736|1656.77|-1600629.58|-4804769067544.612305|true|religion|luke ovid|2019-02-21 09:30:51|2051-08-22 00:41:24.173555438|2041-04-15
+114|-28840|1335803002|3094|-22269.41|1188501.65|-2026377024912.235840|true|mathematics|david young|2054-03-31 08:13:24|2037-11-06 13:20:44.525490319|2093-08-21
+28|27340|-1117019030|-8858063395050110976|17746.57|-370973.48|3665311301420.927734|true|joggying|calvin xylophone|2016-06-21 09:03:55|2038-11-18 02:43:13.946089513|1996-04-09
+87|6875|-980869630|1981|-8269.53|-680305.04|-2046202768950.067383|false|forestry|oscar falkner|2054-03-28 23:22:20|2057-06-06 19:13:12.719272020|1984-12-16
+-38|19894||-8140349174954893312|42897.60|-895172.87|-2612690969145.852051|false|religion|irene ichabod|2029-06-18 22:32:24|2069-05-05 07:02:12.743691084|1993-05-09
+-94|9948|-455114104|-7041362811802148864|35927.66|2125413.96|928631125022.398438|false|american history|sarah zipper|2075-08-11 09:15:18|2025-01-16 11:03:58.820006581|2047-04-09
+90||1709983738|8972161729142095872|27727.80|3584555.68|2487331275679.380859|true|kindergarten|wendy zipper|2075-07-22 03:37:27|2070-11-18 17:07:35.641105448|2005-06-22
+-55|-4877||7989119273552158720|49881.20|-729941.08|2604782698166.197266|true|education|mike van buren||2059-09-21 17
<TRUNCATED>
[08/50] [abbrv] hive git commit: HIVE-10710 Delete
GenericUDF.getConstantLongValue (Alexander Pivovarov,
reviewed by Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10710 Delete GenericUDF.getConstantLongValue (Alexander Pivovarov, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/55b28a0a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/55b28a0a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/55b28a0a
Branch: refs/heads/spark
Commit: 55b28a0a93240270956688ec4bb5a98337bceb00
Parents: d362985
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Thu May 14 10:12:06 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Mon Jun 1 14:35:48 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 23 --------------------
1 file changed, 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/55b28a0a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
index b043bdc..e8df5d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
@@ -538,29 +538,6 @@ public abstract class GenericUDF implements Closeable {
return v;
}
- protected Long getConstantLongValue(ObjectInspector[] arguments, int i)
- throws UDFArgumentTypeException {
- Object constValue = ((ConstantObjectInspector) arguments[i]).getWritableConstantValue();
- if (constValue == null) {
- return null;
- }
- long v;
- if (constValue instanceof IntWritable) {
- v = ((LongWritable) constValue).get();
- } else if (constValue instanceof IntWritable) {
- v = ((IntWritable) constValue).get();
- } else if (constValue instanceof ShortWritable) {
- v = ((ShortWritable) constValue).get();
- } else if (constValue instanceof ByteWritable) {
- v = ((ByteWritable) constValue).get();
- } else {
- throw new UDFArgumentTypeException(i, getFuncName()
- + " only takes LONG/INT/SHORT/BYTE types as " + getArgOrder(i) + " argument, got "
- + constValue.getClass());
- }
- return v;
- }
-
protected String getArgOrder(int i) {
i++;
switch (i % 100) {
[46/50] [abbrv] hive git commit: HIVE-10955 : CliDriver leaves tables
behind at end of test run (Ashutosh Chauhan via Hari Sankar Sivarama
Subramaniyan)
Posted by xu...@apache.org.
HIVE-10955 : CliDriver leaves tables behind at end of test run (Ashutosh Chauhan via Hari Sankar Sivarama Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3863be0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3863be0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3863be0
Branch: refs/heads/spark
Commit: d3863be028ea7cba46f80dd7937d632e849a65fb
Parents: cbeee78
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Sat Jun 6 15:56:13 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 6 15:57:44 2015 -0700
----------------------------------------------------------------------
data/scripts/q_test_cleanup.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d3863be0/data/scripts/q_test_cleanup.sql
----------------------------------------------------------------------
diff --git a/data/scripts/q_test_cleanup.sql b/data/scripts/q_test_cleanup.sql
index 29659a2..4620dcd 100644
--- a/data/scripts/q_test_cleanup.sql
+++ b/data/scripts/q_test_cleanup.sql
@@ -16,7 +16,7 @@ DROP TABLE IF EXISTS dest_j1;
DROP TABLE IF EXISTS dest_g1;
DROP TABLE IF EXISTS dest_g2;
DROP TABLE IF EXISTS fetchtask_ioexception;
-
+DROP TABLE IF EXISTS alltypesorc;
DROP TABLE IF EXISTS cbo_t1;
DROP TABLE IF EXISTS cbo_t2;
DROP TABLE IF EXISTS cbo_t3;
[24/50] [abbrv] hive git commit: HIVE-10892 : TestHCatClient should
not accept external metastore param from -Dhive.metastore.uris (Sushanth
Sowmyan, reviewed by Thejas Nair)
Posted by xu...@apache.org.
HIVE-10892 : TestHCatClient should not accept external metastore param from -Dhive.metastore.uris (Sushanth Sowmyan, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4140da59
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4140da59
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4140da59
Branch: refs/heads/spark
Commit: 4140da59658d0a7d3b59d827fe0ba582b1e2816e
Parents: b4f57d8
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Wed Jun 3 14:58:22 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Wed Jun 3 14:59:05 2015 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4140da59/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
index 5f94b13..8992d552c 100644
--- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
+++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
@@ -131,7 +131,7 @@ public class TestHCatClient {
public static void startMetaStoreServer() throws Exception {
hcatConf = new HiveConf(TestHCatClient.class);
- String metastoreUri = System.getProperty(HiveConf.ConfVars.METASTOREURIS.varname);
+ String metastoreUri = System.getProperty("test."+HiveConf.ConfVars.METASTOREURIS.varname);
if (metastoreUri != null) {
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metastoreUri);
useExternalMS = true;
[07/50] [abbrv] hive git commit: HIVE-10302: Load small tables (for
map join) in executor memory only once [Spark Branch],
rebased with master (reviewed by Xuefu)
Posted by xu...@apache.org.
HIVE-10302: Load small tables (for map join) in executor memory only once [Spark Branch], rebased with master (reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d3629859
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d3629859
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d3629859
Branch: refs/heads/spark
Commit: d3629859a2cdc3f45a83582db1e100cd9f3a482f
Parents: 441d4c0
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Mon Jun 1 14:09:08 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:09:08 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/MapJoinOperator.java | 6 ++++-
.../hive/ql/exec/spark/HashTableLoader.java | 23 +++++++++++++++++---
.../ql/exec/spark/HivePairFlatMapFunction.java | 1 +
.../hive/ql/exec/spark/SparkUtilities.java | 6 +++++
4 files changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index b1352f3..15cafdd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.HashTableLoaderFactory;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
@@ -50,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.Reusable
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
+import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -521,7 +523,9 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
// in mapreduce case, we need to always clear up as mapreduce doesn't have object registry.
if ((this.getExecContext() != null) && (this.getExecContext().getLocalWork() != null)
- && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())) {
+ && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())
+ && !(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")
+ && SparkUtilities.isDedicatedCluster(hconf))) {
if (isLogInfoEnabled) {
LOG.info("MR: Clearing all map join table containers.");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
index 043f1f7..1d674e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
@@ -113,15 +113,32 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
}
String fileName = localWork.getBucketFileName(bigInputPath);
Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte) pos, fileName);
- LOG.info("\tLoad back all hashtable files from tmp folder uri:" + path);
- mapJoinTables[pos] = mapJoinTableSerdes[pos].load(fs, path);
+ mapJoinTables[pos] = load(fs, path, mapJoinTableSerdes[pos]);
}
} catch (Exception e) {
throw new HiveException(e);
}
}
- @SuppressWarnings("unchecked")
+ private MapJoinTableContainer load(FileSystem fs, Path path,
+ MapJoinTableContainerSerDe mapJoinTableSerde) throws HiveException {
+ LOG.info("\tLoad back all hashtable files from tmp folder uri:" + path);
+ if (!SparkUtilities.isDedicatedCluster(hconf)) {
+ return mapJoinTableSerde.load(fs, path);
+ }
+ MapJoinTableContainer mapJoinTable = SmallTableCache.get(path);
+ if (mapJoinTable == null) {
+ synchronized (path.toString().intern()) {
+ mapJoinTable = SmallTableCache.get(path);
+ if (mapJoinTable == null) {
+ mapJoinTable = mapJoinTableSerde.load(fs, path);
+ SmallTableCache.cache(path, mapJoinTable);
+ }
+ }
+ }
+ return mapJoinTable;
+ }
+
private void loadDirectly(MapJoinTableContainer[] mapJoinTables, String inputFileName)
throws Exception {
MapredLocalWork localWork = context.getLocalWork();
http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
index 2f137f9..7df626b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
@@ -48,6 +48,7 @@ public abstract class HivePairFlatMapFunction<T, K, V> implements PairFlatMapFun
protected void initJobConf() {
if (jobConf == null) {
jobConf = KryoSerializer.deserializeJobConf(this.buffer);
+ SmallTableCache.initialize(jobConf);
setupMRLegacyConfigs();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d3629859/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index ef612bd..ca0ffb6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.UUID;
import org.apache.commons.io.FilenameUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -108,6 +109,11 @@ public class SparkUtilities {
return name;
}
+ public static boolean isDedicatedCluster(Configuration conf) {
+ String master = conf.get("spark.master");
+ return master.startsWith("yarn-") || master.startsWith("local");
+ }
+
public static SparkSession getSparkSession(HiveConf conf,
SparkSessionManager sparkSessionManager) throws HiveException {
SparkSession sparkSession = SessionState.get().getSparkSession();
[06/50] [abbrv] hive git commit: HIVE-10550: Dynamic RDD caching
optimization for HoS [Spark Branch] (Chengxiang reviewed by Xuefu)
Posted by xu...@apache.org.
HIVE-10550: Dynamic RDD caching optimization for HoS [Spark Branch] (Chengxiang reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/441d4c08
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/441d4c08
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/441d4c08
Branch: refs/heads/spark
Commit: 441d4c081b8bc4fa0b7d7f65ef34816d005d41b4
Parents: 37cccbc
Author: chengxiang <ch...@apache.com>
Authored: Thu May 28 13:31:26 2015 +0800
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:04:41 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/spark/CacheTran.java | 54 ++++++++++++++++++++
.../hadoop/hive/ql/exec/spark/MapTran.java | 17 +++---
.../hadoop/hive/ql/exec/spark/ReduceTran.java | 17 +++---
.../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 ++
.../hive/ql/exec/spark/SparkPlanGenerator.java | 25 +++++++--
.../hive/ql/exec/spark/SparkUtilities.java | 36 +++++++++++++
6 files changed, 134 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
new file mode 100644
index 0000000..5ec27ec
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.spark;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.storage.StorageLevel;
+
+public abstract class CacheTran<KI extends WritableComparable, VI, KO extends WritableComparable, VO>
+ implements SparkTran<KI, VI, KO, VO> {
+ // whether to cache current RDD.
+ private boolean caching = false;
+ private JavaPairRDD<KO, VO> cachedRDD;
+
+ protected CacheTran(boolean cache) {
+ this.caching = cache;
+ }
+
+ @Override
+ public JavaPairRDD<KO, VO> transform(
+ JavaPairRDD<KI, VI> input) {
+ if (caching) {
+ if (cachedRDD == null) {
+ cachedRDD = doTransform(input);
+ cachedRDD.persist(StorageLevel.MEMORY_AND_DISK());
+ }
+ return cachedRDD;
+ } else {
+ return doTransform(input);
+ }
+ }
+
+ public Boolean isCacheEnable() {
+ return caching;
+ }
+
+ protected abstract JavaPairRDD<KO, VO> doTransform(JavaPairRDD<KI, VI> input);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
index 2170243..2a18991 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/MapTran.java
@@ -22,12 +22,20 @@ import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.io.BytesWritable;
import org.apache.spark.api.java.JavaPairRDD;
-public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey, BytesWritable> {
+public class MapTran extends CacheTran<BytesWritable, BytesWritable, HiveKey, BytesWritable> {
private HiveMapFunction mapFunc;
private String name = "MapTran";
+ public MapTran() {
+ this(false);
+ }
+
+ public MapTran(boolean cache) {
+ super(cache);
+ }
+
@Override
- public JavaPairRDD<HiveKey, BytesWritable> transform(
+ public JavaPairRDD<HiveKey, BytesWritable> doTransform(
JavaPairRDD<BytesWritable, BytesWritable> input) {
return input.mapPartitionsToPair(mapFunc);
}
@@ -42,11 +50,6 @@ public class MapTran implements SparkTran<BytesWritable, BytesWritable, HiveKey,
}
@Override
- public Boolean isCacheEnable() {
- return null;
- }
-
- @Override
public void setName(String name) {
this.name = name;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
index e60dfac..3d56876 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ReduceTran.java
@@ -22,12 +22,20 @@ import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.io.BytesWritable;
import org.apache.spark.api.java.JavaPairRDD;
-public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, HiveKey, BytesWritable> {
+public class ReduceTran extends CacheTran<HiveKey, Iterable<BytesWritable>, HiveKey, BytesWritable> {
private HiveReduceFunction reduceFunc;
private String name = "Reduce";
+ public ReduceTran() {
+ this(false);
+ }
+
+ public ReduceTran(boolean caching) {
+ super(caching);
+ }
+
@Override
- public JavaPairRDD<HiveKey, BytesWritable> transform(
+ public JavaPairRDD<HiveKey, BytesWritable> doTransform(
JavaPairRDD<HiveKey, Iterable<BytesWritable>> input) {
return input.mapPartitionsToPair(reduceFunc);
}
@@ -42,11 +50,6 @@ public class ReduceTran implements SparkTran<HiveKey, Iterable<BytesWritable>, H
}
@Override
- public Boolean isCacheEnable() {
- return null;
- }
-
- @Override
public void setName(String name) {
this.name = name;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
index ee5c78a..762f734 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java
@@ -88,6 +88,9 @@ public class SparkPlan {
}
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_BUILD_RDD_GRAPH);
+ if (LOG.isDebugEnabled()) {
+ LOG.info("print generated spark rdd graph:\n" + SparkUtilities.rddGraphToString(finalRDD));
+ }
return finalRDD;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
index 3f240f5..3a1eff8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
@@ -102,7 +102,7 @@ public class SparkPlanGenerator {
try {
for (BaseWork work : sparkWork.getAllWork()) {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName());
- SparkTran tran = generate(work);
+ SparkTran tran = generate(work, sparkWork);
SparkTran parentTran = generateParentTran(sparkPlan, sparkWork, work);
sparkPlan.addTran(tran);
sparkPlan.connect(parentTran, tran);
@@ -206,18 +206,19 @@ public class SparkPlanGenerator {
return new ShuffleTran(sparkPlan, shuffler, edge.getNumPartitions(), toCache);
}
- private SparkTran generate(BaseWork work) throws Exception {
+ private SparkTran generate(BaseWork work, SparkWork sparkWork) throws Exception {
initStatsPublisher(work);
JobConf newJobConf = cloneJobConf(work);
checkSpecs(work, newJobConf);
byte[] confBytes = KryoSerializer.serializeJobConf(newJobConf);
+ boolean caching = isCachingWork(work, sparkWork);
if (work instanceof MapWork) {
- MapTran mapTran = new MapTran();
+ MapTran mapTran = new MapTran(caching);
HiveMapFunction mapFunc = new HiveMapFunction(confBytes, sparkReporter);
mapTran.setMapFunction(mapFunc);
return mapTran;
} else if (work instanceof ReduceWork) {
- ReduceTran reduceTran = new ReduceTran();
+ ReduceTran reduceTran = new ReduceTran(caching);
HiveReduceFunction reduceFunc = new HiveReduceFunction(confBytes, sparkReporter);
reduceTran.setReduceFunction(reduceFunc);
return reduceTran;
@@ -227,6 +228,22 @@ public class SparkPlanGenerator {
}
}
+ private boolean isCachingWork(BaseWork work, SparkWork sparkWork) {
+ boolean caching = true;
+ List<BaseWork> children = sparkWork.getChildren(work);
+ if (children.size() < 2) {
+ caching = false;
+ } else {
+ // do not cache this if its child RDD is intend to be cached.
+ for (BaseWork child : children) {
+ if (cloneToWork.containsKey(child)) {
+ caching = false;
+ }
+ }
+ }
+ return caching;
+ }
+
private void checkSpecs(BaseWork work, JobConf jc) throws Exception {
Set<Operator<?>> opList = work.getAllOperators();
for (Operator<?> op : opList) {
http://git-wip-us.apache.org/repos/asf/hive/blob/441d4c08/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index c012af8..ef612bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Collection;
import java.util.UUID;
import org.apache.commons.io.FilenameUtils;
@@ -33,7 +34,12 @@ import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.spark.Dependency;
import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.rdd.UnionRDD;
+import scala.collection.JavaConversions;
/**
* Contains utilities methods used as part of Spark tasks.
@@ -116,4 +122,34 @@ public class SparkUtilities {
SessionState.get().setSparkSession(sparkSession);
return sparkSession;
}
+
+
+ public static String rddGraphToString(JavaPairRDD rdd) {
+ StringBuilder sb = new StringBuilder();
+ rddToString(rdd.rdd(), sb, "");
+ return sb.toString();
+ }
+
+ private static void rddToString(RDD rdd, StringBuilder sb, String offset) {
+ sb.append(offset).append(rdd.getClass().getCanonicalName()).append("[").append(rdd.hashCode()).append("]");
+ if (rdd.getStorageLevel().useMemory()) {
+ sb.append("(cached)");
+ }
+ sb.append("\n");
+ Collection<Dependency> dependencies = JavaConversions.asJavaCollection(rdd.dependencies());
+ if (dependencies != null) {
+ offset += "\t";
+ for (Dependency dependency : dependencies) {
+ RDD parentRdd = dependency.rdd();
+ rddToString(parentRdd, sb, offset);
+ }
+ } else if (rdd instanceof UnionRDD) {
+ UnionRDD unionRDD = (UnionRDD) rdd;
+ offset += "\t";
+ Collection<RDD> parentRdds = JavaConversions.asJavaCollection(unionRDD.rdds());
+ for (RDD parentRdd : parentRdds) {
+ rddToString(parentRdd, sb, offset);
+ }
+ }
+ }
}
[27/50] [abbrv] hive git commit: HIVE-10761 : Create codahale-based
metrics system for Hive (Szehon, reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e60bb35..9b40909 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,7 @@
<commons-pool.version>1.5.4</commons-pool.version>
<commons-dbcp.version>1.4</commons-dbcp.version>
<derby.version>10.11.1.1</derby.version>
+ <dropwizard.version>3.1.0</dropwizard.version>
<guava.version>14.0.1</guava.version>
<groovy.version>2.1.6</groovy.version>
<hadoop-20S.version>1.2.1</hadoop-20S.version>
@@ -128,6 +129,8 @@
<httpcomponents.core.version>4.4</httpcomponents.core.version>
<ivy.version>2.4.0</ivy.version>
<jackson.version>1.9.2</jackson.version>
+ <!-- jackson 1 and 2 lines can coexist without issue, as they have different artifactIds -->
+ <jackson.new.version>2.4.2</jackson.new.version>
<javaewah.version>0.3.2</javaewah.version>
<javolution.version>5.5.1</javolution.version>
<jdo-api.version>3.0.1</jdo-api.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/service/src/java/org/apache/hive/service/server/HiveServer2.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 58e8e49..7820ed5 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -42,14 +42,15 @@ import org.apache.curator.framework.api.CuratorEvent;
import org.apache.curator.framework.api.CuratorEventType;
import org.apache.curator.framework.recipes.nodes.PersistentEphemeralNode;
import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.hive.common.JvmPauseMonitor;
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.HiveStringUtils;
@@ -305,6 +306,15 @@ public class HiveServer2 extends CompositeService {
LOG.info("Shutting down HiveServer2");
HiveConf hiveConf = this.getHiveConf();
super.stop();
+ // Shutdown Metrics
+ if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) {
+ try {
+ MetricsFactory.getMetricsInstance().deInit();
+ } catch (Exception e) {
+ LOG.error("error in Metrics deinit: " + e.getClass().getName() + " "
+ + e.getMessage(), e);
+ }
+ }
// Remove this server instance from ZooKeeper if dynamic service discovery is set
if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
try {
@@ -344,7 +354,18 @@ public class HiveServer2 extends CompositeService {
server = new HiveServer2();
server.init(hiveConf);
server.start();
- ShimLoader.getHadoopShims().startPauseMonitor(hiveConf);
+
+ if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) {
+ MetricsFactory.getMetricsInstance().init(hiveConf);
+ }
+ try {
+ JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(hiveConf);
+ pauseMonitor.start();
+ } catch (Throwable t) {
+ LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
+ "warned upon.", t);
+ }
+
// If we're supporting dynamic service discovery, we'll add the service uri for this
// HiveServer2 instance to Zookeeper as a znode.
if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index 6d8166c..ffffcb7 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -159,11 +159,6 @@ public class Hadoop20SShims extends HadoopShimsSecure {
}
@Override
- public void startPauseMonitor(Configuration conf) {
- /* no supported */
- }
-
- @Override
public boolean isLocalMode(Configuration conf) {
return "local".equals(getJobLauncherRpcAddress(conf));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 19324b8..5ddab98 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -212,19 +212,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
@Override
- public void startPauseMonitor(Configuration conf) {
- try {
- Class.forName("org.apache.hadoop.util.JvmPauseMonitor");
- org.apache.hadoop.util.JvmPauseMonitor pauseMonitor = new org.apache.hadoop.util
- .JvmPauseMonitor(conf);
- pauseMonitor.start();
- } catch (Throwable t) {
- LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
- "warned upon.", t);
- }
- }
-
- @Override
public boolean isLocalMode(Configuration conf) {
return "local".equals(conf.get("mapreduce.framework.name"));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 5a6bc44..5b7e7f6 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -146,8 +146,6 @@ public interface HadoopShims {
public JobContext newJobContext(Job job);
- public void startPauseMonitor(Configuration conf);
-
/**
* Check wether MR is configured to run in local-mode
* @param conf
[13/50] [abbrv] hive git commit: HIVE-10878 : Add tests to cover
avg() function for 'x preceding and y preceding' windowing spec (Aihua Xu via
Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10878 : Add tests to cover avg() function for 'x preceding and y preceding' windowing spec (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/715871a0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/715871a0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/715871a0
Branch: refs/heads/spark
Commit: 715871a0e66c9686a707986109316dba358b5c88
Parents: e22aaf6
Author: Aihua Xu <ai...@gmail.com>
Authored: Mon Jun 1 07:56:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Jun 2 09:55:56 2015 -0700
----------------------------------------------------------------------
.../clientpositive/windowing_windowspec2.q | 6 +
.../clientpositive/windowing_windowspec2.q.out | 434 +++++++++++++++++++
2 files changed, 440 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/715871a0/ql/src/test/queries/clientpositive/windowing_windowspec2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec2.q b/ql/src/test/queries/clientpositive/windowing_windowspec2.q
index e270a86..3c5bc3d 100644
--- a/ql/src/test/queries/clientpositive/windowing_windowspec2.q
+++ b/ql/src/test/queries/clientpositive/windowing_windowspec2.q
@@ -23,6 +23,12 @@ select ts, f, sum(f) over (partition by ts order by f rows between unbounded pre
select ts, f, sum(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100;
select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100;
+-- avg
+select ts, f, avg(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100;
+select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100;
+select ts, f, avg(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100;
+select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100;
+
-- count
select ts, f, count(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100;
select ts, f, count(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/715871a0/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_windowspec2.q.out b/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
index a318398..e71a03f 100644
--- a/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
+++ b/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
@@ -478,6 +478,440 @@ POSTHOOK: Input: default@over10k
2013-03-01 09:11:58.703072 71.68 802.1099938452244
2013-03-01 09:11:58.703072 79.46 882.1299904882908
2013-03-01 09:11:58.703072 80.02 963.4199914038181
+PREHOOK: query: -- avg
+select ts, f, avg(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: -- avg
+select ts, f, avg(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 NULL
+2013-03-01 09:11:58.70307 10.89 3.1700000762939453
+2013-03-01 09:11:58.70307 14.54 7.03000020980835
+2013-03-01 09:11:58.70307 14.78 12.71500015258789
+2013-03-01 09:11:58.70307 17.85 14.65999984741211
+2013-03-01 09:11:58.70307 20.61 16.31500005722046
+2013-03-01 09:11:58.70307 28.69 19.230000495910645
+2013-03-01 09:11:58.70307 29.22 24.65000057220459
+2013-03-01 09:11:58.70307 31.17 28.954999923706055
+2013-03-01 09:11:58.70307 38.35 30.19499969482422
+2013-03-01 09:11:58.70307 38.61 34.75999927520752
+2013-03-01 09:11:58.70307 39.48 38.47999954223633
+2013-03-01 09:11:58.70307 40.54 39.045000076293945
+2013-03-01 09:11:58.70307 41.6 40.010000228881836
+2013-03-01 09:11:58.70307 46.08 41.06999969482422
+2013-03-01 09:11:58.70307 54.36 43.84000015258789
+2013-03-01 09:11:58.70307 56.94 50.220001220703125
+2013-03-01 09:11:58.70307 64.96 55.64999961853027
+2013-03-01 09:11:58.70307 73.52 60.94999885559082
+2013-03-01 09:11:58.70307 78.58 69.23999786376953
+2013-03-01 09:11:58.70307 81.41 76.04999923706055
+2013-03-01 09:11:58.70307 84.71 79.99500274658203
+2013-03-01 09:11:58.70307 87.43 83.06000137329102
+2013-03-01 09:11:58.70307 91.36 86.06999969482422
+2013-03-01 09:11:58.70307 92.96 89.39500045776367
+2013-03-01 09:11:58.70307 95.04 92.15999984741211
+2013-03-01 09:11:58.703071 0.83 NULL
+2013-03-01 09:11:58.703071 1.99 0.8299999833106995
+2013-03-01 09:11:58.703071 3.73 1.4099999964237213
+2013-03-01 09:11:58.703071 8.86 2.8600000143051147
+2013-03-01 09:11:58.703071 10.62 6.294999837875366
+2013-03-01 09:11:58.703071 11.32 9.739999771118164
+2013-03-01 09:11:58.703071 12.83 10.96999979019165
+2013-03-01 09:11:58.703071 14.7 12.074999809265137
+2013-03-01 09:11:58.703071 14.96 13.764999866485596
+2013-03-01 09:11:58.703071 17.58 14.829999923706055
+2013-03-01 09:11:58.703071 19.1 16.269999980926514
+2013-03-01 09:11:58.703071 21.01 18.34000015258789
+2013-03-01 09:11:58.703071 26.95 20.05500030517578
+2013-03-01 09:11:58.703071 27.23 23.980000495910645
+2013-03-01 09:11:58.703071 29.07 27.09000015258789
+2013-03-01 09:11:58.703071 29.71 28.149999618530273
+2013-03-01 09:11:58.703071 31.84 29.389999389648438
+2013-03-01 09:11:58.703071 31.94 30.774999618530273
+2013-03-01 09:11:58.703071 35.32 31.890000343322754
+2013-03-01 09:11:58.703071 37.32 33.63000011444092
+2013-03-01 09:11:58.703071 38.5 36.31999969482422
+2013-03-01 09:11:58.703071 42.08 37.90999984741211
+2013-03-01 09:11:58.703071 44.3 40.290000915527344
+2013-03-01 09:11:58.703071 44.66 43.19000053405762
+2013-03-01 09:11:58.703071 46.84 44.47999954223633
+2013-03-01 09:11:58.703071 48.89 45.75
+2013-03-01 09:11:58.703071 49.64 47.864999771118164
+2013-03-01 09:11:58.703071 50.28 49.26499938964844
+2013-03-01 09:11:58.703071 52.09 49.959999084472656
+2013-03-01 09:11:58.703071 53.26 51.18499946594238
+2013-03-01 09:11:58.703071 54.09 52.67499923706055
+2013-03-01 09:11:58.703071 56.45 53.67499923706055
+2013-03-01 09:11:58.703071 56.76 55.27000045776367
+2013-03-01 09:11:58.703071 61.41 56.60499954223633
+2013-03-01 09:11:58.703071 61.88 59.084999084472656
+2013-03-01 09:11:58.703071 63.03 61.64500045776367
+2013-03-01 09:11:58.703071 64.55 62.454999923706055
+2013-03-01 09:11:58.703071 68.62 63.790000915527344
+2013-03-01 09:11:58.703071 76.13 66.58500289916992
+2013-03-01 09:11:58.703071 79.05 72.375
+2013-03-01 09:11:58.703071 80.43 77.59000015258789
+2013-03-01 09:11:58.703071 81.41 79.7400016784668
+2013-03-01 09:11:58.703071 82.85 80.92000198364258
+2013-03-01 09:11:58.703071 83.98 82.13000106811523
+2013-03-01 09:11:58.703071 84.21 83.41500091552734
+2013-03-01 09:11:58.703071 85.55 84.09500122070312
+2013-03-01 09:11:58.703071 87.93 84.88000106811523
+2013-03-01 09:11:58.703071 88.93 86.7400016784668
+2013-03-01 09:11:58.703071 94.27 88.43000030517578
+2013-03-01 09:11:58.703071 99.45 91.5999984741211
+2013-03-01 09:11:58.703072 0.36 NULL
+2013-03-01 09:11:58.703072 0.48 0.36000001430511475
+2013-03-01 09:11:58.703072 0.79 0.42000000178813934
+2013-03-01 09:11:58.703072 1.27 0.635000005364418
+2013-03-01 09:11:58.703072 4.48 1.030000001192093
+2013-03-01 09:11:58.703072 9.0 2.875
+2013-03-01 09:11:58.703072 23.27 6.740000009536743
+2013-03-01 09:11:58.703072 25.13 16.135000228881836
+2013-03-01 09:11:58.703072 25.34 24.199999809265137
+2013-03-01 09:11:58.703072 25.91 25.234999656677246
+2013-03-01 09:11:58.703072 29.01 25.625
+2013-03-01 09:11:58.703072 30.47 27.460000038146973
+2013-03-01 09:11:58.703072 37.95 29.739999771118164
+2013-03-01 09:11:58.703072 39.3 34.21000003814697
+2013-03-01 09:11:58.703072 45.91 38.625
+2013-03-01 09:11:58.703072 52.44 42.60499954223633
+2013-03-01 09:11:58.703072 54.1 49.17499923706055
+2013-03-01 09:11:58.703072 56.7 53.26999855041504
+2013-03-01 09:11:58.703072 58.77 55.39999961853027
+2013-03-01 09:11:58.703072 62.09 57.73500061035156
+2013-03-01 09:11:58.703072 68.2 60.43000030517578
+2013-03-01 09:11:58.703072 71.68 65.14499855041504
+2013-03-01 09:11:58.703072 79.46 69.93999862670898
+2013-03-01 09:11:58.703072 80.02 75.56999969482422
+PREHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 NULL
+2013-03-01 09:11:58.70307 10.89 3.1700000762939453
+2013-03-01 09:11:58.70307 14.54 7.03000020980835
+2013-03-01 09:11:58.70307 14.78 9.533333460489908
+2013-03-01 09:11:58.70307 17.85 10.84500002861023
+2013-03-01 09:11:58.70307 20.61 12.24600009918213
+2013-03-01 09:11:58.70307 28.69 13.640000184377035
+2013-03-01 09:11:58.70307 29.22 15.790000234331403
+2013-03-01 09:11:58.70307 31.17 17.46875011920929
+2013-03-01 09:11:58.70307 38.35 18.99111122555203
+2013-03-01 09:11:58.70307 38.61 20.926999950408934
+2013-03-01 09:11:58.70307 39.48 22.534545464949176
+2013-03-01 09:11:58.70307 40.54 23.94666663805644
+2013-03-01 09:11:58.70307 41.6 25.22307696709266
+2013-03-01 09:11:58.70307 46.08 26.39285707473755
+2013-03-01 09:11:58.70307 54.36 27.70533339182536
+2013-03-01 09:11:58.70307 56.94 29.371250092983246
+2013-03-01 09:11:58.70307 64.96 30.992941183202408
+2013-03-01 09:11:58.70307 73.52 32.8799999554952
+2013-03-01 09:11:58.70307 78.58 35.01894714957789
+2013-03-01 09:11:58.70307 81.41 37.196999883651735
+2013-03-01 09:11:58.70307 84.71 39.30238101595924
+2013-03-01 09:11:58.70307 87.43 41.36636365543712
+2013-03-01 09:11:58.70307 91.36 43.369130466295324
+2013-03-01 09:11:58.70307 92.96 45.368750055631004
+2013-03-01 09:11:58.70307 95.04 47.272400016784665
+2013-03-01 09:11:58.703071 0.83 NULL
+2013-03-01 09:11:58.703071 1.99 0.8299999833106995
+2013-03-01 09:11:58.703071 3.73 1.4099999964237213
+2013-03-01 09:11:58.703071 8.86 2.1833333373069763
+2013-03-01 09:11:58.703071 10.62 3.8524999171495438
+2013-03-01 09:11:58.703071 11.32 5.205999910831451
+2013-03-01 09:11:58.703071 12.83 6.224999874830246
+2013-03-01 09:11:58.703071 14.7 7.168571310383933
+2013-03-01 09:11:58.703071 14.96 8.109999872744083
+2013-03-01 09:11:58.703071 17.58 8.871111002233294
+2013-03-01 09:11:58.703071 19.1 9.74199989438057
+2013-03-01 09:11:58.703071 21.01 10.592727211388675
+2013-03-01 09:11:58.703071 26.95 11.460833296179771
+2013-03-01 09:11:58.703071 27.23 12.652307716699747
+2013-03-01 09:11:58.703071 29.07 13.693571418523788
+2013-03-01 09:11:58.703071 29.71 14.718666636943817
+2013-03-01 09:11:58.703071 31.84 15.65562491491437
+2013-03-01 09:11:58.703071 31.94 16.607646987718695
+2013-03-01 09:11:58.703071 35.32 17.459444406959747
+2013-03-01 09:11:58.703071 37.32 18.399473632636823
+2013-03-01 09:11:58.703071 38.5 19.345499935746194
+2013-03-01 09:11:58.703071 42.08 20.257618986424944
+2013-03-01 09:11:58.703071 44.3 21.249545479362663
+2013-03-01 09:11:58.703071 44.66 22.251739121001698
+2013-03-01 09:11:58.703071 46.84 23.1854166512688
+2013-03-01 09:11:58.703071 48.89 24.131599991321565
+2013-03-01 09:11:58.703071 49.64 25.083846122026443
+2013-03-01 09:11:58.703071 50.28 25.993333280086517
+2013-03-01 09:11:58.703071 52.09 26.8607141907726
+2013-03-01 09:11:58.703071 53.26 27.73068956876623
+2013-03-01 09:11:58.703071 54.09 28.581666527191796
+2013-03-01 09:11:58.703071 56.45 29.40451599897877
+2013-03-01 09:11:58.703071 56.76 30.24968739785254
+2013-03-01 09:11:58.703071 61.41 31.05303015311559
+2013-03-01 09:11:58.703071 61.88 31.945882202947843
+2013-03-01 09:11:58.703071 63.03 32.801142741952624
+2013-03-01 09:11:58.703071 64.55 33.64083318743441
+2013-03-01 09:11:58.703071 68.62 34.47621615674045
+2013-03-01 09:11:58.703071 76.13 35.37473685647312
+2013-03-01 09:11:58.703071 79.05 36.419743533317856
+2013-03-01 09:11:58.703071 80.43 37.48550002127886
+2013-03-01 09:11:58.703071 81.41 38.532926857471466
+2013-03-01 09:11:58.703071 82.85 39.553809638534275
+2013-03-01 09:11:58.703071 83.98 40.56069775098978
+2013-03-01 09:11:58.703071 84.21 41.547500151124865
+2013-03-01 09:11:58.703071 85.55 42.49555568297704
+2013-03-01 09:11:58.703071 87.93 43.43152193012445
+2013-03-01 09:11:58.703071 88.93 44.378298065763836
+2013-03-01 09:11:58.703071 94.27 45.30645852908492
+2013-03-01 09:11:58.703071 99.45 46.305714408962096
+2013-03-01 09:11:58.703072 0.36 NULL
+2013-03-01 09:11:58.703072 0.48 0.36000001430511475
+2013-03-01 09:11:58.703072 0.79 0.42000000178813934
+2013-03-01 09:11:58.703072 1.27 0.5433333416779836
+2013-03-01 09:11:58.703072 4.48 0.7250000014901161
+2013-03-01 09:11:58.703072 9.0 1.47600000500679
+2013-03-01 09:11:58.703072 23.27 2.730000004172325
+2013-03-01 09:11:58.703072 25.13 5.664285783256803
+2013-03-01 09:11:58.703072 25.34 8.097499955445528
+2013-03-01 09:11:58.703072 25.91 10.013333310683569
+2013-03-01 09:11:58.703072 29.01 11.602999964356423
+2013-03-01 09:11:58.703072 30.47 13.185454533858733
+2013-03-01 09:11:58.703072 37.95 14.62583326548338
+2013-03-01 09:11:58.703072 39.3 16.419999996056923
+2013-03-01 09:11:58.703072 45.91 18.05428565612861
+2013-03-01 09:11:58.703072 52.44 19.911333268880846
+2013-03-01 09:11:58.703072 54.1 21.944374853745103
+2013-03-01 09:11:58.703072 56.7 23.835882125531924
+2013-03-01 09:11:58.703072 58.77 25.661666494276787
+2013-03-01 09:11:58.703072 62.09 27.404210387091887
+2013-03-01 09:11:58.703072 68.2 29.138499875366687
+2013-03-01 09:11:58.703072 71.68 30.998571164551237
+2013-03-01 09:11:58.703072 79.46 32.84772703457963
+2013-03-01 09:11:58.703072 80.02 34.87434755848802
+PREHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 12.71500015258789
+2013-03-01 09:11:58.70307 10.89 14.65999984741211
+2013-03-01 09:11:58.70307 14.54 16.31500005722046
+2013-03-01 09:11:58.70307 14.78 19.230000495910645
+2013-03-01 09:11:58.70307 17.85 24.65000057220459
+2013-03-01 09:11:58.70307 20.61 28.954999923706055
+2013-03-01 09:11:58.70307 28.69 30.19499969482422
+2013-03-01 09:11:58.70307 29.22 34.75999927520752
+2013-03-01 09:11:58.70307 31.17 38.47999954223633
+2013-03-01 09:11:58.70307 38.35 39.045000076293945
+2013-03-01 09:11:58.70307 38.61 40.010000228881836
+2013-03-01 09:11:58.70307 39.48 41.06999969482422
+2013-03-01 09:11:58.70307 40.54 43.84000015258789
+2013-03-01 09:11:58.70307 41.6 50.220001220703125
+2013-03-01 09:11:58.70307 46.08 55.64999961853027
+2013-03-01 09:11:58.70307 54.36 60.94999885559082
+2013-03-01 09:11:58.70307 56.94 69.23999786376953
+2013-03-01 09:11:58.70307 64.96 76.04999923706055
+2013-03-01 09:11:58.70307 73.52 79.99500274658203
+2013-03-01 09:11:58.70307 78.58 83.06000137329102
+2013-03-01 09:11:58.70307 81.41 86.06999969482422
+2013-03-01 09:11:58.70307 84.71 89.39500045776367
+2013-03-01 09:11:58.70307 87.43 92.15999984741211
+2013-03-01 09:11:58.70307 91.36 94.0
+2013-03-01 09:11:58.70307 92.96 95.04000091552734
+2013-03-01 09:11:58.70307 95.04 NULL
+2013-03-01 09:11:58.703071 0.83 2.8600000143051147
+2013-03-01 09:11:58.703071 1.99 6.294999837875366
+2013-03-01 09:11:58.703071 3.73 9.739999771118164
+2013-03-01 09:11:58.703071 8.86 10.96999979019165
+2013-03-01 09:11:58.703071 10.62 12.074999809265137
+2013-03-01 09:11:58.703071 11.32 13.764999866485596
+2013-03-01 09:11:58.703071 12.83 14.829999923706055
+2013-03-01 09:11:58.703071 14.7 16.269999980926514
+2013-03-01 09:11:58.703071 14.96 18.34000015258789
+2013-03-01 09:11:58.703071 17.58 20.05500030517578
+2013-03-01 09:11:58.703071 19.1 23.980000495910645
+2013-03-01 09:11:58.703071 21.01 27.09000015258789
+2013-03-01 09:11:58.703071 26.95 28.149999618530273
+2013-03-01 09:11:58.703071 27.23 29.389999389648438
+2013-03-01 09:11:58.703071 29.07 30.774999618530273
+2013-03-01 09:11:58.703071 29.71 31.890000343322754
+2013-03-01 09:11:58.703071 31.84 33.63000011444092
+2013-03-01 09:11:58.703071 31.94 36.31999969482422
+2013-03-01 09:11:58.703071 35.32 37.90999984741211
+2013-03-01 09:11:58.703071 37.32 40.290000915527344
+2013-03-01 09:11:58.703071 38.5 43.19000053405762
+2013-03-01 09:11:58.703071 42.08 44.47999954223633
+2013-03-01 09:11:58.703071 44.3 45.75
+2013-03-01 09:11:58.703071 44.66 47.864999771118164
+2013-03-01 09:11:58.703071 46.84 49.26499938964844
+2013-03-01 09:11:58.703071 48.89 49.959999084472656
+2013-03-01 09:11:58.703071 49.64 51.18499946594238
+2013-03-01 09:11:58.703071 50.28 52.67499923706055
+2013-03-01 09:11:58.703071 52.09 53.67499923706055
+2013-03-01 09:11:58.703071 53.26 55.27000045776367
+2013-03-01 09:11:58.703071 54.09 56.60499954223633
+2013-03-01 09:11:58.703071 56.45 59.084999084472656
+2013-03-01 09:11:58.703071 56.76 61.64500045776367
+2013-03-01 09:11:58.703071 61.41 62.454999923706055
+2013-03-01 09:11:58.703071 61.88 63.790000915527344
+2013-03-01 09:11:58.703071 63.03 66.58500289916992
+2013-03-01 09:11:58.703071 64.55 72.375
+2013-03-01 09:11:58.703071 68.62 77.59000015258789
+2013-03-01 09:11:58.703071 76.13 79.7400016784668
+2013-03-01 09:11:58.703071 79.05 80.92000198364258
+2013-03-01 09:11:58.703071 80.43 82.13000106811523
+2013-03-01 09:11:58.703071 81.41 83.41500091552734
+2013-03-01 09:11:58.703071 82.85 84.09500122070312
+2013-03-01 09:11:58.703071 83.98 84.88000106811523
+2013-03-01 09:11:58.703071 84.21 86.7400016784668
+2013-03-01 09:11:58.703071 85.55 88.43000030517578
+2013-03-01 09:11:58.703071 87.93 91.5999984741211
+2013-03-01 09:11:58.703071 88.93 96.8599967956543
+2013-03-01 09:11:58.703071 94.27 99.44999694824219
+2013-03-01 09:11:58.703071 99.45 NULL
+2013-03-01 09:11:58.703072 0.36 0.635000005364418
+2013-03-01 09:11:58.703072 0.48 1.030000001192093
+2013-03-01 09:11:58.703072 0.79 2.875
+2013-03-01 09:11:58.703072 1.27 6.740000009536743
+2013-03-01 09:11:58.703072 4.48 16.135000228881836
+2013-03-01 09:11:58.703072 9.0 24.199999809265137
+2013-03-01 09:11:58.703072 23.27 25.234999656677246
+2013-03-01 09:11:58.703072 25.13 25.625
+2013-03-01 09:11:58.703072 25.34 27.460000038146973
+2013-03-01 09:11:58.703072 25.91 29.739999771118164
+2013-03-01 09:11:58.703072 29.01 34.21000003814697
+2013-03-01 09:11:58.703072 30.47 38.625
+2013-03-01 09:11:58.703072 37.95 42.60499954223633
+2013-03-01 09:11:58.703072 39.3 49.17499923706055
+2013-03-01 09:11:58.703072 45.91 53.26999855041504
+2013-03-01 09:11:58.703072 52.44 55.39999961853027
+2013-03-01 09:11:58.703072 54.1 57.73500061035156
+2013-03-01 09:11:58.703072 56.7 60.43000030517578
+2013-03-01 09:11:58.703072 58.77 65.14499855041504
+2013-03-01 09:11:58.703072 62.09 69.93999862670898
+2013-03-01 09:11:58.703072 68.2 75.56999969482422
+2013-03-01 09:11:58.703072 71.68 79.73999786376953
+2013-03-01 09:11:58.703072 79.46 80.65499877929688
+2013-03-01 09:11:58.703072 80.02 84.13500213623047
+PREHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, avg(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 7.03000020980835
+2013-03-01 09:11:58.70307 10.89 9.533333460489908
+2013-03-01 09:11:58.70307 14.54 10.84500002861023
+2013-03-01 09:11:58.70307 14.78 12.24600009918213
+2013-03-01 09:11:58.70307 17.85 13.640000184377035
+2013-03-01 09:11:58.70307 20.61 15.790000234331403
+2013-03-01 09:11:58.70307 28.69 17.46875011920929
+2013-03-01 09:11:58.70307 29.22 18.99111122555203
+2013-03-01 09:11:58.70307 31.17 20.926999950408934
+2013-03-01 09:11:58.70307 38.35 22.534545464949176
+2013-03-01 09:11:58.70307 38.61 23.94666663805644
+2013-03-01 09:11:58.70307 39.48 25.22307696709266
+2013-03-01 09:11:58.70307 40.54 26.39285707473755
+2013-03-01 09:11:58.70307 41.6 27.70533339182536
+2013-03-01 09:11:58.70307 46.08 29.371250092983246
+2013-03-01 09:11:58.70307 54.36 30.992941183202408
+2013-03-01 09:11:58.70307 56.94 32.8799999554952
+2013-03-01 09:11:58.70307 64.96 35.01894714957789
+2013-03-01 09:11:58.70307 73.52 37.196999883651735
+2013-03-01 09:11:58.70307 78.58 39.30238101595924
+2013-03-01 09:11:58.70307 81.41 41.36636365543712
+2013-03-01 09:11:58.70307 84.71 43.369130466295324
+2013-03-01 09:11:58.70307 87.43 45.368750055631004
+2013-03-01 09:11:58.70307 91.36 47.272400016784665
+2013-03-01 09:11:58.70307 92.96 49.10961543596708
+2013-03-01 09:11:58.70307 95.04 49.10961543596708
+2013-03-01 09:11:58.703071 0.83 1.4099999964237213
+2013-03-01 09:11:58.703071 1.99 2.1833333373069763
+2013-03-01 09:11:58.703071 3.73 3.8524999171495438
+2013-03-01 09:11:58.703071 8.86 5.205999910831451
+2013-03-01 09:11:58.703071 10.62 6.224999874830246
+2013-03-01 09:11:58.703071 11.32 7.168571310383933
+2013-03-01 09:11:58.703071 12.83 8.109999872744083
+2013-03-01 09:11:58.703071 14.7 8.871111002233294
+2013-03-01 09:11:58.703071 14.96 9.74199989438057
+2013-03-01 09:11:58.703071 17.58 10.592727211388675
+2013-03-01 09:11:58.703071 19.1 11.460833296179771
+2013-03-01 09:11:58.703071 21.01 12.652307716699747
+2013-03-01 09:11:58.703071 26.95 13.693571418523788
+2013-03-01 09:11:58.703071 27.23 14.718666636943817
+2013-03-01 09:11:58.703071 29.07 15.65562491491437
+2013-03-01 09:11:58.703071 29.71 16.607646987718695
+2013-03-01 09:11:58.703071 31.84 17.459444406959747
+2013-03-01 09:11:58.703071 31.94 18.399473632636823
+2013-03-01 09:11:58.703071 35.32 19.345499935746194
+2013-03-01 09:11:58.703071 37.32 20.257618986424944
+2013-03-01 09:11:58.703071 38.5 21.249545479362663
+2013-03-01 09:11:58.703071 42.08 22.251739121001698
+2013-03-01 09:11:58.703071 44.3 23.1854166512688
+2013-03-01 09:11:58.703071 44.66 24.131599991321565
+2013-03-01 09:11:58.703071 46.84 25.083846122026443
+2013-03-01 09:11:58.703071 48.89 25.993333280086517
+2013-03-01 09:11:58.703071 49.64 26.8607141907726
+2013-03-01 09:11:58.703071 50.28 27.73068956876623
+2013-03-01 09:11:58.703071 52.09 28.581666527191796
+2013-03-01 09:11:58.703071 53.26 29.40451599897877
+2013-03-01 09:11:58.703071 54.09 30.24968739785254
+2013-03-01 09:11:58.703071 56.45 31.05303015311559
+2013-03-01 09:11:58.703071 56.76 31.945882202947843
+2013-03-01 09:11:58.703071 61.41 32.801142741952624
+2013-03-01 09:11:58.703071 61.88 33.64083318743441
+2013-03-01 09:11:58.703071 63.03 34.47621615674045
+2013-03-01 09:11:58.703071 64.55 35.37473685647312
+2013-03-01 09:11:58.703071 68.62 36.419743533317856
+2013-03-01 09:11:58.703071 76.13 37.48550002127886
+2013-03-01 09:11:58.703071 79.05 38.532926857471466
+2013-03-01 09:11:58.703071 80.43 39.553809638534275
+2013-03-01 09:11:58.703071 81.41 40.56069775098978
+2013-03-01 09:11:58.703071 82.85 41.547500151124865
+2013-03-01 09:11:58.703071 83.98 42.49555568297704
+2013-03-01 09:11:58.703071 84.21 43.43152193012445
+2013-03-01 09:11:58.703071 85.55 44.378298065763836
+2013-03-01 09:11:58.703071 87.93 45.30645852908492
+2013-03-01 09:11:58.703071 88.93 46.305714408962096
+2013-03-01 09:11:58.703071 94.27 47.3686000597477
+2013-03-01 09:11:58.703071 99.45 47.3686000597477
+2013-03-01 09:11:58.703072 0.36 0.42000000178813934
+2013-03-01 09:11:58.703072 0.48 0.5433333416779836
+2013-03-01 09:11:58.703072 0.79 0.7250000014901161
+2013-03-01 09:11:58.703072 1.27 1.47600000500679
+2013-03-01 09:11:58.703072 4.48 2.730000004172325
+2013-03-01 09:11:58.703072 9.0 5.664285783256803
+2013-03-01 09:11:58.703072 23.27 8.097499955445528
+2013-03-01 09:11:58.703072 25.13 10.013333310683569
+2013-03-01 09:11:58.703072 25.34 11.602999964356423
+2013-03-01 09:11:58.703072 25.91 13.185454533858733
+2013-03-01 09:11:58.703072 29.01 14.62583326548338
+2013-03-01 09:11:58.703072 30.47 16.419999996056923
+2013-03-01 09:11:58.703072 37.95 18.05428565612861
+2013-03-01 09:11:58.703072 39.3 19.911333268880846
+2013-03-01 09:11:58.703072 45.91 21.944374853745103
+2013-03-01 09:11:58.703072 52.44 23.835882125531924
+2013-03-01 09:11:58.703072 54.1 25.661666494276787
+2013-03-01 09:11:58.703072 56.7 27.404210387091887
+2013-03-01 09:11:58.703072 58.77 29.138499875366687
+2013-03-01 09:11:58.703072 62.09 30.998571164551237
+2013-03-01 09:11:58.703072 68.2 32.84772703457963
+2013-03-01 09:11:58.703072 71.68 34.87434755848802
+2013-03-01 09:11:58.703072 79.46 36.75541627034545
+2013-03-01 09:11:58.703072 80.02 38.53679965615272
PREHOOK: query: -- count
select ts, f, count(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
PREHOOK: type: QUERY
[42/50] [abbrv] hive git commit: HIVE-10906 : Value based UDAF
function without orderby expression throws NPE (Aihua Xu via Ashutosh
Chauhan)
Posted by xu...@apache.org.
HIVE-10906 : Value based UDAF function without orderby expression throws NPE (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3626a795
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3626a795
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3626a795
Branch: refs/heads/spark
Commit: 3626a7951772a8e205bd33c01cb817af7e8223e2
Parents: 7983593
Author: Aihua Xu <ai...@gmail.com>
Authored: Fri Jun 5 18:01:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 6 11:10:49 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/WindowingSpec.java | 38 ++++++++++++++------
.../clientpositive/windowing_windowspec3.q | 6 ++--
.../clientpositive/windowing_windowspec3.q.out | 26 ++++++++++++++
3 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3626a795/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
index 50f86df..953f3ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
@@ -145,7 +145,7 @@ public class WindowingSpec {
}
// 2. A Window Spec with no Parition Spec, is Partitioned on a Constant(number 0)
- applyContantPartition(wdwSpec);
+ applyConstantPartition(wdwSpec);
// 3. For missing Wdw Frames or for Frames with only a Start Boundary, completely
// specify them by the rules in {@link effectiveWindowFrame}
@@ -154,8 +154,8 @@ public class WindowingSpec {
// 4. Validate the effective Window Frames with the rules in {@link validateWindowFrame}
validateWindowFrame(wdwSpec);
- // 5. If there is no Order, then add the Partition expressions as the Order.
- wdwSpec.ensureOrderSpec();
+ // 5. Add the Partition expressions as the Order if there is no Order and validate Order spec.
+ setAndValidateOrderSpec(wdwSpec);
}
}
@@ -196,7 +196,7 @@ public class WindowingSpec {
}
}
- private void applyContantPartition(WindowSpec wdwSpec) {
+ private void applyConstantPartition(WindowSpec wdwSpec) {
PartitionSpec partSpec = wdwSpec.getPartition();
if ( partSpec == null ) {
partSpec = new PartitionSpec();
@@ -277,20 +277,36 @@ public class WindowingSpec {
end.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT ) {
throw new SemanticException("End of a WindowFrame cannot be UNBOUNDED PRECEDING");
}
-
- validateValueBoundary(wFrame.getStart(), wdwSpec.getOrder());
- validateValueBoundary(wFrame.getEnd(), wdwSpec.getOrder());
}
- private void validateValueBoundary(BoundarySpec bs, OrderSpec order) throws SemanticException {
- if ( bs instanceof ValueBoundarySpec ) {
- ValueBoundarySpec vbs = (ValueBoundarySpec) bs;
+ /**
+ * Add default order spec if there is no order and validate order spec for valued based
+ * windowing since only one sort key is allowed.
+ * @param wdwSpec
+ * @throws SemanticException
+ */
+ private void setAndValidateOrderSpec(WindowSpec wdwSpec) throws SemanticException {
+ wdwSpec.ensureOrderSpec();
+
+ WindowFrameSpec wFrame = wdwSpec.getWindowFrame();
+ OrderSpec order = wdwSpec.getOrder();
+
+ BoundarySpec start = wFrame.getStart();
+ BoundarySpec end = wFrame.getEnd();
+
+ if (start instanceof ValueBoundarySpec || end instanceof ValueBoundarySpec) {
if ( order != null ) {
if ( order.getExpressions().size() > 1 ) {
throw new SemanticException("Range based Window Frame can have only 1 Sort Key");
}
+
+ if (start instanceof ValueBoundarySpec) {
+ ((ValueBoundarySpec)start).setExpression(order.getExpressions().get(0).getExpression());
+ }
+ if (end instanceof ValueBoundarySpec) {
+ ((ValueBoundarySpec)end).setExpression(order.getExpressions().get(0).getExpression());
+ }
}
- vbs.setExpression(order.getExpressions().get(0).getExpression());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3626a795/ql/src/test/queries/clientpositive/windowing_windowspec3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec3.q b/ql/src/test/queries/clientpositive/windowing_windowspec3.q
index d00e939..c87aaff 100644
--- a/ql/src/test/queries/clientpositive/windowing_windowspec3.q
+++ b/ql/src/test/queries/clientpositive/windowing_windowspec3.q
@@ -16,6 +16,10 @@ create table emp(empno smallint,
load data local inpath '../../data/files/emp2.txt' into table emp;
+-- No order by
+select hirets, salary, sum(salary) over (partition by hirets range between current row and unbounded following) from emp;
+
+
-- Support date datatype
select deptno, empno, hiredate, salary,
sum(salary) over (partition by deptno order by hiredate range 90 preceding),
@@ -25,5 +29,3 @@ select deptno, empno, hiredate, salary,
sum(salary) over (partition by deptno order by hiredate range between 10 following and unbounded following),
sum(salary) over (partition by deptno order by hiredate range between unbounded preceding and 10 following)
from emp;
-
-
http://git-wip-us.apache.org/repos/asf/hive/blob/3626a795/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_windowspec3.q.out b/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
index a5eae5b..bf7797a 100644
--- a/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
+++ b/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
@@ -42,6 +42,32 @@ POSTHOOK: query: load data local inpath '../../data/files/emp2.txt' into table e
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@emp
+PREHOOK: query: -- No order by
+select hirets, salary, sum(salary) over (partition by hirets range between current row and unbounded following) from emp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emp
+#### A masked pattern was here ####
+POSTHOOK: query: -- No order by
+select hirets, salary, sum(salary) over (partition by hirets range between current row and unbounded following) from emp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emp
+#### A masked pattern was here ####
+NULL 1500.0 3000.0
+NULL 1500.0 3000.0
+1980-12-17 00:00:00 800.0 800.0
+1981-02-20 00:00:00 1600.0 1600.0
+1981-02-22 00:00:00 1250.0 1250.0
+1981-04-02 00:00:00 2975.0 2975.0
+1981-05-01 00:00:00 2850.0 2850.0
+1981-06-09 00:00:00 2450.0 2450.0
+1981-09-08 00:00:00 1500.0 1500.0
+1981-09-28 00:00:00 1250.0 1250.0
+1981-11-17 00:00:00 5000.0 5000.0
+1981-12-03 00:00:00 3000.0 3950.0
+1981-12-03 00:00:00 950.0 3950.0
+1982-01-23 00:00:00 1300.0 1300.0
+1982-12-09 00:00:00 3000.0 3000.0
+1983-01-12 00:00:00 1100.0 1100.0
PREHOOK: query: -- Support date datatype
select deptno, empno, hiredate, salary,
sum(salary) over (partition by deptno order by hiredate range 90 preceding),
[11/50] [abbrv] hive git commit: HIVE-10877 : - adding missing
UtilsForTest from previous patch
Posted by xu...@apache.org.
HIVE-10877 : - adding missing UtilsForTest from previous patch
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b6b638a8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b6b638a8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b6b638a8
Branch: refs/heads/spark
Commit: b6b638a86d33077e42d71ab22ad36d117a79c369
Parents: 9bc8ea2
Author: Thejas Nair <th...@hortonworks.com>
Authored: Mon Jun 1 16:41:13 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Mon Jun 1 16:41:21 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/UtilsForTest.java | 41 ++++++++++++++++++++
1 file changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b6b638a8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java
new file mode 100644
index 0000000..699d446
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/UtilsForTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+/**
+ * Test utilities
+ */
+public class UtilsForTest {
+ /**
+ * Use this if you want a fresh metastore for your test, without any existing entries.
+ * It updates the configuration to point to new derby dir location
+ * @param conf HiveConf to be updated
+ * @param newloc new location within test temp dir for the metastore db
+ */
+ public static void setNewDerbyDbLocation(HiveConf conf, String newloc) {
+ String newDbLoc = System.getProperty("test.tmp.dir") + newloc + "metastore_db";
+
+ conf.setVar(ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:;databaseName=" + newDbLoc
+ + ";create=true");
+ }
+
+}
[49/50] [abbrv] hive git commit: HIVE-10929: In Tez mode,
dynamic partitioning query with union all fails at moveTask,
Invalid partition key & values (Vikram Dixit K reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-10929: In Tez mode,dynamic partitioning query with union all fails at moveTask,Invalid partition key & values (Vikram Dixit K reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4d592303
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4d592303
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4d592303
Branch: refs/heads/spark
Commit: 4d592303ad0f925c7d0815c003cedd45ab4f0064
Parents: d3863be
Author: Vaibhav Gumashta <vg...@apache.org>
Authored: Sun Jun 7 11:38:10 2015 -0700
Committer: Vaibhav Gumashta <vg...@apache.org>
Committed: Sun Jun 7 11:39:12 2015 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/parse/GenTezUtils.java | 8 +
.../tez_union_dynamic_partition.q | 21 +
.../clientpositive/tez/explainuser_2.q.out | 63 +-
.../tez/tez_union_dynamic_partition.q.out | 148 +++
.../results/clientpositive/tez/union4.q.out | 4 +
.../results/clientpositive/tez/union6.q.out | 4 +
.../tez/vector_leftsemi_mapjoin.q.out | 1032 +++++++++---------
.../tez/vector_multi_insert.q.out | 20 +-
.../clientpositive/tez/vector_outer_join1.q.out | 48 +-
.../clientpositive/tez/vector_outer_join2.q.out | 22 +-
.../clientpositive/tez/vector_outer_join3.q.out | 60 +-
.../clientpositive/tez/vector_outer_join4.q.out | 48 +-
13 files changed, 855 insertions(+), 624 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 47a1107..784b502 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -336,6 +336,7 @@ minitez.query.files=bucket_map_join_tez1.q,\
tez_schema_evolution.q,\
tez_union.q,\
tez_union2.q,\
+ tez_union_dynamic_partition.q,\
tez_union_view.q,\
tez_union_decimal.q,\
tez_union_group_by.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 0edfc5d..11c1df6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorUtils;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.UnionOperator;
@@ -238,6 +239,11 @@ public class GenTezUtils {
Iterator<Operator<?>> it = newRoots.iterator();
for (Operator<?> orig: roots) {
+ Set<FileSinkOperator> fsOpSet = OperatorUtils.findOperators(orig, FileSinkOperator.class);
+ for (FileSinkOperator fsOp : fsOpSet) {
+ context.fileSinkSet.remove(fsOp);
+ }
+
Operator<?> newRoot = it.next();
replacementMap.put(orig, newRoot);
@@ -301,6 +307,8 @@ public class GenTezUtils {
linked.add(desc);
desc.setDirName(new Path(path, ""+linked.size()));
+ desc.setLinkedFileSink(true);
+ desc.setParentDir(path);
desc.setLinkedFileSinkDesc(linked);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/queries/clientpositive/tez_union_dynamic_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_union_dynamic_partition.q b/ql/src/test/queries/clientpositive/tez_union_dynamic_partition.q
new file mode 100644
index 0000000..1c44a6c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_union_dynamic_partition.q
@@ -0,0 +1,21 @@
+create table dummy(i int);
+insert into table dummy values (1);
+select * from dummy;
+
+create table partunion1(id1 int) partitioned by (part1 string);
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+explain insert into table partunion1 partition(part1)
+select temps.* from (
+select 1 as id1, '2014' as part1 from dummy
+union all
+select 2 as id1, '2014' as part1 from dummy ) temps;
+
+insert into table partunion1 partition(part1)
+select temps.* from (
+select 1 as id1, '2014' as part1 from dummy
+union all
+select 2 as id1, '2014' as part1 from dummy ) temps;
+
+select * from partunion1;
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
index 0340714..222e89e 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
@@ -3960,11 +3960,11 @@ Map 5 <- Union 2 (CONTAINS)
Map 7 <- Map 6 (BROADCAST_EDGE)
Map 8 <- Union 9 (CONTAINS)
-Stage-7
+Stage-15
Stats-Aggr Operator
- Stage-2
+ Stage-1
Move Operator
- table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.c","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
+ table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.b","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
Stage-4
Dependency Collection{}
Stage-3
@@ -4344,18 +4344,63 @@ Stage-7
Statistics:Num rows: 1705 Data size: 18038 Basic stats: COMPLETE Column stats: NONE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.c","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
Please refer to the previous Select Operator [SEL_17]
-Stage-6
+Stage-14
Stats-Aggr Operator
- Stage-1
+ Stage-0
Move Operator
- table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.b","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
+ table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.a","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
Please refer to the previous Stage-4
-Stage-5
+Stage-13
Stats-Aggr Operator
- Stage-0
+ Stage-2
Move Operator
- table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.a","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
+ table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","name:":"default.c","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
Please refer to the previous Stage-4
+Stage-12
+ Stats-Aggr Operator
+ Please refer to the previous Stage-1
+Stage-19
+ Stats-Aggr Operator
+ Please refer to the previous Stage-2
+Stage-18
+ Stats-Aggr Operator
+ Please refer to the previous Stage-1
+Stage-17
+ Stats-Aggr Operator
+ Please refer to the previous Stage-0
+Stage-16
+ Stats-Aggr Operator
+ Please refer to the previous Stage-2
+Stage-20
+ Stats-Aggr Operator
+ Please refer to the previous Stage-0
+Stage-9
+ Stats-Aggr Operator
+ Please refer to the previous Stage-1
+Stage-22
+ Stats-Aggr Operator
+ Please refer to the previous Stage-2
+Stage-8
+ Stats-Aggr Operator
+ Please refer to the previous Stage-0
+Stage-21
+ Stats-Aggr Operator
+ Please refer to the previous Stage-1
+Stage-7
+ Stats-Aggr Operator
+ Please refer to the previous Stage-2
+Stage-6
+ Stats-Aggr Operator
+ Please refer to the previous Stage-1
+Stage-10
+ Stats-Aggr Operator
+ Please refer to the previous Stage-2
+Stage-11
+ Stats-Aggr Operator
+ Please refer to the previous Stage-0
+Stage-5
+ Stats-Aggr Operator
+ Please refer to the previous Stage-0
PREHOOK: query: explain
FROM
(
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out b/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
new file mode 100644
index 0000000..68a7531
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out
@@ -0,0 +1,148 @@
+PREHOOK: query: create table dummy(i int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dummy
+POSTHOOK: query: create table dummy(i int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dummy
+PREHOOK: query: insert into table dummy values (1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@dummy
+POSTHOOK: query: insert into table dummy values (1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@dummy
+POSTHOOK: Lineage: dummy.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: select * from dummy
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dummy
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dummy
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dummy
+#### A masked pattern was here ####
+1
+PREHOOK: query: create table partunion1(id1 int) partitioned by (part1 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@partunion1
+POSTHOOK: query: create table partunion1(id1 int) partitioned by (part1 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@partunion1
+PREHOOK: query: explain insert into table partunion1 partition(part1)
+select temps.* from (
+select 1 as id1, '2014' as part1 from dummy
+union all
+select 2 as id1, '2014' as part1 from dummy ) temps
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert into table partunion1 partition(part1)
+select temps.* from (
+select 1 as id1, '2014' as part1 from dummy
+union all
+select 2 as id1, '2014' as part1 from dummy ) temps
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Map 3 <- Union 2 (CONTAINS)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: dummy
+ Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 1 (type: int), '2014' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: dummy
+ Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 2 (type: int), '2014' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ part1
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.partunion1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-4
+ Stats-Aggr Operator
+
+PREHOOK: query: insert into table partunion1 partition(part1)
+select temps.* from (
+select 1 as id1, '2014' as part1 from dummy
+union all
+select 2 as id1, '2014' as part1 from dummy ) temps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dummy
+PREHOOK: Output: default@partunion1
+POSTHOOK: query: insert into table partunion1 partition(part1)
+select temps.* from (
+select 1 as id1, '2014' as part1 from dummy
+union all
+select 2 as id1, '2014' as part1 from dummy ) temps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dummy
+POSTHOOK: Output: default@partunion1@part1=2014
+POSTHOOK: Lineage: partunion1 PARTITION(part1=2014).id1 EXPRESSION []
+PREHOOK: query: select * from partunion1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partunion1
+PREHOOK: Input: default@partunion1@part1=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select * from partunion1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partunion1
+POSTHOOK: Input: default@partunion1@part1=2014
+#### A masked pattern was here ####
+1 2014
+2 2014
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/union4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/union4.q.out b/ql/src/test/results/clientpositive/tez/union4.q.out
index 7cd6099..9d079ad 100644
--- a/ql/src/test/results/clientpositive/tez/union4.q.out
+++ b/ql/src/test/results/clientpositive/tez/union4.q.out
@@ -33,6 +33,7 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
@@ -139,6 +140,9 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator
+ Stage: Stage-4
+ Stats-Aggr Operator
+
PREHOOK: query: insert overwrite table tmptable
select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1
UNION ALL
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/union6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/union6.q.out b/ql/src/test/results/clientpositive/tez/union6.q.out
index 77f1d0e..4647278 100644
--- a/ql/src/test/results/clientpositive/tez/union6.q.out
+++ b/ql/src/test/results/clientpositive/tez/union6.q.out
@@ -31,6 +31,7 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
@@ -111,6 +112,9 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator
+ Stage: Stage-4
+ Stats-Aggr Operator
+
PREHOOK: query: insert overwrite table tmptable
select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
UNION ALL
[44/50] [abbrv] hive git commit: HIVE-10853 : Create ExplainTask in
ATS hook through ExplainWork (Pengcheng Xiong via Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-10853 : Create ExplainTask in ATS hook through ExplainWork (Pengcheng Xiong via Gunther Hagleitner)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4f8de20a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4f8de20a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4f8de20a
Branch: refs/heads/spark
Commit: 4f8de20ace77036a212debaf5aeed0ec7e4b479a
Parents: 3ea0359
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Fri May 29 11:19:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 6 14:22:04 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 27 +++++++++++++++++---
1 file changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4f8de20a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 53d169d..87638da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.hooks;
+import java.io.Serializable;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -25,11 +26,16 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.plan.ExplainWork;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -113,12 +119,25 @@ public class ATSHook implements ExecuteWithHookContext {
switch(hookContext.getHookType()) {
case PRE_EXEC_HOOK:
- ExplainTask explain = new ExplainTask();
+ ExplainWork work = new ExplainWork(null,// resFile
+ null,// pCtx
+ plan.getRootTasks(),// RootTasks
+ plan.getFetchTask(),// FetchTask
+ null,// astStringTree
+ null,// analyzer
+ false,// extended
+ true,// formatted
+ false,// dependency
+ false,// logical
+ false,// authorize
+ false,// userLevelExplain
+ null// cboInfo
+ );
+ @SuppressWarnings("unchecked")
+ ExplainTask explain = (ExplainTask) TaskFactory.get(work, conf);
explain.initialize(conf, plan, null);
String query = plan.getQueryStr();
- List<Task<?>> rootTasks = plan.getRootTasks();
- JSONObject explainPlan = explain.getJSONPlan(null, null, rootTasks,
- plan.getFetchTask(), true, false, false);
+ JSONObject explainPlan = explain.getJSONPlan(null, work);
fireAndForget(conf, createPreHookEvent(queryId, query,
explainPlan, queryStartTime, user, requestuser, numMrJobs, numTezJobs, opId));
break;
[26/50] [abbrv] hive git commit: HIVE-10735: Cached plan race
condition - VectorMapJoinCommonOperator has no closeOp() (Matt McCline,
reviewed by Sergey Shelukhin)
Posted by xu...@apache.org.
HIVE-10735: Cached plan race condition - VectorMapJoinCommonOperator has no closeOp() (Matt McCline, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82beb2ba
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82beb2ba
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82beb2ba
Branch: refs/heads/spark
Commit: 82beb2ba3d0b2fa7ef690c4a51d641ced161fce2
Parents: 949ddaa
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Wed Jun 3 21:34:33 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Wed Jun 3 21:34:33 2015 -0700
----------------------------------------------------------------------
.../persistence/BytesBytesMultiHashMap.java | 48 +++++++-------------
.../persistence/MapJoinBytesTableContainer.java | 4 +-
.../ql/exec/vector/VectorFilterOperator.java | 5 +-
3 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/82beb2ba/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
index 2ba622e..3bba890 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
@@ -184,19 +184,6 @@ public final class BytesBytesMultiHashMap {
this(initialCapacity, loadFactor, wbSize, -1);
}
- public class ThreadSafeGetter {
- private WriteBuffers.Position position = new WriteBuffers.Position();
- public byte getValueResult(byte[] key, int offset, int length,
- BytesBytesMultiHashMap.Result hashMapResult) {
- return BytesBytesMultiHashMap.this.getValueResult(key, offset, length, hashMapResult, position);
- }
-
- public void populateValue(WriteBuffers.ByteSegmentRef valueRef) {
- // Convenience method, populateValue is thread-safe.
- BytesBytesMultiHashMap.this.populateValue(valueRef);
- }
- }
-
/**
* The result of looking up a key in the multi-hash map.
*
@@ -232,6 +219,14 @@ public final class BytesBytesMultiHashMap {
public Result() {
hasRows = false;
byteSegmentRef = new WriteBuffers.ByteSegmentRef();
+ readPos = new WriteBuffers.Position();
+ }
+
+ /**
+ * Return the thread-safe read position.
+ */
+ public WriteBuffers.Position getReadPos() {
+ return readPos;
}
/**
@@ -260,14 +255,11 @@ public final class BytesBytesMultiHashMap {
* Whether there are multiple values (true) or just a single value (false).
* @param offsetAfterListRecordKeyLen
* The offset of just after the key length in the list record. Or, 0 when single row.
- * @param readPos
- * Holds mutable read position for thread safety.
*/
public void set(BytesBytesMultiHashMap hashMap, long firstOffset, boolean hasList,
- long offsetAfterListRecordKeyLen, WriteBuffers.Position readPos) {
+ long offsetAfterListRecordKeyLen) {
this.hashMap = hashMap;
- this.readPos = readPos;
this.firstOffset = firstOffset;
this.hasList = hasList;
@@ -410,7 +402,6 @@ public final class BytesBytesMultiHashMap {
*/
public void forget() {
hashMap = null;
- readPos = null;
byteSegmentRef.reset(0, 0);
hasRows = false;
readIndex = 0;
@@ -481,29 +472,23 @@ public final class BytesBytesMultiHashMap {
++numValues;
}
- public ThreadSafeGetter createGetterForThread() {
- return new ThreadSafeGetter();
- }
-
- /** Not thread-safe! Use createGetterForThread. */
- public byte getValueResult(byte[] key, int offset, int length, Result hashMapResult) {
- return getValueResult(key, offset, length, hashMapResult, writeBuffers.getReadPosition());
- }
-
/**
* Finds a key. Values can be read with the supplied result object.
*
+ * Important Note: The caller is expected to pre-allocate the hashMapResult and not
+ * share it among other threads.
+ *
* @param key Key buffer.
* @param offset the offset to the key in the buffer
* @param hashMapResult The object to fill in that can read the values.
- * @param readPos Holds mutable read position for thread safety.
* @return The state byte.
*/
- private byte getValueResult(byte[] key, int offset, int length, Result hashMapResult,
- WriteBuffers.Position readPos) {
+ public byte getValueResult(byte[] key, int offset, int length, Result hashMapResult) {
hashMapResult.forget();
+ WriteBuffers.Position readPos = hashMapResult.getReadPos();
+
// First, find first record for the key.
long ref = findKeyRefToRead(key, offset, length, readPos);
if (ref == 0) {
@@ -515,8 +500,7 @@ public final class BytesBytesMultiHashMap {
// This relies on findKeyRefToRead doing key equality check and leaving read ptr where needed.
long offsetAfterListRecordKeyLen = hasList ? writeBuffers.getReadPoint(readPos) : 0;
- hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen,
- readPos);
+ hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen);
return Ref.getStateByte(ref);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82beb2ba/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
index 6deaafc..34b3aa9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
@@ -500,7 +500,6 @@ public class MapJoinBytesTableContainer
private byte aliasFilter;
/** Hash table wrapper specific to the container. */
- private final BytesBytesMultiHashMap.ThreadSafeGetter threadSafeHashMapGetter;
private BytesBytesMultiHashMap.Result hashMapResult;
/**
@@ -520,14 +519,13 @@ public class MapJoinBytesTableContainer
valueStruct = null; // No rows?
}
uselessIndirection = new ByteArrayRef();
- threadSafeHashMapGetter = hashMap.createGetterForThread();
hashMapResult = new BytesBytesMultiHashMap.Result();
clearRows();
}
public JoinUtil.JoinResult setFromOutput(Output output) {
- aliasFilter = threadSafeHashMapGetter.getValueResult(
+ aliasFilter = hashMap.getValueResult(
output.getData(), 0, output.getLength(), hashMapResult);
dummyRow = null;
if (hashMapResult.hasRows()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/82beb2ba/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
index d1b8939..b40564c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
@@ -41,7 +41,7 @@ public class VectorFilterOperator extends FilterOperator {
private VectorExpression conditionEvaluator = null;
// Temporary selected vector
- private final int[] temporarySelected = new int [VectorizedRowBatch.DEFAULT_SIZE];
+ private transient int[] temporarySelected;
// filterMode is 1 if condition is always true, -1 if always false
// and 0 if condition needs to be computed.
@@ -77,6 +77,9 @@ public class VectorFilterOperator extends FilterOperator {
filterMode = -1;
}
}
+
+ temporarySelected = new int [VectorizedRowBatch.DEFAULT_SIZE];
+
return result;
}
[16/50] [abbrv] hive git commit: HIVE-10667 : Upgrade Calcite
dependency to newest version (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10667 : Upgrade Calcite dependency to newest version (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7556361c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7556361c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7556361c
Branch: refs/heads/spark
Commit: 7556361c2a43b47547a9ccdf9f9bec42964a75ae
Parents: d5fdf14
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Jun 2 03:59:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Jun 2 13:04:58 2015 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../ql/optimizer/calcite/HiveCalciteUtil.java | 2 +-
.../ql/optimizer/calcite/HiveRelOptUtil.java | 414 -------------------
.../calcite/reloperators/HiveProject.java | 5 +-
.../calcite/reloperators/HiveSemiJoin.java | 79 ++++
.../calcite/translator/HiveOpConverter.java | 6 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 10 +-
7 files changed, 91 insertions(+), 427 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b21d894..e60bb35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,7 @@
<antlr.version>3.4</antlr.version>
<avro.version>1.7.7</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
- <calcite.version>1.2.0-incubating</calcite.version>
+ <calcite.version>1.3.0-incubating</calcite.version>
<datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
<datanucleus-core.version>3.2.10</datanucleus-core.version>
<datanucleus-rdbms.version>3.2.9</datanucleus-rdbms.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index f4e7c45..199a358 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -504,7 +504,7 @@ public class HiveCalciteUtil {
}
// 1. Split leaf join predicate to expressions from left, right
- HiveRelOptUtil.splitJoinCondition(systemFieldList, inputs, pe,
+ RelOptUtil.splitJoinCondition(systemFieldList, inputs, pe,
joinKeyExprs, filterNulls, null);
// 2. Collect child projection indexes used
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
deleted file mode 100644
index 2827c3b..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.optimizer.calcite;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlOperator;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.Util;
-
-import com.google.common.collect.ImmutableList;
-
-
-public class HiveRelOptUtil extends RelOptUtil {
-
- /**
- * Splits out the equi-join (and optionally, a single non-equi) components
- * of a join condition, and returns what's left. Projection might be
- * required by the caller to provide join keys that are not direct field
- * references.
- *
- * @param sysFieldList list of system fields
- * @param leftRel left join input
- * @param rightRel right join input
- * @param condition join condition
- * @param leftJoinKeys The join keys from the left input which are equi-join
- * keys
- * @param rightJoinKeys The join keys from the right input which are
- * equi-join keys
- * @param filterNulls The join key positions for which null values will not
- * match. null values only match for the "is not distinct
- * from" condition.
- * @param rangeOp if null, only locate equi-joins; otherwise, locate a
- * single non-equi join predicate and return its operator
- * in this list; join keys associated with the non-equi
- * join predicate are at the end of the key lists
- * returned
- * @return What's left, never null
- */
- public static RexNode splitJoinCondition(
- List<RelDataTypeField> sysFieldList,
- RelNode leftRel,
- RelNode rightRel,
- RexNode condition,
- List<RexNode> leftJoinKeys,
- List<RexNode> rightJoinKeys,
- List<Integer> filterNulls,
- List<SqlOperator> rangeOp) {
- return splitJoinCondition(
- sysFieldList,
- ImmutableList.of(leftRel, rightRel),
- condition,
- ImmutableList.of(leftJoinKeys, rightJoinKeys),
- filterNulls,
- rangeOp);
- }
-
- /**
- * Splits out the equi-join (and optionally, a single non-equi) components
- * of a join condition, and returns what's left. Projection might be
- * required by the caller to provide join keys that are not direct field
- * references.
- *
- * @param sysFieldList list of system fields
- * @param inputs join inputs
- * @param condition join condition
- * @param joinKeys The join keys from the inputs which are equi-join
- * keys
- * @param filterNulls The join key positions for which null values will not
- * match. null values only match for the "is not distinct
- * from" condition.
- * @param rangeOp if null, only locate equi-joins; otherwise, locate a
- * single non-equi join predicate and return its operator
- * in this list; join keys associated with the non-equi
- * join predicate are at the end of the key lists
- * returned
- * @return What's left, never null
- */
- public static RexNode splitJoinCondition(
- List<RelDataTypeField> sysFieldList,
- List<RelNode> inputs,
- RexNode condition,
- List<List<RexNode>> joinKeys,
- List<Integer> filterNulls,
- List<SqlOperator> rangeOp) {
- final List<RexNode> nonEquiList = new ArrayList<>();
-
- splitJoinCondition(
- sysFieldList,
- inputs,
- condition,
- joinKeys,
- filterNulls,
- rangeOp,
- nonEquiList);
-
- // Convert the remainders into a list that are AND'ed together.
- return RexUtil.composeConjunction(
- inputs.get(0).getCluster().getRexBuilder(), nonEquiList, false);
- }
-
- private static void splitJoinCondition(
- List<RelDataTypeField> sysFieldList,
- List<RelNode> inputs,
- RexNode condition,
- List<List<RexNode>> joinKeys,
- List<Integer> filterNulls,
- List<SqlOperator> rangeOp,
- List<RexNode> nonEquiList) {
- final int sysFieldCount = sysFieldList.size();
- final RelOptCluster cluster = inputs.get(0).getCluster();
- final RexBuilder rexBuilder = cluster.getRexBuilder();
- final RelDataTypeFactory typeFactory = cluster.getTypeFactory();
-
- int[] firstFieldInputs = new int[inputs.size()];
- int totalFieldCount = 0;
- for (int i = 0; i < inputs.size(); i++) {
- firstFieldInputs[i] = totalFieldCount + sysFieldCount;
- totalFieldCount += sysFieldCount
- + inputs.get(i).getRowType().getFieldCount();
- }
-
- // adjustment array
- int[] adjustments = new int[totalFieldCount];
- for (int i = 0; i < inputs.size(); i++) {
- int limit = i == inputs.size() - 1
- ? totalFieldCount : firstFieldInputs[i + 1];
- for (int j = firstFieldInputs[i]; j < limit; j++) {
- adjustments[j] = -firstFieldInputs[i];
- }
- }
-
- if (condition instanceof RexCall) {
- RexCall call = (RexCall) condition;
- if (call.getOperator() == SqlStdOperatorTable.AND) {
- for (RexNode operand : call.getOperands()) {
- splitJoinCondition(
- sysFieldList,
- inputs,
- operand,
- joinKeys,
- filterNulls,
- rangeOp,
- nonEquiList);
- }
- return;
- }
-
- RexNode leftKey = null;
- RexNode rightKey = null;
- int leftInput = 0;
- int rightInput = 0;
- List<RelDataTypeField> leftFields = null;
- List<RelDataTypeField> rightFields = null;
- boolean reverse = false;
-
- SqlKind kind = call.getKind();
-
- // Only consider range operators if we haven't already seen one
- if ((kind == SqlKind.EQUALS)
- || (filterNulls != null
- && kind == SqlKind.IS_NOT_DISTINCT_FROM)
- || (rangeOp != null
- && rangeOp.isEmpty()
- && (kind == SqlKind.GREATER_THAN
- || kind == SqlKind.GREATER_THAN_OR_EQUAL
- || kind == SqlKind.LESS_THAN
- || kind == SqlKind.LESS_THAN_OR_EQUAL))) {
- final List<RexNode> operands = call.getOperands();
- RexNode op0 = operands.get(0);
- RexNode op1 = operands.get(1);
-
- final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
- final ImmutableBitSet projRefs1 = InputFinder.bits(op1);
-
- boolean foundBothInputs = false;
- for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
- final int lowerLimit = firstFieldInputs[i];
- final int upperLimit = i == inputs.size() - 1
- ? totalFieldCount : firstFieldInputs[i + 1];
- if (projRefs0.nextSetBit(lowerLimit) != -1
- && projRefs0.nextSetBit(upperLimit) == -1
- && projRefs0.nextSetBit(0) == projRefs0.nextSetBit(lowerLimit)
- && projRefs0.nextSetBit(lowerLimit) < upperLimit) {
- if (leftKey == null) {
- leftKey = op0;
- leftInput = i;
- leftFields = inputs.get(leftInput).getRowType().getFieldList();
- } else {
- rightKey = op0;
- rightInput = i;
- rightFields = inputs.get(rightInput).getRowType().getFieldList();
- reverse = true;
- foundBothInputs = true;
- }
- } else if (projRefs1.nextSetBit(lowerLimit) != -1
- && projRefs1.nextSetBit(upperLimit) == -1
- && projRefs1.nextSetBit(0) == projRefs1.nextSetBit(lowerLimit)
- && projRefs1.nextSetBit(lowerLimit) < upperLimit) {
- if (leftKey == null) {
- leftKey = op1;
- leftInput = i;
- leftFields = inputs.get(leftInput).getRowType().getFieldList();
- } else {
- rightKey = op1;
- rightInput = i;
- rightFields = inputs.get(rightInput).getRowType().getFieldList();
- foundBothInputs = true;
- }
- }
- }
-
- if ((leftKey != null) && (rightKey != null)) {
- // replace right Key input ref
- rightKey =
- rightKey.accept(
- new RelOptUtil.RexInputConverter(
- rexBuilder,
- rightFields,
- rightFields,
- adjustments));
-
- // left key only needs to be adjusted if there are system
- // fields, but do it for uniformity
- leftKey =
- leftKey.accept(
- new RelOptUtil.RexInputConverter(
- rexBuilder,
- leftFields,
- leftFields,
- adjustments));
-
- RelDataType leftKeyType = leftKey.getType();
- RelDataType rightKeyType = rightKey.getType();
-
- if (leftKeyType != rightKeyType) {
- // perform casting
- RelDataType targetKeyType =
- typeFactory.leastRestrictive(
- ImmutableList.of(leftKeyType, rightKeyType));
-
- if (targetKeyType == null) {
- throw Util.newInternal(
- "Cannot find common type for join keys "
- + leftKey + " (type " + leftKeyType + ") and "
- + rightKey + " (type " + rightKeyType + ")");
- }
-
- if (leftKeyType != targetKeyType) {
- leftKey =
- rexBuilder.makeCast(targetKeyType, leftKey);
- }
-
- if (rightKeyType != targetKeyType) {
- rightKey =
- rexBuilder.makeCast(targetKeyType, rightKey);
- }
- }
- }
- }
-
- if ((rangeOp == null)
- && ((leftKey == null) || (rightKey == null))) {
- // no equality join keys found yet:
- // try transforming the condition to
- // equality "join" conditions, e.g.
- // f(LHS) > 0 ===> ( f(LHS) > 0 ) = TRUE,
- // and make the RHS produce TRUE, but only if we're strictly
- // looking for equi-joins
- final ImmutableBitSet projRefs = InputFinder.bits(condition);
- leftKey = null;
- rightKey = null;
-
- boolean foundInput = false;
- for (int i = 0; i < inputs.size() && !foundInput; i++) {
- final int lowerLimit = firstFieldInputs[i];
- final int upperLimit = i == inputs.size() - 1
- ? totalFieldCount : firstFieldInputs[i + 1];
- if (projRefs.nextSetBit(lowerLimit) < upperLimit) {
- leftInput = i;
- leftFields = inputs.get(leftInput).getRowType().getFieldList();
-
- leftKey = condition.accept(
- new RelOptUtil.RexInputConverter(
- rexBuilder,
- leftFields,
- leftFields,
- adjustments));
-
- rightKey = rexBuilder.makeLiteral(true);
-
- // effectively performing an equality comparison
- kind = SqlKind.EQUALS;
-
- foundInput = true;
- }
- }
- }
-
- if ((leftKey != null) && (rightKey != null)) {
- // found suitable join keys
- // add them to key list, ensuring that if there is a
- // non-equi join predicate, it appears at the end of the
- // key list; also mark the null filtering property
- addJoinKey(
- joinKeys.get(leftInput),
- leftKey,
- (rangeOp != null) && !rangeOp.isEmpty());
- addJoinKey(
- joinKeys.get(rightInput),
- rightKey,
- (rangeOp != null) && !rangeOp.isEmpty());
- if (filterNulls != null
- && kind == SqlKind.EQUALS) {
- // nulls are considered not matching for equality comparison
- // add the position of the most recently inserted key
- filterNulls.add(joinKeys.get(leftInput).size() - 1);
- }
- if (rangeOp != null
- && kind != SqlKind.EQUALS
- && kind != SqlKind.IS_DISTINCT_FROM) {
- if (reverse) {
- kind = reverse(kind);
- }
- rangeOp.add(op(kind, call.getOperator()));
- }
- return;
- } // else fall through and add this condition as nonEqui condition
- }
-
- // The operator is not of RexCall type
- // So we fail. Fall through.
- // Add this condition to the list of non-equi-join conditions.
- nonEquiList.add(condition);
- }
-
- private static SqlKind reverse(SqlKind kind) {
- switch (kind) {
- case GREATER_THAN:
- return SqlKind.LESS_THAN;
- case GREATER_THAN_OR_EQUAL:
- return SqlKind.LESS_THAN_OR_EQUAL;
- case LESS_THAN:
- return SqlKind.GREATER_THAN;
- case LESS_THAN_OR_EQUAL:
- return SqlKind.GREATER_THAN_OR_EQUAL;
- default:
- return kind;
- }
- }
-
- private static SqlOperator op(SqlKind kind, SqlOperator operator) {
- switch (kind) {
- case EQUALS:
- return SqlStdOperatorTable.EQUALS;
- case NOT_EQUALS:
- return SqlStdOperatorTable.NOT_EQUALS;
- case GREATER_THAN:
- return SqlStdOperatorTable.GREATER_THAN;
- case GREATER_THAN_OR_EQUAL:
- return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL;
- case LESS_THAN:
- return SqlStdOperatorTable.LESS_THAN;
- case LESS_THAN_OR_EQUAL:
- return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
- case IS_DISTINCT_FROM:
- return SqlStdOperatorTable.IS_DISTINCT_FROM;
- case IS_NOT_DISTINCT_FROM:
- return SqlStdOperatorTable.IS_NOT_DISTINCT_FROM;
- default:
- return operator;
- }
- }
-
- private static void addJoinKey(
- List<RexNode> joinKeyList,
- RexNode key,
- boolean preserveLastElementInList) {
- if (!joinKeyList.isEmpty() && preserveLastElementInList) {
- joinKeyList.add(joinKeyList.size() - 1, key);
- } else {
- joinKeyList.add(key);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
index cf0c462..4b7887a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
@@ -39,10 +39,10 @@ import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.MappingType;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
-import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCost;
+
import com.google.common.collect.ImmutableList;
public class HiveProject extends Project implements HiveRelNode {
@@ -68,6 +68,7 @@ public class HiveProject extends Project implements HiveRelNode {
public HiveProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode child,
List<? extends RexNode> exps, RelDataType rowType, int flags) {
super(cluster, traitSet, child, exps, rowType, flags);
+ assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
virtualCols = ImmutableList.copyOf(HiveCalciteUtil.getVirtualCols(exps));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
new file mode 100644
index 0000000..dd1691c
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory;
+import org.apache.calcite.rel.core.SemiJoin;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.ImmutableIntList;
+
+public class HiveSemiJoin extends SemiJoin implements HiveRelNode {
+
+ public static final SemiJoinFactory HIVE_SEMIJOIN_FACTORY = new HiveSemiJoinFactoryImpl();
+
+ public HiveSemiJoin(RelOptCluster cluster,
+ RelTraitSet traitSet,
+ RelNode left,
+ RelNode right,
+ RexNode condition,
+ ImmutableIntList leftKeys,
+ ImmutableIntList rightKeys) {
+ super(cluster, traitSet, left, right, condition, leftKeys, rightKeys);
+ }
+
+ @Override
+ public SemiJoin copy(RelTraitSet traitSet, RexNode condition,
+ RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) {
+ final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
+ return new HiveSemiJoin(getCluster(), traitSet, left, right, condition,
+ joinInfo.leftKeys, joinInfo.rightKeys);
+ }
+
+ @Override
+ public void implement(Implementor implementor) {
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ return RelMetadataQuery.getNonCumulativeCost(this);
+ }
+
+ /**
+ * Implementation of {@link SemiJoinFactory} that returns
+ * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin}
+ * .
+ */
+ private static class HiveSemiJoinFactoryImpl implements SemiJoinFactory {
+ @Override
+ public RelNode createSemiJoin(RelNode left, RelNode right,
+ RexNode condition) {
+ final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
+ final RelOptCluster cluster = left.getCluster();
+ return new HiveSemiJoin(cluster, left.getTraitSet(), left, right, condition,
+ joinInfo.leftKeys, joinInfo.rightKeys);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index 4f19caf..9c21238 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -32,7 +32,6 @@ import org.apache.calcite.rel.RelDistribution;
import org.apache.calcite.rel.RelDistribution.Type;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.SemiJoin;
import org.apache.calcite.rel.rules.MultiJoin;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
@@ -63,6 +62,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
@@ -156,8 +156,8 @@ public class HiveOpConverter {
return visit((MultiJoin) rn);
} else if (rn instanceof HiveJoin) {
return visit((HiveJoin) rn);
- } else if (rn instanceof SemiJoin) {
- SemiJoin sj = (SemiJoin) rn;
+ } else if (rn instanceof HiveSemiJoin) {
+ HiveSemiJoin sj = (HiveSemiJoin) rn;
HiveJoin hj = HiveJoin.getJoin(sj.getCluster(), sj.getLeft(), sj.getRight(),
sj.getCondition(), sj.getJoinType(), true);
return visit(hj);
http://git-wip-us.apache.org/repos/asf/hive/blob/7556361c/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index d545bf0..02fc796 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -59,8 +59,6 @@ import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.core.SemiJoin;
import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
@@ -121,7 +119,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.Unsu
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfigContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
@@ -133,6 +130,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
@@ -956,7 +954,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// 5. Projection Pruning
HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
- RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
+ HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
basePlan = fieldTrimmer.trim(basePlan);
@@ -1196,7 +1194,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
List<RexNode> leftJoinKeys = new ArrayList<RexNode>();
List<RexNode> rightJoinKeys = new ArrayList<RexNode>();
- RexNode nonEquiConds = HiveRelOptUtil.splitJoinCondition(sysFieldList, leftRel, rightRel,
+ RexNode nonEquiConds = RelOptUtil.splitJoinCondition(sysFieldList, leftRel, rightRel,
calciteJoinCond, leftJoinKeys, rightJoinKeys, null, null);
if (!nonEquiConds.isAlwaysTrue()) {
@@ -1211,7 +1209,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
HiveProject.DEFAULT_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0,
leftKeys, rightKeys);
- joinRel = new SemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0],
+ joinRel = new HiveSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0],
inputRels[1], calciteJoinCond, ImmutableIntList.copyOf(leftKeys),
ImmutableIntList.copyOf(rightKeys));
} else {
[05/50] [abbrv] hive git commit: HIVE-10458: Enable parallel order by
for spark [Spark Branch] (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
HIVE-10458: Enable parallel order by for spark [Spark Branch] (Rui reviewed by Xuefu)
Signed-off-by: Rui Li <ru...@intel.com>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37cccbc5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37cccbc5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37cccbc5
Branch: refs/heads/spark
Commit: 37cccbc58fdee764743bda6ba2819e94e8d9a83f
Parents: b3214f9
Author: Rui Li <ru...@intel.com>
Authored: Thu May 21 10:13:17 2015 +0800
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:04:24 2015 -0700
----------------------------------------------------------------------
.../correlation/ReduceSinkDeDuplication.java | 12 ++++-----
.../spark/SetSparkReducerParallelism.java | 28 +++++++++++++++++++-
.../hive/ql/parse/spark/GenSparkUtils.java | 2 +-
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 15 +++++------
.../clientpositive/spark/parallel_orderby.q.out | 6 ++---
5 files changed, 44 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
index 404b759..8ac9ca7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
@@ -489,7 +489,7 @@ public class ReduceSinkDeDuplication implements Transform {
if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
CorrelationUtilities.replaceReduceSinkWithSelectOperator(
cRS, dedupCtx.getPctx(), dedupCtx);
- pRS.getConf().setEnforceSort(true);
+ pRS.getConf().setDeduplicated(true);
return true;
}
return false;
@@ -512,7 +512,7 @@ public class ReduceSinkDeDuplication implements Transform {
if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
CorrelationUtilities.removeReduceSinkForGroupBy(
cRS, cGBY, dedupCtx.getPctx(), dedupCtx);
- pRS.getConf().setEnforceSort(true);
+ pRS.getConf().setDeduplicated(true);
return true;
}
return false;
@@ -535,7 +535,7 @@ public class ReduceSinkDeDuplication implements Transform {
CorrelationUtilities.findPossibleParent(
pJoin, ReduceSinkOperator.class, dedupCtx.trustScript());
if (pRS != null) {
- pRS.getConf().setEnforceSort(true);
+ pRS.getConf().setDeduplicated(true);
}
return true;
}
@@ -559,7 +559,7 @@ public class ReduceSinkDeDuplication implements Transform {
CorrelationUtilities.findPossibleParent(
pJoin, ReduceSinkOperator.class, dedupCtx.trustScript());
if (pRS != null) {
- pRS.getConf().setEnforceSort(true);
+ pRS.getConf().setDeduplicated(true);
}
return true;
}
@@ -579,7 +579,7 @@ public class ReduceSinkDeDuplication implements Transform {
if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
CorrelationUtilities.replaceReduceSinkWithSelectOperator(
cRS, dedupCtx.getPctx(), dedupCtx);
- pRS.getConf().setEnforceSort(true);
+ pRS.getConf().setDeduplicated(true);
return true;
}
return false;
@@ -596,7 +596,7 @@ public class ReduceSinkDeDuplication implements Transform {
start, ReduceSinkOperator.class, dedupCtx.trustScript());
if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
CorrelationUtilities.removeReduceSinkForGroupBy(cRS, cGBY, dedupCtx.getPctx(), dedupCtx);
- pRS.getConf().setEnforceSort(true);
+ pRS.getConf().setDeduplicated(true);
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
index f9ef474..5f9225c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer.spark;
+import java.util.List;
import java.util.Stack;
import org.apache.commons.logging.Log;
@@ -26,6 +27,7 @@ import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.LimitOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -75,7 +77,7 @@ public class SetSparkReducerParallelism implements NodeProcessor {
context.getVisitedReduceSinks().add(sink);
- if (desc.getNumReducers() <= 0) {
+ if (needSetParallelism(sink, context.getConf())) {
if (constantReducers > 0) {
LOG.info("Parallelism for reduce sink " + sink + " set by user to " + constantReducers);
desc.setNumReducers(constantReducers);
@@ -158,4 +160,28 @@ public class SetSparkReducerParallelism implements NodeProcessor {
return false;
}
+ // tests whether the RS needs automatic setting parallelism
+ private boolean needSetParallelism(ReduceSinkOperator reduceSink, HiveConf hiveConf) {
+ ReduceSinkDesc desc = reduceSink.getConf();
+ if (desc.getNumReducers() <= 0) {
+ return true;
+ }
+ if (desc.getNumReducers() == 1 && desc.hasOrderBy() &&
+ hiveConf.getBoolVar(HiveConf.ConfVars.HIVESAMPLINGFORORDERBY) && !desc.isDeduplicated()) {
+ List<Operator<? extends OperatorDesc>> children = reduceSink.getChildOperators();
+ while (children != null && children.size() > 0) {
+ if (children.size() != 1 || children.get(0) instanceof LimitOperator) {
+ return false;
+ }
+ if (children.get(0) instanceof ReduceSinkOperator ||
+ children.get(0) instanceof FileSinkOperator) {
+ break;
+ }
+ children = children.get(0).getChildOperators();
+ }
+ return true;
+ }
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
index e27ce0d..7992c88 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
@@ -399,7 +399,7 @@ public class GenSparkUtils {
*/
private static boolean groupByNeedParLevelOrder(ReduceSinkOperator reduceSinkOperator) {
// whether we have to enforce sort anyway, e.g. in case of RS deduplication
- if (reduceSinkOperator.getConf().isEnforceSort()) {
+ if (reduceSinkOperator.getConf().isDeduplicated()) {
return true;
}
List<Operator<? extends OperatorDesc>> children = reduceSinkOperator.getChildOperators();
http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index 1891dff..b4316ec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
-import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -114,8 +113,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
// Write type, since this needs to calculate buckets differently for updates and deletes
private AcidUtils.Operation writeType;
- // whether we'll enforce the sort order of the RS
- private transient boolean enforceSort = false;
+ // whether this RS is deduplicated
+ private transient boolean isDeduplicated = false;
// used by spark mode to decide whether global order is needed
private transient boolean hasOrderBy = false;
@@ -174,7 +173,7 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
desc.setStatistics(this.getStatistics());
desc.setSkipTag(skipTag);
desc.reduceTraits = reduceTraits.clone();
- desc.setEnforceSort(enforceSort);
+ desc.setDeduplicated(isDeduplicated);
desc.setHasOrderBy(hasOrderBy);
return desc;
}
@@ -434,12 +433,12 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return writeType;
}
- public boolean isEnforceSort() {
- return enforceSort;
+ public boolean isDeduplicated() {
+ return isDeduplicated;
}
- public void setEnforceSort(boolean isDeduplicated) {
- this.enforceSort = isDeduplicated;
+ public void setDeduplicated(boolean isDeduplicated) {
+ this.isDeduplicated = isDeduplicated;
}
public boolean hasOrderBy() {
http://git-wip-us.apache.org/repos/asf/hive/blob/37cccbc5/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out b/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
index 0194dbb..03314ea 100644
--- a/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
+++ b/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
@@ -38,7 +38,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (SORT, 1)
+ Reducer 2 <- Map 1 (SORT, 4)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -117,7 +117,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE true
- numFiles 1
+ numFiles 4
numRows 48
rawDataSize 512
totalSize 560
@@ -231,7 +231,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE true
- numFiles 1
+ numFiles 4
numRows 48
rawDataSize 512
totalSize 560
[10/50] [abbrv] hive git commit: HIVE-10302 Load small tables (for
map join) in executor memory only once [Spark Branch] (added new file missed
in rebasing)
Posted by xu...@apache.org.
HIVE-10302 Load small tables (for map join) in executor memory only once [Spark Branch] (added new file missed in rebasing)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9bc8ea2a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9bc8ea2a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9bc8ea2a
Branch: refs/heads/spark
Commit: 9bc8ea2ae25e2ef766604068ac143eeeaa376929
Parents: 3817bb5
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Mon Jun 1 15:59:32 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Mon Jun 1 15:59:32 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/spark/SmallTableCache.java | 73 ++++++++++++++++++++
1 file changed, 73 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9bc8ea2a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java
new file mode 100644
index 0000000..1992d16
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SmallTableCache.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.spark;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+
+public class SmallTableCache {
+ private static final Log LOG = LogFactory.getLog(SmallTableCache.class.getName());
+
+ private static final ConcurrentHashMap<Path, MapJoinTableContainer>
+ tableContainerMap = new ConcurrentHashMap<Path, MapJoinTableContainer>();
+ private static volatile String queryId;
+
+ /**
+ * Check if this is a new query. If so, clean up the cache
+ * that is for the previous query, and reset the current query id.
+ */
+ public static void initialize(Configuration conf) {
+ String currentQueryId = conf.get(HiveConf.ConfVars.HIVEQUERYID.varname);
+ if (!currentQueryId.equals(queryId)) {
+ if (!tableContainerMap.isEmpty()) {
+ synchronized (tableContainerMap) {
+ if (!currentQueryId.equals(queryId) && !tableContainerMap.isEmpty()) {
+ for (MapJoinTableContainer tableContainer: tableContainerMap.values()) {
+ tableContainer.clear();
+ }
+ tableContainerMap.clear();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cleaned up small table cache for query " + queryId);
+ }
+ }
+ }
+ }
+ queryId = currentQueryId;
+ }
+ }
+
+ public static void cache(Path path, MapJoinTableContainer tableContainer) {
+ if (tableContainerMap.putIfAbsent(path, tableContainer) == null && LOG.isDebugEnabled()) {
+ LOG.debug("Cached small table file " + path + " for query " + queryId);
+ }
+ }
+
+ public static MapJoinTableContainer get(Path path) {
+ MapJoinTableContainer tableContainer = tableContainerMap.get(path);
+ if (tableContainer != null && LOG.isDebugEnabled()) {
+ LOG.debug("Loaded small table file " + path + " from cache for query " + queryId);
+ }
+ return tableContainer;
+ }
+}
[14/50] [abbrv] hive git commit: HIVE-10748 Replace StringBuffer with StringBuilder where possible (Alexander Pivovarov, reviewed by Ashutosh Chauhan, Sergio Peña and Swarnim Kulkarni)
Posted by xu...@apache.org.
HIVE-10748 Replace StringBuffer with StringBuilder where possible (Alexander Pivovarov, reviewed by Ashutosh Chauhan, Sergio Peña and Swarnim Kulkarni)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aaf3bd5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aaf3bd5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aaf3bd5
Branch: refs/heads/spark
Commit: 8aaf3bd5b7702b4f92503dbf7ffcccad82633d6d
Parents: 715871a
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Wed May 20 19:05:42 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue Jun 2 11:35:51 2015 -0700
----------------------------------------------------------------------
.../common/jsonexplain/tez/TezJsonParser.java | 2 +-
.../hive/common/type/TestHiveBaseChar.java | 2 +-
.../hive/common/type/TestHiveVarchar.java | 22 ++++++++++----------
.../hive/hcatalog/common/HCatException.java | 2 +-
.../hive/hcatalog/data/TestJsonSerDe.java | 2 +-
.../streaming/DelimitedInputWriter.java | 2 +-
.../hive/hcatalog/streaming/HiveEndPoint.java | 2 +-
.../hive/hcatalog/streaming/TestStreaming.java | 2 +-
.../apache/hive/jdbc/HivePreparedStatement.java | 2 +-
.../metastore/txn/CompactionTxnHandler.java | 9 ++++----
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 4 ++--
.../apache/hadoop/hive/ql/exec/Utilities.java | 9 ++++----
.../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 4 ++--
.../ql/exec/vector/VectorizedBatchUtil.java | 2 +-
.../hive/ql/optimizer/GenMRTableScan1.java | 2 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 5 ++++-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 2 +-
.../ql/optimizer/calcite/RelOptHiveTable.java | 2 +-
.../spark/SparkReduceSinkMapJoinProc.java | 2 +-
.../hive/ql/parse/ProcessAnalyzeTable.java | 4 ++--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 2 +-
.../apache/hadoop/hive/ql/plan/FilterDesc.java | 4 ++--
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 19 ++++++++++-------
.../hadoop/hive/ql/plan/TableScanDesc.java | 2 +-
.../hive/ql/txn/compactor/CompactorMR.java | 9 ++++----
.../hadoop/hive/ql/txn/compactor/Initiator.java | 2 +-
.../hadoop/hive/ql/txn/compactor/Worker.java | 3 +--
.../objectinspector/ObjectInspectorUtils.java | 2 +-
.../hive/serde2/lazy/TestLazySimpleSerDe.java | 9 +-------
.../apache/hive/service/auth/HttpAuthUtils.java | 2 +-
32 files changed, 72 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
index 6d6bbc2..43ddff3 100644
--- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
+++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java
@@ -125,7 +125,7 @@ public class TezJsonParser implements JsonParser {
if (candidate.tezStageDependency != null && candidate.tezStageDependency.size() > 0) {
outputStream.println("Vertex dependency in root stage");
for (Entry<Vertex, List<Connection>> entry : candidate.tezStageDependency.entrySet()) {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
sb.append(entry.getKey().name);
sb.append(" <- ");
boolean printcomma = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
index 012c28b..2848465 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
@@ -55,7 +55,7 @@ public class TestHiveBaseChar extends TestCase {
}
public static String createRandomSupplementaryCharString(int len) {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
for (int idx = 0; idx < len; ++idx) {
sb.appendCodePoint(getRandomCodePoint(' '));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
index 309d042..44e2338 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
@@ -18,19 +18,19 @@
package org.apache.hadoop.hive.common.type;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.LogUtils;
-import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
import java.util.Random;
-import com.google.code.tempusfugit.concurrency.annotations.*;
-import com.google.code.tempusfugit.concurrency.*;
-import org.junit.*;
-import static org.junit.Assert.*;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.code.tempusfugit.concurrency.ConcurrentRule;
+import com.google.code.tempusfugit.concurrency.RepeatingRule;
+import com.google.code.tempusfugit.concurrency.annotations.Concurrent;
+import com.google.code.tempusfugit.concurrency.annotations.Repeating;
public class TestHiveVarchar {
@Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
@@ -80,7 +80,7 @@ public class TestHiveVarchar {
// Try with supplementary characters
for (int idx1 = 0; idx1 < lengths.length; ++idx1) {
// Create random test string
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
int curLen = lengths[idx1];
for (int idx2 = 0; idx2 < curLen; ++idx2) {
sb.appendCodePoint(getRandomCodePoint(' '));
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
index 265d08d..3749da6 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatException.java
@@ -104,7 +104,7 @@ public class HCatException extends IOException {
public static String buildErrorMessage(ErrorType type, String extraMessage, Throwable cause) {
//Initial message is just the error type message
- StringBuffer message = new StringBuffer(HCatException.class.getName());
+ StringBuilder message = new StringBuilder(HCatException.class.getName());
message.append(" : " + type.getErrorCode());
message.append(" : " + type.getErrorMessage());
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
index 2947c43..618f39b 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
@@ -227,7 +227,7 @@ public class TestJsonSerDe extends TestCase {
return "";
}
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
int numStrings = columnNames.split(",").length;
sb.append("_col0");
for (int i = 1; i < numStrings; i++) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
index eae91cb..6dc69f0 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/DelimitedInputWriter.java
@@ -186,7 +186,7 @@ public class DelimitedInputWriter extends AbstractRecordWriter {
// handles nulls in items[]
// TODO: perhaps can be made more efficient by creating a byte[] directly
private static byte[] join(String[] items, char separator) {
- StringBuffer buff = new StringBuffer(100);
+ StringBuilder buff = new StringBuilder(100);
if(items.length == 0)
return "".getBytes();
int i=0;
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
index 3c25486..7e99008 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
@@ -418,7 +418,7 @@ public class HiveEndPoint {
throw new IllegalArgumentException("Partition values:" + partVals +
", does not match the partition Keys in table :" + partKeys );
}
- StringBuffer buff = new StringBuffer(partKeys.size()*20);
+ StringBuilder buff = new StringBuilder(partKeys.size()*20);
buff.append(" ( ");
int i=0;
for (FieldSchema schema : partKeys) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index 329e5da..c0af533 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -794,7 +794,7 @@ public class TestStreaming {
throw new IllegalArgumentException("Partition values:" + partVals
+ ", does not match the partition Keys in table :" + partKeys );
}
- StringBuffer buff = new StringBuffer(partKeys.size()*20);
+ StringBuilder buff = new StringBuilder(partKeys.size()*20);
buff.append(" ( ");
int i=0;
for (FieldSchema schema : partKeys) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
index 8a0671f..2625155 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java
@@ -132,7 +132,7 @@ public class HivePreparedStatement extends HiveStatement implements PreparedStat
return sql;
}
- StringBuffer newSql = new StringBuffer(sql);
+ StringBuilder newSql = new StringBuilder(sql);
int paramLoc = 1;
while (getCharIndexFromSqlByParamLocation(sql, '?', paramLoc) > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 52147bc..26e72be 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -340,7 +340,7 @@ public class CompactionTxnHandler extends TxnHandler {
if (txnids.size() > 0) {
// Remove entries from txn_components, as there may be aborted txn components
- StringBuffer buf = new StringBuffer();
+ StringBuilder buf = new StringBuilder();
buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
boolean first = true;
for (long id : txnids) {
@@ -404,7 +404,7 @@ public class CompactionTxnHandler extends TxnHandler {
Set<Long> txnids = new HashSet<Long>();
while (rs.next()) txnids.add(rs.getLong(1));
if (txnids.size() > 0) {
- StringBuffer buf = new StringBuffer("delete from TXNS where txn_id in (");
+ StringBuilder buf = new StringBuilder("delete from TXNS where txn_id in (");
boolean first = true;
for (long tid : txnids) {
if (first) first = false;
@@ -412,8 +412,9 @@ public class CompactionTxnHandler extends TxnHandler {
buf.append(tid);
}
buf.append(")");
- LOG.debug("Going to execute update <" + buf.toString() + ">");
- int rc = stmt.executeUpdate(buf.toString());
+ String bufStr = buf.toString();
+ LOG.debug("Going to execute update <" + bufStr + ">");
+ int rc = stmt.executeUpdate(bufStr);
LOG.debug("Removed " + rc + " records from txns");
LOG.debug("Going to commit");
dbConn.commit();
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index ada79bd..35c4cfc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -854,7 +854,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
}
}
else {
- StringBuffer s = new StringBuffer();
+ StringBuilder s = new StringBuilder();
first = true;
for (Task<?> parent : task.getParentTasks()) {
if (!first) {
@@ -886,7 +886,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
if (task instanceof ConditionalTask
&& ((ConditionalTask) task).getListTasks() != null) {
- StringBuffer s = new StringBuffer();
+ StringBuilder s = new StringBuilder();
first = true;
for (Task<?> con : ((ConditionalTask) task).getListTasks()) {
if (!first) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 5d58839..5f92d11 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1811,11 +1811,12 @@ public final class Utilities {
private static String adjustBucketNumLen(String bucketNum, String taskId) {
int bucketNumLen = bucketNum.length();
int taskIdLen = taskId.length();
- StringBuffer s = new StringBuffer();
+ StringBuilder s = new StringBuilder();
for (int i = 0; i < taskIdLen - bucketNumLen; i++) {
s.append("0");
}
- return s.toString() + bucketNum;
+ s.append(bucketNum);
+ return s.toString();
}
/**
@@ -1836,7 +1837,7 @@ public final class Utilities {
return filename.replaceAll(oldTaskId, newTaskId);
}
- StringBuffer snew = new StringBuffer();
+ StringBuilder snew = new StringBuilder();
for (int idx = 0; idx < spl.length - 1; idx++) {
if (idx > 0) {
snew.append(oldTaskId);
@@ -3169,7 +3170,7 @@ public final class Utilities {
* @return a string with escaped '_' and '%'.
*/
public static String escapeSqlLike(String key) {
- StringBuffer sb = new StringBuffer(key.length());
+ StringBuilder sb = new StringBuilder(key.length());
for (char c: key.toCharArray()) {
switch(c) {
case '_':
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index a9d2dbf..3fceeb0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -62,7 +62,7 @@ abstract class SparkJobMonitor {
return;
}
- StringBuffer reportBuffer = new StringBuffer();
+ StringBuilder reportBuffer = new StringBuilder();
SimpleDateFormat dt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
String currentDate = dt.format(new Date());
reportBuffer.append(currentDate + "\t");
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index 78caba8..aa84850 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -596,7 +596,7 @@ public class TezJobMonitor {
private void printStatusInPlace(Map<String, Progress> progressMap, long startTime,
boolean vextexStatusFromAM, DAGClient dagClient) {
- StringBuffer reportBuffer = new StringBuffer();
+ StringBuilder reportBuffer = new StringBuilder();
int sumComplete = 0;
int sumTotal = 0;
@@ -781,7 +781,7 @@ public class TezJobMonitor {
}
private String getReport(Map<String, Progress> progressMap) {
- StringBuffer reportBuffer = new StringBuffer();
+ StringBuilder reportBuffer = new StringBuilder();
SortedSet<String> keys = new TreeSet<String>(progressMap.keySet());
for (String s: keys) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index 4a16b4c..99cb620 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -643,7 +643,7 @@ public class VectorizedBatchUtil {
}
public static void debugDisplayOneRow(VectorizedRowBatch batch, int index, String prefix) {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
sb.append(prefix + " row " + index + " ");
for (int column = 0; column < batch.cols.length; column++) {
ColumnVector colVector = batch.cols[column];
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
index c5f03d9..eed1d7c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
@@ -187,7 +187,7 @@ public class GenMRTableScan1 implements NodeProcessor {
ParseContext parseCtx, Task<? extends Serializable> currTask,
StatsWork statsWork, Task<StatsWork> statsTask) throws SemanticException {
String aggregationKey = op.getConf().getStatsAggPrefix();
- StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
+ StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(op, aggregationKeyBuffer);
aggregationKey = aggregationKeyBuffer.toString();
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index acd9bf5..29854d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer;
+import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
@@ -1814,7 +1815,7 @@ public final class GenMapRedUtils {
}
public static List<Path> getInputPathsForPartialScan(TableScanOperator tableScanOp,
- StringBuffer aggregationKey) throws SemanticException {
+ Appendable aggregationKey) throws SemanticException {
List<Path> inputPaths = new ArrayList<Path>();
switch (tableScanOp.getConf().getTableMetadata().getTableSpec().specType) {
case TABLE_ONLY:
@@ -1829,6 +1830,8 @@ public final class GenMapRedUtils {
} catch (MetaException e) {
throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AGGKEY.getMsg(
part.getDataLocation().toString() + e.getMessage()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
inputPaths.add(part.getDataLocation());
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
index f7e1dbc..bca91dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
@@ -283,7 +283,7 @@ public class ReduceSinkMapJoinProc implements NodeProcessor {
Map<Byte, List<ExprNodeDesc>> keyExprMap = mapJoinOp.getConf().getKeys();
List<ExprNodeDesc> keyCols = keyExprMap.get(Byte.valueOf((byte) 0));
- StringBuffer keyOrder = new StringBuffer();
+ StringBuilder keyOrder = new StringBuilder();
for (ExprNodeDesc k: keyCols) {
keyOrder.append("+");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
index 43882e7..6c0bd25 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
@@ -221,7 +221,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
}
private String getColNamesForLogging(Set<String> colLst) {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
boolean firstEntry = true;
for (String colName : colLst) {
if (firstEntry) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
index e477f04..fd42959 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java
@@ -207,7 +207,7 @@ public class SparkReduceSinkMapJoinProc implements NodeProcessor {
Map<Byte, List<ExprNodeDesc>> keyExprMap = mapJoinOp.getConf().getKeys();
List<ExprNodeDesc> keyCols = keyExprMap.get(Byte.valueOf((byte) 0));
- StringBuffer keyOrder = new StringBuffer();
+ StringBuilder keyOrder = new StringBuilder();
for (int i = 0; i < keyCols.size(); i++) {
keyOrder.append("+");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
index 7108a47..f8d6905 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
@@ -168,12 +168,12 @@ public class ProcessAnalyzeTable implements NodeProcessor {
*
* It is composed of PartialScanTask followed by StatsTask.
*/
- private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
+ private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
StatsWork statsWork, GenTezProcContext context, Task<StatsWork> statsTask)
throws SemanticException {
String aggregationKey = tableScan.getConf().getStatsAggPrefix();
- StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
+ StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
aggregationKey = aggregationKeyBuffer.toString();
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index d609732..428ba38 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -955,7 +955,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
*
*/
private ASTNode findCTEFromName(QB qb, String cteName) {
- StringBuffer qId = new StringBuffer();
+ StringBuilder qId = new StringBuilder();
if (qb.getId() != null) {
qId.append(qb.getId());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
index 9e19733..66e148f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
@@ -168,7 +168,7 @@ public class SparkProcessAnalyzeTable implements NodeProcessor {
StatsWork statsWork, GenSparkProcContext context, Task<StatsWork> statsTask)
throws SemanticException {
String aggregationKey = tableScan.getConf().getStatsAggPrefix();
- StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
+ StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
aggregationKey = aggregationKeyBuffer.toString();
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
index 8dff2fc..3a1a4af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
@@ -104,11 +104,11 @@ public class FilterDesc extends AbstractOperatorDesc {
@Explain(displayName = "predicate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getPredicateString() {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
PlanUtils.addExprToStringBuffer(predicate, sb);
return sb.toString();
}
-
+
public org.apache.hadoop.hive.ql.plan.ExprNodeDesc getPredicate() {
return predicate;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index cb0b680..76926e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.plan;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -413,7 +414,7 @@ public final class PlanUtils {
// We basically use ReduceSinkOperators and set the transfer to
// be broadcast (instead of partitioned). As a consequence we use
// a different SerDe than in the MR mapjoin case.
- StringBuffer order = new StringBuffer();
+ StringBuilder order = new StringBuilder();
for (FieldSchema f: fieldSchemas) {
order.append("+");
}
@@ -930,7 +931,7 @@ public final class PlanUtils {
}
public static String getExprListString(Collection<? extends ExprNodeDesc> exprs) {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
boolean first = true;
for (ExprNodeDesc expr: exprs) {
if (!first) {
@@ -944,11 +945,15 @@ public final class PlanUtils {
return sb.length() == 0 ? null : sb.toString();
}
- public static void addExprToStringBuffer(ExprNodeDesc expr, StringBuffer sb) {
- sb.append(expr.getExprString());
- sb.append(" (type: ");
- sb.append(expr.getTypeString());
- sb.append(")");
+ public static void addExprToStringBuffer(ExprNodeDesc expr, Appendable sb) {
+ try {
+ sb.append(expr.getExprString());
+ sb.append(" (type: ");
+ sb.append(expr.getTypeString());
+ sb.append(")");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
public static void addInputsForView(ParseContext parseCtx) throws HiveException {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index aa291b9..6282380 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -134,7 +134,7 @@ public class TableScanDesc extends AbstractOperatorDesc {
@Explain(displayName = "filterExpr")
public String getFilterExprString() {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
PlanUtils.addExprToStringBuffer(filterExpr, sb);
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index b8e18ea..c5f2d4d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -585,7 +585,7 @@ public class CompactorMR {
@Override
public String toString() {
- StringBuffer buf = new StringBuffer();
+ StringBuilder buf = new StringBuilder();
buf.append(size());
buf.append(':');
if (size() > 0) {
@@ -631,14 +631,15 @@ public class CompactorMR {
@Override
public String toString() {
- StringBuffer buf = new StringBuffer();
+ StringBuilder buf = new StringBuilder();
buf.append(size());
buf.append(':');
if (size() > 0) {
for (Path p : this) {
- buf.append(p.toString().length());
+ String pStr = p.toString();
+ buf.append(pStr.length());
buf.append(':');
- buf.append(p.toString());
+ buf.append(pStr);
}
}
return buf.toString();
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 847d751..32a9ef8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -259,7 +259,7 @@ public class Initiator extends CompactorThread {
HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_PCT_THRESHOLD);
boolean bigEnough = (float)deltaSize/(float)baseSize > deltaPctThreshold;
if (LOG.isDebugEnabled()) {
- StringBuffer msg = new StringBuffer("delta size: ");
+ StringBuilder msg = new StringBuilder("delta size: ");
msg.append(deltaSize);
msg.append(" base size: ");
msg.append(baseSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index f26225a..e164661 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.CompactionTxnHandler;
-import org.apache.hadoop.hive.metastore.txn.TxnHandler;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -137,7 +136,7 @@ public class Worker extends CompactorThread {
final ValidTxnList txns =
CompactionTxnHandler.createValidCompactTxnList(txnHandler.getOpenTxnsInfo());
LOG.debug("ValidCompactTxnList: " + txns.writeToString());
- final StringBuffer jobName = new StringBuffer(name);
+ final StringBuilder jobName = new StringBuilder(name);
jobName.append("-compactor-");
jobName.append(ci.getFullPartitionName());
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
index f3f7d95..041d218 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
@@ -475,7 +475,7 @@ public final class ObjectInspectorUtils {
return result.toString();
}
case UNION: {
- StringBuffer result = new StringBuffer();
+ StringBuilder result = new StringBuilder();
result.append(oi.getClass().getSimpleName() + "<");
UnionObjectInspector uoi = (UnionObjectInspector)oi;
List<ObjectInspector> ois = uoi.getObjectInspectors();
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
index 19fe952..b11ce32 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.serde2.lazy;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -33,14 +32,8 @@ import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.SimpleMapEqualComparer;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
-import org.apache.hadoop.hive.serde2.objectinspector.TestSimpleMapEqualComparer.TextStringMapHolder;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
@@ -72,7 +65,7 @@ public class TestLazySimpleSerDe extends TestCase {
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\tNULL\t");
t.append(new byte[]{(byte)Integer.parseInt("10111111", 2)}, 0, 1);
- StringBuffer sb = new StringBuffer("123\t456\t789\t1000\t5.3\thive and hadoop\t1\tNULL\t");
+ StringBuilder sb = new StringBuilder("123\t456\t789\t1000\t5.3\thive and hadoop\t1\tNULL\t");
String s = sb.append(new String(Base64.encodeBase64(new byte[]{(byte)Integer.parseInt("10111111", 2)}))).toString();
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
http://git-wip-us.apache.org/repos/asf/hive/blob/8aaf3bd5/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java b/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
index 3ef5577..a58db9c 100644
--- a/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
+++ b/service/src/java/org/apache/hive/service/auth/HttpAuthUtils.java
@@ -92,7 +92,7 @@ public final class HttpAuthUtils {
* cu=<username>&rn=<randomNumber>&s=<cookieSignature>
*/
public static String createCookieToken(String clientUserName) {
- StringBuffer sb = new StringBuffer();
+ StringBuilder sb = new StringBuilder();
sb.append(COOKIE_CLIENT_USER_NAME).append(COOKIE_KEY_VALUE_SEPARATOR).append(clientUserName).
append(COOKIE_ATTR_SEPARATOR);
sb.append(COOKIE_CLIENT_RAND_NUMBER).append(COOKIE_KEY_VALUE_SEPARATOR).
[41/50] [abbrv] hive git commit: HIVE-10911 : Add support for date
datatype in the value based windowing function (Aihua Xu via Ashutosh
Chauhan)
Posted by xu...@apache.org.
HIVE-10911 : Add support for date datatype in the value based windowing function (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7983593f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7983593f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7983593f
Branch: refs/heads/spark
Commit: 7983593fbc094a90679f8b1fc775b0dd4d5feab3
Parents: dbd89ac
Author: Aihua Xu <ai...@gmail.com>
Authored: Fri Jun 5 13:31:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 6 11:02:59 2015 -0700
----------------------------------------------------------------------
data/files/emp2.txt | 16 ++++
.../hadoop/hive/ql/parse/PTFTranslator.java | 1 +
.../hive/ql/udf/ptf/WindowingTableFunction.java | 31 ++++++++
.../clientpositive/windowing_windowspec3.q | 29 +++++++
.../clientpositive/windowing_windowspec3.q.out | 84 ++++++++++++++++++++
5 files changed, 161 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7983593f/data/files/emp2.txt
----------------------------------------------------------------------
diff --git a/data/files/emp2.txt b/data/files/emp2.txt
new file mode 100644
index 0000000..650aff7
--- /dev/null
+++ b/data/files/emp2.txt
@@ -0,0 +1,16 @@
+7369|SMITH|CLERK|7902|1980-12-17|1980-12-17 00:00:00|800|NULL|20
+7499|ALLEN|SALESMAN|7698|1981-02-20|1981-02-20 00:00:00|1600|300|30
+7521|WARD|SALESMAN|7698|1981-02-22|1981-02-22 00:00:00|1250|500|30
+7566|JONES|MANAGER|7839|1981-04-02|1981-04-02 00:00:00|2975|NULL|20
+7654|MARTIN|SALESMAN|7698|1981-09-28|1981-09-28 00:00:00|1250|1400|30
+7698|BLAKE|MANAGER|7839|1981-05-01|1981-05-01 00:00:00|2850|NULL|30
+7782|CLARK|MANAGER|7839|1981-06-09|1981-06-09 00:00:00|2450|NULL|10
+7788|SCOTT|ANALYST|7566|1982-12-09|1982-12-09 00:00:00|3000|NULL|20
+7839|KING|PRESIDENT|NULL|1981-11-17|1981-11-17 00:00:00|5000|NULL|10
+7844|TURNER|SALESMAN|7698|1981-09-08|1981-09-08 00:00:00|1500|0|30
+7876|ADAMS|CLERK|7788|1983-01-12|1983-01-12 00:00:00|1100|NULL|20
+7900|JAMES|CLERK|7698|1981-12-03|1981-12-03 00:00:00|950|NULL|30
+7902|FORD|ANALYST|7566|1981-12-03|1981-12-03 00:00:00|3000|NULL|20
+7934|MILLER|CLERK|7782|1982-01-23|1982-01-23 00:00:00|1300|NULL|10
+7988|KATY|ANALYST|7566|NULL|NULL|1500|NULL|10
+7987|JULIA|ANALYST|7566|NULL|NULL|1500|NULL|10
http://git-wip-us.apache.org/repos/asf/hive/blob/7983593f/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
index c1b8a1d..e0cd398 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
@@ -610,6 +610,7 @@ public class PTFTranslator {
case SHORT:
case DECIMAL:
case TIMESTAMP:
+ case DATE:
case STRING:
break;
default:
http://git-wip-us.apache.org/repos/asf/hive/blob/7983593f/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
index 40fd6a4..4635ba2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf.ptf;
import java.util.AbstractList;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -1141,6 +1142,8 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
return new DoubleValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
case DECIMAL:
return new HiveDecimalValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
+ case DATE:
+ return new DateValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
case STRING:
return new StringValueBoundaryScanner(vbDef, order, vbDef.getExpressionDef());
}
@@ -1231,6 +1234,34 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
}
}
+ public static class DateValueBoundaryScanner extends ValueBoundaryScanner {
+ public DateValueBoundaryScanner(BoundaryDef bndDef, Order order,
+ PTFExpressionDef expressionDef) {
+ super(bndDef,order,expressionDef);
+ }
+
+ @Override
+ public boolean isGreater(Object v1, Object v2, int amt) {
+ Date l1 = PrimitiveObjectInspectorUtils.getDate(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ Date l2 = PrimitiveObjectInspectorUtils.getDate(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ if (l1 != null && l2 != null) {
+ return (double)(l1.getTime() - l2.getTime())/1000 > (long)amt * 24 * 3600; // Converts amt days to milliseconds
+ }
+ return l1 != l2; // True if only one date is null
+ }
+
+ @Override
+ public boolean isEqual(Object v1, Object v2) {
+ Date l1 = PrimitiveObjectInspectorUtils.getDate(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ Date l2 = PrimitiveObjectInspectorUtils.getDate(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ return (l1 == null && l2 == null) || (l1 != null && l1.equals(l2));
+ }
+ }
+
public static class StringValueBoundaryScanner extends ValueBoundaryScanner {
public StringValueBoundaryScanner(BoundaryDef bndDef, Order order,
PTFExpressionDef expressionDef) {
http://git-wip-us.apache.org/repos/asf/hive/blob/7983593f/ql/src/test/queries/clientpositive/windowing_windowspec3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec3.q b/ql/src/test/queries/clientpositive/windowing_windowspec3.q
new file mode 100644
index 0000000..d00e939
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/windowing_windowspec3.q
@@ -0,0 +1,29 @@
+-- Test value based windowing spec
+
+drop table if exists emp;
+
+create table emp(empno smallint,
+ ename varchar(10),
+ job varchar(10),
+ manager smallint,
+ hiredate date,
+ hirets timestamp,
+ salary double,
+ bonus double,
+ deptno tinyint)
+ row format delimited
+ fields terminated by '|';
+
+load data local inpath '../../data/files/emp2.txt' into table emp;
+
+-- Support date datatype
+select deptno, empno, hiredate, salary,
+ sum(salary) over (partition by deptno order by hiredate range 90 preceding),
+ sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 90 following),
+ sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 10 preceding),
+ sum(salary) over (partition by deptno order by hiredate range between 10 following and 90 following),
+ sum(salary) over (partition by deptno order by hiredate range between 10 following and unbounded following),
+ sum(salary) over (partition by deptno order by hiredate range between unbounded preceding and 10 following)
+from emp;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/7983593f/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_windowspec3.q.out b/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
new file mode 100644
index 0000000..a5eae5b
--- /dev/null
+++ b/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
@@ -0,0 +1,84 @@
+PREHOOK: query: -- Test value based windowing spec
+
+drop table if exists emp
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- Test value based windowing spec
+
+drop table if exists emp
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table emp(empno smallint,
+ ename varchar(10),
+ job varchar(10),
+ manager smallint,
+ hiredate date,
+ hirets timestamp,
+ salary double,
+ bonus double,
+ deptno tinyint)
+ row format delimited
+ fields terminated by '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@emp
+POSTHOOK: query: create table emp(empno smallint,
+ ename varchar(10),
+ job varchar(10),
+ manager smallint,
+ hiredate date,
+ hirets timestamp,
+ salary double,
+ bonus double,
+ deptno tinyint)
+ row format delimited
+ fields terminated by '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@emp
+PREHOOK: query: load data local inpath '../../data/files/emp2.txt' into table emp
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@emp
+POSTHOOK: query: load data local inpath '../../data/files/emp2.txt' into table emp
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@emp
+PREHOOK: query: -- Support date datatype
+select deptno, empno, hiredate, salary,
+ sum(salary) over (partition by deptno order by hiredate range 90 preceding),
+ sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 90 following),
+ sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 10 preceding),
+ sum(salary) over (partition by deptno order by hiredate range between 10 following and 90 following),
+ sum(salary) over (partition by deptno order by hiredate range between 10 following and unbounded following),
+ sum(salary) over (partition by deptno order by hiredate range between unbounded preceding and 10 following)
+from emp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emp
+#### A masked pattern was here ####
+POSTHOOK: query: -- Support date datatype
+select deptno, empno, hiredate, salary,
+ sum(salary) over (partition by deptno order by hiredate range 90 preceding),
+ sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 90 following),
+ sum(salary) over (partition by deptno order by hiredate range between 90 preceding and 10 preceding),
+ sum(salary) over (partition by deptno order by hiredate range between 10 following and 90 following),
+ sum(salary) over (partition by deptno order by hiredate range between 10 following and unbounded following),
+ sum(salary) over (partition by deptno order by hiredate range between unbounded preceding and 10 following)
+from emp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emp
+#### A masked pattern was here ####
+10 7988 NULL 1500.0 3000.0 3000.0 NULL NULL 8750.0 3000.0
+10 7987 NULL 1500.0 3000.0 3000.0 NULL NULL 8750.0 3000.0
+10 7782 1981-06-09 2450.0 2450.0 2450.0 NULL NULL 6300.0 5450.0
+10 7839 1981-11-17 5000.0 5000.0 6300.0 NULL 1300.0 1300.0 10450.0
+10 7934 1982-01-23 1300.0 6300.0 6300.0 5000.0 NULL NULL 11750.0
+20 7369 1980-12-17 800.0 800.0 800.0 NULL NULL 10075.0 800.0
+20 7566 1981-04-02 2975.0 2975.0 2975.0 NULL NULL 7100.0 3775.0
+20 7902 1981-12-03 3000.0 3000.0 3000.0 NULL NULL 4100.0 6775.0
+20 7788 1982-12-09 3000.0 3000.0 4100.0 NULL 1100.0 1100.0 9775.0
+20 7876 1983-01-12 1100.0 4100.0 4100.0 3000.0 NULL NULL 10875.0
+30 7499 1981-02-20 1600.0 1600.0 5700.0 NULL 2850.0 6550.0 2850.0
+30 7521 1981-02-22 1250.0 2850.0 5700.0 NULL 2850.0 6550.0 2850.0
+30 7698 1981-05-01 2850.0 5700.0 5700.0 2850.0 NULL 3700.0 5700.0
+30 7844 1981-09-08 1500.0 1500.0 3700.0 NULL 2200.0 2200.0 7200.0
+30 7654 1981-09-28 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0
+30 7900 1981-12-03 950.0 3700.0 3700.0 2750.0 NULL NULL 9400.0
[25/50] [abbrv] hive git commit: HIVE-10919: Windows: create table
with JsonSerDe failed via beeline unless you add hcatalog core jar to
classpath (Hari Sankar Sivarama Subramaniyan, reviewed by Sushanth Sowmyan
Posted by xu...@apache.org.
HIVE-10919: Windows: create table with JsonSerDe failed via beeline unless you add hcatalog core jar to classpath (Hari Sankar Sivarama Subramaniyan, reviewed by Sushanth Sowmyan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/949ddaae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/949ddaae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/949ddaae
Branch: refs/heads/spark
Commit: 949ddaae125a110d08e9cd6d565b7193e4941a37
Parents: aec98e9
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Wed Jun 3 21:17:01 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Wed Jun 3 21:17:01 2015 -0700
----------------------------------------------------------------------
bin/hive.cmd | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/949ddaae/bin/hive.cmd
----------------------------------------------------------------------
diff --git a/bin/hive.cmd b/bin/hive.cmd
index ba7e443..9080796 100644
--- a/bin/hive.cmd
+++ b/bin/hive.cmd
@@ -158,7 +158,7 @@ if not exist %HIVE_LIB%\hive-cli-*.jar (
exit /b 1
)
-set CLASSPATH=%CLASSPATH%;%HIVE_LIB%\*
+set CLASSPATH=%CLASSPATH%;%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\*;
@rem maybe we should just make users set HADOOP_HOME env variable as a prereq
@rem in the next iteration, use "where" command to find directory of hadoop install from path
[15/50] [abbrv] hive git commit: HIVE-10752: Revert HIVE-5193 (Aihua
Xu via Chaoyu Tang)
Posted by xu...@apache.org.
HIVE-10752: Revert HIVE-5193 (Aihua Xu via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5fdf143
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5fdf143
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5fdf143
Branch: refs/heads/spark
Commit: d5fdf143d29344b6728e1489937a87e073aa9894
Parents: 8aaf3bd
Author: ctang <ct...@cloudera.com>
Authored: Tue Jun 2 15:24:20 2015 -0400
Committer: ctang <ct...@cloudera.com>
Committed: Tue Jun 2 15:24:36 2015 -0400
----------------------------------------------------------------------
.../apache/hive/hcatalog/pig/HCatLoader.java | 8 ----
.../hive/hcatalog/pig/TestHCatLoader.java | 44 --------------------
2 files changed, 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdf143/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
index c951847..0685790 100644
--- a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
+++ b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
@@ -19,7 +19,6 @@
package org.apache.hive.hcatalog.pig;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
@@ -32,7 +31,6 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.security.Credentials;
@@ -163,12 +161,6 @@ public class HCatLoader extends HCatBaseLoader {
if (requiredFieldsInfo != null) {
// convert to hcatschema and pass to HCatInputFormat
try {
- //push down projections to columnar store works for RCFile and ORCFile
- ArrayList<Integer> list = new ArrayList<Integer>(requiredFieldsInfo.getFields().size());
- for (RequiredField rf : requiredFieldsInfo.getFields()) {
- list.add(rf.getIndex());
- }
- ColumnProjectionUtils.appendReadColumns(job.getConfiguration(), list);
outputSchema = phutil.getHCatSchema(requiredFieldsInfo.getFields(), signature, this.getClass());
HCatInputFormat.setOutputSchema(job, outputSchema);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdf143/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
index fc18a3b..45a219c 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
@@ -19,8 +19,6 @@
package org.apache.hive.hcatalog.pig;
import java.io.File;
-import java.io.FileWriter;
-import java.io.PrintWriter;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.sql.Date;
@@ -36,10 +34,7 @@ import java.util.Properties;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
@@ -49,7 +44,6 @@ import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.StorageFormats;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Shell;
@@ -66,10 +60,6 @@ import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-import org.apache.pig.PigRunner;
-import org.apache.pig.tools.pigstats.OutputStats;
-import org.apache.pig.tools.pigstats.PigStats;
-
import org.joda.time.DateTime;
import org.junit.After;
@@ -492,40 +482,6 @@ public class TestHCatLoader {
}
@Test
- public void testColumnarStorePushdown() throws Exception {
- assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
- String PIGOUTPUT_DIR = TEST_DATA_DIR+ "/colpushdownop";
- String PIG_FILE = "test.pig";
- String expectedCols = "0,1";
- PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
- w.println("A = load '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
- w.println("B = foreach A generate name,studentid;");
- w.println("C = filter B by name is not null;");
- w.println("store C into '" + PIGOUTPUT_DIR + "' using PigStorage();");
- w.close();
-
- try {
- String[] args = { "-x", "local", PIG_FILE };
- PigStats stats = PigRunner.run(args, null);
- //Pig script was successful
- assertTrue(stats.isSuccessful());
- //Single MapReduce job is launched
- OutputStats outstats = stats.getOutputStats().get(0);
- assertTrue(outstats!= null);
- assertEquals(expectedCols,outstats.getConf()
- .get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR));
- //delete output file on exit
- FileSystem fs = FileSystem.get(outstats.getConf());
- if (fs.exists(new Path(PIGOUTPUT_DIR)))
- {
- fs.delete(new Path(PIGOUTPUT_DIR), true);
- }
- } finally {
- new File(PIG_FILE).delete();
- }
- }
-
- @Test
public void testGetInputBytes() throws Exception {
assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
File file = new File(TEST_WAREHOUSE_DIR + "/" + SPECIFIC_SIZE_TABLE + "/part-m-00000");
[18/50] [abbrv] hive git commit: HIVE-10855: with vectorization
enabled join operation involving interval_day_time fails (Matt McCline
reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fe6ee17..47a1107 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -226,6 +226,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
vector_inner_join.q,\
vector_interval_1.q,\
vector_interval_2.q,\
+ vector_interval_mapjoin.q,\
vector_join30.q,\
vector_join_filters.q,\
vector_join_nulls.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index 6b95360..6654166 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -35,6 +35,16 @@ import org.apache.hadoop.io.Writable;
public abstract class ColumnVector {
/*
+ * The current kinds of column vectors.
+ */
+ public static enum Type {
+ LONG,
+ DOUBLE,
+ BYTES,
+ DECIMAL
+ }
+
+ /*
* If hasNulls is true, then this array contains true if the value
* is null, otherwise false. The array is always allocated, so a batch can be re-used
* later and nulls added.
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
index 8c4b6ea..6673509 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
@@ -20,7 +20,12 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.util.Arrays;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* Class to keep information on a set of typed vector columns. Used by
@@ -117,28 +122,41 @@ public class VectorColumnSetInfo {
protected void addKey(String outputType) throws HiveException {
indexLookup[addIndex] = new KeyLookupHelper();
- if (VectorizationContext.isIntFamily(outputType) ||
- VectorizationContext.isDatetimeFamily(outputType)) {
+
+ String typeName = VectorizationContext.mapTypeNameSynonyms(outputType);
+
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
+
+ switch (columnVectorType) {
+ case LONG:
longIndices[longIndicesIndex] = addIndex;
indexLookup[addIndex].setLong(longIndicesIndex);
++longIndicesIndex;
- } else if (VectorizationContext.isFloatFamily(outputType)) {
+ break;
+
+ case DOUBLE:
doubleIndices[doubleIndicesIndex] = addIndex;
indexLookup[addIndex].setDouble(doubleIndicesIndex);
++doubleIndicesIndex;
- } else if (VectorizationContext.isStringFamily(outputType) ||
- outputType.equalsIgnoreCase("binary")) {
+ break;
+
+ case BYTES:
stringIndices[stringIndicesIndex]= addIndex;
indexLookup[addIndex].setString(stringIndicesIndex);
++stringIndicesIndex;
- } else if (VectorizationContext.isDecimalFamily(outputType)) {
- decimalIndices[decimalIndicesIndex]= addIndex;
- indexLookup[addIndex].setDecimal(decimalIndicesIndex);
- ++decimalIndicesIndex;
- }
- else {
- throw new HiveException("Unsuported vector output type: " + outputType);
+ break;
+
+ case DECIMAL:
+ decimalIndices[decimalIndicesIndex]= addIndex;
+ indexLookup[addIndex].setDecimal(decimalIndicesIndex);
+ ++decimalIndicesIndex;
+ break;
+
+ default:
+ throw new HiveException("Unexpected column vector type " + columnVectorType);
}
+
addIndex++;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index 0058141..f12bfde 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -20,6 +20,12 @@ package org.apache.hadoop.hive.ql.exec.vector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* This class copies specified columns of a row from one VectorizedRowBatch to another.
@@ -186,7 +192,7 @@ public class VectorCopyRow {
private CopyRow[] subRowToBatchCopiersByValue;
private CopyRow[] subRowToBatchCopiersByReference;
- public void init(VectorColumnMapping columnMapping) {
+ public void init(VectorColumnMapping columnMapping) throws HiveException {
int count = columnMapping.getCount();
subRowToBatchCopiersByValue = new CopyRow[count];
subRowToBatchCopiersByReference = new CopyRow[count];
@@ -194,24 +200,35 @@ public class VectorCopyRow {
for (int i = 0; i < count; i++) {
int inputColumn = columnMapping.getInputColumns()[i];
int outputColumn = columnMapping.getOutputColumns()[i];
- String typeName = columnMapping.getTypeNames()[i];
+ String typeName = columnMapping.getTypeNames()[i].toLowerCase();
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
CopyRow copyRowByValue = null;
CopyRow copyRowByReference = null;
- if (VectorizationContext.isIntFamily(typeName) ||
- VectorizationContext.isDatetimeFamily(typeName)) {
+ switch (columnVectorType) {
+ case LONG:
copyRowByValue = new LongCopyRow(inputColumn, outputColumn);
- } else if (VectorizationContext.isFloatFamily(typeName)) {
+ break;
+
+ case DOUBLE:
copyRowByValue = new DoubleCopyRow(inputColumn, outputColumn);
- } else if (VectorizationContext.isStringFamily(typeName)) {
+ break;
+
+ case BYTES:
copyRowByValue = new BytesCopyRowByValue(inputColumn, outputColumn);
copyRowByReference = new BytesCopyRowByReference(inputColumn, outputColumn);
- } else if (VectorizationContext.decimalTypePattern.matcher(typeName).matches()){
+ break;
+
+ case DECIMAL:
copyRowByValue = new DecimalCopyRow(inputColumn, outputColumn);
- } else {
- throw new RuntimeException("Cannot allocate vector copy row for " + typeName);
+ break;
+
+ default:
+ throw new HiveException("Unexpected column vector type " + columnVectorType);
}
+
subRowToBatchCopiersByValue[i] = copyRowByValue;
if (copyRowByReference == null) {
subRowToBatchCopiersByReference[i] = copyRowByValue;
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 61d2972..392e56d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.hive.ql.udf.generic.*;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
@@ -2044,6 +2045,51 @@ public class VectorizationContext {
}
}
+ public static String mapTypeNameSynonyms(String typeName) {
+ typeName = typeName.toLowerCase();
+ if (typeName.equals("long")) {
+ return "bigint";
+ } else if (typeName.equals("string_family")) {
+ return "string";
+ } else {
+ return typeName;
+ }
+ }
+
+ public static ColumnVector.Type getColumnVectorTypeFromTypeInfo(TypeInfo typeInfo) throws HiveException {
+ PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
+ PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
+
+ switch (primitiveCategory) {
+ case BOOLEAN:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ case DATE:
+ case TIMESTAMP:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_DAY_TIME:
+ return ColumnVector.Type.LONG;
+
+ case FLOAT:
+ case DOUBLE:
+ return ColumnVector.Type.DOUBLE;
+
+ case STRING:
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ return ColumnVector.Type.BYTES;
+
+ case DECIMAL:
+ return ColumnVector.Type.DECIMAL;
+
+ default:
+ throw new HiveException("Unexpected primitive type category " + primitiveCategory);
+ }
+ }
+
// TODO: When we support vectorized STRUCTs and can handle more in the reduce-side (MERGEPARTIAL):
// TODO: Write reduce-side versions of AVG. Currently, only map-side (HASH) versions are in table.
// TODO: And, investigate if different reduce-side versions are needed for var* and std*, or if map-side aggregate can be used.. Right now they are conservatively
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index af78776..4c8c4b1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.optimized.VectorMapJoinOptimizedCreateHashTable;
@@ -68,8 +69,12 @@ import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc;
import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableImplementationType;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* This class is common operator class for native vectorized map join.
@@ -658,7 +663,7 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
* Setup our 2nd batch with the same "column schema" as the big table batch that can be used to
* build join output results in.
*/
- protected VectorizedRowBatch setupOverflowBatch() {
+ protected VectorizedRowBatch setupOverflowBatch() throws HiveException {
VectorizedRowBatch overflowBatch;
Map<Integer, String> scratchColumnTypeMap = vOutContext.getScratchColumnTypeMap();
@@ -701,23 +706,38 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
* Allocate overflow batch columns by hand.
*/
private void allocateOverflowBatchColumnVector(VectorizedRowBatch overflowBatch, int outputColumn,
- String typeName) {
+ String typeName) throws HiveException {
if (overflowBatch.cols[outputColumn] == null) {
- String vectorTypeName;
- if (VectorizationContext.isIntFamily(typeName) ||
- VectorizationContext.isDatetimeFamily(typeName)) {
- vectorTypeName = "long";
- } else if (VectorizationContext.isFloatFamily(typeName)) {
- vectorTypeName = "double";
- } else if (VectorizationContext.isStringFamily(typeName)) {
- vectorTypeName = "string";
- } else if (VectorizationContext.decimalTypePattern.matcher(typeName).matches()){
- vectorTypeName = typeName; // Keep precision and scale.
- } else {
- throw new RuntimeException("Cannot determine vector type for " + typeName);
+ typeName = VectorizationContext.mapTypeNameSynonyms(typeName);
+
+ String columnVectorTypeName;
+
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
+
+ switch (columnVectorType) {
+ case LONG:
+ columnVectorTypeName = "long";
+ break;
+
+ case DOUBLE:
+ columnVectorTypeName = "double";
+ break;
+
+ case BYTES:
+ columnVectorTypeName = "string";
+ break;
+
+ case DECIMAL:
+ columnVectorTypeName = typeName; // Keep precision and scale.
+ break;
+
+ default:
+ throw new HiveException("Unexpected column vector type " + columnVectorType);
}
- overflowBatch.cols[outputColumn] = VectorizedRowBatchCtx.allocateColumnVector(vectorTypeName, VectorizedRowBatch.DEFAULT_SIZE);
+
+ overflowBatch.cols[outputColumn] = VectorizedRowBatchCtx.allocateColumnVector(columnVectorTypeName, VectorizedRowBatch.DEFAULT_SIZE);
if (LOG.isDebugEnabled()) {
LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator initializeOp overflowBatch outputColumn " + outputColumn + " class " + overflowBatch.cols[outputColumn].getClass().getSimpleName());
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q b/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q
new file mode 100644
index 0000000..9a58658
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_interval_mapjoin.q
@@ -0,0 +1,87 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.auto.convert.join=true;
+
+create table vectortab_a_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k;
+
+CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k;
+
+create table vectortab_b_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k;
+
+CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k;
+
+explain
+select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s;
+
+select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out
new file mode 100644
index 0000000..b4d3477
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_interval_mapjoin.q.out
@@ -0,0 +1,281 @@
+PREHOOK: query: create table vectortab_a_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_a_1k
+POSTHOOK: query: create table vectortab_a_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_a_1k
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@vectortab_a_1k
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@vectortab_a_1k
+PREHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@vectortab_a_1k
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_a_1korc
+POSTHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@vectortab_a_1k
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_a_1korc
+PREHOOK: query: create table vectortab_b_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_b_1k
+POSTHOOK: query: create table vectortab_b_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_b_1k
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@vectortab_b_1k
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@vectortab_b_1k
+PREHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@vectortab_b_1k
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_b_1korc
+POSTHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@vectortab_b_1k
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_b_1korc
+PREHOOK: query: explain
+select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: vectortab_a_1korc
+ Statistics: Num rows: 1000 Data size: 460264 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: s is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col1 is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 115066 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: interval_day_time), _col0 (type: string)
+ 1 _col1 (type: interval_day_time), _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: vectortab_b_1korc
+ Statistics: Num rows: 1000 Data size: 458448 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: s is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col1 is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 114612 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: interval_day_time), _col0 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: interval_day_time), _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 114612 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vectortab_a_1korc
+PREHOOK: Input: default@vectortab_b_1korc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vectortab_a_1korc
+POSTHOOK: Input: default@vectortab_b_1korc
+#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out b/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
new file mode 100644
index 0000000..976091b
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_interval_mapjoin.q.out
@@ -0,0 +1,281 @@
+PREHOOK: query: create table vectortab_a_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_a_1k
+POSTHOOK: query: create table vectortab_a_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_a_1k
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@vectortab_a_1k
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_a_1k' OVERWRITE INTO TABLE vectortab_a_1k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@vectortab_a_1k
+PREHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@vectortab_a_1k
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_a_1korc
+POSTHOOK: query: CREATE TABLE vectortab_a_1korc STORED AS ORC AS SELECT * FROM vectortab_a_1k
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@vectortab_a_1k
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_a_1korc
+PREHOOK: query: create table vectortab_b_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_b_1k
+POSTHOOK: query: create table vectortab_b_1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ dc decimal(38,18),
+ bo boolean,
+ s string,
+ s2 string,
+ ts timestamp,
+ ts2 timestamp,
+ dt date)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_b_1k
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@vectortab_b_1k
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/vectortab_b_1k' OVERWRITE INTO TABLE vectortab_b_1k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@vectortab_b_1k
+PREHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@vectortab_b_1k
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vectortab_b_1korc
+POSTHOOK: query: CREATE TABLE vectortab_b_1korc STORED AS ORC AS SELECT * FROM vectortab_b_1k
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@vectortab_b_1k
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vectortab_b_1korc
+PREHOOK: query: explain
+select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-4
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ v2:vectortab_b_1korc
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ v2:vectortab_b_1korc
+ TableScan
+ alias: vectortab_b_1korc
+ Statistics: Num rows: 1000 Data size: 458448 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: s is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 229224 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col1 is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 114612 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col1 (type: interval_day_time), _col0 (type: string)
+ 1 _col1 (type: interval_day_time), _col0 (type: string)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: vectortab_a_1korc
+ Statistics: Num rows: 1000 Data size: 460264 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: s is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: s (type: string), (dt - CAST( ts AS DATE)) (type: interval_day_time)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 230132 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col1 is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 115066 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: interval_day_time), _col0 (type: string)
+ 1 _col1 (type: interval_day_time), _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string), _col1 (type: interval_day_time)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 126572 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vectortab_a_1korc
+PREHOOK: Input: default@vectortab_b_1korc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ v1.s,
+ v2.s,
+ v1.intrvl1
+from
+ ( select
+ s,
+ (cast(dt as date) - cast(ts as date)) as intrvl1
+ from
+ vectortab_a_1korc ) v1
+join
+ (
+ select
+ s ,
+ (cast(dt as date) - cast(ts as date)) as intrvl2
+ from
+ vectortab_b_1korc
+ ) v2
+ on v1.intrvl1 = v2.intrvl2
+ and v1.s = v2.s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vectortab_a_1korc
+POSTHOOK: Input: default@vectortab_b_1korc
+#### A masked pattern was here ####
[09/50] [abbrv] hive git commit: HIVE-10877 : TestUtil class name
confuses ptest2 (Thejas Nair, reviewed by Hari Sankar Sivarama Subramaniyan )
Posted by xu...@apache.org.
HIVE-10877 : TestUtil class name confuses ptest2 (Thejas Nair, reviewed by Hari Sankar Sivarama Subramaniyan )
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3817bb53
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3817bb53
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3817bb53
Branch: refs/heads/spark
Commit: 3817bb5363948c74da99c15c65c4935cc7356f08
Parents: 55b28a0
Author: Thejas Nair <th...@hortonworks.com>
Authored: Mon Jun 1 15:47:25 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Mon Jun 1 15:47:25 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/TestUtil.java | 41 --------------------
.../plugin/TestHiveAuthorizerShowFilters.java | 4 +-
2 files changed, 2 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3817bb53/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java
deleted file mode 100644
index 304fe5b..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/TestUtil.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-
-/**
- * Test utilities
- */
-public class TestUtil {
- /**
- * Use this if you want a fresh metastore for your test, without any existing entries.
- * It updates the configuration to point to new derby dir location
- * @param conf HiveConf to be updated
- * @param newloc new location within test temp dir for the metastore db
- */
- public static void setNewDerbyDbLocation(HiveConf conf, String newloc) {
- String newDbLoc = System.getProperty("test.tmp.dir") + newloc + "metastore_db";
-
- conf.setVar(ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:;databaseName=" + newDbLoc
- + ";create=true");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/3817bb53/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
index 266d888..0c03a00 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
@@ -27,7 +27,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.hadoop.hive.TestUtil;
+import org.apache.hadoop.hive.UtilsForTest;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
@@ -120,7 +120,7 @@ public class TestHiveAuthorizerShowFilters {
conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true);
conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false);
conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- TestUtil.setNewDerbyDbLocation(conf, TestHiveAuthorizerShowFilters.class.getSimpleName());
+ UtilsForTest.setNewDerbyDbLocation(conf, TestHiveAuthorizerShowFilters.class.getSimpleName());
SessionState.start(conf);
driver = new Driver(conf);
[47/50] [abbrv] hive git commit: HIVE-10929: In Tez mode,
dynamic partitioning query with union all fails at moveTask,
Invalid partition key & values (Vikram Dixit K reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/vector_multi_insert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_multi_insert.q.out b/ql/src/test/results/clientpositive/tez/vector_multi_insert.q.out
index b54f496..1080bf4 100644
--- a/ql/src/test/results/clientpositive/tez/vector_multi_insert.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_multi_insert.q.out
@@ -85,17 +85,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 154 Data size: 616 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (rn < 100) (type: boolean)
- Statistics: Num rows: 51 Data size: 204 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: rn (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 51 Data size: 204 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 51 Data size: 204 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -103,14 +103,14 @@ STAGE PLANS:
name: default.orc_rn1
Filter Operator
predicate: ((rn >= 100) and (rn < 1000)) (type: boolean)
- Statistics: Num rows: 17 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: rn (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 17 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 17 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -118,14 +118,14 @@ STAGE PLANS:
name: default.orc_rn2
Filter Operator
predicate: (rn >= 1000) (type: boolean)
- Statistics: Num rows: 51 Data size: 204 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: rn (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 51 Data size: 204 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 51 Data size: 204 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
index a2f4458..4101c35 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
@@ -111,11 +111,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 13 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 13 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -125,11 +125,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
input vertices:
1 Map 2
- Statistics: Num rows: 14 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 14 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -139,16 +139,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 13 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 13 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int)
sort order: +
Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 13 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
Execution mode: vectorized
@@ -218,11 +218,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -232,11 +232,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 2
- Statistics: Num rows: 1149 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 1149 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -246,16 +246,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Stage: Stage-0
@@ -357,11 +357,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 522 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 522 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -371,7 +371,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 1149 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -382,7 +382,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 4
- Statistics: Num rows: 1263 Data size: 5061 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count(), sum(_col0)
@@ -398,31 +398,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
index 8c8f54e..b8e18e0 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
@@ -118,11 +118,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 348 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 348 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -132,7 +132,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 1149 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -143,12 +143,12 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 4
- Statistics: Num rows: 1263 Data size: 5061 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col1 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 1263 Data size: 5061 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
mode: hash
@@ -163,31 +163,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 522 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 522 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
Map-reduce partition columns: _col0 (type: bigint)
- Statistics: Num rows: 522 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
index fa47a03..b5619a0 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
@@ -118,11 +118,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 40 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 40 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -132,7 +132,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 1149 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -142,7 +142,7 @@ STAGE PLANS:
1 _col0 (type: string)
input vertices:
1 Map 4
- Statistics: Num rows: 1263 Data size: 5061 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -158,31 +158,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1045 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -265,11 +265,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 20 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 20 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -279,7 +279,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 45 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -289,7 +289,7 @@ STAGE PLANS:
1 _col0 (type: string)
input vertices:
1 Map 4
- Statistics: Num rows: 49 Data size: 5061 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -305,31 +305,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring2 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 41 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -412,11 +412,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 19 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 19 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -426,7 +426,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col2
input vertices:
1 Map 3
- Statistics: Num rows: 41 Data size: 4601 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -436,7 +436,7 @@ STAGE PLANS:
1 _col1 (type: string), _col0 (type: int)
input vertices:
1 Map 4
- Statistics: Num rows: 45 Data size: 5061 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -452,31 +452,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 38 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 38 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: bigint)
sort order: ++
Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint)
- Statistics: Num rows: 38 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 40 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 40 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
- Statistics: Num rows: 40 Data size: 4183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/4d592303/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
index a73cd06..2c9a9cd 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
@@ -111,11 +111,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 12 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 12 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -125,11 +125,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
input vertices:
1 Map 2
- Statistics: Num rows: 13 Data size: 4385 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 13 Data size: 4385 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -139,16 +139,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 12 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 12 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int)
sort order: +
Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 12 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
Execution mode: vectorized
@@ -613,11 +613,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -627,11 +627,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 2
- Statistics: Num rows: 1095 Data size: 4385 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 1095 Data size: 4385 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -641,16 +641,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Stage: Stage-0
@@ -749,11 +749,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 498 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 498 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -763,7 +763,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 1095 Data size: 4385 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -773,7 +773,7 @@ STAGE PLANS:
1 _col0 (type: tinyint)
input vertices:
1 Map 4
- Statistics: Num rows: 1204 Data size: 4823 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 36 Data size: 7165 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -789,31 +789,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 996 Data size: 3987 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
[31/50] [abbrv] hive git commit: HIVE-9069: Simplify filter
predicates for CBO (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_udf_col.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_udf_col.q.out b/ql/src/test/results/clientpositive/ppd_udf_col.q.out
index 8f064c0..7c963fb 100644
--- a/ql/src/test/results/clientpositive/ppd_udf_col.q.out
+++ b/ql/src/test/results/clientpositive/ppd_udf_col.q.out
@@ -20,19 +20,24 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) = 100.0) and (rand() <= 0.1)) (type: boolean)
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 100.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '100' (type: string), rand() (type: double)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (rand() <= 0.1) (type: boolean)
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Select Operator
+ expressions: '100' (type: string), rand() (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -66,22 +71,30 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) = 100.0) and ((rand() <= 0.1) and (rand() > 0.1))) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 100.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '100' (type: string), rand() (type: double)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 20
- Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (rand() <= 0.1) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (rand() > 0.1) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: '100' (type: string), rand() (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -113,17 +126,20 @@ STAGE PLANS:
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Select Operator
- expressions: '100' (type: string), rand() (type: double), '4' (type: string)
- outputColumnNames: _col0, _col1, _col2
+ Filter Operator
+ predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: '100' (type: string), rand() (type: double), '4' (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -153,19 +169,26 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((UDFToDouble(value) * 10.0) <= 200.0) and (UDFToDouble(key) = 100.0)) (type: boolean)
+ predicate: ((UDFToDouble(key) = 100.0) and ((UDFToDouble(value) * 10.0) <= 200.0)) (type: boolean)
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '100' (type: string), rand() (type: double), (UDFToDouble(value) * 10.0) (type: double)
- outputColumnNames: _col0, _col1, _col2
+ expressions: value (type: string)
+ outputColumnNames: _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Filter Operator
+ predicate: ((UDFToDouble(_col1) * 10.0) <= 200.0) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: '100' (type: string), rand() (type: double), (UDFToDouble(_col1) * 10.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -195,19 +218,24 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) = 100.0) and (rand() <= 0.1)) (type: boolean)
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 100.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '100' (type: string), rand() (type: double)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (rand() <= 0.1) (type: boolean)
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Select Operator
+ expressions: '100' (type: string), rand() (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -241,22 +269,30 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) = 100.0) and ((rand() <= 0.1) and (rand() > 0.1))) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 100.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '100' (type: string), rand() (type: double)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 20
- Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (rand() <= 0.1) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (rand() > 0.1) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: '100' (type: string), rand() (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_union.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_union.q.out b/ql/src/test/results/clientpositive/ppd_union.q.out
index 4b0c6a4..166eff1 100644
--- a/ql/src/test/results/clientpositive/ppd_union.q.out
+++ b/ql/src/test/results/clientpositive/ppd_union.q.out
@@ -34,15 +34,18 @@ STAGE PLANS:
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Union
- Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Filter Operator
+ predicate: ((_col0 > '4') and (_col1 > 'val_4')) (type: boolean)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
+ Union
+ Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -53,15 +56,18 @@ STAGE PLANS:
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Union
- Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Filter Operator
+ predicate: ((_col0 > '4') and (_col1 > 'val_4')) (type: boolean)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
+ Union
+ Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/ppd_vc.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_vc.q.out b/ql/src/test/results/clientpositive/ppd_vc.q.out
index f058526..a16058c 100644
--- a/ql/src/test/results/clientpositive/ppd_vc.q.out
+++ b/ql/src/test/results/clientpositive/ppd_vc.q.out
@@ -680,10 +680,10 @@ STAGE PLANS:
name: default.srcpart
Truncated Path -> Alias:
/src [$hdt$_0:a]
- /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:srcpart]
- /srcpart/ds=2008-04-08/hr=12 [$hdt$_1:srcpart]
- /srcpart/ds=2008-04-09/hr=11 [$hdt$_1:srcpart]
- /srcpart/ds=2008-04-09/hr=12 [$hdt$_1:srcpart]
+ /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:$hdt$_1:srcpart]
+ /srcpart/ds=2008-04-08/hr=12 [$hdt$_1:$hdt$_1:srcpart]
+ /srcpart/ds=2008-04-09/hr=11 [$hdt$_1:$hdt$_1:srcpart]
+ /srcpart/ds=2008-04-09/hr=12 [$hdt$_1:$hdt$_1:srcpart]
Needs Tagging: true
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/spark/ppd_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join2.q.out b/ql/src/test/results/clientpositive/spark/ppd_join2.q.out
index dd4d129..cf81423 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join2.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join2.q.out
@@ -47,17 +47,24 @@ STAGE PLANS:
predicate: (((key <> '306') and (sqrt(key) <> 13.0)) and value is not null) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ predicate: (sqrt(_col0) <> 13.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -71,17 +78,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '302') and ((_col0 <> '311') and (_col0 < '400'))) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '14') (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ predicate: ((_col0 <> '302') and ((_col0 <> '311') and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 5
Map Operator Tree:
TableScan
@@ -95,17 +105,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '305') and (_col0 <> '14')) (type: boolean)
- Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col0 <> '311') and (((_col1 <> 'val_50') or (_col0 > '1')) and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ predicate: ((_col0 <> '305') and (_col0 <> '14')) (type: boolean)
+ Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -136,26 +149,26 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: string), _col3 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col2 <> '10') or (_col0 <> '10')) (type: boolean)
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col1 is not null (type: boolean)
- Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col3 (type: string)
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/spark/ppd_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_join3.q.out
index c93994c..d2343c4 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join3.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join3.q.out
@@ -51,16 +51,19 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '11') and ((_col0 > '0') and ((_col0 < '400') and ((_col0 <> '12') and (_col0 <> '4'))))) (type: boolean)
- Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '1') (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ predicate: ((_col0 <> '11') and ((_col0 > '0') and ((_col0 < '400') and ((_col0 <> '12') and (_col0 <> '4'))))) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
@@ -74,17 +77,20 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '11') and ((_col0 > '0') and (_col0 < '400'))) (type: boolean)
- Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col0 <> '4') (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ predicate: ((_col0 <> '11') and ((_col0 > '0') and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 5
Map Operator Tree:
TableScan
@@ -94,20 +100,27 @@ STAGE PLANS:
predicate: (((((((((key <> '11') and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key < '400')) and (key <> '12')) and (key <> '4')) and key is not null) and (key <> '13')) and (key <> '1')) (type: boolean)
Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 <> '12') and (_col0 <> '4')) (type: boolean)
- Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 19 Data size: 201 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 19 Data size: 201 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col0 > '0') and (((_col1 <> 'val_500') or (_col0 > '1')) and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 <> '12') and (_col0 <> '4')) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -117,14 +130,14 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col1, _col3
- Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -138,29 +151,29 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col1 > '10') or (_col0 <> '10')) (type: boolean)
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col0 <> '13') and (_col0 <> '1')) (type: boolean)
- Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: string)
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out b/ql/src/test/results/clientpositive/subquery_notin.q.out
index 0b6f33f..fd6d53b 100644
--- a/ql/src/test/results/clientpositive/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notin.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: -- non agg, non corr
explain
select *
@@ -151,7 +151,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select *
from src
where src.key not in ( select key from src s1 where s1.key > '2')
@@ -285,7 +285,7 @@ POSTHOOK: Input: default@src
199 val_199
199 val_199
2 val_2
-Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: -- non agg, corr
explain
select p_mfgr, b.p_name, p_size
@@ -528,7 +528,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select p_mfgr, b.p_name, p_size
from part b
where b.p_name not in
@@ -1243,7 +1243,7 @@ Manufacturer#5 almond antique medium spring khaki 6
Manufacturer#5 almond azure blanched chiffon midnight 23
Manufacturer#5 almond antique blue firebrick mint 31
Manufacturer#5 almond aquamarine dodger light gainsboro 46
-Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: -- non agg, non corr, Group By in Parent Query
select li.l_partkey, count(*)
from lineitem li
@@ -1278,7 +1278,7 @@ POSTHOOK: Input: default@lineitem
139636 1
175839 1
182052 1
-Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: -- alternate not in syntax
select *
from src
@@ -1442,7 +1442,7 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@t1_v
POSTHOOK: Output: database:default
POSTHOOK: Output: default@T2_v
-Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain
select *
from T1_v where T1_v.key not in (select T2_v.key from T2_v)
@@ -1587,7 +1587,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select *
from T1_v where T1_v.key not in (select T2_v.key from T2_v)
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
index 24c56bd..c32504e 100644
--- a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: -- non agg, non corr
-- JAVA_VERSION_SPECIFIC_OUTPUT
@@ -188,7 +188,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: -- non agg, corr
explain
select b.p_mfgr, min(p_retailprice)
@@ -335,7 +335,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(_col1), min(_col1)
+ aggregations: min(_col1), max(_col1)
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
@@ -348,13 +348,13 @@ STAGE PLANS:
value expressions: _col1 (type: double), _col2 (type: double)
Reduce Operator Tree:
Group By Operator
- aggregations: max(VALUE._col0), min(VALUE._col1)
+ aggregations: min(VALUE._col0), max(VALUE._col1)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 is null or _col2 is null) and ((_col1 - _col2) > 600.0)) (type: boolean)
+ predicate: ((_col0 is null or _col1 is null) and ((_col2 - _col1) > 600.0)) (type: boolean)
Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
@@ -445,7 +445,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product
PREHOOK: query: select b.p_mfgr, min(p_retailprice)
from part b
group by b.p_mfgr
@@ -470,7 +470,7 @@ POSTHOOK: Input: default@part
#### A masked pattern was here ####
Manufacturer#1 1173.15
Manufacturer#2 1690.68
-Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product
PREHOOK: query: -- agg, non corr
explain
select b.p_mfgr, min(p_retailprice)
@@ -729,7 +729,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product
PREHOOK: query: select b.p_mfgr, min(p_retailprice)
from part b
group by b.p_mfgr
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
index c393e4b..2256f6e 100644
--- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
+++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
@@ -781,7 +781,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: -- non agg, corr
explain
select p_mfgr, b.p_name, p_size
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/subquery_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_views.q.out b/ql/src/test/results/clientpositive/subquery_views.q.out
index bdab9ea..41834a3 100644
--- a/ql/src/test/results/clientpositive/subquery_views.q.out
+++ b/ql/src/test/results/clientpositive/subquery_views.q.out
@@ -69,8 +69,8 @@ POSTHOOK: type: CREATEVIEW
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@cv2
-Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
-Warning: Shuffle Join JOIN[46][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product
PREHOOK: query: explain
select *
from cv2 where cv2.key in (select key from cv2 c where c.key < '11')
@@ -378,8 +378,8 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
-Warning: Shuffle Join JOIN[46][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product
PREHOOK: query: select *
from cv2 where cv2.key in (select key from cv2 c where c.key < '11')
PREHOOK: type: QUERY
[37/50] [abbrv] hive git commit: HIVE-10925: Non-static threadlocals
in metastore code can potentially cause memory leak (Vaibhav Gumashta
reviewed by Sushanth Sowmyan, Alan Gates, Thejas Nair)
Posted by xu...@apache.org.
HIVE-10925: Non-static threadlocals in metastore code can potentially cause memory leak (Vaibhav Gumashta reviewed by Sushanth Sowmyan, Alan Gates, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c9b7d253
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c9b7d253
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c9b7d253
Branch: refs/heads/spark
Commit: c9b7d253853573339f54536a552d2564a7a2eadb
Parents: bef5266
Author: Vaibhav Gumashta <vg...@apache.org>
Authored: Thu Jun 4 16:20:00 2015 -0700
Committer: Vaibhav Gumashta <vg...@apache.org>
Committed: Thu Jun 4 16:20:00 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 4 ++--
.../authorization/AuthorizationPreEventListener.java | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c9b7d253/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 1688920..85a734c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -299,7 +299,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
};
- private final ThreadLocal<TxnHandler> threadLocalTxn = new ThreadLocal<TxnHandler>() {
+ private static final ThreadLocal<TxnHandler> threadLocalTxn = new ThreadLocal<TxnHandler>() {
@Override
protected synchronized TxnHandler initialValue() {
return null;
@@ -316,7 +316,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// Thread local configuration is needed as many threads could make changes
// to the conf using the connection hook
- private final ThreadLocal<Configuration> threadLocalConf =
+ private static final ThreadLocal<Configuration> threadLocalConf =
new ThreadLocal<Configuration>() {
@Override
protected synchronized Configuration initialValue() {
http://git-wip-us.apache.org/repos/asf/hive/blob/c9b7d253/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
index 63ba327..84f3f76 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
@@ -68,14 +68,14 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener {
public static final Log LOG = LogFactory.getLog(
AuthorizationPreEventListener.class);
- private final ThreadLocal<Configuration> tConfig = new ThreadLocal<Configuration>() {
+ private static final ThreadLocal<Configuration> tConfig = new ThreadLocal<Configuration>() {
@Override
protected Configuration initialValue() {
return new HiveConf(AuthorizationPreEventListener.class);
}
};
- private final ThreadLocal<HiveMetastoreAuthenticationProvider> tAuthenticator
+ private static final ThreadLocal<HiveMetastoreAuthenticationProvider> tAuthenticator
= new ThreadLocal<HiveMetastoreAuthenticationProvider>() {
@Override
protected HiveMetastoreAuthenticationProvider initialValue() {
@@ -88,7 +88,7 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener {
}
};
- private final ThreadLocal<List<HiveMetastoreAuthorizationProvider>> tAuthorizers
+ private static final ThreadLocal<List<HiveMetastoreAuthorizationProvider>> tAuthorizers
= new ThreadLocal<List<HiveMetastoreAuthorizationProvider>>() {
@Override
protected List<HiveMetastoreAuthorizationProvider> initialValue() {
@@ -101,7 +101,7 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener {
}
};
- private final ThreadLocal<Boolean> tConfigSetOnAuths = new ThreadLocal<Boolean>() {
+ private static final ThreadLocal<Boolean> tConfigSetOnAuths = new ThreadLocal<Boolean>() {
@Override
protected Boolean initialValue() {
return false;
[20/50] [abbrv] hive git commit: HIVE-10855: with vectorization
enabled join operation involving interval_day_time fails (Matt McCline
reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/09100831/data/files/vectortab_a_1k
----------------------------------------------------------------------
diff --git a/data/files/vectortab_a_1k b/data/files/vectortab_a_1k
new file mode 100644
index 0000000..1b25ac1
--- /dev/null
+++ b/data/files/vectortab_a_1k
@@ -0,0 +1,1000 @@
+111|||2607|5765.39|3882592.64|3785719054585.463867|true|mathematics|nick xylophone|2065-08-13 19:03:52|2018-11-14 17:26:37.322428829|2086-05-13
+-87|21091|-1012329052|9182828596851990528|22311.90|2883584.50|3563455992720.427734|true|quiet hour|david quirinius|2047-08-30 02:59:28|2062-07-07 09:32:33.387727593|2083-11-21
+-58|-31395|-1730740504|302|11322.18|-2081693.61|-1639547871112.334473|true|industrial engineering|xavier laertes|2057-09-02 10:26:29|2074-04-07 03:59:19.110762829|
+-19|133|95356298|78|-15490.77|-642668.92||true|american history|tom ovid|2059-06-14 15:58:37|2021-07-06 11:56:50.11392413|
+95|27922|-2027812975|3764|-38980.40|-497550.81|4616197451686.900391|false|education|wendy davidson|2031-12-06 02:13:55|2037-04-01 15:48:02.707720905|2013-05-01
+23|-9171|-805288503|7432428551399669760|-35391.71|-1387292.76|-4384441919303.938477||nap time|xavier hernandez|2069-02-22 06:47:15|2068-04-02 03:24:11.192596144|2048-06-21
+7|725||3118|-12519.93|-1305193.84|3984781579512.529297|true|forestry|david davidson|2071-01-16 20:21:17|2049-05-02 13:53:42.230477366|2030-02-20
+52|8918|-772236518|7250237407877382144|32666.20|-3997512.40|-4361210532142.081055|true|religion||2068-10-07 03:23:30||2010-12-13
+54|-30304|-146961490|8100036735858401280|38097.34|300734.43|-101907680156.507812|false|wind surfing|alice ellison|2076-07-17 05:35:58|2017-10-19 13:37:33.31074556|1985-06-25
+36|-10317|-538812082|7792036342592348160|36810.38|1666710.20|1617263763890.299805|true|debate|xavier king||2049-11-23 23:25:42.906731900|2025-10-11
+-92|21849|868714547|-7496839341561954304|32711.55|-2738225.86|-946920531679.183105|false|biology|zach falkner|2064-05-11 04:54:01|2041-04-22 19:12:27.123637103|2020-04-05
+-127|7353|1008698636|1719|4983.60|-2636583.28|2290285251062.656250|false|american history|ethan polk|2029-05-28 08:57:08|2071-11-04 02:44:19.546016087|2048-06-29
+73|16195|737149747|8283099811330506752|5758.00|2121137.53|-3867208889795.392578|true|philosophy|sarah davidson|2061-05-23 23:39:02|2064-12-20 12:52:14.436538366|2053-04-28
+-46|-28501|810157660|2241|-7769.30|-3860829.99|4232305555838.919922|true|biology|wendy falkner|2054-06-13 14:12:16|2044-03-24 22:32:03.722424719|
+31|-9609|1541249928|7659279803863146496|-32124.85|1848079.75|-1733147315988.326172|true|philosophy|calvin miller|2058-02-09 03:37:29|2039-10-26 21:00:53.243928394|2030-11-17
+-35|||1075|21713.21|-4409416.25|-433003793989.392578|false|opthamology||2026-04-19 20:19:23|2060-01-21 11:16:28.358392262|1983-12-12
+48|7401|476704350|-7911421221625077760|-23364.57|393045.55|-3706576226699.773438|false|zync studies|sarah polk|2064-11-08 08:38:31|2047-09-13 00:05:07.657220269|2073-09-23
+-29|14773|-1974777102|1774|-41874.29|360339.96|4724948953312.201172|false|undecided|oscar ellison|2058-08-24 22:30:53|2023-10-22 21:22:34.254262972|2064-04-02
+-57|30921|-522450861|8895174927321243648|-7382.06|-4926003.80|4192159288015.525391|true|topology|david white|2027-03-09 17:30:45|2032-03-22 06:16:46.756458466|2077-08-03
+|26557|-407089271|661|-38119.53|-51958.20|-4230513318073.718750||quiet hour|wendy carson|2016-08-23 22:18:05|2079-03-05 14:50:42.389412815|2102-03-14
+-4||273256071|-9014145341570203648|-14145.11|-3116102.10|2704318642566.079102|false||ulysses white|2050-06-25 15:49:09|2013-10-28 19:25:59.70638641|2032-03-19
+47|23441|-978892011|-7049618574399692800|1466.49|-2506000.67|-1261885101463.384766|false|study skills|xavier underhill|2053-03-08 05:42:09|2059-11-24 17:10:42.836803274|2012-02-18
+55|-1928|174310705|-6935038507792801792|-4574.16|-2096084.32||false|topology|tom king|2068-11-10 06:35:37|2075-12-17 14:54:53.484250504|2010-11-13
+-43|2671|851975276|504|35338.95|2409466.90|-4397204039138.387207|false|industrial engineering|xavier xylophone|2044-09-15 08:07:20|2066-11-14 20:08:56.457818450|2097-09-25
+40|-21772|-1832606512|-8831091081349758976|8799.89|1118606.84|868268330921.963867|false|wind surfing|jessica van buren|2031-08-02 13:03:07|2066-10-02 06:56:23.275227665|2089-06-05
+-37|20704|-1506324615|923||-4894306.72|3900883250375.564453|false|education|alice ellison|2078-01-19 06:32:53|2034-07-13 13:55:48.524380282|
+||-419335927|3263|-9973.58|379816.01|-4708070356227.338867|true|religion|sarah van buren|2061-02-13 08:24:35|2074-03-23 07:11:45.666149042|1994-04-22
+114|-2828|-1144976744|289|-21745.11|1431965.10|-654012413621.529785|true|history|katie robinson|2075-12-28 00:49:24|2028-08-01 10:04:22.203514303|2047-09-09
+-103|-20934|1107757211|8509508263705477120||507403.73|-4227549209611.147949||zync studies|zach robinson|2062-09-02 06:50:21||2084-08-14
+-87|||345|-31059.05|-4822074.38|2868093144922.561523|true|education|zach miller|2055-05-09 16:32:47|2015-11-28 12:47:47.540035115|2049-05-06
+83|-5635|-42151403|281|5719.35|-1928099.64|3951352468527.070312|false|philosophy|calvin young|2016-07-13 04:11:42|2025-05-12 16:31:50.778958822|2036-08-04
+-51|-20411|929560791|3637|48956.95|-1601399.55|1334219192158.307617|true|values clariffication|david polk|2032-08-10 12:27:00|2059-05-03 02:46:31.520651892|2048-12-21
+-86|8488|-425196209|-9066993118333706240|28889.32|3917802.65||true|opthamology|sarah nixon|2050-01-29 20:33:45|2052-07-20 02:09:30.619526537|2029-03-12
+77|-11232|1426152053|7354813692542304256|2007.63|-4307343.30|4686486642663.113281|false|study skills|bob falkner|2018-03-02 05:13:42|2070-06-20 06:01:30.717133325|2042-09-29
+||546555204|2029|6253.69|2528579.30|-3261089774688.485840|false||victor thompson||2060-02-01 14:16:38.477867192|2081-09-16
+-44|-16218||-8807361476639629312|36511.51|286248.72|2844066824227.447266|false|religion|xavier van buren|2039-09-01 16:08:42|2065-11-28 01:54:34.188054521|2039-07-18
+-23|27169|821316302|1371|-21871.99|-3722153.69|-545562668559.717773|false|undecided|mike brown|2063-04-07 13:36:45|2069-02-11 21:42:06.394121082|2054-04-18
+-75|23220|-1421860505|-7221474017515347968|-11848.57|-4271448.15|-1871570979952.359375|false|mathematics|yuri nixon|2025-10-02 14:38:27|2014-08-18 11:46:05.790786201|2037-05-04
+23||-677778959|8871707618793996288|-37545.98|2442119.38|-555577054032.596680|false|kindergarten|calvin quirinius|2041-09-15 03:23:20|2017-07-16 08:23:28.728239344|2014-06-03
+14|-20192|563507584|7198687580227043328|36052.76|-1306614.08|-1346869656422.100586|true|chemistry|katie robinson|2063-06-16 08:30:34|2057-04-22 20:48:38.16105051|2080-07-26
+-71|30090|1050809633|1990|32100.40|1500446.78|3795668572538.915039|false|wind surfing|tom garcia|2053-07-13 17:41:24|2019-12-19 14:37:16.693138309|2018-01-30
+-96||-1565671389|8984935029383389184|28720.33|-3180309.64|4280974782610.193359|false|xylophone band|bob underhill|2078-08-06 00:23:13|2034-07-26 23:30:44.175867724|1989-03-16
+74|23177|1910930064|815|45521.78|-4243565.50|-2179991053030.186035|true|religion|jessica steinbeck|2027-08-11 19:58:53|2025-12-23 12:56:48.785116575|2076-03-16
+-6|21162|1941527322|2056|18542.60|96721.07|60247412756.062500|true|values clariffication|katie ellison|2027-03-11 08:49:50|2014-05-26 15:11:45.90866024|2099-05-09
+20|-24115|684561551|8290944180915871744|-32692.85|-793214.39|158268139075.410156|true|philosophy|oscar van buren|2071-11-06 04:15:33|2078-07-06 07:11:57.809667477|2002-07-08
+121|-18533|167432368|228|-49580.62|-1199504.10|-4410893311038.144531|true|wind surfing|priscilla hernandez|2049-03-14 00:55:26|2015-08-24 11:31:28.973482642|2038-10-27
+-93|-28769|-463071187||29111.14|3936159.86|855085840126.683594|false|yard duty|quinn young|2074-10-25 23:48:59|2068-09-06 20:05:31.931744370|2092-10-19
+-95|5736|1204325852|3460|35891.97|3800244.68|-1017339551408.831055|true|philosophy|calvin ovid|2014-10-27 17:15:46|2074-07-15 23:20:20.972096957|1983-04-05
+-25|-11760|-2069439395||-44985.09|132699.25|1886345168609.331055|true|quiet hour|nick steinbeck|2029-03-23 20:58:01|2031-01-08 05:13:27.828333067|1980-03-09
+9|-18796|654939016|-8345065519816695808|32691.31|-2340544.58||false|yard duty|katie steinbeck|2056-01-19 20:32:40|2068-12-31 18:53:27.897916012|2095-05-24
+-115|28393|2080249726|-7576194692683563008|4313.15|311917.45|-2770003968115.233398|false|nap time|ethan robinson|2030-10-10 13:39:56|2044-09-10 23:59:14.562909594|2055-12-02
+100|6743|1565313938|-8782213262837530624|-964.34|-4407852.66|-773857309922.618652|true|biology|oscar carson|2044-11-17 07:46:19|2053-01-01 14:45:15.764196054|
+80|-25780|1017953606|7220131672176058368|-35258.17|1908273.76|1088509311003.029297|true|values clariffication|gabriella brown|2075-10-25 05:21:06|2031-10-26 08:37:10.578092307|
+110|-11008|-2009569943|7452756603516190720|3399.76|1741339.95|-2575772562204.221680|false|education|alice laertes|2058-10-01 22:58:51|2081-02-24 17:48:22.155034035|2032-11-07
+-38|-15115|-916344293|7410872053689794560|-26858.87|-818719.64|2489707283313.293945|true|wind surfing|nick thompson|2014-06-23 11:31:25|2038-07-05 23:22:59.392777497|2054-03-11
+0|27116||-7357888618985873408|42003.58|1662438.31|-2435727088707.188965|false|linguistics|mike zipper|2033-07-29 19:41:57|2079-03-13 01:50:52.671710709|
+|2955|-1463884101|-7409317158045442048|-8186.07|-2572292.78|-2934889488954.759766|false|yard duty|katie ichabod|2015-01-24 13:02:54|2042-06-16 20:51:00.958638266|1994-06-02
+-30|-12628|868717604|3472|-38487.44|-4162489.36|-4161258810464.367676|true|industrial engineering||2023-12-05 22:32:04|2022-06-14 23:40:08.41754293|2094-06-06
+59|27852|-1830870295|2494|-24567.97|-929319.13|-3421625759809.039062|true||david quirinius|2074-01-14 13:16:59|2028-01-07 18:05:38.773104432|2002-07-10
+78|-7850|1677494300|-8992599250893979648|44585.81|-2982899.63|-597466124925.460938|true|forestry|victor van buren|2017-03-29 22:51:02|2068-08-17 17:51:55.371885346|2028-09-02
+-117|-21922|-799249885|8014986215157530624||2470347.06||true|chemistry|quinn ichabod|2029-05-06 15:39:17|2024-11-10 13:44:54.55064479|1994-03-27
+48|28558|-78240945|3159|-33993.69|889632.88|3293895628016.016602|false|opthamology|yuri underhill|2020-10-08 08:30:26|2019-11-20 03:42:51.136962161|1988-05-08
+-76|-3008|-1904737684|||4481669.11|-2023909790198.992188|false|joggying|ethan nixon|2026-11-16 23:20:05|2034-08-07 18:06:43.803487119|1985-04-16
+112|-6874|203688965|7436133434239229952|7968.70|4867019.14|-1079601478463.523438|true|quiet hour|yuri garcia|2022-06-09 22:16:54|2030-01-13 10:59:10.162327595|2025-08-01
+-23|-3520|1464695860||41789.11|-3616352.29|3476169263477.153320|false|forestry|alice hernandez|2045-06-14 13:53:38|2027-07-20 19:10:56.885557429|2072-09-25
+97|-14537|-1358159222|-8845239510002753536|30582.49|772376.99|-2651348117038.914062|false|forestry|quinn king|2016-07-21 22:10:27|2063-04-22 11:22:26.510916644|2044-01-22
+-88|198|323919214|8345435427356090368|35725.39|530232.57|4000300362554.191406|true|debate|sarah davidson|2052-02-29 16:23:06|2065-11-02 15:21:22.820623860|2016-12-30
+-19|-32480|2126491387|9094945190752903168|-24257.03|-4364624.36|1971729105497.779297|||oscar underhill|2044-04-04 07:26:25|2033-02-14 16:22:07.768207723|2105-12-20
+80|-22426|-235819331|-9078662294976061440|-13063.45|-639370.11|-2899905469957.140625|false|study skills|yuri xylophone|2054-03-25 06:38:11|2023-07-21 09:30:19.636575309|2043-03-22
+33|20884|-909127123|7689489436826804224|-20048.83|-4996960.35|2669260655452.099609|true|geology|rachel xylophone|2076-08-27 00:00:51|2031-07-31 16:48:51.285046883|2037-03-17
+-20|15688|-1967660827|7620183559667081216|-31608.43|-3778691.05|-253958738593.276367|false|linguistics||2065-07-05 19:36:50|2029-01-01 03:06:13.533276880|2105-01-24
+-1|-5593|2058640744|-9071565764086521856|-25936.66||-2624029717623.415039|true|forestry|wendy polk|2034-11-26 03:16:39|2073-01-17 06:48:04.722179257|1970-02-07
+-21|-23540|1678261510|2092|-44267.84|-3266915.53|136230451259.180664|false|yard duty|wendy davidson|2050-05-05 09:31:02|2070-04-03 08:45:42.349508679|1976-07-06
+-15|-24576|1022707418|1481|46316.04|825604.05|-4275490669201.076172|true|yard duty||2014-06-03 17:24:32|2014-11-29 21:31:04.183698999|2015-04-13
+-108|-18292|-934008333|8939431770838810624|-19049.65|-3634517.80|308587454891.693359|||ethan ovid|2066-11-25 08:03:39|2046-11-27 13:27:02.11530030|1992-12-03
+96|-11047|-2096425960|803|4967.48|-2777770.11|4037681204968.351562|false|linguistics|gabriella polk|2019-09-16 21:26:32|2047-10-24 22:32:11.31036942|1984-08-10
+71|||-8730803262481580032|16660.42|3752718.33|-311540696207.970703|true|history|ethan van buren|2043-08-10 22:21:54|2074-06-06 06:33:09.229039028|1976-08-29
+-52|-18385|-774406989|8201303040648052736|7836.59|-2532286.37|1913657229799.596680|true|quiet hour|oscar zipper|2039-08-18 17:59:50|2024-02-20 12:49:38.156348041|
+-11|1847|22308780|-7953426740065312768||-2185323.72|-4536408641834.341797|false|philosophy|mike ovid|2046-04-01 23:23:25|2030-11-15 02:16:54.219567283|1995-07-03
+-103|-3091|-392713245|-8099313480512716800|40508.97|-2375645.47|-3996919734918.407227|false|chemistry|tom carson|2038-02-20 05:19:48|2057-02-04 09:09:17.378101856|2076-01-14
+|3694|-1404921781|1972|-31368.73|3279707.14|-2805368749215.979492|true|yard duty|rachel miller|2066-10-28 12:26:53|2065-05-14 17:44:00.934322881|2055-10-15
+121|-30677|-1061222139|7933040277013962752|-46396.26|958738.91|501053268792.510742|true|topology|xavier garcia|2066-03-24 23:40:52|2047-12-05 05:59:03.709642655|2041-01-13
+18|-1660|-1955545912|-9189155542884474880|-7714.06|881042.86|2065189794939.748047|false|topology|irene ellison|2026-11-08 06:16:55|2025-05-17 11:03:47.185223340|2074-07-12
+-42|9028|1621606222|2434|4438.76|-2964984.97|-2918353603075.267578|true|topology|priscilla young|2033-05-06 00:00:14|2023-10-29 17:43:13.522556162|2028-03-26
+124||978044705|7888238729321496576|-24208.95||-4850732296584.077148|false|undecided|jessica white|2020-12-07 05:16:00|2072-01-10 09:27:24.503516138|2019-05-19
+56|5599|-665623523|-8240034910581153792|-2335.60|202813.82|3851526875785.361328|true|nap time|sarah johnson|2021-03-07 18:22:05|2049-10-01 06:32:49.357282514|2018-10-27
+-102|-13722|259524903||-16413.09|-625420.83|3535592833060.938477||yard duty|bob nixon|2066-09-13 11:21:42|2066-04-06 02:06:21.730154464|
+-56|11238|410340192|3567|29934.62|-2105876.12|2566162905785.595703|false|quiet hour|calvin brown|2018-12-30 00:00:33|2053-12-25 09:43:49.544264273|2095-02-28
+-111|-797|1412102605|7296164580491075584|31824.08|-886591.25|-2874808713941.199219|false|education|ulysses hernandez|2055-02-28 04:35:16|2032-02-26 21:07:51.12262410|2098-06-15
+-42|10479|860658464|268|32754.56|-2369331.68|654615772986.453125|true|forestry|fred brown|2024-01-02 15:21:18|2023-03-29 02:26:33.829299976|2062-03-23
+|30052|-1728171376|-8654433008222797824|35386.65|-1875563.10|-3891050509015.840820|true|forestry|rachel carson|2018-04-15 00:41:38|2016-03-28 02:03:24.513570827|2060-09-17
+-101|-32455|516843026|2229|-36732.79|-4985474.88||true|nap time|victor ichabod|2051-12-24 02:43:52|2069-12-02 02:14:56.589108032|2030-01-26
+-75|-15431|877053605|-8783777723063099392|-10499.63|678272.19|3041525159920.972656|false|history|holly falkner|2080-07-05 04:43:29|2034-10-26 21:04:12.771747509|2075-09-17
+4|8948|1211873318|2897|-17416.18|-4925512.63|-4514053790870.740234|true|opthamology|oscar ellison|2019-04-11 13:20:29|2026-07-07 02:42:24.833351047|2048-12-23
+-58|1393|1848935036|7989160253372817408|47928.52|-853795.31|-1657076687161.556152|true|forestry|mike van buren|2061-05-06 21:57:53|2058-06-06 16:07:39.838916621|2065-11-16
+-87|22557|873035819|-8028910243475038208||-2816366.10|-769631198270.062012|true|debate|calvin johnson|2054-12-07 01:08:19|2080-09-25 11:27:16.9717306|2080-12-04
+83|-10781|-480058682|1286|-17175.04||-2199239017358.216797|true|chemistry|jessica nixon|2040-07-07 10:30:33|2066-09-29 14:31:17.34936376|2096-02-14
+21|30632|2040926345|8677632093825916928|-45759.70|-2898003.58|-4156272991252.514160|false|geology|yuri underhill|2035-02-11 06:39:18|2017-02-20 17:17:37.184850313|2042-05-26
+-73|-6948|1773417290|8011181697250631680|11623.01|-958775.93|-1866254854407.820801|false|joggying|oscar brown|2039-02-19 15:50:12|2018-01-08 03:08:57.403292535|2100-01-30
+60|-19517|-884109192|-7966960765508280320|-10884.65|-3876912.53|-2066003780139.915527|true|yard duty|alice zipper||2062-02-01 03:32:34.383978859|2011-05-28
+-68|-14315|889733679|9185952983951343616|27500.67|3188754.82|4438755017449.623047|false|philosophy|priscilla zipper|2076-07-13 10:52:27|2016-05-03 07:39:40.516896330|2069-09-26
+-95|31125|1036391201|7319711402123149312|31964.96|-1757277.11|-1496962050139.103516|true|kindergarten|victor ovid|2032-12-06 07:37:49|2062-07-26 22:38:29.146178601|2032-04-09
+-94|-6088|1604076720|9119046173224370176|19740.52|-4678968.17|3557302643795.097656|true|american history|david xylophone|2073-01-23 03:17:41|2040-11-18 04:31:12.709589656|2004-09-28
+19|-9243|-1249011023|-8760655406971863040|-32655.55|-1649087.83|932399155849.453125|false|forestry|alice van buren|2018-07-11 06:15:11|2027-04-13 10:06:40.902651522|2019-12-25
+12|7343|1418228573|8190539859890601984|-20276.81|-4235586.20|-1196769263706.656250|true|values clariffication||2070-05-15 09:16:30|2078-03-22 13:36:17.940377243|2037-02-13
+-12|6813|-253084551|3974|40698.97|-1115432.48|-2448300846308.981934|false|wind surfing|wendy ichabod|2022-05-20 11:14:25|2053-10-25 16:58:08.223136198|2097-11-18
+-125|-22641|1597303154|2106|4874.58||-30490934435.612305|false|religion||2015-03-28 09:25:39|2013-03-04 23:58:15.71378943|2078-11-03
+39|8410|1640445482|2579|-14853.40|-3366426.92|1755470105685.397461|false|mathematics|jessica xylophone|||2081-02-20
+|-12755|-1353470095|7210160489915236352||-1899846.57|-653238714138.346680|true|undecided|priscilla xylophone|2022-03-08 18:50:03|2059-09-27 11:04:07.90814681|2011-06-29
+102|28247|794783516|-7907355742053883904|20683.15|2965661.82|-624867269213.389648|false|chemistry|mike van buren|2064-01-12 01:22:43|2055-03-27 10:02:55.174795513|2085-03-01
+114|19513|418182899|1846|-24886.47|-3030589.60|4183756295927.345703|true|education|luke xylophone|2017-10-15 17:18:43|2063-06-10 14:18:48.104797455|2044-04-24
+108|27636|215508794|1189|-25835.00|4453603.41|4405275049808.744141|true|nap time|ulysses van buren|2015-07-02 03:45:28|2013-12-31 08:25:18.118248757|1981-11-06
+23|-19623|-1039040287||-6512.80|-660440.82|-1642262021182.041504|false|geology|xavier quirinius|2080-12-09 03:55:10||1998-04-16
+-53|-4169|879290165|68|34884.11|-1267493.50|-3279094975952.290039|false|industrial engineering|mike young|2075-12-04 18:22:09|2054-10-01 19:12:15.766024626|1977-12-27
+-88|16796|1151752586|-6975459232300236800|-4784.15|-4864373.89|3221553706534.408203|true|topology|tom zipper|2064-04-09 08:01:25|2070-08-17 06:01:00.968011813|2027-04-10
+13|17531|1805308672|-7700203302632210432|-37763.56|3704107.83|3425256239457.610352|false|undecided|xavier thompson|2027-04-18 16:26:01|2024-07-18 19:36:10.425538948|2092-04-17
+-61|32019|1420099773|-8021859935185928192|38974.63|1673755.73|2102524829645.560547|true|chemistry|ulysses falkner|2029-02-04 06:51:54|2046-05-19 15:05:25.732127788|2062-03-10
+7|19302|538268118|7339426767877390336|20672.29|4555902.70||false|topology|fred laertes|2017-01-13 01:23:09|2016-05-16 23:25:47.63891547|1998-12-23
+-114|-28736|449788961|774|-18622.98|-1268938.21|2666440101920.493164|false||priscilla ichabod|2063-12-27 01:29:50|2035-03-18 21:03:10.648253783|2062-10-07
+101||-1914072976|2295|1339.82|4112381.85|-1984449836425.916504|false|study skills|alice ovid||2075-09-26 04:56:48.660923039|2050-07-10
+44|-18090|259204652|-8559252110266564608|-9038.73|-4718571.75|3963517992293.716797|true|forestry|victor white|2066-12-24 15:59:32|2038-03-27 02:23:06.834673196|2096-07-12
+-24|10161|-882028850|3079|-45314.95|1053922.25|1574414367849.214844|false|history|xavier ovid|2052-09-14 19:24:26|2034-04-25 18:26:08.505694709|2040-08-03
+89|-24801|-158233823|1346|42990.33|4421723.61|2978558145914.504883|true|geology||2028-06-15 20:46:41|2071-05-17 17:16:54.148951459|1989-11-05
+31|-31582|1219616145|-8007017894942638080|-44727.28|-4114788.64|-4378058802654.081055|true|american history|quinn quirinius|2060-09-20 08:49:06|2023-11-29 06:13:18.611946024|2019-12-11
+|-19159|1807877618|987|-5972.19|-119465.75|1452475353908.404297|false|quiet hour|holly ichabod|2062-09-02 18:18:39|2070-04-17 08:28:26.5503386|2075-09-26
+27|17023|1747664003|-9049720998034137088|22254.70|3306808.08|3177841668890.231445|false|biology|priscilla ellison|2015-07-10 05:27:46|2027-06-11 15:58:30.658173981|2024-08-23
+|-25344||-8158011642485825536|-48220.79|4820070.80|3955085450779.839844|true|chemistry|victor polk|2055-12-20 16:02:07|2024-09-03 18:32:19.912271231|2011-06-12
+-96|-6950|-1070951602|7697541332524376064|-23329.71|4539092.16|-1684983372492.716797|false|undecided||2069-02-08 07:45:14|2068-03-26 18:14:15.165533561|
+-74|4261|217823040|8773222500321361920|-11741.38|1094988.55|3709069978104.153320|true|topology|rachel xylophone|2044-11-15 10:42:51|2059-08-26 20:14:15.307180062|2022-10-12
+-72|-7178|1814570016|-7915999634274369536|30236.61|2437780.95|2666538537304.746094|false|biology|fred carson||2079-01-28 01:30:38.861840968|2077-05-11
+-64|-1725|1493555718|6974475559697768448|-16776.52|-4105739.18|-2442617490401.950195|true|education|katie thompson|2044-03-29 00:20:37|2015-04-18 09:51:54.990681181|2078-01-13
+41|-19270|1119976718|7052226236896256000|28023.88|2110273.07|-4751768490466.296875|false|biology|yuri laertes|2079-09-26 20:39:48|2068-11-17 19:45:06.599632615|1983-05-31
+-58|-6494|1222217404|650|37717.83|107717.64|3643292384671.887695|true|industrial engineering|luke steinbeck|2028-02-25 14:55:40|2074-07-02 03:20:40.417209896|1973-03-04
+-127|-4252||3079|39550.67|-3279977.09|18225457799.800781|false|zync studies|holly zipper|2045-08-14 20:11:34|2024-05-11 15:11:48.691514072|2074-05-09
+-26|10261|914062370|1039||2283246.90|-1345490060709.941895|false|religion|luke robinson|2039-04-22 14:44:00|2053-03-18 12:47:22.448344815|1979-11-27
+-46|7356|683320224|-7610137349734883328|23741.83|-4065786.49|4723266191273.687500|true|chemistry|nick carson|2022-07-31 17:56:09|2024-03-03 22:59:52.779904666|2022-08-11
+48|28309|-570632618|-7526793959592140800|-4768.36|-4329360.25|3073439981743.923828|false|joggying|victor laertes|2034-03-18 16:11:32||2086-05-21
+39|3228|-535056977|8897901899039473664|3865.37|-206033.60|3887729126314.027344||study skills|wendy laertes|2018-03-11 05:09:22|2034-09-09 08:05:25.434342520|1982-04-12
+-73|-1011|-1058166020|8793387410919038976|-9572.10|2650235.31|-3040241790788.478516|false|quiet hour|oscar robinson|2062-03-17 08:07:25|2042-08-08 14:24:00.510838590|2078-05-19
+-65|17073|1111985530|9112400579327483904|46710.33|130023.02|||industrial engineering|wendy falkner|2051-09-14 21:51:35||1994-06-09
+-127|29365|895945459|-8559008501282832384|19094.69||-4601795255939.329102|true|study skills|katie underhill|2062-08-08 00:34:44|2035-03-17 18:33:10.848870183|2087-04-30
+42||-997463353|1788|-7.25|-1624411.04|2061492866787.446289|false|debate|xavier white|2074-08-12 02:19:10|2024-07-12 04:37:59.670091691|2079-05-31
+-24|-13978|-491882534|8554899472487596032|-42196.85|2380170.34|995819157271.000977|true|debate|ethan garcia|2067-04-29 08:24:05|2033-02-23 23:45:43.224368846|2026-11-24
+45|-7865|829101712|-8070535484085895168|-8695.03|1393839.55|-2930125300660.009277|true|religion|calvin garcia|2069-10-10 07:46:10|2030-09-03 02:47:09.999512465|2094-08-04
+34|-32240||-7858505678035951616|21913.94|-4848416.07|-4417461834328.939453|true|yard duty|tom miller|2020-08-15 02:15:04|2049-01-07 01:23:16.815364884|2041-04-17
+82|-4550|-1497098905|1058|19349.22|-3207956.11|2622882069011.147461|false|education|gabriella nixon|2048-06-12 05:15:59|2070-06-08 15:10:15.103401622|2047-07-22
+90|12762|-983874694|-7598782894648565760|-6545.09|4223917.78|-2724893143053.712891|true|biology|wendy carson|2064-09-28 23:47:20|2057-05-10 17:22:29.44470118|2009-10-04
+-52|-15501|-1184620079|7947544013461512192|17444.50|-4019474.98|-4273574634879.970703|true|debate|wendy falkner|2033-04-24 23:46:39|2064-12-09 12:41:18.605098238|1999-04-04
+-121|-19282|787925706||-39608.35|-3490480.46|-1396998641989.632812|true|opthamology|xavier robinson|2050-05-29 18:50:17|2061-07-15 16:24:02.41944450|2064-10-08
+-104|-6736|272086526|8625937019655200768|-42455.39|-2155542.09|-3341536888638.039062|true|undecided|wendy xylophone|2052-10-28 04:50:15||2103-12-05
+-105|-4059|760466914|3701|12507.09|2661135.97|1085001013184.765625|true|history||2056-10-06 10:43:05|2024-10-19 16:47:43.958389682|1991-09-28
+90|-25683|745725681|-7692192232238678016|364.62|-695924.61|-918004138590.973145|false|yard duty|mike polk|2052-05-18 02:04:21||2015-02-25
+-90|9110|1398486099|1291|-18652.98|4330260.30|-2988564000800.586426|false|chemistry|sarah robinson|2030-12-19 06:51:08|2073-06-18 12:33:18.634781669|1999-02-06
+53|24715|476919973|3887|-33746.36|-1812815.81|2248167382355.781250||values clariffication|calvin garcia|2028-06-28 08:30:31|2039-03-21 22:46:13.628314022|1994-05-06
+-112|14286|287239980|1462|23269.50|-3421045.14|-4825619680711.810547|false|geology|katie johnson|2023-07-03 15:05:18|2018-11-09 18:46:47.759800359|1983-09-25
+-36|3213|-1528033060|-8485389240529354752|33155.26|3831347.85|-2604012470736.947266|false|study skills|ulysses ichabod|2039-07-17 08:06:17|2032-05-11 08:35:57.629571826|2033-04-05
+-122|-16872|1520375588|8660248367767076864|-46409.01|1435619.95|-301617019681.543945|true|nap time|david steinbeck|2028-09-21 14:32:03|2053-05-02 22:50:04.195045396|1977-04-22
+|32092|-951728053|-8115963579415650304|17565.73|-4855596.00|-1567705616965.078125|false|wind surfing|ethan ovid|2079-03-11 06:37:16|2080-10-21 08:32:40.577382791|2021-08-16
+108|-6933|-66112513|388|-23820.11|-2660889.99|4673408308800.226562|true|values clariffication|calvin brown|2013-07-21 19:03:27||2078-03-23
+79|-6142|1295073553|-7262049693594943488|34920.29|-2775037.93|-3850646756153.040039||chemistry|sarah king|2062-11-28 06:34:11|2064-12-02 14:20:25.689540381|1970-10-19
+114|17164|-217785690|1343|-22425.89|-4109302.98|1542043552284.902344|true|biology|mike miller|2043-01-06 00:29:33|2027-09-11 18:50:09.224829213|2084-05-01
+79|22366|1632769786|1508|-16193.28|-4581299.04|-4048240005182.783691|true|opthamology|sarah zipper|2027-06-15 04:28:56|2024-11-18 13:11:30.807086190|2094-07-09
+-65|5785|-648766606|1701|-25497.04|3706795.80|4406779957872.437500|true|study skills|tom polk||2047-01-20 19:02:44.296212906|2079-09-24
+-23|-29475|2052773366|8837420822750314496|-44389.81|-3244548.09|-3320189294654.056641|true|xylophone band|katie carson|2014-08-14 04:46:15|2032-01-06 02:52:13.841451923|2008-01-10
+25|24014|-1318045616|-8704234107608203264|-25094.94||1313758909596.565430|false|study skills|irene young|2034-08-06 10:56:24|2050-11-22 00:49:32.763404963|
+67|-9614|1124269631|3680|29490.38|-1942538.00|-2214662106936.273926|false|topology|katie ovid||2028-06-06 03:25:20.277859556|2088-10-27
+23|-21708|-94709066|7271887863395459072|-43725.17|-582351.40|-1251943808107.505371|true|mathematics|oscar garcia||2019-08-07 09:24:28.969847804|2091-10-12
+-49|-12071|1384071499|8156018594610790400|44500.74|-423299.81||false|mathematics|zach garcia|2062-01-17 23:46:46|2035-06-22 17:50:21.193860499|2068-12-27
+-34|-27553|1860113703|8708845895460577280||4443055.22|-3152882360663.084961|false|geology|priscilla young|2032-12-02 17:45:28|2030-02-07 12:05:50.442818058|1981-07-26
+||1634441052|2847|-30620.44|-4164391.70|4767856866280.625000|true|kindergarten|rachel steinbeck||2015-02-23 17:38:53.735735894|2038-02-01
+-126|-31537|693331761|2509|-25833.15|-4801160.87|2804616317735.586914|false|kindergarten|priscilla thompson|2046-02-05 15:38:03|2035-04-25 16:30:39.959832372|1983-09-26
+121|-6219|65956045|1987|-13776.59||3855241018435.791016|false|joggying|nick laertes|2076-07-22 18:55:23|2045-05-01 12:49:24.875445|2104-07-12
+-8|30861||8470141334513098752|6021.11|3057994.15||true|undecided|gabriella thompson|2056-03-17 16:36:40|2076-09-17 19:51:03.971998194|2019-06-03
+108|-10687|977292235|2108|34147.75|3835815.78|-3242583557065.046387|true|xylophone band|luke van buren||2026-07-29 12:13:07.350517258|2004-10-09
+-103|5235|283322761|-7623405558242500608|-9121.56|4969099.99|1418203690420.333984|true|history|ethan ichabod|2052-01-27 14:18:09|2076-09-25 20:35:14.535725735|1995-08-27
+-61|3063|-870624802|8687042963221159936|3943.90|-209028.71|-2183718197982.749023|false|joggying|calvin davidson|2034-06-08 16:02:13|2029-09-24 14:25:09.37764381|2105-01-30
+-96|-10516||3568|-19009.76|3239989.12|-4784778822088.213867|true|topology|oscar nixon||2036-02-03 04:58:16.203066567|2065-12-22
+-50|-30748|-1735287250|7746402369011277824|-4179.91|1292492.11|2324680845107.813477|false|mathematics|david allen|2016-06-22 21:51:17|2069-09-12 15:00:57.362462590|2072-09-24
+-103|9969|-1703620970|-8387536830476820480||-3049943.90|4709582975677.548828|false|zync studies|calvin allen||2079-11-27 18:43:43.659151983|2044-09-10
+34|-21818|818313200|3060|-38828.51|-2063369.12|-3583867768957.524902|false|debate|sarah johnson|2064-02-16 11:03:07|2019-10-12 04:27:17.822098134|2004-10-26
+-85|19350|1409872356|-7512289590991544320|-10184.90|3793945.90|-923624349228.369141|false|industrial engineering|katie white|2032-02-14 12:11:48|2061-04-03 12:08:01.5162482|2081-03-17
+-78|-28932|187893585|8235179243092090880|-23609.39|962981.03|2966147671023.726562|false|zync studies|luke ichabod|2017-01-05 14:49:04|2059-12-23 13:06:00.979846594|1993-11-14
+92|2728|398960205|8540237852367446016|-8750.79|-4703249.19|-501603049952.486328|false|nap time|tom king|2030-04-28 01:59:00|2018-08-22 06:45:33.15399240|1977-02-18
+8|6144|1437057145|-7265998318110711808|-21666.49|361449.07|-3148061620108.357422|true|quiet hour|luke johnson|2065-11-30 23:06:00|2052-07-18 07:53:41.806082172|2062-06-04
+-120|26859|-1554325042|583|15171.32|-2158930.81|-301298241024.774414|false|undecided|victor van buren|2050-07-22 13:12:51|2063-08-25 00:41:53.424629732|1979-04-17
+-124|-10095|1033609549|-8318886086186213376||3677625.78|4672389589988.128906|false|history|irene johnson|2056-03-14 14:13:51|2077-11-14 16:16:46.603310412|2084-07-26
+75|-23638||2410||-968637.64|1698116680328.283203|true|debate|rachel allen|2042-10-06 19:18:58|2057-08-09 04:27:05.138333574|1993-11-01
+-59|1361|-360113158|3021|10379.72|-2330239.84|2488834645794.498047|false|opthamology|bob king|2063-06-09 19:51:43|2037-08-18 09:27:00.913006136|2011-10-15
+|3523|661380540|-8018511948141748224|31005.68|-1939975.71|561437334048.342773|true|wind surfing|quinn thompson||1970-01-01 00:00:00.212340798|2086-01-31
+40|-32296|84231802|2886|-4154.25|4389868.65|378820126909.822266|false|american history|holly laertes||2065-08-04 06:59:17.824387481|2014-11-15
+-32|-6637|-1726479726|-7831320202242228224|39531.99|2377538.79|1099330213298.444336|false|debate|victor davidson|2050-04-05 23:38:10|2022-03-12 21:56:07.234951116|2018-10-08
+84|-18387|346562088|7998357471114969088|44620.73|4045062.26|4748679788424.253906|true|kindergarten|zach quirinius|2020-07-07 16:26:44|2047-03-31 09:15:12.887634795|2084-02-05
+-27|-10815|1769324649|7936149988210212864|11543.56|227674.23|-812606022147.335449|true|zync studies|alice johnson|2048-05-19 07:13:08|2062-03-12 06:53:03.331725965|2076-03-18
+81|2643|2140632003|-6970396058557005824|19854.94|2938603.02|562510896793.071289|true|education|david davidson|2068-04-12 18:18:24|2041-04-04 08:06:20.143031301|1972-02-24
+-24||780938234|-7083646746411720704|-16507.75|-3851733.33|1863907193854.252930|true|joggying|luke falkner|2049-02-04 12:28:05|2057-01-09 07:22:38.778950773|2100-08-23
+-101|-8857|-1918651448|-8675661101615489024|43481.62|149175.06|-2178760699903.925293|false|joggying|mike davidson|2058-06-04 16:19:32|2072-08-14 09:56:17.194554282|1976-03-18
+|-10821|770574055|-9126793997498957824|-36806.07|94051.69|3234847750887.297852|true|yard duty||2051-04-08 16:12:47|2075-11-21 21:55:19.18153321|2066-08-28
+42|9158|776606164|2506|-24639.80|-4869417.64|3079389364956.313477|true|industrial engineering|alice laertes|2044-10-22 22:17:52|2074-08-21 15:27:43.96892721|2058-05-29
+|7180|-51612681|7249443195032985600|-7568.74|1202707.37|-2119064600783.993164|false|undecided|calvin ichabod||2022-04-25 10:39:59.377128631|1982-07-06
+29|10891|1153811197|-8243487285852766208|11569.34|4212709.34|1034657320312.746094|false|american history|victor brown|2031-09-30 15:01:06|2025-09-04 14:13:00.976690553|1993-05-12
+-122|7654|-734921821|3941|7256.71|517028.01|-1114345601613.280273|false|topology|zach carson||2070-05-25 06:36:39.297148492|2001-05-07
+-50|-28968|-1460613213|8875745082589929472|-7442.32|-2702250.82||false|philosophy|priscilla brown|2032-06-20 09:31:24|2052-09-03 16:21:52.208215796|2087-08-18
+-112|-29907|206121314|-8705403811649355776|-44201.12|-897624.69|-2173857521374.103027|true|xylophone band|xavier brown|2059-05-21 10:37:20|2065-01-02 03:32:19.285960165|1998-03-18
+-6|17701|-1606567895|8410599906334097408|23194.74|4572852.32|-1064163809823.100098|true|xylophone band|holly nixon|2044-04-30 10:00:19||2011-12-07
+-117|-5869|1523657918|-8400045653258444800|-37316.59|-1140903.56|3668214733218.100586||study skills|fred brown|2066-10-11 19:46:58|2057-04-21 18:12:07.438157094|2072-06-21
+-58|-2118|-37876543|-8859107121649893376|-2682.16|-2318215.28|4457864918964.134766|false|zync studies|bob van buren|2065-12-18 19:50:19|2043-05-16 22:15:58.560013699|2000-07-15
+19|-26060|1425362689|2201|35741.32|-635373.52|-1660120664738.187012|false||mike quirinius|2055-11-26 10:21:07|2043-11-02 23:23:13.175695849|1978-08-05
+-115|1755|596045726|-7000925438663041024|1169.13|1224590.95|4349386575138.148438|true|xylophone band|calvin quirinius|2014-03-11 02:26:36|2019-09-03 01:49:36.181933289|2088-02-28
+-95|-2617|1229172951|490|-18461.73|1170976.95|1480054571327.649414|false|mathematics|quinn hernandez|2021-05-20 06:00:47|2014-10-28 09:16:31.610844395|2080-12-12
+30|-28244|1978200605|1910|7799.30|-123266.43|-1103541868445.567871|false|undecided|quinn underhill|2019-08-08 22:46:59|2059-05-28 04:51:35.498984116|1981-10-31
+-83|-15497|1000106109|8900351886974279680|46362.81|-4358428.80|3426770165622.876953|false||ethan brown|2027-01-01 03:13:01|2063-08-29 02:30:48.169621762|1992-08-21
+-93|4032|879289168|7343171468838567936|29799.26|-3707213.21|-3700102015935.797852||education||2080-10-01 22:58:13|2042-10-22 03:29:04.44238849|2038-02-21
+7|-29895|-464804906|2131|-21839.05|4389713.57|-1713151304480.397949|true|quiet hour|luke ellison|2027-10-30 14:21:42||2066-10-25
+-106|-22071|-1247229632||-38427.52|-2639190.56|-755064468729.002930|true|religion|victor robinson|2044-02-05 12:48:23|2058-01-26 21:26:48.253653944|2010-09-07
+-72|-5907|-211669740|3208|-38197.98|781541.24|-2726366048183.592773|false|industrial engineering|alice allen|2036-11-15 10:48:44|2074-12-03 23:58:03.654571719|2076-11-09
+-8|10600|-1155174991|-7420448501073051648|-48598.14|-2840797.35|966723065681.023438||kindergarten|luke robinson|2075-10-15 20:32:00||1990-02-19
+-89|-8267|786565385|3418|40746.89|-3336502.60|-3655334890183.326172|false|history|quinn van buren|2073-12-08 10:59:27|2059-12-18 19:47:35.135760242|1992-09-21
+58|24847|1115197541|-7030489936116252672|-32967.44|-3216068.70|-4540163781577.817383|true|study skills|ethan quirinius|2057-07-19 17:20:26|2061-01-07 10:20:58.287466414|2058-04-02
+82|||1093|-41542.22|-2852753.19||false|mathematics|oscar ellison|2078-03-12 21:36:59||2040-03-02
+7|17107|-1227085134|454|-45679.81|4280344.76|-4755957353103.070312|true|biology|victor robinson|2075-06-02 02:53:30|2040-12-08 01:16:10.194735899|2020-12-19
+-109|-25692|-714594143|2791|15638.71|4868354.39|-1920412941995.195312|true|yard duty|priscilla brown|2013-03-25 15:38:28|2039-10-25 16:44:04.385364994|2073-11-05
+98|19215|30036142|1252|5512.48|2861312.90|304220501137.706055|true|zync studies|wendy polk|2074-09-18 04:34:04|2042-10-10 04:18:49.518287517|1992-09-29
+-119|-20879|-1900894010|19|-14328.52|3163463.29|-2231521147965.978027||american history|xavier van buren|2051-08-02 17:01:13|2070-06-10 09:08:31.433429764|2049-12-19
+-80|11979|874824958|485|-49403.10|1836049.78|-3891815088473.258301|false|american history|ethan king|2023-03-21 16:09:44|2018-10-09 10:15:27.268042756|
+-122|4568|1505665168|-7453525026342617088|-47688.82||-2370722163282.117188|false|mathematics||2043-02-15 19:11:53|2080-04-06 03:34:04.633286901|2079-05-24
+|2677|-2077771325|1604|4199.81|1851095.86|-3091236716436.114258|true|philosophy|fred brown|2051-05-22 03:56:26|2073-04-07 19:52:50.601742131|2023-01-10
+62|-14280|-2043805661|8365058996333953024|11341.10|4541245.28|4402724332930.460938|true|xylophone band|yuri robinson|2080-04-12 01:21:14|2031-04-06 19:14:59.55187421|2031-05-31
+62|13578|738356485||22454.83|-4779.61|-1857979799639.935059|false|nap time|ethan king|2046-10-08 18:53:18|2058-12-03 23:22:35.944838503|1971-07-29
+-48|-22938|340384179|-7877598807023386624|2517.88|-2373948.65|947585735001.510742|false|opthamology|david thompson|2075-07-24 01:40:11|2036-06-14 00:00:15.278560351|2003-09-18
+-108|-23194|-1198465530|3055|-36853.20|2315107.36|-1344917461892.407715|true||rachel quirinius|2041-04-15 03:29:16|2043-07-29 21:24:09.491072788|
+-112|19350|-705887590|-9079801920509001728|42578.38|2583540.14|885849762897.948242|false|american history|ethan white|2055-05-21 11:56:02|2059-12-29 12:16:34.533626678|1984-03-14
+33|16430|1347876055|-8465978403747037184||2398238.05|3976258329685.542969|false|undecided|david quirinius|2040-12-15 14:20:04|2059-03-26 08:14:53.410162567|2013-04-21
+|-21574|62293025|2426|14978.09|3616509.09|3825560324453.007812|true|history|sarah falkner|2035-04-19 13:32:57|2061-05-10 03:23:57.557730657|2099-03-03
+5|-6114|198017473|135|-14085.31|2000803.23|-2919243579540.449219|true|american history|nick brown|2073-05-02 15:11:49|2047-05-06 13:18:00.344896111|1997-06-17
+35|28128|1286367391|-7488415863027367936|-26817.08|-797714.37|3875352447190.109375|true|study skills|jessica brown|2072-03-19 14:27:46|2018-12-18 23:20:30.63266134|2080-12-24
+78|-22358|376076075|-9136398397785948160|46846.67|-1700725.96|-899635084786.325684|false|industrial engineering|katie king|2017-10-06 08:33:37|2057-10-12 20:55:36.383445468|1972-02-28
+-34|3009|-1804244259|579||-2740315.70|362659766665.806641|true|zync studies|sarah thompson|2032-07-12 22:13:10|2035-11-06 17:59:27.77357393|1996-08-02
+-103|4081|2083836439|8144552446127972352|36570.50|3555782.96|3609640367332.538086|false|philosophy|sarah davidson|2031-08-26 09:58:56|2023-01-11 17:04:23.449037266|2041-06-07
+39|-16367|1776456512|6933451028794925056|-43403.36|-4299638.88|793504824242.568359|false|forestry|ulysses thompson|2028-12-06 04:06:25|2055-06-24 20:19:48.682407175|1971-01-13
+37|20128||1777|35965.37|-1496116.32|-3499447525525.525391|false|linguistics|calvin robinson|2067-08-21 11:00:04|2033-11-12 22:25:15.735331075|2019-10-13
+45|20254|1033836308|-8927968289860370432|-47362.97|3785304.85|-3394951194042.761230|false|forestry|david thompson|2017-04-09 00:02:35|2054-01-29 09:25:44.815093672|
+-99|26910|-571587579|7265141874315517952|-28011.29|-4797236.03|4313736848665.250000|false|wind surfing|katie underhill|2080-01-07 15:47:45|2057-04-03 15:14:56.240582613|2046-03-03
+-118|16437|727802564|4078|32210.62|913010.81|2387469547668.992188|false|undecided|fred ellison|2030-03-31 06:22:45|2061-03-01 05:58:51.835198480|1982-08-19
+98|-11929|596242714|443|-18850.34|-2514812.80|3841757426531.521484|false|undecided|tom miller|2056-04-06 22:21:49|2023-11-11 06:58:15.825259324|2070-01-08
+-1|-22217|-743680989|7391208370547269632|1450.95|3168984.07|422951017333.055664|false|topology|priscilla thompson|2047-05-23 07:19:55|2019-08-24 23:35:52.785182782|2092-08-23
+-23|32589|1575091509|7790728456522784768|43789.16|-3767707.87|449819577744.783203|true|forestry|quinn thompson|2049-08-29 23:03:28|2059-03-17 09:44:04.511510153|2082-09-14
+-48|-22447|669871113|8398862954249560064|45334.49|-168095.10|-95855026425.759766|true|quiet hour|ethan laertes|2026-06-26 02:42:50|2014-08-21 12:14:15.695317873|2075-09-12
+114|27046|397255100|-8836899523028312064|30215.93|-717006.21|-357328163291.050781|true||zach brown|2073-12-21 08:21:58|2061-05-01 00:50:27.746635942|2021-06-09
+9|-11679|-2076886223|-7805985795815342080|19575.50|120994.39|380373641521.103516|true|xylophone band|ethan carson|2076-07-30 01:09:00|2057-09-20 07:35:34.305010925|2065-08-02
+-5|14783|531459992|1439|19351.91|4803315.07|-574114492621.049805|true|topology|jessica thompson|2072-08-04 11:14:51|2025-01-30 01:47:54.411372161|2085-08-12
+-41|22870|177391521|7266437490436341760|12.17|4688982.65|-4069350543654.266602|false|religion|oscar falkner|2016-04-04 12:22:03|2056-09-18 10:09:51.803143104|2104-08-29
+-123|7959|311478497|-8852770376039219200|-44161.63|1078252.98|1685122154602.396484|true|values clariffication|irene hernandez|2022-06-21 13:36:54|2057-10-24 03:22:01.465719271|2102-04-28
+82|-17995|673904922|3119|31236.39|3313603.14|350230667228.643555|true|religion|calvin steinbeck|2035-06-03 08:40:07|2047-07-13 00:30:59.312757955|1979-08-05
+109|-28299|1042237722|2962|-49297.70|2544504.56|1300896197981.842773|false|kindergarten|gabriella falkner|2045-09-01 02:14:09|2049-12-12 02:33:51.210858024|2041-12-08
+|7483|-1974257754|9001907486943993856|-39292.53|2604166.52|2628816035521.843750|false|mathematics|jessica nixon||2024-04-14 05:51:51.799661240|1973-10-11
+37|29639|218917585|-7378096180613840896|2468.85|545061.80|4576650304788.998047|false|joggying|quinn ellison|2030-03-18 10:56:59|2064-04-10 13:26:19.857498352|1973-05-29
+54|13145||7091300332052062208|-17351.53|4747854.89|3817727970940.189453|true|topology|oscar thompson|2065-09-11 19:52:17|2038-02-26 14:34:48.662632111|2098-03-08
+79|-4491|-559270035|-7818454479651135488|32811.79|-2589530.66|3650613746347.026367|false|forestry|zach polk|2062-04-28 22:14:44|2053-08-03 04:16:15.344431898|
+123|31427|-1131684944|-8431492599012163584|48223.45|-4797838.11|4766118630785.527344|false|philosophy|quinn white|2035-12-20 20:20:18|2055-03-06 04:00:57.170922096|2071-06-23
+-57|-3602|-373034494|-8521578237232529408|35310.58|523737.60|4138846175212.621094||industrial engineering|ethan polk||2070-05-20 09:18:23.921779415|2089-12-19
+-6|-16722|-490337498|1926|43246.64|-22165.47|-624395488497.526855|false|nap time|jessica falkner|2070-09-14 21:01:42|2024-10-17 22:24:01.153750276|2069-10-18
+64|-3219|-1829691116|7490717730239250432|46586.47|-1915228.77|1287655745842.997070|true|wind surfing|fred ellison|2060-10-10 10:17:38|2080-01-01 16:19:34.640061653|2095-02-04
+-26|-20949||-9187662685618348032|43398.02|-49310.50|1863250559721.372070|true|study skills|katie carson|2048-04-09 07:40:02|2073-09-18 13:35:45.258178237|1984-12-02
+-50|13078|135341845|2016|47991.75||-3445060192851.059570|false|zync studies|rachel young|2024-12-31 09:01:02||1970-08-12
+-41|27758|1646811064|1153|-4313.26|-3049537.40|-69377778943.611328|true|topology|katie johnson|2041-02-02 14:22:59|2041-06-19 10:09:46.112258643|2041-07-24
+-70|26232|-1447263708|2277|41062.58|-2943975.09|-2631454750987.565430|false|linguistics|yuri miller|2051-01-04 12:34:39|2075-09-08 01:57:00.107926096|2070-02-10
+|-22858|1743696703|3300|-4380.97|-2069211.91|-4596153868922.030273|false|study skills|rachel van buren|2080-08-26 15:15:51|2030-04-01 16:16:55.346669591|2015-09-27
+-40|3354|1303632852|3244|-21821.23|3976574.38|-4697749109705.853516|false|zync studies|quinn robinson|2071-08-03 03:03:18|2033-11-19 04:14:03.372021127|2093-10-02
+-56|11120|1949494660|7107604675626008576|11034.18|3802098.40|-3918244512545.996094|false|wind surfing|calvin nixon|2022-03-17 12:57:17|2020-11-30 18:42:55.661360529|2038-10-05
+-57|-20329|-1857500489|1198|13491.58|-2693679.28|755404222558.269531|true|mathematics|oscar ellison||2033-01-29 05:36:21.574403565|2092-09-14
+|-6735|-2138343289|3401|-22767.90|2032418.93|2618674044147.155273|false|xylophone band|xavier davidson|2019-01-23 07:48:23|2018-12-23 15:42:16.611413141|2046-12-15
+-22|10761|-1226425562|7718825401976684544|8162.40|1047714.74|-2476610981011.973145|false|opthamology|mike robinson|2057-11-26 17:53:27|2069-08-23 05:01:36.124435431|2092-08-24
+-40|29502|-809805200|-8518258741831680000||1144686.92|-4590760977006.702148|true|mathematics||2036-10-09 18:06:29|2064-10-14 13:41:49.171097321|2081-04-05
+50|24081|-728541537||48961.12|-4012888.15|1070384818656.826172|true|topology|victor xylophone|2016-05-04 04:35:48|2038-11-09 14:41:33.447548472|2051-02-08
+102|-6073|1464703053|2004|-38318.30|3968580.02||false|yard duty|david white|2046-02-20 04:57:04|2060-04-05 20:06:25.19768150|2102-06-28
+-41|-11863|-116484575|7801697837312884736|38211.68|-480570.59|-1334633602495.500977|true|nap time|ulysses quirinius|2036-02-23 15:54:13|2068-06-07 20:02:06.356997102|2088-09-29
+-59|-28646|1797164732|3949|18310.99|4375064.78|2366334406338.168945|true|american history|holly thompson|2027-01-27 17:59:41|2045-11-11 14:23:14.608629099|1987-09-11
+123|16767|1198701102|7762823913046556672||1643132.07|2312892585063.259766|true|linguistics|sarah underhill|2017-01-27 09:35:20|2035-06-28 17:21:09.561662434|2086-02-05
+127|-14936|1772349172|-7884460946615984128|-20111.34|-2887093.65|-2318194071335.035645|false|history|victor van buren|2018-11-16 12:15:48|2065-12-31 08:22:47.150581738|2100-05-20
+18|2612|1373871781|-7255010240787030016|49301.50|3253928.69|-4736789216462.362305|true|biology|ethan ellison|2049-01-22 13:36:56|2015-03-06 18:11:17.822359715|2103-01-05
+109||1647411522|-7262384251828518912|3972.39|154588.69||false||mike ellison|2015-10-15 23:12:55|2063-04-26 00:01:09.31563180|2026-05-22
+107|-2027|-1856034030|9132009829414584320|36330.70|2284543.68|-3148453530441.202148|true|study skills|gabriella nixon|2031-01-12 14:14:17|2050-02-14 06:21:51.178052619|2016-01-05
+7|-12626||7573530789362262016||-1103225.79|-3059007558357.649414|true|industrial engineering|katie robinson|2032-10-19 18:30:33|2043-10-22 16:20:40.509237844|1984-06-13
+49||-36038293|7157247449513484288|||3344159968330.622070|true|geology|holly allen|2072-05-19 04:19:02|2064-02-17 13:02:26.377162854|2082-10-30
+37|-26666|-890552359|3493|-41018.63|-3192687.31|610319388952.653320|false|kindergarten|sarah young|2066-06-27 21:59:53|2041-09-23 12:15:37.914672228|2101-03-11
+-43|-12811|1265528735|7491898395977523200|-17199.24|2696923.64|3366546613003.044922|true|undecided|ethan robinson|2029-10-16 04:28:37|2069-03-21 18:01:09.359881056|2039-11-24
+126|2683|-1462604138|-7276111129363046400|12788.08|-3553203.52|2029170799165.562500|true|linguistics||2061-01-10 10:34:14|2036-06-23 07:16:14.299460550|2072-11-12
+-109|23031|1517488324|275|30188.31|-3437436.90|72393656889.993164|false|forestry|luke quirinius|2073-09-17 10:02:35|2058-07-18 11:06:36.866728379|2072-05-24
+-119|-21412|962712814|2715|-42015.52|-3416767.70|2730111077795.705078|true|nap time|ulysses brown|2031-03-11 11:56:59|2035-09-11 18:57:20.545053849|2009-02-02
+-109|-32022|793047956|8150115791664340992|-13856.97|3208380.07|3846344885594.810547|false|education|mike laertes|2037-10-15 17:01:28|2070-05-09 00:38:50.111446261|2025-09-01
+-1|-20591|972835688|7844258063629852672|-30899.74|2941710.70|2520339817545.540039|true|joggying|victor quirinius|2053-05-10 11:05:39|2025-06-02 14:48:41.118807009|
+-86|-28097|291866793|1095|-13029.79|2395802.44|4638765048183.728516|false|debate|nick hernandez|2064-08-23 06:30:52||2103-11-18
+|16439|1972940844|8759089349412847616||2775872.41|2078988628132.627930|true|philosophy|nick quirinius|2020-10-01 01:39:49|2046-10-08 04:45:36.155827749|2046-06-24
+8|-7230|-2065080832|950|3517.45|-355907.54|-3463587144624.789062|true|biology|priscilla falkner|2077-07-28 16:36:07|2034-01-22 01:11:00.675336432|2051-01-03
+50|-11769|-558456218|6960137166475911168|1455.89|-3580235.81|-2225358531562.626953|true|undecided|yuri xylophone|2028-03-16 13:31:23|2036-10-04 05:21:52.917455779|1983-01-13
+-92|1558||-7612455481940246528|20027.51|4940501.37|-2309132393323.310059|true|american history|alice king|2023-02-04 21:40:55|2072-10-28 16:02:05.785926481|1980-05-03
+-110|18346|-1603374745|7471208109437304832|2980.46|-4262671.69|-3375577964698.521484|true|philosophy|alice zipper|2061-06-20 11:36:22|2080-05-22 04:06:26.257944476|2080-04-10
+-28|-21648|1081920048|1501|-39377.99|3185244.69|1714008069386.992188|false|values clariffication|tom thompson|2063-04-15 02:39:09|2075-09-24 04:37:07.641525192|2084-05-31
+26|29245|-817093900|-7145585429014888448|29563.03|2338619.47|-47947487681.199219||geology|priscilla polk|2015-11-23 03:51:26|2048-05-27 06:59:27.15089547|2053-05-23
+90|-3222|1301426600|7794244032613703680|43143.95|4941656.20|3845904537462.523438|true|chemistry|irene hernandez|2054-10-20 19:50:57|2075-08-30 11:18:34.922622163|2047-02-03
+-35|27983|975932228|-7510418793070075904|-31952.67|-547797.70|4033293860789.123047|true|geology|victor carson|2042-12-09 03:28:14|2065-01-16 03:24:08.971263541|2010-06-30
+-14|-29239|230954385|7983789401706094592|28755.14|2806157.04|-3764755036302.638184|true||wendy robinson|2022-06-26 13:18:28|2033-05-18 03:11:50.871005511|2007-03-28
+-86|-15393|-1392487784|3990|25365.72|-3122775.81|-3136333430552.511719|true|study skills|irene carson|2075-08-29 11:01:05|2058-01-25 11:54:54.863071401|1985-03-10
+58|-31551|1443426396|3830|-44944.38||-1901078252690.346680|false|mathematics||2069-02-25 02:44:41|2026-03-16 18:07:16.176024280|2087-02-16
+-79|-12463|-37773326|-7152177800841502720|15979.17|136523.07|4328797369585.437500|false|zync studies|fred hernandez|2074-01-21 22:42:03|2063-04-17 17:43:27.97971264|1986-09-21
+-125|-29323|44009986|1856|36983.95|841608.25||false|history|rachel young|2052-06-19 09:14:41|2025-12-14 16:53:02.735673012|2076-10-09
+-62|-28146|-901778330|2712|-29272.48|495232.86|-4971306749304.673828|false|values clariffication|david young|2023-08-22 20:04:18|2018-02-08 01:53:45.121138509|2067-01-28
+26|-24538|1767359228|-8746702976270385152|-10262.64|3827137.43|738589831875.913086|false|wind surfing|zach brown|2013-07-02 04:00:34|2039-08-29 12:57:02.351355581|2047-07-31
+21|7128|-217930632|-7903158849011843072|777.32|-3589619.96|424579951542.725586|true|philosophy|rachel robinson|2046-01-04 01:02:07|2066-12-14 05:45:23.618651463|2006-08-23
+-87|19739|-1699049982|2244|4616.60|2733769.47|2986145815124.956055|true|linguistics|yuri carson|2043-05-07 16:06:28|2076-01-07 23:49:22.432671324|2089-08-31
+66|29834|1307148254|2735|42117.25|1129420.19|4620474372066.283203|false|kindergarten|jessica xylophone|2051-08-08 09:05:51|2048-06-06 23:50:37.973877723|2060-05-08
+77|21511||1168|47905.76|-2302110.36|-2332600454045.881348|false|debate|mike van buren||2046-10-07 14:52:17.37357221|2001-12-10
+-31|-9482|1330219997|-9080956291212132352|26375.90|-1329189.97|1655214180391.766602|false|chemistry|holly xylophone|2051-07-18 20:15:33|2058-08-16 06:28:29.740534617|1982-12-07
+-43|27454|219415594|945|-12127.36|4216836.73|3646917581760.433594|false|education|calvin steinbeck|2076-07-10 08:53:35|2018-10-06 03:52:24.596896729|1983-09-04
+55|-24267|1458051497|8417381121663746048|49672.46|2222816.47|-3645327331789.474121|true|yard duty|wendy garcia|2036-06-01 21:18:52|2018-11-11 07:00:13.274801916|1996-09-30
+-13|-32541|1563120121|3823|28411.76|1170745.89|2011473770962.695312|false|undecided|nick polk|2023-03-23 06:07:51|2076-10-16 02:23:25.407392909|1985-12-24
+103|18395|-2144138362|-7868306678534193152|21455.88|-4870777.55|-2770023875366.263184|false|american history|victor laertes|2047-01-10 23:37:37|2032-02-23 13:39:32.775684381|2052-11-07
+-23|17964|-1100641049|-8518060755719585792|42499.60||-3519839102751.412109|false|chemistry|luke miller|2015-03-11 11:14:33|2068-07-24 11:17:54.14270195|2080-02-08
+20|32096|1796950944|3703|6669.34|141075.74|-2930607649235.206055|true|wind surfing|ulysses garcia|2013-02-18 21:06:48|2015-01-14 11:10:53.143208292|2030-08-16
+-60|-4804|-1988508336|7027529814236192768|36634.88|292742.80|-4785713903272.927734|false|kindergarten|bob young|2028-02-08 13:16:09|2058-06-30 12:21:12.174769417|2103-08-30
+-31|-15633|1109664665|-9105701280936501248|-7200.23|-3290965.94|-2858696088737.069336|false|wind surfing|nick king|2071-07-06 14:12:56|2072-05-07 22:23:28.909242175|
+50|28118|-432218419|7570474972934488064|-23669.50|4714268.20|3257421256704.163086|false|undecided|nick white|2016-04-08 07:34:48|2041-04-06 04:08:01.271192129|2075-07-29
+-64|23229||-8430370933326536704|34019.68|1961397.66|2639849725646.250977|true|religion|luke carson|2079-03-12 02:12:51|2077-01-08 09:54:43.763841247|2004-07-25
+-49||152891873|417|||4308469366197.300781|false|forestry|irene underhill|2079-06-08 05:52:56|2029-11-07 18:19:49.927077461|2033-09-20
+-47|-20729|1673218677|49|37640.59||-713961786513.870117|false|debate|xavier falkner|2020-04-17 16:54:30|2028-06-03 04:57:19.609929160|2082-08-25
+-16||-1733560816||24701.68|-4273424.19|-2584537633556.181152|true|biology|david davidson|2053-08-30 09:50:24|2051-05-28 08:21:28.136207105|1991-03-05
+55|-24186|-1871209811|7068517339681259520|46423.34|2538160.45|-3739128624481.580078|false|yard duty|fred polk|2038-03-02 19:52:08|2066-12-04 13:20:32.335880942|1994-06-21
+-8|26625|-906986958|2187||2742196.26|-4507463460711.384766|true|history|wendy johnson|2064-06-20 11:56:49|2025-07-17 23:27:04.166297070|2052-06-20
+30|-12880|-40407627|2348|-6268.82|2266879.88|3002655520364.605469|true|topology|quinn underhill|2063-07-19 09:45:06|2045-02-16 17:39:10.146220452|2075-03-15
+-85|-15762|-1556127172|-7139677575412686848||-4483436.56|-1265174168415.015137|||fred nixon|2061-03-21 05:44:18|2014-10-08 04:02:23.556239597|2071-03-05
+-6|15862|1012696613|888|-22200.62|3639449.27|2198173643342.400391|false|biology|david hernandez|2051-03-03 20:56:40|2058-12-04 08:26:03.321045989|2027-01-07
+92|-23550|1238986437|921|49411.28|1421037.00|-4845225760189.179688|true|history|ulysses underhill|2036-05-02 17:20:37|2018-07-15 18:49:56.958391676|2092-11-15
+4|18984|-267130580|-8034414142083170304|34672.91|-4128287.64|766368935100.487305|true|education|fred steinbeck|2079-07-18 08:49:16|2045-03-01 15:48:34.880096673|2053-04-30
+54|29015|-1065248998|-7330203470474985472||-3027947.92|3399052323913.345703|true|biology|sarah ellison|2026-03-03 11:12:47|2057-08-18 20:13:54.240194383|2059-10-09
+-119|31140||789||-940868.71|-4363934228329.320312|true|quiet hour|zach falkner|2049-03-25 02:38:35|2041-11-22 05:46:48.4658675|2006-04-06
+-127|-31998|-71305062|-7158472098920390656|20591.34|3729248.23|-526374280413.826172|false|mathematics|ethan young|2066-06-15 04:16:02|2030-06-20 07:00:32.309007556|2006-02-26
+51|-1034|-103219371|-7881262505761710080|29627.76|712784.66||true|nap time|xavier zipper|2066-05-06 03:31:19|2030-08-29 00:39:40.299687444|2071-11-21
+-113|26241|-971203543|8451612303224520704|-29948.96|3327680.13|-18523066115.392578|false|nap time|quinn miller|2065-09-30 09:42:56|2037-03-26 02:34:03.815937235|2026-03-17
+104||-996953616|3541|18791.19|-462147.25|2541474056133.629883|false|philosophy|victor zipper|2060-05-20 14:26:32|2068-07-18 13:21:40.858914884|2085-03-29
+37|2784|644934949|8143462899383345152|19598.55|-4115146.61|-1524392874934.043945|false||irene hernandez|2047-08-20 09:34:21|2025-03-27 03:58:40.290018987|2096-08-25
+|-19276|605141554|8383159090746204160|45522.67|-4427581.33|-3551912779492.703125|true|linguistics|luke xylophone|2030-07-10 13:00:24|2048-12-11 23:41:59.785955383|2035-02-18
+-32|-24178|1148500740|-8923529803981905920|-36565.06|3529368.28|2380664601539.273438|true|topology|luke thompson|2073-01-23 21:54:44|2017-08-29 15:18:43.706057935|2022-08-28
+|-3813|1992977592|7885697257930588160|40610.97|4059042.75|-614948955390.488770|true|study skills|quinn garcia|2071-12-14 10:05:04|2022-09-08 09:37:06.875565982|1975-07-07
+30|10615|-847235873|9107991000536498176|-9874.16|-4892978.32|-830926430811.871582|true|values clariffication|oscar young|2031-07-31 13:47:34||2060-07-26
+13|-15389|865013617|1409|-45007.46|1282299.74|-3272482088828.603516|true|joggying|fred miller|2070-08-11 19:19:39|2075-09-01 02:16:50.347835856|2078-10-27
+-96|11809|-1644966759||-14002.93|-4469818.97|1917126159337.136719|false|biology|calvin quirinius|2045-03-07 23:39:15|2032-04-12 02:27:12.657670943|2009-05-05
+20|6372|1478365409|-8675892979328212992|38991.14|-2146326.91|-1669697999836.649902|false|joggying|calvin white|2027-05-03 14:40:13|2065-12-20 12:05:02.853602253|2009-06-25
+81|7299|-1126628450|8920344895701393408|-21909.86|-1392050.93|-3312751733215.968750|false|opthamology|sarah hernandez|2015-04-12 04:15:23|2080-04-01 07:17:45.74683010|2103-02-26
+-61|26540|-968377273|-6974654664348033024|-26186.32|3944430.54|1407237264653.069336|true|zync studies||2049-02-15 00:08:46|2021-07-04 09:11:18.595023146|2043-04-25
+|-13973|1912175355|-8879742387365429248|-24520.99|1718891.92|990917459853.477539|true|yard duty|alice nixon|2022-01-20 13:23:39|2035-08-16 20:58:12.262393960|2095-12-31
+-80|-15299|1318606691|-8938849835283677184|-8804.48|-4963660.13|-4051044335853.006836|false||quinn young|2069-08-28 15:39:57|2031-10-23 14:35:30.465215236|2027-10-23
+-33|-32462|1222935237|-7213775605408178176|9353.59|-2865973.47|-29816572801.699219|false|study skills|gabriella garcia|2073-11-24 15:12:21|2020-10-06 02:54:29.617493343|2018-02-28
+-125|22704|-1096771844|9199741683232399360|28595.90|3955962.84|4910989073004.792969|false|philosophy|ethan van buren|2062-12-30 06:36:02|2054-01-25 08:11:43.748458487|2081-06-29
+-22|-18659|1416850873|8489584373231919104|34567.10|-2690728.31|-4902808539333.527344|false|linguistics|bob polk|2071-07-20 03:43:45|2059-05-11 00:01:57.809402055|
+-77|-19912|-340951385|-7344947507044466688|12056.25|-2933320.14|4292999497091.330078|true|philosophy|xavier robinson|2042-03-04 00:33:10|2081-01-29 00:05:29.43863329|2014-02-25
+-101|-13743|-158420748|-8756989568739835904|30210.95|1930694.98|-4980390206441.782227||mathematics|ethan davidson|2075-07-09 02:02:31||2004-11-05
+97|-2152|-1616030844||6923.19|1009414.39|-4803916908307.214844|false|opthamology|irene falkner|2019-05-12 23:35:04|2034-10-21 07:24:20.465428653|2083-09-11
+96|22603|925032386|-8046189486447017984|-8322.37|-3646620.83|3514393344286.617188|false|study skills|david polk|2061-10-25 22:52:55|2047-10-20 20:43:26.170125012|2094-08-30
+70|4491|-1362178985|1287|-45257.20|734363.29|-1819407181812.877441|true|values clariffication|quinn laertes|2015-06-04 07:45:25|2078-03-22 02:23:35.763695497|2012-10-25
+-98|-18140|-835198551|982|39755.57|1372628.14|-1714648472726.310059|false|kindergarten|rachel van buren|2072-04-30 23:06:10|2076-04-03 10:07:19.979852554|2058-01-21
+50|23956|2097519027|615|-24059.46|631764.92|-2495415685760.988770|true|forestry|sarah laertes|2028-12-22 13:28:33|2017-11-07 21:48:52.848530518|1993-12-02
+79|20872|1495575878|-7542857121910046720|-11668.81|-4226007.73|-2260276582420.459473|true|american history|quinn king||2044-03-22 10:07:42.593619051|1970-06-19
+65|-24591|-496870819|1648|-200.89|-3053285.53|-3512079952779.068359|true|mathematics|tom quirinius|2050-08-20 15:12:10|2020-12-29 05:36:09.651652502|2053-03-08
+-68|-29685|-1127100849|-8203008052020879360|37283.21|-1596933.52|4832262437578.585938||geology|oscar white|2048-02-17 07:53:44|2033-11-12 06:12:45.238117579|1973-05-13
+|-3206|-1741895392|-8688153842294595584|-2854.09|-1738342.95|-4557913856473.696289|true||rachel underhill|2077-05-20 02:03:40|2033-08-05 01:49:40.305347119|2027-10-12
+-98|16734|-596963345|-7707867749256445952|-47352.27|-4521296.31|3236784579167.704102|true|education|yuri ovid||2077-01-28 08:26:13.323779922|2093-03-26
+-66|-24368|1832650234|-7894382303337832448|2348.21|46896.52|-888063795956.135254|true|mathematics|alice garcia|2061-05-29 01:02:47|2018-08-11 08:18:59.362818129|2057-03-18
+-5|9716|-499533481|1667|44403.36||2674989060827.001953|false|history|oscar underhill|2055-10-25 00:02:32|2069-02-01 10:23:16.84520652|2053-06-24
+48|-9183|-607285491|8780196485890555904|75.56|-3870004.21|662312551004.902344|true|yard duty|sarah thompson|2076-06-19 08:17:36|2013-11-07 19:36:51.34765657|2045-10-15
+55|-15109|56316391|-7404057145074712576|2893.22|-4203203.26|-3201041913797.339844|true|kindergarten|sarah polk|2044-09-05 22:10:18||2093-11-05
+-69|-20946||-7433265617153343488|-47857.01|1123959.85|-4751032804232.125000|true|religion|gabriella allen|2046-02-04 08:22:40|2021-11-15 18:22:18.608780184|2069-04-29
+7|-1180|-423378447|1127|-30027.48|-1414107.87|4302554900117.384766|true|zync studies|oscar white|2020-12-23 10:18:38|2050-09-21 09:03:11.214720790|2099-04-24
+11|14376|234452496|-9149719074367946752|41986.01|4484955.64|-1431661810573.672363|false|wind surfing|victor robinson|2077-06-15 17:08:48|2020-07-26 05:43:36.11096510|2083-06-06
+-31|-408|-1933374662|763|5026.31|2220778.49|-1880721142952.310547|true|values clariffication|priscilla ovid|2079-11-27 10:31:13|2078-12-02 09:20:02.449941920|2020-09-15
+-124||206942178|3249|44696.56|-2844962.02||true|industrial engineering|zach miller||2014-01-24 20:41:26.351041828|1972-03-05
+74|-17840|307333276|-7708932208121225216|33649.41|-2209443.33|-3174835614765.178711|false|opthamology|wendy polk|2049-02-15 09:49:07|2021-09-17 10:39:35.795144931|1975-02-26
+35|6015|-1343327|2689|-34762.90|-3352811.58||false|nap time|oscar king|2061-03-02 03:38:49|2054-04-17 12:47:53.423961154|2027-06-04
+14|-20180|720703232|-7881351200983613440|-14311.83|1718525.86|4502617863309.339844|false|xylophone band|oscar garcia|2035-03-09 12:34:23|2019-05-05 02:57:22.203595007|2072-11-12
+-17|26296|344989592|-7515996202498473984|-29958.06|2161214.73|-209259678055.531250|false|philosophy||2068-02-10 07:56:40|2042-06-02 13:15:00.744949321|2074-05-17
+-111|13488|929751599|1681|15142.21|2124532.47|-4344059773498.542969|true|zync studies||2061-05-05 09:51:27|2035-01-03 11:31:13.490764419|2067-09-16
+-119|28064|631711489|-8507279516485566464|46368.28|1381513.63|-3198785922561.029297|false|philosophy|victor johnson|2053-09-16 22:56:54|2064-09-13 02:14:41.203564754|2052-09-23
+-3|-5897|997193329|1316|-36632.56||-4035360861641.272949|false|xylophone band|david johnson|2033-08-30 22:43:18|2021-08-20 07:30:06.17481138|2093-04-19
+22|-22922|626941809|7892026679115554816|-24481.02|4819862.86|3630292090950.355469|true|philosophy|victor ellison|2072-04-15 13:45:41|2069-12-27 19:14:43.4196486|1970-10-03
+|-22534|-1412187081|2279|35125.32|-1204413.83|13572776973.802734|true|history|gabriella robinson|2035-09-28 15:15:33|2079-06-21 21:50:11.269734623|2021-08-31
+8||842283345|873|44738.40|4982540.22|2545702719622.049805|false|chemistry|gabriella underhill|2044-03-23 02:31:44|2063-03-07 17:44:02.4074809|2059-06-02
+96|-5699|-163859725|8142241016679735296|-21581.78|-1551671.29|-2922841514857.416992|true|wind surfing|zach laertes|2014-12-09 06:23:35|2026-11-05 11:43:02.534377887|2023-06-06
+-2|-26128|1513689502|-7811060170911375360|5804.81|4404088.96|-3923411784468.591309|true||bob allen|2075-07-07 20:40:56||2096-05-07
+13|28551|-1568646283|8411494452500930560|-34399.96|2787253.76|-1592892075417.431152|true|topology|rachel brown||2025-05-06 23:15:59.141775309|1993-08-22
+52|40|587206979|2803|-38274.78|-3013975.64|4769880713478.898438|false|joggying|alice zipper||2062-08-30 07:04:29.405219012|2075-12-10
+-77|-13238|-866304147|1247|30977.58|-1455951.83|-379569461462.274414|false|forestry|holly young||2060-02-22 09:39:53.875517714|2054-12-02
+-16|13154|1141303816|-7673901622181953536|32011.03|-2072819.72|2331383540354.394531|true||quinn xylophone|2013-06-15 10:55:00|2066-12-15 17:24:21.774901994|2038-05-15
+-94|-21451|-203416622|268|33257.23|3858254.19|4841902350857.658203|true|geology|nick miller|2030-02-09 07:34:05|2061-02-09 17:09:30.792720771|2026-01-05
+32|13150|2142592987||21739.73|443657.37|1617671196062.850586|false|values clariffication||2055-04-27 17:42:09|2035-04-30 09:03:34.834154256|2048-07-11
+85|-1767|-1977762695|-7195217207163166720||2897773.78|4178958976210.437500|false|values clariffication|oscar quirinius|2071-10-09 18:35:28|2039-08-28 00:08:15.223157462|1976-06-27
+-1|-19464|538766635|-9012093603044245504|16783.69|102999.04|-3761561711229.433105|false|yard duty|irene laertes|2039-01-07 08:49:02|2077-12-04 00:37:45.588821251|2044-03-15
+84|7569|1739911574|8920533610804609024|39584.58|-2087360.22|2573136281954.120117|false|quiet hour|ulysses ovid|2058-06-30 06:37:18|2028-03-12 18:17:11.549298490|2073-09-23
+93|-254|1159353899|455|-26164.13|-2925253.46|-3708898652238.446289|false|debate|ulysses polk|2071-12-06 03:25:24|2027-01-09 04:39:00.318567529|2103-06-02
+-77|30619|1370723240|9148071980848742400||2115468.25|-2676483804058.200684||education||2076-06-26 22:59:22|2068-09-02 06:39:15.162172013|1981-02-07
+-76|24512|1316931||43602.63|-4235241.77|-2225959074468.946289|true|study skills|priscilla brown|2069-04-16 14:07:02|2059-05-03 07:04:57.531362893|2040-08-20
+|-17607|-628790799|-7093825013581979648|41798.75|789629.23|4045302222731.130859|true|topology|holly nixon|2034-09-29 04:12:38|2028-09-01 15:46:24.993528329|2040-12-14
+-48|18142|1203482872|1342|-48524.80|1488860.19|1080708792025.255859|false|geology|david underhill|2038-12-12 20:41:41|2054-06-04 10:17:24.350972371|2048-11-22
+-35|-28366|1182646662|7295926343524163584|37147.82|-17339.38|850711115335.975586|true|religion|fred ellison|2056-12-16 18:57:16|2030-08-08 13:20:23.87227693|2061-10-03
+|-11187|2127682701|7874764415950176256|21152.51|-1228307.04|-3933963121027.943359|true|quiet hour|sarah garcia|2061-05-01 14:39:07|2039-11-30 14:02:00.787382842|2063-08-30
+96|-25282|2009215103|8682955459667951616|-30776.76|-951920.19|339774798412.620117||philosophy|bob thompson|2018-01-25 07:52:23||2072-09-01
+75|12802|872554087|8825059717746376704|-23535.33|-3969818.94|-2914303137233.673828|false|wind surfing|jessica xylophone|2024-10-12 09:03:29|2080-11-22 08:10:16.820114627|2023-11-21
+-91|-31335|-1575588203|1651|530.17|-2153555.94|-2020396257019.356934|true|history|calvin zipper|2036-06-05 07:17:08|2077-07-11 08:39:35.486363899|2097-02-12
+123|-25183|69110370|1187|18383.16|-1514304.51|2213587444204.989258|true|undecided|fred ichabod|2027-01-18 08:53:58|2053-06-12 11:25:39.320312625|1998-09-27
+-66|21168|976870621|3197|-21478.88|1822183.11|-2601260542275.482910|false||calvin polk|2030-09-09 10:31:01|2071-07-11 10:12:44.480141278|2040-12-04
+108||-144862954|1115|37537.67|-3636489.74|-2775808925857.248047|true|values clariffication|sarah ovid||2045-02-17 17:37:23.51474731|2048-11-26
+-116|22118|1117805438|8945302550165004288|37582.57||501497065597.860352|true||calvin zipper|2066-05-15 01:52:49|2046-09-19 21:50:44.293786457|1977-03-05
+107|-23153|1571267481|9114850402293882880|-30178.38|1989319.92|826991484976.567383|true|nap time|gabriella underhill|2046-10-28 01:02:40|2045-01-12 05:45:37.921528799|2014-07-18
+97|12327|1603612975|8935252708196999168|16046.26|-67740.16|2793889545913.791992|true|opthamology|gabriella steinbeck|2043-09-23 14:40:09|2078-01-01 15:35:48.152129849|2031-05-27
+39|-16820|43983130|2663|-27389.47|775624.70|-1775814976651.695312|true|chemistry|luke allen|2021-03-30 12:39:17|2043-02-02 21:34:00.252214409|2080-09-28
+-73|-15866|-99205196|-8535957064499879936|-48198.87|-2414154.16|349441023585.761719|false|study skills|priscilla hernandez|2055-03-14 23:42:22|2027-01-12 03:52:14.47123348|2080-01-02
+70|-25463|748185058|9091082386452684800|48356.67|-3526156.73|1250977516314.754883|true|industrial engineering|alice quirinius|2015-09-13 22:17:45|2072-08-09 11:39:30.710221441|2052-04-27
+45|20124|-470798506|2915|43588.39|-762457.78||true|forestry|wendy carson|2072-06-06 06:46:18|2014-12-01 04:17:26.513106497|2055-12-01
+28|-9196|-1947868215|2183|49404.36|-3414112.11|2109147570161.909180|true|geology||2027-01-27 11:49:05|2061-05-01 06:27:32.791222399|1990-10-25
+14|25381|659397992|-8147405381260345344|47303.15|-3806020.32|997601888271.546875|false|values clariffication|fred brown|2074-06-25 18:50:23|2057-10-25 22:57:33.478783182|2105-09-25
+82|-1367|895763504|-8654797319350927360|1232.14|851336.52|-3820905742244.491211|true|religion|bob steinbeck|2032-12-27 16:52:15|2041-10-19 14:13:28.458891032|2104-03-31
+18|-13466|-343173797|1261|-24504.59|-2671371.43|4817498142971.957031||linguistics|fred ellison|2053-12-16 04:23:07|2021-12-04 13:24:23.478621300|
+113|-29988|-213198503|8775009214012456960|29118.01|-2672928.16|3597735313714.380859|false||holly allen|2038-04-29 17:39:45|2035-02-10 07:52:53.120396795|2007-11-09
+126||1844415080|2810|-43710.73|4357967.19|-3835582850357.113281|true|history|holly xylophone|2056-04-30 00:14:07|2037-07-23 02:43:53.584900523|2038-06-02
+126||43672187|9067985867711291392|8608.12|-3919341.08|-1433926422321.541016|false|linguistics|alice johnson|2040-06-19 19:13:38|2032-01-20 11:14:11.710454857|
+18|-31663|-1721368386|-8240684139569233920|-41884.25|-4050155.29|-3799361430049.685547||linguistics|wendy brown|2067-02-20 18:47:51|2075-11-01 10:27:41.797075523|1973-08-26
+-115|-20828|-1477897348|1719|19183.72|3667748.43|196163723446.253906|false|philosophy|nick ovid|2038-03-16 23:59:43|2031-04-04 05:56:43.142902096|2094-02-12
+-61|20411|2076370203|7917494645725765632|34656.95|-1033805.16|-2293878051389.661621|false|education|bob thompson|2041-05-09 00:13:16|2064-12-30 17:41:02.427765133|2096-07-09
+100|410|2057486961|3622|49893.07|4038206.70|-434822164725.155273||linguistics|alice laertes|2059-12-30 17:21:06|2067-10-16 22:22:23.909102811|2036-12-30
+68|26697|-1079633326|-8866442231663067136|-43306.72|-616547.34|-892751806585.678711||opthamology|katie robinson|2066-09-01 03:27:00|2059-11-14 17:00:11.431792872|2052-04-27
+6|28146|-309571354|7871554728617025536|10210.31|2684254.99|717298056153.232422|false|nap time|gabriella steinbeck||2079-10-05 02:45:16.165020579|1988-02-01
+-94|-22554|-1114208576|-7380731416973295616||4566778.78|4025279026650.033203|true|xylophone band|jessica hernandez|2060-08-11 01:49:22|2019-01-16 09:25:47.428200312|2038-11-22
+9|11059|1114521964|-8603817012434198528|38309.84|-2121451.24|-1723505323178.535645|true|values clariffication|victor garcia|2076-02-21 16:02:29|2024-12-31 21:24:49.835872918|
+-34|10425|-45439614|-7192529627893858304|21468.82|666967.79|-1710357667596.511719|true|history|david ichabod|2051-07-14 18:03:02|2017-02-27 04:57:13.944522320|2050-01-25
+36|31099|-1108723753|-7287583262310350848|-61.04|-2662929.33|3709449843031.081055|false|industrial engineering|luke quirinius|2036-01-13 20:52:17|2062-10-11 17:14:59.629263445|2045-02-24
+113|-22915|-1352545619|7149417430082027520|-6486.11|-3357464.95|-1319020386301.517090|false|chemistry|yuri carson||2051-08-31 14:41:56.890230740|1992-11-06
+-45|15011|-1061859761|7237310132329488384|-39128.71|319025.53|-1778680361202.553711|true|mathematics|victor nixon|||1998-02-21
+-18|31370|1204834275|-8659643752269242368|39007.50|1149036.85|-2714226684056.946289|true|american history|katie thompson|2027-06-17 03:56:45|2070-08-03 17:42:35.323901256|2099-12-02
+|20969|-469749219|-8051395538179063808|32233.91|2215432.29|-3014965054465.008789|false|wind surfing|calvin underhill|2014-12-17 19:52:13|2022-11-29 17:49:15.855148488|2066-10-20
+-68|-4142|1488440165|1674|-20812.84|253603.39|-2105480401864.810059|false|quiet hour|irene steinbeck|2067-02-13 02:16:25|2022-12-16 07:14:39.23664527|2046-07-15
+-68|27630|1575300276|-8813211231120031744|-14758.39|-1926059.11|-2833925266574.155273||chemistry|ulysses allen||2032-05-08 18:36:19.400564422|1988-01-19
+-106|25835|-533227056|-8103788088118018048|-549.34|4959162.61|4308090638361.007812|true|kindergarten||2073-10-28 10:37:18|2079-07-06 02:03:22.207944103|2028-04-09
+-57|-4726|814544198|7049773031131283456|10513.01|2178130.57|-2416479446500.758789|true|biology|wendy johnson|2020-03-30 21:56:57|2029-12-16 16:34:42.572182352|2097-11-09
+-80|21784||-7035132060308643840|-6419.44|-4383105.22|-2020072615248.374023|true|philosophy|xavier hernandez|2018-09-25 15:18:36|2031-06-28 02:55:11.31963432|2068-08-05
+-78|13795|-1769423338|-7303847963918393344|43895.68|-3084959.79|-4175764579458.167969|true|opthamology|gabriella johnson|2043-10-13 05:10:09|2048-05-19 05:40:47.176953399|
+-69|-16027|819069589|-8046238369820344320|-22162.98|-345358.25|674713800507.689453|false|linguistics|zach thompson|2015-10-24 08:01:09|2020-06-08 12:08:21.511690843|2019-10-03
+47||2068018858|-8503573595507761152|-10699.81|-2421607.62|2965768864894.222656|false|undecided|rachel ichabod|2069-06-04 05:16:15|2061-09-03 01:12:25.916465543|2030-10-14
+85|26579|-1371840597|8822384228057604096|-41991.30|-138391.14|1063729386321.732422|false|debate|jessica xylophone|2054-11-01 01:40:23|2069-12-06 00:11:49.576880733|2090-07-08
+110|-9759|-136514115|-7998947380180819968|-17463.46|1862978.01|-3370842772950.304688|true|study skills|irene ovid|2059-11-03 20:21:53|2061-07-03 17:32:49.424965303|2015-10-17
+-26|-10316|-71449585|-7057750467944931328|7153.05|-3062092.43|2726424274449.748047|true|american history|fred brown|2051-03-07 14:50:11|2027-05-29 12:10:20.647953531|2096-04-06
+113|-22215|674547678|1862|-27979.49|2034087.73|1306104776811.629883|false|industrial engineering|tom johnson||2021-03-25 16:24:12.665579887|2101-08-21
+56|8551|-733239404|8583916402383601664|47565.67|-4620642.81|-2470480735156.428223|false|undecided||2027-03-26 08:37:58|2079-04-18 01:15:58.35100799|2056-01-22
+98|-26361|1489169773|2398|42357.85|-3934751.72|-4429578287144.977539|false|mathematics||2048-03-15 04:39:44|2033-11-06 05:22:43.819682351|1986-02-17
+-32|-19330|2068538934|7487538600082554880|7664.98|4132039.93|-2902910424869.702148|false|geology|quinn polk|2041-11-01 21:08:54|2035-01-03 23:06:46.765787876|2071-02-14
+123|-20409|115470151|8677794924343164928|12145.54|-4842892.80|400760479300.927734|true|xylophone band|mike ellison|2080-02-28 20:34:51|2014-02-14 03:41:40.325566176|1989-11-16
+-13|31135|6526476|8708232769657815040|-14997.43|4694403.83|333471522081.652344|true|xylophone band|sarah brown|2021-01-12 23:04:21|2067-12-27 14:26:36.287488601|2062-07-17
+51|-27259|1829544791|-7623359796281999360|-34844.45|-3126581.66|-2421986983728.814453|true|education|tom miller|2044-10-28 16:20:50|2018-07-05 05:25:15.270012699|2071-07-26
+-95|-17690||8983912573761167360|31719.06|-3442868.07|-1937958357952.531738|false|kindergarten|irene ovid|2058-03-03 15:38:13|2069-02-04 13:07:32.271143352|1983-06-19
+10|-30669|1590744669|-7263060340185194496|-30819.18|1534635.31|4148791909569.630859|false|kindergarten|quinn white|2027-07-01 19:44:37|2056-02-18 15:10:02.841254943|2091-01-11
+93||-971698865|-7873753603299540992|21852.31|-2581251.56|1539283723979.209961|true|quiet hour|bob carson|2026-11-09 05:31:12||
+-28|-18659||-7720966287634112512|20392.44|-2222926.64|1939030156067.396484|false|american history|irene allen|2032-07-18 12:44:48|2026-12-26 16:48:27.776120795|1987-11-12
+-45||-40284975|1806|35653.78|1321025.73|898563412192.885742|true|mathematics||2070-01-12 03:23:36|2034-01-24 11:57:42.385238227|2057-09-03
+82|-13539|1456367662|-8260340354454503424||1093063.49|4557861543446.826172||religion|quinn xylophone|2078-09-07 14:49:27|2025-08-16 09:10:36.887036096|2011-11-10
+125|5469|889772203|2842|38632.60|-4727107.13|2741920281773.531250|false|joggying||2038-11-21 14:39:38|2040-01-28 08:27:19.830854683|2085-04-27
+21|-20517||7069729473166090240|-14418.84||735618454615.902344|true|opthamology|yuri ichabod|2027-06-30 14:53:30|2016-01-03 18:53:34.457261740|1970-04-12
+-96|-25734|-273937943|8688483860094599168|927.24|931681.37|4368902606237.980469|false|debate|sarah steinbeck|2023-10-17 00:33:13|2043-10-07 00:08:12.855001312|2038-03-10
+-82|-23719|-318206520||-23783.97|-837555.18|1762959989546.857422|false|study skills|nick xylophone|2034-11-18 23:41:57|2052-11-30 19:49:07.414111749|2033-11-10
+58|7697|-1437126017|7517159036469575680|-1037.26|-3799864.48|-1963988180877.791504||wind surfing|yuri garcia|2070-10-26 10:29:55|2046-12-29 13:30:24.655176498|2075-11-28
+26|16312||2218|46235.86|-3813446.01|-4186362730051.698242|true|topology|priscilla brown|2055-05-14 21:50:40|2037-10-04 05:24:46.310622445|1998-02-21
+-95|28923|-1769037737|-8996954350906294272|5131.31|-3335839.19|892560184430.806641|false|values clariffication|victor underhill||2051-04-05 17:19:55.107473640|1979-01-11
+89|20014|945683736|2089||2623267.83|-3717730752750.725586|true|debate|gabriella quirinius|2078-08-08 13:57:39|2062-07-26 14:12:36.59165411|1999-03-25
+-52|-12809|-1533934649|-8651641150831362048|48413.99|345690.54|2773050456079.104492|false|linguistics|calvin thompson|2045-08-26 14:34:17|2017-09-25 09:26:40.856875502|1979-12-19
+-107|13792|-1212524805|-8322751250650218496|26905.38|2307830.54|-3744767732184.754883|true|study skills|tom steinbeck|2027-12-07 07:23:38|2030-09-13 21:30:52.282498698|2061-01-04
+-65|21299|-1222897252|1495|-31920.07|-1723567.09|2996766542753.092773|false|wind surfing|zach white|2025-09-09 20:08:36|2066-06-07 03:56:15.873165726|1992-10-17
+101|30222|1667594394|-7848043121524228096|4374.75|1994312.07|4481698153938.253906|false|wind surfing|jessica zipper|2050-04-22 23:54:13|2047-05-21 07:47:57.777923762|1996-11-24
+94|23766|-511198293|-7506254246954500096|20950.55|-748117.50|4939615715200.673828|true|debate|rachel ichabod|2065-05-20 02:43:05|2026-06-12 20:30:30.941431912|2084-07-12
+59|5445|1974939899|7487338208419823616|44886.11|-4889191.55|4911721874172.769531|true|mathematics|gabriella steinbeck|2025-10-07 03:42:36|2045-05-23 22:17:04.9265626|1972-10-12
+-99|-29601|816439627|-7875953567586451456|41944.06|-1349023.71|-1099429516405.692383|true|opthamology|jessica polk|2039-01-01 04:30:38|2076-02-14 07:29:41.551319075|2097-02-19
+99|-3318|-1017027298|1863|24757.93|4444192.30|-3601627653993.517578|true|joggying|victor young|2031-09-16 06:06:37|2050-03-15 17:15:50.541208509|2079-07-14
+-27|-3540|133276416|3835|-506.62|1920067.41|412688526446.700195|false|debate|jessica young|2039-09-26 14:48:34|2077-07-07 08:15:18.365301835|2084-11-07
+-68|11572|25644069|1870|24153.48|4976723.43|-137386195040.676758|true|zync studies|holly falkner|2063-06-20 10:43:27|2024-10-19 14:18:31.573357244|2011-02-18
+89|16250|491016124|-7978782649203228672|-46524.54|1265452.96||false|opthamology|luke allen|2056-12-07 20:53:08|2029-09-05 03:42:32.368459710|2083-12-25
+-16|-19213|1509573831|3960||3292048.84|-240494562170.781250|false|biology|wendy hernandez|2051-02-06 00:15:05|2041-03-28 19:17:32.296675612|2083-07-04
+-56|-6518|-1478812842|-7663293054873812992|-24105.64|-1332349.52|-1890560158574.032715|false|chemistry|sarah robinson|2069-08-29 19:26:05|2046-07-10 19:51:40.969328493|1973-05-29
+-24|28077|-234278308|898|6115.34|652033.34|-4372957414734.699219|false||rachel robinson|2043-06-18 12:08:10|2066-02-25 21:32:14.879009176|1986-03-28
+51|-24313|-938342473|3728|4223.26|-4736632.77|-2080633933255.846680|true|biology|wendy thompson|2019-03-20 15:36:37|2025-06-22 23:33:35.838318924|2083-02-02
+-21|-4539|1469775272|-8615168537390571520|-13079.77|-3313208.22|3092499128010.040039|true|topology|quinn falkner|2073-05-16 23:56:42|2017-07-20 03:37:48.334651957|2055-08-30
+121|29828|-1426893312|3579|4812.14||586167842633.002930|false|wind surfing||2022-12-05 16:40:51||2069-05-27
+-79|-5314|-234758376|8111757081791733760|35336.11|4023549.09|4664860898254.835938|false|mathematics|ethan van buren|2072-06-06 05:31:29|2068-02-13 01:16:52.645583397|2065-07-21
+10|-27295|-1299159155|3111|-40569.04|289098.86|-3359213193119.252930|false|kindergarten|alice ellison|2013-04-01 23:26:31|2042-01-18 04:14:18.953737200|2037-02-15
+36|6892|1090344463|2539|21579.89||-2076008548637.773438||biology|sarah van buren|2044-08-16 16:50:11|2078-11-12 05:33:45.47089985|2054-12-18
+-78|10807|550594651|7497306924248834048|-15217.81|3701631.53|-1010442667526.625488|false|religion|nick garcia|2039-02-22 19:12:26|2058-01-16 13:10:53.407263384|2056-02-09
+-42|26236|-2019287179|2619|13960.36|3980665.59|2745518512550.937500|false|wind surfing|alice young|2049-06-17 08:19:54|2042-10-24 00:38:21.665346021|2001-08-01
+-90|-13424|631954352|1423|43085.76|1692557.60|2335092836258.532227|true|values clariffication|xavier ellison|2028-04-08 19:20:25|2050-10-22 21:57:15.666787161|2100-05-23
+66|-1679|472901914|7652123583449161728|-45141.82|-189883.01|2005119271038.431641|false|education|zach miller||2046-04-05 20:49:51.831050123|2081-01-30
+-61|-19291|936752497|7271786885641666560|6407.13|-2622282.33|-1364893278530.487793|false|philosophy|ethan nixon|2064-03-25 01:50:11|2078-07-14 10:27:03.640728827|2024-09-18
+53|-950||1447|-46967.91|-2338643.42|-3010460855327.999023|false|linguistics|wendy underhill|2061-05-09 13:52:02|2052-06-04 16:42:35.115432702|2058-01-11
+12|-19479|1112783661|2553|-13650.84|-1699487.46|-4635813668758.892578|||nick king|2019-03-23 10:58:33|2071-10-07 13:18:51.554325420|1978-08-21
+-8|26869||8148211378319933440|45519.19|1788716.14||false|values clariffication|fred xylophone|2060-10-30 01:15:38|2071-02-14 08:59:08.385880302|2083-02-05
+0|29498|1393262450|7450416810848313344|-47360.59|1197766.74|1948267796553.970703|false|quiet hour|ulysses johnson|2064-05-23 05:56:44|2064-08-16 10:07:06.357883907|2049-04-05
+37|30056|131031898|2979|16137.50|1448922.16||false|zync studies|calvin allen|2078-08-01 20:45:50|2077-07-22 17:11:31.657495112|2077-05-23
+117||1751468853|-7120456708338688000|-45241.05||4658251999119.685547|true|forestry|tom ellison|2069-06-01 15:56:17|2030-05-31 03:22:11.194126870|2033-08-06
+-120|-31659|-2016985611|7242751359672631296|43128.70|-191280.75|-2180975729787.064453|false|undecided|calvin ovid|2081-01-06 19:08:32|2056-06-12 01:13:12.668973447|2007-12-19
+126|-23667|2111462911|9083076230151864320|11873.69|4280564.29|2689053897530.797852|true|chemistry|katie zipper|2041-07-26 09:46:11|2052-08-09 23:10:15.912587493|2004-10-04
+92|7066|1785455842|-8754992450211692544||356819.61|-319780935492.240234|true|mathematics|ulysses robinson|2069-07-31 07:42:34|2025-12-14 11:43:12.789221272|1989-03-12
+-23|11788|278601840|1537|-3884.62|1500759.88||false|geology|ulysses king|2061-08-15 00:45:19|2052-07-01 09:00:11.170504787|2080-01-24
+120|-27232||7845953007588401152|30784.40|3725404.09|-4223940428806.862793|true|zync studies|yuri polk|2046-03-19 03:26:49|2028-12-07 07:12:17.409326696|2103-03-11
+0|22388|1145627305|9023663198045544448|-16680.30|-4179831.25|-4258847848573.274902|true|linguistics|jessica ellison|2056-08-10 14:54:29|2067-07-03 10:13:58.634487256|2039-05-11
+-102|256|352214248|8900545829211299840|-8142.34|1762753.22|2953930180391.744141|true|zync studies|calvin allen|2053-03-31 14:32:12|2054-01-17 02:39:01.769490002|1989-08-23
+104|-8706|-400501472|1866||-4687152.61|-4538289046733.225586|true|opthamology|fred van buren|2045-10-06 23:55:19|2063-11-06 08:47:29.289513742|2067-08-04
+-74|-22241|-1216166764|-8965578088652095488|11351.19|13624.69|4203637057188.273438|false|topology|nick king|2056-03-27 03:36:35|1970-01-01 00:00:00.158770873|2039-09-02
+-12|-10623|-445353909|154|24563.70|4730542.74|-2976281476329.525391|true|history|quinn davidson|2071-04-24 15:00:40|2016-03-02 22:10:50.657625756|1976-06-03
+8|-32491|-1232183416|2905|-45693.51|3407626.79|-2909647656864.500000|true||sarah underhill|2038-12-19 06:40:41|2070-01-02 18:09:04.499401693|2038-01-27
+41||2090496825|8693036785094565888|449.14|2113679.49|-1419899825383.019531|true|chemistry|wendy polk|2068-07-21 04:20:28|2042-05-24 13:18:57.760909140|2050-11-29
+43|-17366|41063276|3555|-23520.95|-3021088.64|-2720359307901.567383|false|philosophy|wendy brown|2080-12-20 01:36:25|2065-03-06 19:37:09.328344492|2090-05-03
+62|26288||3910|-10665.79|4113207.86|-4165972904406.558594|true|values clariffication|gabriella van buren|2028-01-28 10:55:38|2055-04-02 10:27:11.136514457|2005-08-12
+-76|27523|-191899537|-7792903881635938304|-988.63|2575958.83|134807836030.148438|false|quiet hour|rachel thompson|2064-09-13 12:05:33|2068-07-23 02:24:15.415114292|2006-10-22
+38|-15944|-1141801925|-8395998375405912064|37395.60|-2057132.64|-3432804499574.260254|true|geology|oscar young|2070-12-02 02:21:28|2022-02-22 23:45:54.417929187|2101-12-05
+-9|32017|1367179645|-8503342882470019072|-32948.66|2596690.92|-4465574272510.082031|true|american history|fred falkner|2028-01-06 05:29:29|2034-09-22 22:01:59.220097380|2103-12-30
+-37|8727|523289079|3946|-43342.52|2528654.53|-3334121644829.863770|true|history|gabriella ellison|2039-06-14 08:58:40|2023-05-10 22:56:14.107897590|2058-11-03
+-16|-1749|209430502|2662|-27058.30|1085405.78|-478847345395.791992|false|linguistics|victor falkner|2026-08-15 13:38:26||2101-06-16
+29|20270|-985817478|2848|-17564.28|-1085683.98|1245671080660.350586|true|philosophy|nick ovid|2077-11-22 03:06:05|2018-10-17 23:16:57.22955518|2059-05-05
+47|15266|670667262|3974|48495.59|-4950157.96|2359236945727.296875|true|biology|wendy carson|2062-12-04 16:15:27||1991-07-04
+|-19295|1677197847|8201491077550874624|41623.77|1350136.93|-1226174674712.830078|true|study skills|gabriella ellison|2016-02-17 02:12:04|2043-04-04 00:37:27.881714077|2027-09-11
+|12722|930008274|2386|-2270.46|1951711.24|-2269419016241.016602|true|history|priscilla zipper|2023-03-20 02:05:11|2058-08-15 16:21:32.650056302|2082-06-21
+102||-1112062809|690|-44270.30|-967065.47|4944863867873.259766|false|mathematics|victor nixon|2062-01-19 04:23:24|2034-11-05 03:07:33.846605261|1982-09-16
+-100||1091736925|-8962547695651323904|18808.85|-2597624.19|633233154165.040039|true|study skills|yuri nixon|2027-02-06 11:15:50|2068-11-02 21:11:23.741655982|2094-07-19
+55|-26304|1054864168|2227|-25135.64|-2339553.19||false|mathematics|jessica carson|2018-08-30 01:06:23|2078-11-30 02:36:40.607075532|2013-09-08
+-92|29168|-560322190||22456.65|4966767.72|-4492662222754.084961|true|education|ulysses ellison|2026-06-11 07:20:10|2028-10-07 16:00:16.652808441|
+127|-6294|-41242237||-24339.41|-4580152.68|-4150973486962.570312|true|kindergarten|rachel johnson|2025-04-25 21:40:13|2064-06-21 01:47:28.67945256|2026-04-28
+-104|-25115|824836988|-7020852530219171840|2631.26|-1696924.47|2636281196281.515625|true|religion|ethan king|2064-04-06 11:56:42|2079-03-23 13:15:05.713728857|2012-06-08
+82||373031319|6967631925774639104||-3338254.44|500376662732.691406|false|geology|alice johnson|2046-06-05 16:50:34|2033-10-31 22:49:14.847949456|1990-11-23
+-6|-30482|1275228381|8145750910080745472|-32038.63|-2954981.27|2949286209907.953125|false|linguistics|david young|2035-10-06 02:29:21|2031-02-11 07:57:23.673629394|2105-04-02
+113|-23323|-839512271|8994608999945125888|-28055.16|1643018.67||false|education|priscilla steinbeck|2054-10-12 02:43:21|2027-05-27 09:57:29.960419451|2005-01-23
+15|-7024|923353533|3940|7872.92|-2081172.60|-2466850727047.962402|true|wind surfing|ethan king|2027-10-04 14:01:27|2049-07-15 07:58:03.472552275|1988-08-26
+|-6283|-311437801|-8990843030306717696|-41276.81|1722079.14|2070485805872.195312|true|wind surfing|nick miller|2022-01-16 10:36:26|2029-06-16 15:44:00.263282479|1993-07-11
+25|6980|-1892816721|-8182421179156905984|32052.05|2697304.50|-2452231930319.044434|true|mathematics|sarah van buren|2057-11-28 06:45:32|2065-11-02 04:49:45.252309410|1984-05-26
+-25|26744|-1489628668|-7878145001776152576|38864.72|-543066.51|-3989748992325.772461|true|philosophy|tom quirinius|2027-08-03 17:06:47|2071-12-24 20:56:38.549176629|2093-04-14
+-122|25038|100270148|9096395849845194752|16126.36|-625257.45|-3447447963466.885254|true|opthamology|victor ellison|2036-07-06 17:42:38|2023-05-13 17:55:44.745449578|2082-09-17
+69|-16570|-1870912732|-7199983995864711168|-11501.29|-2355822.95|4784415058876.519531|false|american history|holly hernandez|2074-11-26 21:56:49|2076-02-03 03:26:47.100157028|1999-06-03
+-12|14213|||46313.84|-3912157.22|2214476493452.331055|true|linguistics|luke allen|2052-11-30 04:15:17|2068-10-27 11:57:03.945913098|2006-05-04
+56||658008867|7868367829080506368|-34336.86|-467709.59|2729407234427.085938|false|undecided|alice hernandez|2058-12-23 05:19:29|2038-03-11 15:26:58.124711805|2064-06-26
+-30|31795|217476429|2001||-140021.64|2811318250004.752930|false|chemistry|jessica thompson|2040-02-02 16:00:13|2035-08-19 22:22:12.933201371|2082-12-04
+96|9264|-1057522129|-7429331808102899712|10703.15|-1686203.10|-363688684224.463867|true|kindergarten|ethan quirinius|2081-02-04 05:16:49||
+-70|23063|-1511162508|8928133990107881472|22991.39|-2823357.23|-786567683392.114258|false|debate|calvin van buren|2054-10-09 00:55:52|2053-06-14 18:30:38.792560125|2077-10-29
+106|2326|2080412555|-9175279464813223936||3763969.37|-3478279974550.986328||forestry|fred van buren|2079-08-05 12:51:53|2025-05-02 08:19:30.267343837|2007-10-02
+58|4905|596802082|3467|48922.94|-2427646.65|-1956656626084.800781|true|religion|david brown|2059-08-27 02:37:38|2074-02-26 19:05:09.130777914|2104-03-09
+67|-31596||2692|23665.84|-2517144.39|-3266719980073.062500|false|debate|nick xylophone|2018-03-26 01:09:51|2048-01-16 05:58:57.246574643|2053-10-18
+-24|-13948|1063524922|383|-23278.06|961810.86|3461107192485.286133|true|opthamology||2063-12-10 23:49:15|2014-04-14 19:52:11.36942801|2023-10-23
+-51|-19413|-120704505|2180|7673.97|3840971.12|1717287804088.129883|false|philosophy|luke king|2021-11-07 16:27:24||1974-07-19
+8|-12564|172075892|7299197687217856512|-43842.34|-3395074.10|-4225250387427.301758|true|forestry|sarah nixon|2053-04-02 00:45:22|2071-09-12 07:22:30.450556241|
+-107|19917|-1945738830|8295110846998233088|-28431.96|-4322868.81|786529992285.522461
<TRUNCATED>
[12/50] [abbrv] hive git commit: HIVE-10845: TezJobMonitor uses
killedTaskCount instead of killedTaskAttemptCount (Sid Seth via Gunther
Hagleitner)
Posted by xu...@apache.org.
HIVE-10845: TezJobMonitor uses killedTaskCount instead of killedTaskAttemptCount (Sid Seth via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e22aaf63
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e22aaf63
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e22aaf63
Branch: refs/heads/spark
Commit: e22aaf6381a607a39bb238e5376ca3928790fc08
Parents: b6b638a
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Mon Jun 1 17:05:27 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Mon Jun 1 17:05:27 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e22aaf63/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index 4423cd1..78caba8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -496,7 +496,7 @@ public class TezJobMonitor {
if (progress != null) {
final int totalTasks = progress.getTotalTaskCount();
final int failedTaskAttempts = progress.getFailedTaskAttemptCount();
- final int killedTasks = progress.getKilledTaskCount();
+ final int killedTaskAttempts = progress.getKilledTaskAttemptCount();
final double duration =
perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName) / 1000.0;
VertexStatus vertexStatus = null;
@@ -583,7 +583,7 @@ public class TezJobMonitor {
vertexName,
totalTasks,
failedTaskAttempts,
- killedTasks,
+ killedTaskAttempts,
secondsFormat.format((duration)),
commaFormat.format(cpuTimeMillis),
commaFormat.format(gcTimeMillis),
@@ -623,7 +623,7 @@ public class TezJobMonitor {
final int failed = progress.getFailedTaskAttemptCount();
final int pending = progress.getTotalTaskCount() - progress.getSucceededTaskCount() -
progress.getRunningTaskCount();
- final int killed = progress.getKilledTaskCount();
+ final int killed = progress.getKilledTaskAttemptCount();
// To get vertex status we can use DAGClient.getVertexStatus(), but it will be expensive to
// get status from AM for every refresh of the UI. Lets infer the state from task counts.
[30/50] [abbrv] hive git commit: HIVE-9069: Simplify filter
predicates for CBO (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/991bb454/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
index fe2a0ee..d39da20 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
@@ -687,157 +687,154 @@ Stage-0
limit:-1
Stage-1
Reducer 6
- File Output Operator [FS_42]
+ File Output Operator [FS_45]
compressed:false
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
- Select Operator [SEL_41]
+ Select Operator [SEL_44]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 5 [SIMPLE_EDGE]
- Reduce Output Operator [RS_40]
+ Reduce Output Operator [RS_43]
key expressions:(UDFToLong(_col0) % _col1) (type: bigint), _col0 (type: int)
sort order:+-
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: bigint), _col2 (type: bigint)
- Group By Operator [GBY_38]
+ Group By Operator [GBY_41]
| aggregations:["count(VALUE._col0)"]
| keys:KEY._col0 (type: int), KEY._col1 (type: bigint)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 4 [SIMPLE_EDGE]
- Reduce Output Operator [RS_37]
+ Reduce Output Operator [RS_40]
key expressions:_col0 (type: int), _col1 (type: bigint)
Map-reduce partition columns:_col0 (type: int), _col1 (type: bigint)
sort order:++
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col2 (type: bigint)
- Group By Operator [GBY_36]
+ Group By Operator [GBY_39]
aggregations:["count()"]
keys:_col0 (type: int), _col1 (type: bigint)
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_34]
+ Select Operator [SEL_35]
outputColumnNames:["_col0","_col1"]
Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_33]
- predicate:(((_col1 > 0) or (_col6 >= 0)) and (((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0))) (type: boolean)
+ Filter Operator [FIL_49]
+ predicate:((((_col6 > 0) and ((_col6 >= 1) or (_col2 >= 1))) and ((UDFToLong(_col6) + _col2) >= 0)) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_52]
- | condition map:[{"":"Inner Join 0 to 1"}]
+ Merge Join Operator [MERGEJOIN_55]
+ | condition map:[{"":"Left Outer Join0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col6"]
- | Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ | Statistics:Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 10 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_31]
+ | Reduce Output Operator [RS_32]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
- | Statistics:Num rows: 5 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ | Statistics:Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Select Operator [SEL_27]
+ | Select Operator [SEL_30]
| outputColumnNames:["_col0","_col1"]
- | Statistics:Num rows: 5 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_50]
- | predicate:((c_int > 0) and key is not null) (type: boolean)
- | Statistics:Num rows: 5 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_25]
- | alias:cbo_t3
- | Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
+ | Statistics:Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ | TableScan [TS_29]
+ | alias:cbo_t3
+ | Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 3 [SIMPLE_EDGE]
- Reduce Output Operator [RS_29]
+ Reduce Output Operator [RS_31]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Select Operator [SEL_22]
+ Select Operator [SEL_26]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_47]
+ Filter Operator [FIL_50]
predicate:((_col3 + _col1) >= 0) (type: boolean)
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_51]
+ Merge Join Operator [MERGEJOIN_54]
| condition map:[{"":"Right Outer Join0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 2 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_19]
+ | Reduce Output Operator [RS_23]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Select Operator [SEL_7]
+ | Select Operator [SEL_9]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_6]
+ | Group By Operator [GBY_8]
| | keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| | outputColumnNames:["_col0","_col1","_col2"]
| | Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| |<-Map 1 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_5]
+ | Reduce Output Operator [RS_7]
| key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| sort order:+++
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_4]
+ | Group By Operator [GBY_6]
| keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_48]
- | predicate:(((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) and key is not null) (type: boolean)
+ | Filter Operator [FIL_51]
+ | predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
| alias:cbo_t2
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 9 [SIMPLE_EDGE]
- Reduce Output Operator [RS_20]
+ Reduce Output Operator [RS_24]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Select Operator [SEL_17]
+ Select Operator [SEL_21]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 8 [SIMPLE_EDGE]
- Reduce Output Operator [RS_16]
+ Reduce Output Operator [RS_20]
key expressions:_col3 (type: bigint), _col1 (type: int)
sort order:+-
Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col0 (type: string), _col2 (type: bigint)
- Select Operator [SEL_15]
+ Select Operator [SEL_19]
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator [GBY_14]
+ Group By Operator [GBY_18]
| aggregations:["sum(VALUE._col0)"]
| keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 7 [SIMPLE_EDGE]
- Reduce Output Operator [RS_13]
+ Reduce Output Operator [RS_17]
key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
sort order:+++
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col3 (type: bigint)
- Group By Operator [GBY_12]
+ Group By Operator [GBY_16]
aggregations:["sum(_col1)"]
keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_10]
+ Select Operator [SEL_12]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_49]
- predicate:(((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) and key is not null) (type: boolean)
+ Filter Operator [FIL_52]
+ predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan [TS_8]
+ TableScan [TS_10]
alias:cbo_t1
Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) cbo_t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 2) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c
@@ -859,142 +856,142 @@ Stage-0
limit:-1
Stage-1
Reducer 5
- File Output Operator [FS_35]
+ File Output Operator [FS_39]
compressed:false
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
- Group By Operator [GBY_33]
+ Group By Operator [GBY_37]
| aggregations:["count(VALUE._col0)"]
| keys:KEY._col0 (type: int), KEY._col1 (type: bigint)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 4 [SIMPLE_EDGE]
- Reduce Output Operator [RS_32]
+ Reduce Output Operator [RS_36]
key expressions:_col0 (type: int), _col1 (type: bigint)
Map-reduce partition columns:_col0 (type: int), _col1 (type: bigint)
sort order:++
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col2 (type: bigint)
- Group By Operator [GBY_31]
+ Group By Operator [GBY_35]
aggregations:["count()"]
keys:_col0 (type: int), _col1 (type: bigint)
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_29]
+ Select Operator [SEL_33]
outputColumnNames:["_col0","_col1"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_39]
+ Filter Operator [FIL_43]
predicate:(((_col1 + _col4) >= 2) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_44]
+ Merge Join Operator [MERGEJOIN_48]
| condition map:[{"":"Right Outer Join0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col4","_col6"]
| Statistics:Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 9 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_26]
+ | Reduce Output Operator [RS_30]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Select Operator [SEL_24]
+ | Select Operator [SEL_28]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_23]
+ | TableScan [TS_27]
| alias:cbo_t3
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 3 [SIMPLE_EDGE]
- Reduce Output Operator [RS_25]
+ Reduce Output Operator [RS_29]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint), _col4 (type: int)
- Select Operator [SEL_22]
+ Select Operator [SEL_26]
outputColumnNames:["_col0","_col1","_col2","_col4"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_43]
+ Merge Join Operator [MERGEJOIN_47]
| condition map:[{"":"Left Outer Join0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 2 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_19]
+ | Reduce Output Operator [RS_23]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Select Operator [SEL_7]
+ | Select Operator [SEL_9]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_6]
+ | Group By Operator [GBY_8]
| | keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| | outputColumnNames:["_col0","_col1","_col2"]
| | Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| |<-Map 1 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_5]
+ | Reduce Output Operator [RS_7]
| key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| sort order:+++
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_4]
+ | Group By Operator [GBY_6]
| keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_41]
+ | Filter Operator [FIL_45]
| predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
| alias:cbo_t2
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 8 [SIMPLE_EDGE]
- Reduce Output Operator [RS_20]
+ Reduce Output Operator [RS_24]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Select Operator [SEL_17]
+ Select Operator [SEL_21]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 7 [SIMPLE_EDGE]
- Reduce Output Operator [RS_16]
+ Reduce Output Operator [RS_20]
key expressions:_col3 (type: bigint), _col0 (type: string)
sort order:+-
Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Select Operator [SEL_15]
+ Select Operator [SEL_19]
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator [GBY_14]
+ Group By Operator [GBY_18]
| aggregations:["sum(VALUE._col0)"]
| keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 6 [SIMPLE_EDGE]
- Reduce Output Operator [RS_13]
+ Reduce Output Operator [RS_17]
key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
sort order:+++
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col3 (type: bigint)
- Group By Operator [GBY_12]
+ Group By Operator [GBY_16]
aggregations:["sum(_col1)"]
keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_10]
+ Select Operator [SEL_12]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_42]
+ Filter Operator [FIL_46]
predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan [TS_8]
+ TableScan [TS_10]
alias:cbo_t1
Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int
@@ -1018,155 +1015,152 @@ Stage-0
limit:-1
Stage-1
Reducer 7
- File Output Operator [FS_41]
+ File Output Operator [FS_46]
compressed:false
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
- Select Operator [SEL_40]
+ Select Operator [SEL_45]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 6 [SIMPLE_EDGE]
- Reduce Output Operator [RS_39]
+ Reduce Output Operator [RS_44]
key expressions:_col0 (type: int)
sort order:+
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: bigint), _col2 (type: bigint)
- Group By Operator [GBY_37]
+ Group By Operator [GBY_42]
| aggregations:["count(VALUE._col0)"]
| keys:KEY._col0 (type: int), KEY._col1 (type: bigint)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 5 [SIMPLE_EDGE]
- Reduce Output Operator [RS_36]
+ Reduce Output Operator [RS_41]
key expressions:_col0 (type: int), _col1 (type: bigint)
Map-reduce partition columns:_col0 (type: int), _col1 (type: bigint)
sort order:++
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col2 (type: bigint)
- Group By Operator [GBY_35]
+ Group By Operator [GBY_40]
aggregations:["count()"]
keys:_col0 (type: int), _col1 (type: bigint)
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_33]
+ Select Operator [SEL_36]
outputColumnNames:["_col0","_col1"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_44]
- predicate:(((((_col1 + _col4) >= 0) and ((_col1 > 0) or (_col6 >= 0))) and ((_col6 >= 1) or (_col2 >= 1))) and ((UDFToLong(_col6) + _col2) >= 0)) (type: boolean)
+ Filter Operator [FIL_48]
+ predicate:(((((_col6 > 0) and ((_col6 >= 1) or (_col2 >= 1))) and ((UDFToLong(_col6) + _col2) >= 0)) and ((_col1 + _col4) >= 0)) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_50]
- | condition map:[{"":"Right Outer Join0 to 1"}]
+ Merge Join Operator [MERGEJOIN_52]
+ | condition map:[{"":"Outer Join 0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col4","_col6"]
- | Statistics:Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
+ | Statistics:Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 11 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_30]
+ | Reduce Output Operator [RS_33]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
- | Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
+ | Statistics:Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Select Operator [SEL_28]
+ | Select Operator [SEL_31]
| outputColumnNames:["_col0","_col1"]
- | Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_48]
- | predicate:(c_int > 0) (type: boolean)
- | Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_26]
- | alias:cbo_t3
- | Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
+ | Statistics:Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ | TableScan [TS_30]
+ | alias:cbo_t3
+ | Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 4 [SIMPLE_EDGE]
- Reduce Output Operator [RS_29]
+ Reduce Output Operator [RS_32]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint), _col4 (type: int)
- Select Operator [SEL_25]
+ Select Operator [SEL_29]
outputColumnNames:["_col0","_col1","_col2","_col4"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_49]
+ Merge Join Operator [MERGEJOIN_51]
| condition map:[{"":"Outer Join 0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 10 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_23]
+ | Reduce Output Operator [RS_27]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int), _col2 (type: bigint)
- | Select Operator [SEL_20]
+ | Select Operator [SEL_24]
| | outputColumnNames:["_col0","_col1","_col2"]
| | Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
| |<-Reducer 9 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_19]
+ | Reduce Output Operator [RS_23]
| key expressions:_col3 (type: double)
| sort order:-
| Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col0 (type: string), _col1 (type: int), _col2 (type: bigint)
- | Select Operator [SEL_18]
+ | Select Operator [SEL_22]
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_17]
+ | Group By Operator [GBY_21]
| | aggregations:["sum(VALUE._col0)"]
| | keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| | outputColumnNames:["_col0","_col1","_col2","_col3"]
| | Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
| |<-Map 8 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_16]
+ | Reduce Output Operator [RS_20]
| key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| sort order:+++
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col3 (type: bigint)
- | Group By Operator [GBY_15]
+ | Group By Operator [GBY_19]
| aggregations:["sum(_col1)"]
| keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- | Select Operator [SEL_13]
+ | Select Operator [SEL_15]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_47]
+ | Filter Operator [FIL_50]
| predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_11]
+ | TableScan [TS_13]
| alias:cbo_t1
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 3 [SIMPLE_EDGE]
- Reduce Output Operator [RS_22]
+ Reduce Output Operator [RS_26]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int)
- Select Operator [SEL_9]
+ Select Operator [SEL_11]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 2 [SIMPLE_EDGE]
- Reduce Output Operator [RS_8]
+ Reduce Output Operator [RS_10]
key expressions:_col3 (type: double), _col2 (type: bigint)
sort order:-+
Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col0 (type: string), _col1 (type: int)
- Select Operator [SEL_7]
+ Select Operator [SEL_9]
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator [GBY_6]
+ Group By Operator [GBY_8]
| aggregations:["sum(VALUE._col0)"]
| keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 1 [SIMPLE_EDGE]
- Reduce Output Operator [RS_5]
+ Reduce Output Operator [RS_7]
key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
sort order:+++
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col3 (type: bigint)
- Group By Operator [GBY_4]
+ Group By Operator [GBY_6]
aggregations:["sum(_col1)"]
keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
outputColumnNames:["_col0","_col1","_col2","_col3"]
@@ -1174,7 +1168,7 @@ Stage-0
Select Operator [SEL_2]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_46]
+ Filter Operator [FIL_49]
predicate:((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_0]
@@ -1198,40 +1192,40 @@ Stage-0
limit:-1
Stage-1
Reducer 3
- File Output Operator [FS_39]
+ File Output Operator [FS_43]
compressed:false
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
- Group By Operator [GBY_37]
+ Group By Operator [GBY_41]
| aggregations:["count(VALUE._col0)"]
| keys:KEY._col0 (type: int), KEY._col1 (type: bigint)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 2 [SIMPLE_EDGE]
- Reduce Output Operator [RS_36]
+ Reduce Output Operator [RS_40]
key expressions:_col0 (type: int), _col1 (type: bigint)
Map-reduce partition columns:_col0 (type: int), _col1 (type: bigint)
sort order:++
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col2 (type: bigint)
- Group By Operator [GBY_35]
+ Group By Operator [GBY_39]
aggregations:["count()"]
keys:_col0 (type: int), _col1 (type: bigint)
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_31]
+ Select Operator [SEL_35]
outputColumnNames:["_col0","_col1"]
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_45]
+ Filter Operator [FIL_49]
predicate:((_col3 > 0) or (_col1 >= 0)) (type: boolean)
Statistics:Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_51]
+ Merge Join Operator [MERGEJOIN_55]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col3","_col4"]
| Statistics:Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 1 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_27]
+ | Reduce Output Operator [RS_31]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
@@ -1240,97 +1234,97 @@ Stage-0
| Select Operator [SEL_1]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_46]
+ | Filter Operator [FIL_50]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
| alias:cbo_t3
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 6 [SIMPLE_EDGE]
- Reduce Output Operator [RS_29]
+ Reduce Output Operator [RS_33]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Select Operator [SEL_23]
+ Select Operator [SEL_27]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_47]
+ Filter Operator [FIL_51]
predicate:((_col3 + _col1) >= 0) (type: boolean)
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_50]
+ Merge Join Operator [MERGEJOIN_54]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 5 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_19]
+ | Reduce Output Operator [RS_23]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: int)
- | Select Operator [SEL_9]
+ | Select Operator [SEL_11]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_8]
+ | Group By Operator [GBY_10]
| | keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| | outputColumnNames:["_col0","_col1","_col2"]
| | Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| |<-Map 4 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_7]
+ | Reduce Output Operator [RS_9]
| key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| sort order:+++
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_6]
+ | Group By Operator [GBY_8]
| keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| Select Operator [SEL_4]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_48]
+ | Filter Operator [FIL_52]
| predicate:(((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) and key is not null) (type: boolean)
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_2]
| alias:cbo_t2
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 8 [SIMPLE_EDGE]
- Reduce Output Operator [RS_21]
+ Reduce Output Operator [RS_25]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: bigint)
- Select Operator [SEL_17]
+ Select Operator [SEL_21]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator [GBY_16]
+ Group By Operator [GBY_20]
| aggregations:["sum(VALUE._col0)"]
| keys:KEY._col0 (type: float), KEY._col1 (type: int), KEY._col2 (type: string)
| outputColumnNames:["_col0","_col1","_col2","_col3"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 7 [SIMPLE_EDGE]
- Reduce Output Operator [RS_15]
+ Reduce Output Operator [RS_19]
key expressions:_col0 (type: float), _col1 (type: int), _col2 (type: string)
Map-reduce partition columns:_col0 (type: float), _col1 (type: int), _col2 (type: string)
sort order:+++
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col3 (type: bigint)
- Group By Operator [GBY_14]
+ Group By Operator [GBY_18]
aggregations:["sum(_col1)"]
keys:_col0 (type: float), _col1 (type: int), _col2 (type: string)
outputColumnNames:["_col0","_col1","_col2","_col3"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_12]
+ Select Operator [SEL_14]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_49]
+ Filter Operator [FIL_53]
predicate:(((((((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0))) and (c_float > 0.0)) and ((c_int >= 1) or (c_float >= 1.0))) and ((UDFToFloat(c_int) + c_float) >= 0.0)) and key is not null) (type: boolean)
Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan [TS_10]
+ TableScan [TS_12]
alias:cbo_t1
Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
PREHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc
@@ -1952,17 +1946,17 @@ Stage-0
limit:-1
Stage-1
Reducer 3
- File Output Operator [FS_20]
+ File Output Operator [FS_22]
compressed:false
- Statistics:Num rows: 8 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics:Num rows: 4 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
Select Operator [SEL_19]
outputColumnNames:["_col0","_col1","_col2","_col3","_col4"]
- Statistics:Num rows: 8 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics:Num rows: 4 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator [FIL_18]
- predicate:((_col1 > 0) or (_col6 >= 0)) (type: boolean)
- Statistics:Num rows: 8 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_29]
+ predicate:(((_col4 + 1) = 2) and ((_col1 > 0) or (_col6 >= 0))) (type: boolean)
+ Statistics:Num rows: 4 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
+ Merge Join Operator [MERGEJOIN_32]
| condition map:[{"":"Inner Join 0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4","_col6"]
@@ -1977,7 +1971,7 @@ Stage-0
| Select Operator [SEL_12]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_27]
+ | Filter Operator [FIL_30]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_11]
@@ -1990,10 +1984,10 @@ Stage-0
sort order:+
Statistics:Num rows: 4 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col1 (type: int), _col2 (type: float), _col3 (type: string), _col4 (type: int)
- Filter Operator [FIL_24]
- predicate:((((_col1 + _col4) = 2) and ((_col4 + 1) = 2)) and _col0 is not null) (type: boolean)
+ Filter Operator [FIL_27]
+ predicate:((((_col1 + _col4) = 2) and _col0 is not null) and ((_col4 + 1) = 2)) (type: boolean)
Statistics:Num rows: 4 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_28]
+ Merge Join Operator [MERGEJOIN_31]
| condition map:[{"":"Outer Join 0 to 1"}]
| keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2","_col3","_col4"]
@@ -2008,7 +2002,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_25]
+ | Filter Operator [FIL_28]
| predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -2024,7 +2018,7 @@ Stage-0
Select Operator [SEL_5]
outputColumnNames:["_col0","_col1"]
Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_26]
+ Filter Operator [FIL_29]
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
TableScan [TS_3]
@@ -2044,17 +2038,17 @@ Stage-0
limit:-1
Stage-1
Reducer 2
- File Output Operator [FS_14]
+ File Output Operator [FS_16]
compressed:false
Statistics:Num rows: 12 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
Select Operator [SEL_13]
outputColumnNames:["_col0","_col1","_col2","_col3","_col4"]
Statistics:Num rows: 12 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_17]
- predicate:((((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0))) and ((_col4 + 1) = 2)) (type: boolean)
+ Filter Operator [FIL_20]
+ predicate:((((_col4 + 1) = 2) and ((_col1 > 0) or (_col6 >= 0))) and ((_col1 + _col4) = 2)) (type: boolean)
Statistics:Num rows: 12 Data size: 1212 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_20]
+ Merge Join Operator [MERGEJOIN_23]
| condition map:[{"":"Right Outer Join0 to 1"},{"":"Right Outer Join0 to 2"}]
| keys:{"2":"_col0 (type: string)","1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col1","_col2","_col3","_col4","_col6"]
@@ -2069,7 +2063,7 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_18]
+ | Filter Operator [FIL_21]
| predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
@@ -2085,7 +2079,7 @@ Stage-0
| Select Operator [SEL_5]
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 6 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_19]
+ | Filter Operator [FIL_22]
| predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) (type: boolean)
| Statistics:Num rows: 6 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_3]
@@ -2633,20 +2627,20 @@ Stage-0
limit:-1
Stage-1
Reducer 2
- File Output Operator [FS_20]
+ File Output Operator [FS_24]
compressed:false
Statistics:Num rows: 12 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
- Select Operator [SEL_19]
+ Select Operator [SEL_23]
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 12 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_30]
+ Merge Join Operator [MERGEJOIN_34]
| condition map:[{"":"Left Semi Join 0 to 1"},{"":"Left Semi Join 0 to 2"}]
| keys:{"2":"_col0 (type: string)","1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 12 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 1 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_13]
+ | Reduce Output Operator [RS_17]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
@@ -2655,48 +2649,48 @@ Stage-0
| Select Operator [SEL_2]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 5 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_27]
+ | Filter Operator [FIL_31]
| predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null) (type: boolean)
| Statistics:Num rows: 5 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
| TableScan [TS_0]
| alias:cbo_t1
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 3 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_15]
+ | Reduce Output Operator [RS_19]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_9]
+ | Group By Operator [GBY_13]
| keys:_col0 (type: string)
| outputColumnNames:["_col0"]
| Statistics:Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE
- | Select Operator [SEL_5]
+ | Select Operator [SEL_9]
| outputColumnNames:["_col0"]
| Statistics:Num rows: 5 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_28]
+ | Filter Operator [FIL_32]
| predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null) (type: boolean)
| Statistics:Num rows: 5 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_3]
+ | TableScan [TS_7]
| alias:cbo_t2
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 4 [SIMPLE_EDGE]
- Reduce Output Operator [RS_17]
+ Reduce Output Operator [RS_21]
key expressions:_col0 (type: string)
Map-reduce partition columns:_col0 (type: string)
sort order:+
Statistics:Num rows: 6 Data size: 425 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator [GBY_11]
+ Group By Operator [GBY_15]
keys:_col0 (type: string)
outputColumnNames:["_col0"]
Statistics:Num rows: 6 Data size: 425 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator [SEL_7]
+ Select Operator [SEL_11]
outputColumnNames:["_col0"]
Statistics:Num rows: 18 Data size: 1360 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator [FIL_29]
+ Filter Operator [FIL_33]
predicate:key is not null (type: boolean)
Statistics:Num rows: 18 Data size: 1360 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan [TS_6]
+ TableScan [TS_10]
alias:cbo_t3
Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
PREHOOK: query: explain select a, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) cbo_t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a
@@ -2719,98 +2713,98 @@ Stage-0
limit:-1
Stage-1
Reducer 6
- File Output Operator [FS_44]
+ File Output Operator [FS_50]
compressed:false
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}
- Select Operator [SEL_43]
+ Select Operator [SEL_49]
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 5 [SIMPLE_EDGE]
- Reduce Output Operator [RS_42]
+ Reduce Output Operator [RS_48]
key expressions:_col1 (type: bigint), _col0 (type: string)
sort order:++
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col2 (type: bigint)
- Group By Operator [GBY_40]
+ Group By Operator [GBY_46]
| aggregations:["count(VALUE._col0)"]
| keys:KEY._col0 (type: string), KEY._col1 (type: bigint)
| outputColumnNames:["_col0","_col1","_col2"]
| Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 4 [SIMPLE_EDGE]
- Reduce Output Operator [RS_39]
+ Reduce Output Operator [RS_45]
key expressions:_col0 (type: string), _col1 (type: bigint)
Map-reduce partition columns:_col0 (type: string), _col1 (type: bigint)
sort order:++
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
value expressions:_col2 (type: bigint)
- Group By Operator [GBY_38]
+ Group By Operator [GBY_44]
aggregations:["count()"]
keys:_col0 (type: string), _col1 (type: bigint)
outputColumnNames:["_col0","_col1","_col2"]
Statistics:Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator [MERGEJOIN_56]
+ Merge Join Operator [MERGEJOIN_62]
| condition map:[{"":"Left Semi Join 0 to 1"},{"":"Left Semi Join 0 to 2"}]
| keys:{"2":"_col0 (type: string)","1":"_col0 (type: string)","0":"_col0 (type: string)"}
| outputColumnNames:["_col0","_col1"]
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
|<-Map 10 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_35]
+ | Reduce Output Operator [RS_41]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 6 Data size: 425 Basic stats: COMPLETE Column stats: COMPLETE
- | Group By Operator [GBY_29]
+ | Group By Operator [GBY_35]
| keys:_col0 (type: string)
| outputColumnNames:["_col0"]
| Statistics:Num rows: 6 Data size: 425 Basic stats: COMPLETE Column stats: COMPLETE
- | Select Operator [SEL_25]
+ | Select Operator [SEL_31]
| outputColumnNames:["_col0"]
| Statistics:Num rows: 18 Data size: 1360 Basic stats: COMPLETE Column stats: COMPLETE
- | Filter Operator [FIL_55]
+ | Filter Operator [FIL_61]
| predicate:key is not null (type: boolean)
| Statistics:Num rows: 18 Data size: 1360 Basic stats: COMPLETE Column stats: COMPLETE
- | TableScan [TS_24]
+ | TableScan [TS_30]
| alias:cbo_t3
| Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE
|<-Reducer 3 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_31]
+ | Reduce Output Operator [RS_37]
| key expressions:_col0 (type: string)
| Map-reduce partition columns:_col0 (type: string)
| sort order:+
| Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| value expressions:_col1 (type: bigint)
- | Select Operator [SEL_9]
+ | Select Operator [SEL_11]
| | outputColumnNames:["_col0","_col1"]
| | Statistics:Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE
| |<-Reducer 2 [SIMPLE_EDGE]
- | Reduce Output Operator [RS_8]
+ | Reduce Output Operator [RS_10]
| key expressions:_col3 (type: double), _col2 (type: bigint)
| sort order:-+
| Statistics:Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE
<TRUNCATED>
[17/50] [abbrv] hive git commit: HIVE-10728 : deprecate
unix_timestamp(void) and make it deterministic (Sergey Shelukhin,
reveiwed by Ashutosh Chauhan(
Posted by xu...@apache.org.
HIVE-10728 : deprecate unix_timestamp(void) and make it deterministic (Sergey Shelukhin, reveiwed by Ashutosh Chauhan(
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24d3307b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24d3307b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24d3307b
Branch: refs/heads/spark
Commit: 24d3307be79d35d3a34c49014dfdd597112f9106
Parents: 7556361
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Jun 2 18:37:48 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Jun 2 18:37:48 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/session/SessionState.java | 4 +--
.../udf/generic/GenericUDFToUnixTimeStamp.java | 6 +++-
.../ql/udf/generic/GenericUDFUnixTimeStamp.java | 36 ++++++++++++++------
.../queries/clientpositive/autogen_colalias.q | 4 +--
.../queries/clientpositive/udf_unix_timestamp.q | 8 +++--
.../clientpositive/autogen_colalias.q.out | 12 +++----
ql/src/test/results/clientpositive/udf5.q.out | 30 +++++-----------
.../clientpositive/udf_unix_timestamp.q.out | 32 +++++++++++++++--
8 files changed, 85 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 37b6d6f..7930298 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -903,12 +903,12 @@ public class SessionState {
return ((ss != null) && (ss.out != null)) ? ss.out : System.out;
}
- public PrintStream getInfoStream() {
+ public static PrintStream getInfoStream() {
SessionState ss = SessionState.get();
return ((ss != null) && (ss.info != null)) ? ss.info : getErrStream();
}
- public PrintStream getErrStream() {
+ public static PrintStream getErrStream() {
SessionState ss = SessionState.get();
return ((ss != null) && (ss.err != null)) ? ss.err : System.err;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
index 65a2297..4ab5389 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
@@ -152,10 +152,14 @@ public class GenericUDFToUnixTimeStamp extends GenericUDF {
return retValue;
}
Timestamp timestamp = inputTimestampOI.getPrimitiveJavaObject(arguments[0].get());
- retValue.set(timestamp.getTime() / 1000);
+ setValueFromTs(retValue, timestamp);
return retValue;
}
+ protected static void setValueFromTs(LongWritable value, Timestamp timestamp) {
+ value.set(timestamp.getTime() / 1000);
+ }
+
@Override
public String getDisplayString(String[] children) {
StringBuilder sb = new StringBuilder(32);
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
index 0720c55..c1b2a01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUnixTimeStamp.java
@@ -18,23 +18,43 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.io.PrintStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.LongWritable;
+
-@UDFType(deterministic = false)
+@UDFType(deterministic = true)
@Description(name = "unix_timestamp",
- value = "_FUNC_([date[, pattern]]) - Returns the UNIX timestamp",
- extended = "Converts the current or specified time to number of seconds "
- + "since 1970-01-01.")
+ value = "_FUNC_(date[, pattern]) - Converts the time to a number",
+ extended = "Converts the specified time to number of seconds "
+ + "since 1970-01-01. The _FUNC_(void) overload is deprecated, use current_timestamp.")
public class GenericUDFUnixTimeStamp extends GenericUDFToUnixTimeStamp {
-
+ private static final Log LOG = LogFactory.getLog(GenericUDFUnixTimeStamp.class);
+ private LongWritable currentTimestamp; // retValue is transient so store this separately.
@Override
protected void initializeInput(ObjectInspector[] arguments) throws UDFArgumentException {
if (arguments.length > 0) {
super.initializeInput(arguments);
+ } else {
+ if (currentTimestamp == null) {
+ currentTimestamp = new LongWritable(0);
+ setValueFromTs(currentTimestamp, SessionState.get().getQueryCurrentTimestamp());
+ String msg = "unix_timestamp(void) is deprecated. Use current_timestamp instead.";
+ LOG.warn(msg);
+ PrintStream stream = LogHelper.getInfoStream();
+ if (stream != null) {
+ stream.println(msg);
+ }
+ }
}
}
@@ -45,10 +65,6 @@ public class GenericUDFUnixTimeStamp extends GenericUDFToUnixTimeStamp {
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
- if (arguments.length == 0) {
- retValue.set(System.currentTimeMillis() / 1000);
- return retValue;
- }
- return super.evaluate(arguments);
+ return (arguments.length == 0) ? currentTimestamp : super.evaluate(arguments);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/queries/clientpositive/autogen_colalias.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/autogen_colalias.q b/ql/src/test/queries/clientpositive/autogen_colalias.q
index e247a73..dac0bf7 100644
--- a/ql/src/test/queries/clientpositive/autogen_colalias.q
+++ b/ql/src/test/queries/clientpositive/autogen_colalias.q
@@ -1,6 +1,6 @@
CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax';
-create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
+create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
1,
0)
from src group by src.key;
@@ -12,7 +12,7 @@ describe dest_grouped_old2;
set hive.autogen.columnalias.prefix.label=column_;
set hive.autogen.columnalias.prefix.includefuncname=true;
-create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
+create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
(src.key +5) % 2,
0)
from src group by src.key;
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_unix_timestamp.q b/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
index 1664329..bb598c2 100644
--- a/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
+++ b/ql/src/test/queries/clientpositive/udf_unix_timestamp.q
@@ -21,9 +21,13 @@ SELECT
unix_timestamp('2009 Mar 20 11:30:01 am', 'yyyy MMM dd h:mm:ss a')
FROM oneline;
+create table foo as SELECT
+ 'deprecated' as a,
+ unix_timestamp() as b
+FROM oneline;
+drop table foo;
+
SELECT
'random_string',
unix_timestamp('random_string')
FROM oneline;
-
-
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/results/clientpositive/autogen_colalias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autogen_colalias.q.out b/ql/src/test/results/clientpositive/autogen_colalias.q.out
index 7257aaa..5569b46 100644
--- a/ql/src/test/results/clientpositive/autogen_colalias.q.out
+++ b/ql/src/test/results/clientpositive/autogen_colalias.q.out
@@ -4,7 +4,7 @@ PREHOOK: Output: test_max
POSTHOOK: query: CREATE TEMPORARY FUNCTION test_max AS 'org.apache.hadoop.hive.ql.udf.UDAFTestMax'
POSTHOOK: type: CREATEFUNCTION
POSTHOOK: Output: test_max
-PREHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
+PREHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
1,
0)
from src group by src.key
@@ -12,7 +12,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@dest_grouped_old1
-POSTHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
+POSTHOOK: query: create table dest_grouped_old1 as select 1+1, 2+2 as zz, src.key, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 1,
1,
0)
from src group by src.key
@@ -33,7 +33,7 @@ c3 int
c4 bigint
c5 double
c6 bigint
-c7 bigint
+c7 timestamp
c8 int
c9 int
PREHOOK: query: create table dest_grouped_old2 as select distinct src.key from src
@@ -53,7 +53,7 @@ POSTHOOK: query: describe dest_grouped_old2
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@dest_grouped_old2
key string
-PREHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
+PREHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
(src.key +5) % 2,
0)
from src group by src.key
@@ -61,7 +61,7 @@ PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@dest_grouped_new1
-POSTHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), unix_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
+POSTHOOK: query: create table dest_grouped_new1 as select 1+1, 2+2 as zz, ((src.key % 2)+2)/2, test_max(length(src.value)), count(src.value), sin(count(src.value)), count(sin(src.value)), current_timestamp(), CAST(SUM(IF(value > 10, value, 1)) AS INT), if(src.key > 10,
(src.key +5) % 2,
0)
from src group by src.key
@@ -82,7 +82,7 @@ test_max_length_src__3 int
count_src_value_4 bigint
sin_count_src_value_5 double
count_sin_src_value_6 bigint
-unix_timestamp_7 bigint
+current_timestamp_7 timestamp
sum_if_value_10_valu_8 int
if_src_key_10_src_ke_9 double
PREHOOK: query: create table dest_grouped_new2 as select distinct src.key from src
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/results/clientpositive/udf5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf5.q.out b/ql/src/test/results/clientpositive/udf5.q.out
index 26cf3f1..860ebcb 100644
--- a/ql/src/test/results/clientpositive/udf5.q.out
+++ b/ql/src/test/results/clientpositive/udf5.q.out
@@ -54,33 +54,21 @@ POSTHOOK: query: EXPLAIN
SELECT from_unixtime(unix_timestamp('2010-01-13 11:57:40', 'yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss'), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) from dest1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: dest1
- Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: from_unixtime(unix_timestamp('2010-01-13 11:57:40','yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss') (type: string), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
- ListSink
+ TableScan
+ alias: dest1
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: '01/13/10 11:57:40' (type: string), '2010-01-13 11:57:40' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
+ ListSink
PREHOOK: query: SELECT from_unixtime(unix_timestamp('2010-01-13 11:57:40', 'yyyy-MM-dd HH:mm:ss'), 'MM/dd/yy HH:mm:ss'), from_unixtime(unix_timestamp('2010-01-13 11:57:40')) from dest1
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/24d3307b/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out b/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
index e4ffa1a..c64379d 100644
--- a/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/udf_unix_timestamp.q.out
@@ -2,13 +2,13 @@ PREHOOK: query: DESCRIBE FUNCTION unix_timestamp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION unix_timestamp
POSTHOOK: type: DESCFUNCTION
-unix_timestamp([date[, pattern]]) - Returns the UNIX timestamp
+unix_timestamp(date[, pattern]) - Converts the time to a number
PREHOOK: query: DESCRIBE FUNCTION EXTENDED unix_timestamp
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED unix_timestamp
POSTHOOK: type: DESCFUNCTION
-unix_timestamp([date[, pattern]]) - Returns the UNIX timestamp
-Converts the current or specified time to number of seconds since 1970-01-01.
+unix_timestamp(date[, pattern]) - Converts the time to a number
+Converts the specified time to number of seconds since 1970-01-01. The unix_timestamp(void) overload is deprecated, use current_timestamp.
PREHOOK: query: create table oneline(key int, value string)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -70,6 +70,32 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@oneline
#### A masked pattern was here ####
2009 Mar 20 11:30:01 am 1237573801
+unix_timestamp(void) is deprecated. Use current_timestamp instead.
+unix_timestamp(void) is deprecated. Use current_timestamp instead.
+PREHOOK: query: create table foo as SELECT
+ 'deprecated' as a,
+ unix_timestamp() as b
+FROM oneline
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@oneline
+PREHOOK: Output: database:default
+PREHOOK: Output: default@foo
+POSTHOOK: query: create table foo as SELECT
+ 'deprecated' as a,
+ unix_timestamp() as b
+FROM oneline
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@oneline
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@foo
+PREHOOK: query: drop table foo
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@foo
+PREHOOK: Output: default@foo
+POSTHOOK: query: drop table foo
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@foo
+POSTHOOK: Output: default@foo
PREHOOK: query: SELECT
'random_string',
unix_timestamp('random_string')
[03/50] [abbrv] hive git commit: HIVE-10671: yarn-cluster mode offers
a degraded performance from yarn-client [Spark Branch] (Rui via Xuefu,
reviewed by Chengxiang)
Posted by xu...@apache.org.
HIVE-10671: yarn-cluster mode offers a degraded performance from yarn-client [Spark Branch] (Rui via Xuefu, reviewed by Chengxiang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f27c6ed
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f27c6ed
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f27c6ed
Branch: refs/heads/spark
Commit: 3f27c6ed47bbfd8257c70bca0983a7f2e3e74f4a
Parents: 2df14f9
Author: xzhang <xz...@xzdt>
Authored: Thu May 14 14:20:11 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Jun 1 14:03:54 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/spark/RemoteHiveSparkClient.java | 10 +++++-----
.../hadoop/hive/ql/exec/spark/SparkUtilities.java | 11 +++++++----
.../java/org/apache/hive/spark/client/JobContext.java | 2 +-
.../org/apache/hive/spark/client/JobContextImpl.java | 7 ++++---
.../apache/hive/spark/client/SparkClientUtilities.java | 13 ++++++++-----
5 files changed, 25 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index 059016d..bae30f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -28,6 +28,7 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -167,10 +168,10 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
try {
URI fileUri = SparkUtilities.getURI(addedFile);
if (fileUri != null && !localFiles.contains(fileUri)) {
+ localFiles.add(fileUri);
if (SparkUtilities.needUploadToHDFS(fileUri, sparkConf)) {
fileUri = SparkUtilities.uploadToHDFS(fileUri, hiveConf);
}
- localFiles.add(fileUri);
remoteClient.addFile(fileUri);
}
} catch (URISyntaxException e) {
@@ -184,10 +185,10 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
try {
URI jarUri = SparkUtilities.getURI(addedJar);
if (jarUri != null && !localJars.contains(jarUri)) {
+ localJars.add(jarUri);
if (SparkUtilities.needUploadToHDFS(jarUri, sparkConf)) {
jarUri = SparkUtilities.uploadToHDFS(jarUri, hiveConf);
}
- localJars.add(jarUri);
remoteClient.addJar(jarUri);
}
} catch (URISyntaxException e) {
@@ -224,10 +225,9 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
// Add jar to current thread class loader dynamically, and add jar paths to JobConf as Spark
// may need to load classes from this jar in other threads.
- List<String> addedJars = jc.getAddedJars();
+ Set<String> addedJars = jc.getAddedJars();
if (addedJars != null && !addedJars.isEmpty()) {
- SparkClientUtilities.addToClassPath(addedJars.toArray(new String[addedJars.size()]),
- localJobConf, jc.getLocalTmpDir());
+ SparkClientUtilities.addToClassPath(addedJars, localJobConf, jc.getLocalTmpDir());
localJobConf.set(Utilities.HIVE_ADDED_JARS, StringUtils.join(addedJars, ";"));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index 72ab913..c012af8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.UUID;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.fs.FileSystem;
@@ -76,11 +77,13 @@ public class SparkUtilities {
* @throws IOException
*/
public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException {
- Path tmpDir = SessionState.getHDFSSessionPath(conf);
+ Path localFile = new Path(source.getPath());
+ // give the uploaded file a UUID
+ Path remoteFile = new Path(SessionState.getHDFSSessionPath(conf),
+ UUID.randomUUID() + "-" + getFileName(source));
FileSystem fileSystem = FileSystem.get(conf);
- fileSystem.copyFromLocalFile(new Path(source.getPath()), tmpDir);
- String filePath = tmpDir + File.separator + getFileName(source);
- Path fullPath = fileSystem.getFileStatus(new Path(filePath)).getPath();
+ fileSystem.copyFromLocalFile(localFile, remoteFile);
+ Path fullPath = fileSystem.getFileStatus(remoteFile).getPath();
return fullPath.toUri();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
index 36e252c..af6332e 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
@@ -57,7 +57,7 @@ public interface JobContext {
/**
* Return all added jar path which added through AddJarJob.
*/
- List<String> getAddedJars();
+ Set<String> getAddedJars();
/**
* Returns a local tmp dir specific to the context
http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
index 164d90a..beed8a3 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobContextImpl.java
@@ -18,6 +18,7 @@
package org.apache.hive.spark.client;
import java.io.File;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -34,14 +35,14 @@ class JobContextImpl implements JobContext {
private final JavaSparkContext sc;
private final ThreadLocal<MonitorCallback> monitorCb;
private final Map<String, List<JavaFutureAction<?>>> monitoredJobs;
- private final List<String> addedJars;
+ private final Set<String> addedJars;
private final File localTmpDir;
public JobContextImpl(JavaSparkContext sc, File localTmpDir) {
this.sc = sc;
this.monitorCb = new ThreadLocal<MonitorCallback>();
monitoredJobs = new ConcurrentHashMap<String, List<JavaFutureAction<?>>>();
- addedJars = new CopyOnWriteArrayList<String>();
+ addedJars = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
this.localTmpDir = localTmpDir;
}
@@ -64,7 +65,7 @@ class JobContextImpl implements JobContext {
}
@Override
- public List<String> getAddedJars() {
+ public Set<String> getAddedJars() {
return addedJars;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3f27c6ed/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
index 879f8a4..b079ee2 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -38,9 +39,9 @@ public class SparkClientUtilities {
/**
* Add new elements to the classpath.
*
- * @param newPaths Array of classpath elements
+ * @param newPaths Set of classpath elements
*/
- public static void addToClassPath(String[] newPaths, Configuration conf, File localTmpDir)
+ public static void addToClassPath(Set<String> newPaths, Configuration conf, File localTmpDir)
throws Exception {
ClassLoader cloader = Thread.currentThread().getContextClassLoader();
URLClassLoader loader = (URLClassLoader) cloader;
@@ -74,9 +75,11 @@ public class SparkClientUtilities {
Path remoteFile = new Path(path);
Path localFile =
new Path(localTmpDir.getAbsolutePath() + File.separator + remoteFile.getName());
- LOG.info("Copying " + remoteFile + " to " + localFile);
- FileSystem fs = remoteFile.getFileSystem(conf);
- fs.copyToLocalFile(remoteFile, localFile);
+ if (!new File(localFile.toString()).exists()) {
+ LOG.info("Copying " + remoteFile + " to " + localFile);
+ FileSystem remoteFS = remoteFile.getFileSystem(conf);
+ remoteFS.copyToLocalFile(remoteFile, localFile);
+ }
return urlFromPathString(localFile.toString(), conf, localTmpDir);
} else {
url = new File(path).toURL();
[43/50] [abbrv] hive git commit: HIVE-10900 : Fix the indeterministic
stats for some hive queries (Pengcheng Xiong via Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10900 : Fix the indeterministic stats for some hive queries (Pengcheng Xiong via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3ea0359e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3ea0359e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3ea0359e
Branch: refs/heads/spark
Commit: 3ea0359e2a074111563aa329de7af06459c90074
Parents: 3626a79
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Fri Jun 5 15:32:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 6 11:22:16 2015 -0700
----------------------------------------------------------------------
.../positive/accumulo_predicate_pushdown.q.out | 76 ++++++++++----------
.../results/positive/accumulo_queries.q.out | 70 +++++++++---------
2 files changed, 73 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3ea0359e/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out b/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
index 47e92c2..eb03041 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_predicate_pushdown.q.out
@@ -39,17 +39,17 @@ STAGE PLANS:
TableScan
alias: accumulo_pushdown
filterExpr: (key > '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (key > '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -195,17 +195,17 @@ STAGE PLANS:
TableScan
alias: accumulo_pushdown
filterExpr: (key >= '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (key >= '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -250,17 +250,17 @@ STAGE PLANS:
TableScan
alias: accumulo_pushdown
filterExpr: (key > '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (value like '%9%') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -306,17 +306,17 @@ STAGE PLANS:
TableScan
alias: accumulo_pushdown
filterExpr: (key >= '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: ((value like '%9%') and (UDFToDouble(key) = UDFToDouble(UDFToInteger(value)))) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -359,17 +359,17 @@ STAGE PLANS:
TableScan
alias: accumulo_pushdown
filterExpr: ((key < '80') and (key > '90')) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (value like '%90%') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -409,11 +409,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: accumulo_pushdown
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
PREHOOK: query: -- with a predicate which is not actually part of the filter, so
@@ -438,17 +438,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: accumulo_pushdown
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (CASE WHEN ((key < '90')) THEN (2) ELSE (4) END > 3) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -482,17 +482,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: accumulo_pushdown
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: ((key <= '80') or (value like '%90%')) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -521,17 +521,17 @@ STAGE PLANS:
TableScan
alias: accumulo_pushdown
filterExpr: ((key > '281') and (key < '287')) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: ((key > '281') and (key < '287')) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -576,17 +576,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: accumulo_pushdown
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: (key <= '90') (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/3ea0359e/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
index c99c423..d3c717b 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
@@ -119,19 +119,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: accumulo_table_1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: UDFToDouble(key) is not null (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: UDFToDouble(_col0) (type: double)
sort order: +
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -152,9 +152,9 @@ STAGE PLANS:
Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0
- 1 {VALUE._col0} {VALUE._col1}
+ keys:
+ 0 UDFToDouble(_col0) (type: double)
+ 1 UDFToDouble(_col0) (type: double)
outputColumnNames: _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -270,49 +270,49 @@ STAGE PLANS:
TableScan
alias: accumulo_table_1
filterExpr: (100 < key) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TableScan
alias: accumulo_table_2
filterExpr: (key < 120) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: string)
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0
- 1 {KEY.reducesinkkey0} {VALUE._col0}
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
outputColumnNames: _col2, _col3
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: _col2 (type: int), _col3 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
table:
@@ -327,15 +327,15 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -516,7 +516,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -529,43 +529,43 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: accumulo_table_1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: UDFToDouble(key) is not null (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: UDFToDouble(_col0) (type: double)
sort order: +
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string)
TableScan
Reduce Output Operator
key expressions: UDFToDouble(_col0) (type: double)
sort order: +
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1}
- 1 {VALUE._col1}
+ keys:
+ 0 UDFToDouble(_col0) (type: double)
+ 1 UDFToDouble(_col0) (type: double)
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), UDFToInteger(_col3) (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.accumulo.mr.HiveAccumuloTableInputFormat
output format: org.apache.hadoop.hive.accumulo.mr.HiveAccumuloTableOutputFormat
[23/50] [abbrv] hive git commit: HIVE-10887 : TestCliDriver tests
ordering issues with Mac and CentOS (Hari Subramaniyan,
reviewed by Thejas Nair)
Posted by xu...@apache.org.
HIVE-10887 : TestCliDriver tests ordering issues with Mac and CentOS (Hari Subramaniyan, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4f57d88
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4f57d88
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4f57d88
Branch: refs/heads/spark
Commit: b4f57d8885a85a8d977fa48fa756c5c2b1cd2be7
Parents: 0910083
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Wed Jun 3 13:30:50 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Wed Jun 3 13:30:50 2015 -0700
----------------------------------------------------------------------
.../queries/clientpositive/unionDistinct_2.q | 2 +
.../update_after_multiple_inserts.q | 2 +
.../clientpositive/tez/unionDistinct_2.q.out | 38 ++++++++++--------
.../tez/update_after_multiple_inserts.q.out | 12 ++++--
.../clientpositive/unionDistinct_2.q.out | 42 +++++++++++---------
.../update_after_multiple_inserts.q.out | 12 ++++--
6 files changed, 64 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/queries/clientpositive/unionDistinct_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/unionDistinct_2.q b/ql/src/test/queries/clientpositive/unionDistinct_2.q
index ab78128..324bf68 100644
--- a/ql/src/test/queries/clientpositive/unionDistinct_2.q
+++ b/ql/src/test/queries/clientpositive/unionDistinct_2.q
@@ -1,3 +1,5 @@
+-- SORT_QUERY_RESULTS
+
CREATE TABLE u1 as select key, value from src order by key limit 5;
CREATE TABLE u2 as select key, value from src order by key limit 3;
http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q b/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
index 6ef209f..eb8e5a1 100644
--- a/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
+++ b/ql/src/test/queries/clientpositive/update_after_multiple_inserts.q
@@ -3,6 +3,8 @@ set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.enforce.bucketing=true;
+-- SORT_QUERY_RESULTS
+
create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
index 1ba78b7..6d59369 100644
--- a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out
@@ -1,9 +1,13 @@
-PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@u1
-POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
@@ -60,11 +64,11 @@ POSTHOOK: query: select * from u3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@u3
#### A masked pattern was here ####
-98 val_98
-98 val_98
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -94,16 +98,16 @@ POSTHOOK: Input: default@u3
0 val_0
0 val_0
0 val_0
-10 val_10
-100 val_100
0 val_0
0 val_0
0 val_0
-98 val_98
-98 val_98
+10 val_10
+100 val_100
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -133,11 +137,11 @@ POSTHOOK: Input: default@u3
0 val_0
10 val_10
100 val_100
-98 val_98
-98 val_98
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -167,11 +171,11 @@ POSTHOOK: Input: default@u3
0 val_0
10 val_10
100 val_100
-98 val_98
-98 val_98
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -504,9 +508,9 @@ POSTHOOK: Input: default@u2
POSTHOOK: Input: default@v
#### A masked pattern was here ####
0 val_0
+0 val_0
10 val_10
100 val_100
-0 val_0
PREHOOK: query: select distinct * from
(
select key, value from u1
http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out b/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
index a2ad3af..d536fe9 100644
--- a/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
+++ b/ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out
@@ -1,10 +1,14 @@
-PREHOOK: query: create table acid_uami(i int,
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@acid_uami
-POSTHOOK: query: create table acid_uami(i int,
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
POSTHOOK: type: CREATETABLE
@@ -50,8 +54,8 @@ POSTHOOK: Input: default@acid_uami
#### A masked pattern was here ####
1 109.23 mary had a little lamb
10 119.23 and everywhere that mary went
-65530 823.19 the lamb was sure to go
6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
PREHOOK: type: QUERY
PREHOOK: Input: default@acid_uami
@@ -70,5 +74,5 @@ POSTHOOK: Input: default@acid_uami
#### A masked pattern was here ####
1 3.14 mary had a little lamb
10 3.14 and everywhere that mary went
-65530 823.19 the lamb was sure to go
6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/unionDistinct_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/unionDistinct_2.q.out
index 66df734..6d59369 100644
--- a/ql/src/test/results/clientpositive/unionDistinct_2.q.out
+++ b/ql/src/test/results/clientpositive/unionDistinct_2.q.out
@@ -1,9 +1,13 @@
-PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@u1
-POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE u1 as select key, value from src order by key limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
@@ -60,11 +64,11 @@ POSTHOOK: query: select * from u3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@u3
#### A masked pattern was here ####
-98 val_98
-98 val_98
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -91,19 +95,19 @@ POSTHOOK: Input: default@u1
POSTHOOK: Input: default@u2
POSTHOOK: Input: default@u3
#### A masked pattern was here ####
-98 val_98
-98 val_98
-97 val_97
-97 val_97
-96 val_96
0 val_0
0 val_0
0 val_0
-10 val_10
-100 val_100
0 val_0
0 val_0
0 val_0
+10 val_10
+100 val_100
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -133,11 +137,11 @@ POSTHOOK: Input: default@u3
0 val_0
10 val_10
100 val_100
-98 val_98
-98 val_98
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -167,11 +171,11 @@ POSTHOOK: Input: default@u3
0 val_0
10 val_10
100 val_100
-98 val_98
-98 val_98
+96 val_96
97 val_97
97 val_97
-96 val_96
+98 val_98
+98 val_98
PREHOOK: query: select key, value from
(
select key, value from u1
@@ -504,9 +508,9 @@ POSTHOOK: Input: default@u2
POSTHOOK: Input: default@v
#### A masked pattern was here ####
0 val_0
+0 val_0
10 val_10
100 val_100
-0 val_0
PREHOOK: query: select distinct * from
(
select key, value from u1
http://git-wip-us.apache.org/repos/asf/hive/blob/b4f57d88/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out b/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
index a2ad3af..d536fe9 100644
--- a/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
+++ b/ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out
@@ -1,10 +1,14 @@
-PREHOOK: query: create table acid_uami(i int,
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@acid_uami
-POSTHOOK: query: create table acid_uami(i int,
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
POSTHOOK: type: CREATETABLE
@@ -50,8 +54,8 @@ POSTHOOK: Input: default@acid_uami
#### A masked pattern was here ####
1 109.23 mary had a little lamb
10 119.23 and everywhere that mary went
-65530 823.19 the lamb was sure to go
6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
PREHOOK: type: QUERY
PREHOOK: Input: default@acid_uami
@@ -70,5 +74,5 @@ POSTHOOK: Input: default@acid_uami
#### A masked pattern was here ####
1 3.14 mary had a little lamb
10 3.14 and everywhere that mary went
-65530 823.19 the lamb was sure to go
6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
[21/50] [abbrv] hive git commit: HIVE-10855: with vectorization
enabled join operation involving interval_day_time fails (Matt McCline
reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-10855: with vectorization enabled join operation involving interval_day_time fails (Matt McCline reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09100831
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09100831
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09100831
Branch: refs/heads/spark
Commit: 09100831adff7589ee48e735a4beac6ebb25cb3e
Parents: 24d3307
Author: Matt McCline <mm...@hortonworks.com>
Authored: Tue Jun 2 20:32:14 2015 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Jun 2 20:32:14 2015 -0700
----------------------------------------------------------------------
data/files/vectortab_a_1k | 1000 ++++++++++++++++++
data/files/vectortab_b_1k | 1000 ++++++++++++++++++
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/exec/vector/ColumnVector.java | 10 +
.../ql/exec/vector/VectorColumnSetInfo.java | 42 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 35 +-
.../ql/exec/vector/VectorizationContext.java | 46 +
.../mapjoin/VectorMapJoinCommonOperator.java | 50 +-
.../clientpositive/vector_interval_mapjoin.q | 87 ++
.../tez/vector_interval_mapjoin.q.out | 281 +++++
.../vector_interval_mapjoin.q.out | 281 +++++
11 files changed, 2797 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
[29/50] [abbrv] hive git commit: HIVE-10922: In HS2 doAs=false mode,
file system related errors in one query causes other failures (Thejas
M Nair, reviewed by Vikram Dixit K)
Posted by xu...@apache.org.
HIVE-10922: In HS2 doAs=false mode, file system related errors in one query causes other failures (Thejas M Nair, reviewed by Vikram Dixit K)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0e5d907e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0e5d907e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0e5d907e
Branch: refs/heads/spark
Commit: 0e5d907e57304a67752fe68108483c305f6f66a5
Parents: a96fbde
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Thu Jun 4 00:52:47 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Thu Jun 4 00:55:24 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/metastore/HiveAlterHandler.java | 10 ++++++----
.../apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java | 1 -
.../java/org/apache/hadoop/hive/metastore/Warehouse.java | 2 --
3 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0e5d907e/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 5391171..99d6680 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -59,20 +59,24 @@ public class HiveAlterHandler implements AlterHandler {
private static final Log LOG = LogFactory.getLog(HiveAlterHandler.class
.getName());
+ @Override
public Configuration getConf() {
return hiveConf;
}
+ @Override
@SuppressWarnings("nls")
public void setConf(Configuration conf) {
hiveConf = conf;
}
+ @Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname,
String name, Table newt) throws InvalidOperationException, MetaException {
alterTable(msdb, wh, dbname, name, newt, false);
}
+ @Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname,
String name, Table newt, boolean cascade) throws InvalidOperationException, MetaException {
if (newt == null) {
@@ -200,8 +204,6 @@ public class HiveAlterHandler implements AlterHandler {
+ " already exists : " + destPath);
}
} catch (IOException e) {
- Warehouse.closeFs(srcFs);
- Warehouse.closeFs(destFs);
throw new InvalidOperationException("Unable to access new location "
+ destPath + " for table " + newt.getDbName() + "."
+ newt.getTableName());
@@ -316,6 +318,7 @@ public class HiveAlterHandler implements AlterHandler {
}
return ex.getMessage();
}
+ @Override
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<String> part_vals, final Partition new_part)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
@@ -430,8 +433,6 @@ public class HiveAlterHandler implements AlterHandler {
+ " already exists : " + destPath);
}
} catch (IOException e) {
- Warehouse.closeFs(srcFs);
- Warehouse.closeFs(destFs);
throw new InvalidOperationException("Unable to access new location "
+ destPath + " for partition " + tbl.getDbName() + "."
+ tbl.getTableName() + " " + new_part.getValues());
@@ -491,6 +492,7 @@ public class HiveAlterHandler implements AlterHandler {
return oldPart;
}
+ @Override
public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<Partition> new_parts)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
http://git-wip-us.apache.org/repos/asf/hive/blob/0e5d907e/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
index c3598db..b255090 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
@@ -60,7 +60,6 @@ public class HiveMetaStoreFsImpl implements MetaStoreFS {
} catch (FileNotFoundException e) {
return true; // ok even if there is not data
} catch (Exception e) {
- Warehouse.closeFs(fs);
MetaStoreUtils.logAndThrowMetaException(e);
}
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/0e5d907e/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 011d4d8..bc0f6e3 100755
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -200,7 +200,6 @@ public class Warehouse {
fs = getFs(f);
return FileUtils.mkdir(fs, f, inheritPerms, conf);
} catch (IOException e) {
- closeFs(fs);
MetaStoreUtils.logAndThrowMetaException(e);
}
return false;
@@ -480,7 +479,6 @@ public class Warehouse {
} catch (FileNotFoundException e) {
return false;
} catch (IOException e) {
- closeFs(fs);
MetaStoreUtils.logAndThrowMetaException(e);
}
return true;
[36/50] [abbrv] hive git commit: HIVE-10934: Restore support for DROP
PARTITION PURGE (Eugene Koifman via Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-10934: Restore support for DROP PARTITION PURGE (Eugene Koifman via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bef52661
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bef52661
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bef52661
Branch: refs/heads/spark
Commit: bef52661cdacc22ea6dc06d7b09cad49a63d0bf9
Parents: 17bedcc
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Thu Jun 4 16:14:25 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Thu Jun 4 16:14:25 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/parse/HiveParser.g | 4 +-
.../clientpositive/alter_table_partition_drop.q | 11 +++
.../alter_table_partition_drop.q.out | 74 ++++++++++++++++++++
3 files changed, 87 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bef52661/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index b267bd2..15f1f11 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -1166,8 +1166,8 @@ partitionLocation
alterStatementSuffixDropPartitions[boolean table]
@init { pushMsg("drop partition statement", state); }
@after { popMsg(state); }
- : KW_DROP ifExists? dropPartitionSpec (COMMA dropPartitionSpec)* ignoreProtection? replicationClause?
- -> { table }? ^(TOK_ALTERTABLE_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection? replicationClause?)
+ : KW_DROP ifExists? dropPartitionSpec (COMMA dropPartitionSpec)* ignoreProtection? KW_PURGE? replicationClause?
+ -> { table }? ^(TOK_ALTERTABLE_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection? KW_PURGE? replicationClause?)
-> ^(TOK_ALTERVIEW_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection? replicationClause?)
;
http://git-wip-us.apache.org/repos/asf/hive/blob/bef52661/ql/src/test/queries/clientpositive/alter_table_partition_drop.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alter_table_partition_drop.q b/ql/src/test/queries/clientpositive/alter_table_partition_drop.q
new file mode 100644
index 0000000..fb1d401
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_table_partition_drop.q
@@ -0,0 +1,11 @@
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+DROP TABLE IF EXISTS part_table PURGE;
+CREATE TABLE part_table (key INT, value STRING) partitioned by (p STRING);
+
+INSERT INTO part_table PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24', 2, 'bar');
+SELECT * FROM part_table;
+ALTER TABLE part_table DROP PARTITION (p='2014-09-23');
+SELECT * FROM part_table;
+ALTER TABLE part_table DROP PARTITION (p='2014-09-24') PURGE;
+SELECT * FROM part_table;
http://git-wip-us.apache.org/repos/asf/hive/blob/bef52661/ql/src/test/results/clientpositive/alter_table_partition_drop.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_table_partition_drop.q.out b/ql/src/test/results/clientpositive/alter_table_partition_drop.q.out
new file mode 100644
index 0000000..9a94bb9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alter_table_partition_drop.q.out
@@ -0,0 +1,74 @@
+PREHOOK: query: DROP TABLE IF EXISTS part_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS part_table PURGE
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE part_table (key INT, value STRING) partitioned by (p STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_table
+POSTHOOK: query: CREATE TABLE part_table (key INT, value STRING) partitioned by (p STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_table
+PREHOOK: query: INSERT INTO part_table PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24', 2, 'bar')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@part_table
+POSTHOOK: query: INSERT INTO part_table PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24', 2, 'bar')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@part_table@p=2014-09-23
+POSTHOOK: Output: default@part_table@p=2014-09-24
+POSTHOOK: Lineage: part_table PARTITION(p=2014-09-23).key EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: part_table PARTITION(p=2014-09-23).value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: part_table PARTITION(p=2014-09-24).key EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: part_table PARTITION(p=2014-09-24).value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: SELECT * FROM part_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_table
+PREHOOK: Input: default@part_table@p=2014-09-23
+PREHOOK: Input: default@part_table@p=2014-09-24
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM part_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_table
+POSTHOOK: Input: default@part_table@p=2014-09-23
+POSTHOOK: Input: default@part_table@p=2014-09-24
+#### A masked pattern was here ####
+1 foo 2014-09-23
+2 bar 2014-09-24
+PREHOOK: query: ALTER TABLE part_table DROP PARTITION (p='2014-09-23')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@part_table
+PREHOOK: Output: default@part_table@p=2014-09-23
+POSTHOOK: query: ALTER TABLE part_table DROP PARTITION (p='2014-09-23')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@part_table
+POSTHOOK: Output: default@part_table@p=2014-09-23
+PREHOOK: query: SELECT * FROM part_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_table
+PREHOOK: Input: default@part_table@p=2014-09-24
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM part_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_table
+POSTHOOK: Input: default@part_table@p=2014-09-24
+#### A masked pattern was here ####
+2 bar 2014-09-24
+PREHOOK: query: ALTER TABLE part_table DROP PARTITION (p='2014-09-24') PURGE
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@part_table
+PREHOOK: Output: default@part_table@p=2014-09-24
+POSTHOOK: query: ALTER TABLE part_table DROP PARTITION (p='2014-09-24') PURGE
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@part_table
+POSTHOOK: Output: default@part_table@p=2014-09-24
+PREHOOK: query: SELECT * FROM part_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM part_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_table
+#### A masked pattern was here ####
[28/50] [abbrv] hive git commit: HIVE-10761 : Create codahale-based
metrics system for Hive (Szehon, reviewed by Xuefu)
Posted by xu...@apache.org.
HIVE-10761 : Create codahale-based metrics system for Hive (Szehon, reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a96fbdee
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a96fbdee
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a96fbdee
Branch: refs/heads/spark
Commit: a96fbdeef442fdeecca74d1c3f8a855b5c9d514b
Parents: 82beb2b
Author: Szehon Ho <sz...@cloudera.com>
Authored: Wed Jun 3 23:46:28 2015 -0700
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Wed Jun 3 23:46:28 2015 -0700
----------------------------------------------------------------------
common/pom.xml | 20 +
.../hadoop/hive/common/JvmPauseMonitor.java | 225 ++++++++++++
.../hive/common/metrics/LegacyMetrics.java | 262 +++++++++++++
.../hadoop/hive/common/metrics/Metrics.java | 253 -------------
.../hive/common/metrics/common/Metrics.java | 68 ++++
.../common/metrics/common/MetricsFactory.java | 48 +++
.../metrics/metrics2/CodahaleMetrics.java | 366 +++++++++++++++++++
.../metrics/metrics2/MetricsReporting.java | 27 ++
.../org/apache/hadoop/hive/conf/HiveConf.java | 18 +-
.../hive/common/metrics/TestLegacyMetrics.java | 295 +++++++++++++++
.../hadoop/hive/common/metrics/TestMetrics.java | 286 ---------------
.../metrics/metrics2/TestCodahaleMetrics.java | 138 +++++++
.../hive/metastore/TestMetaStoreMetrics.java | 94 +++++
.../hadoop/hive/metastore/HiveMetaStore.java | 132 ++++---
pom.xml | 3 +
.../apache/hive/service/server/HiveServer2.java | 25 +-
.../hadoop/hive/shims/Hadoop20SShims.java | 5 -
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 13 -
.../apache/hadoop/hive/shims/HadoopShims.java | 2 -
19 files changed, 1665 insertions(+), 615 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index a615c1e..8d4b1ea 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -98,6 +98,26 @@
<artifactId>json</artifactId>
<version>${json.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.new.version}</version>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java b/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
new file mode 100644
index 0000000..c3949f2
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.util.Daemon;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Based on the JvmPauseMonitor from Hadoop.
+ */
+public class JvmPauseMonitor {
+ private static final Log LOG = LogFactory.getLog(
+ JvmPauseMonitor.class);
+
+ /** The target sleep time */
+ private static final long SLEEP_INTERVAL_MS = 500;
+
+ /** log WARN if we detect a pause longer than this threshold */
+ private final long warnThresholdMs;
+ private static final String WARN_THRESHOLD_KEY =
+ "jvm.pause.warn-threshold.ms";
+ private static final long WARN_THRESHOLD_DEFAULT = 10000;
+
+ /** log INFO if we detect a pause longer than this threshold */
+ private final long infoThresholdMs;
+ private static final String INFO_THRESHOLD_KEY =
+ "jvm.pause.info-threshold.ms";
+ private static final long INFO_THRESHOLD_DEFAULT = 1000;
+
+ private long numGcWarnThresholdExceeded = 0;
+ private long numGcInfoThresholdExceeded = 0;
+ private long totalGcExtraSleepTime = 0;
+
+ private Thread monitorThread;
+ private volatile boolean shouldRun = true;
+
+ public JvmPauseMonitor(Configuration conf) {
+ this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT);
+ this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT);
+ }
+
+ public void start() {
+ Preconditions.checkState(monitorThread == null,
+ "JvmPauseMonitor thread is Already started");
+ monitorThread = new Daemon(new Monitor());
+ monitorThread.start();
+ }
+
+ public void stop() {
+ shouldRun = false;
+ if (isStarted()) {
+ monitorThread.interrupt();
+ try {
+ monitorThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ public boolean isStarted() {
+ return monitorThread != null;
+ }
+
+ public long getNumGcWarnThreadholdExceeded() {
+ return numGcWarnThresholdExceeded;
+ }
+
+ public long getNumGcInfoThresholdExceeded() {
+ return numGcInfoThresholdExceeded;
+ }
+
+ public long getTotalGcExtraSleepTime() {
+ return totalGcExtraSleepTime;
+ }
+
+ private String formatMessage(long extraSleepTime,
+ Map<String, GcTimes> gcTimesAfterSleep,
+ Map<String, GcTimes> gcTimesBeforeSleep) {
+
+ Set<String> gcBeanNames = Sets.intersection(
+ gcTimesAfterSleep.keySet(),
+ gcTimesBeforeSleep.keySet());
+ List<String> gcDiffs = Lists.newArrayList();
+ for (String name : gcBeanNames) {
+ GcTimes diff = gcTimesAfterSleep.get(name).subtract(
+ gcTimesBeforeSleep.get(name));
+ if (diff.gcCount != 0) {
+ gcDiffs.add("GC pool '" + name + "' had collection(s): " +
+ diff.toString());
+ }
+ }
+
+ String ret = "Detected pause in JVM or host machine (eg GC): " +
+ "pause of approximately " + extraSleepTime + "ms\n";
+ if (gcDiffs.isEmpty()) {
+ ret += "No GCs detected";
+ } else {
+ ret += Joiner.on("\n").join(gcDiffs);
+ }
+ return ret;
+ }
+
+ private Map<String, GcTimes> getGcTimes() {
+ Map<String, GcTimes> map = Maps.newHashMap();
+ List<GarbageCollectorMXBean> gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ for (GarbageCollectorMXBean gcBean : gcBeans) {
+ map.put(gcBean.getName(), new GcTimes(gcBean));
+ }
+ return map;
+ }
+
+ private static class GcTimes {
+ private GcTimes(GarbageCollectorMXBean gcBean) {
+ gcCount = gcBean.getCollectionCount();
+ gcTimeMillis = gcBean.getCollectionTime();
+ }
+
+ private GcTimes(long count, long time) {
+ this.gcCount = count;
+ this.gcTimeMillis = time;
+ }
+
+ private GcTimes subtract(GcTimes other) {
+ return new GcTimes(this.gcCount - other.gcCount,
+ this.gcTimeMillis - other.gcTimeMillis);
+ }
+
+ @Override
+ public String toString() {
+ return "count=" + gcCount + " time=" + gcTimeMillis + "ms";
+ }
+
+ private long gcCount;
+ private long gcTimeMillis;
+ }
+
+ private class Monitor implements Runnable {
+ @Override
+ public void run() {
+ Stopwatch sw = new Stopwatch();
+ Map<String, GcTimes> gcTimesBeforeSleep = getGcTimes();
+ while (shouldRun) {
+ sw.reset().start();
+ try {
+ Thread.sleep(SLEEP_INTERVAL_MS);
+ } catch (InterruptedException ie) {
+ return;
+ }
+ long extraSleepTime = sw.elapsedMillis() - SLEEP_INTERVAL_MS;
+ Map<String, GcTimes> gcTimesAfterSleep = getGcTimes();
+
+ if (extraSleepTime > warnThresholdMs) {
+ ++numGcWarnThresholdExceeded;
+ LOG.warn(formatMessage(
+ extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
+ incrementMetricsCounter("jvm.pause.warn-threshold", 1);
+ } else if (extraSleepTime > infoThresholdMs) {
+ ++numGcInfoThresholdExceeded;
+ LOG.info(formatMessage(
+ extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
+ incrementMetricsCounter("jvm.pause.info-threshold", 1);
+ }
+ incrementMetricsCounter("jvm.pause.extraSleepTime", extraSleepTime);
+ totalGcExtraSleepTime += extraSleepTime;
+ gcTimesBeforeSleep = gcTimesAfterSleep;
+ }
+ }
+
+ private void incrementMetricsCounter(String name, long count) {
+ try {
+ MetricsFactory.getMetricsInstance().incrementCounter(name, count);
+ } catch (Exception e) {
+ LOG.warn("Error Reporting JvmPauseMonitor to Metrics system", e);
+ }
+ }
+ }
+
+ /**
+ * Simple 'main' to facilitate manual testing of the pause monitor.
+ *
+ * This main function just leaks memory into a list. Running this class
+ * with a 1GB heap will very quickly go into "GC hell" and result in
+ * log messages about the GC pauses.
+ */
+ public static void main(String []args) throws Exception {
+ new JvmPauseMonitor(new Configuration()).start();
+ List<String> list = Lists.newArrayList();
+ int i = 0;
+ while (true) {
+ list.add(String.valueOf(i++));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
new file mode 100644
index 0000000..14f7afb
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics;
+
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+/**
+ * This class may eventually get superseded by org.apache.hadoop.hive.common.metrics2.Metrics.
+ *
+ * Metrics Subsystem - allows exposure of a number of named parameters/counters
+ * via jmx, intended to be used as a static subsystem
+ *
+ * Has a couple of primary ways it can be used:
+ * (i) Using the set and get methods to set and get named parameters
+ * (ii) Using the incrementCounter method to increment and set named
+ * parameters in one go, rather than having to make a get and then a set.
+ * (iii) Using the startScope and endScope methods to start and end
+ * named "scopes" that record the number of times they've been
+ * instantiated and amount of time(in milliseconds) spent inside
+ * the scopes.
+ */
+public class LegacyMetrics implements Metrics {
+
+ private LegacyMetrics() {
+ // block
+ }
+
+ /**
+ * MetricsScope : A class that encapsulates an idea of a metered scope.
+ * Instantiating a named scope and then closing it exposes two counters:
+ * (i) a "number of calls" counter ( <name>.n ), and
+ * (ii) a "number of msecs spent between scope open and close" counter. ( <name>.t)
+ */
+ public static class MetricsScope {
+
+ final LegacyMetrics metrics;
+
+ final String name;
+ final String numCounter;
+ final String timeCounter;
+ final String avgTimeCounter;
+
+ private boolean isOpen = false;
+ private Long startTime = null;
+
+ /**
+ * Instantiates a named scope - intended to only be called by Metrics, so locally scoped.
+ * @param name - name of the variable
+ * @throws IOException
+ */
+ private MetricsScope(String name, LegacyMetrics metrics) throws IOException {
+ this.metrics = metrics;
+ this.name = name;
+ this.numCounter = name + ".n";
+ this.timeCounter = name + ".t";
+ this.avgTimeCounter = name + ".avg_t";
+ open();
+ }
+
+ public Long getNumCounter() throws IOException {
+ return (Long) metrics.get(numCounter);
+ }
+
+ public Long getTimeCounter() throws IOException {
+ return (Long) metrics.get(timeCounter);
+ }
+
+ /**
+ * Opens scope, and makes note of the time started, increments run counter
+ * @throws IOException
+ *
+ */
+ public void open() throws IOException {
+ if (!isOpen) {
+ isOpen = true;
+ startTime = System.currentTimeMillis();
+ } else {
+ throw new IOException("Scope named " + name + " is not closed, cannot be opened.");
+ }
+ }
+
+ /**
+ * Closes scope, and records the time taken
+ * @throws IOException
+ */
+ public void close() throws IOException {
+ if (isOpen) {
+ Long endTime = System.currentTimeMillis();
+ synchronized(metrics) {
+ Long num = metrics.incrementCounter(numCounter);
+ Long time = metrics.incrementCounter(timeCounter, endTime - startTime);
+ if (num != null && time != null) {
+ metrics.set(avgTimeCounter, Double.valueOf(time.doubleValue() / num.doubleValue()));
+ }
+ }
+ } else {
+ throw new IOException("Scope named " + name + " is not open, cannot be closed.");
+ }
+ isOpen = false;
+ }
+
+
+ /**
+ * Closes scope if open, and reopens it
+ * @throws IOException
+ */
+ public void reopen() throws IOException {
+ if(isOpen) {
+ close();
+ }
+ open();
+ }
+
+ }
+
+ private static final MetricsMBean metrics = new MetricsMBeanImpl();
+
+ private static final ObjectName oname;
+ static {
+ try {
+ oname = new ObjectName(
+ "org.apache.hadoop.hive.common.metrics:type=MetricsMBean");
+ } catch (MalformedObjectNameException mone) {
+ throw new RuntimeException(mone);
+ }
+ }
+
+
+ private static final ThreadLocal<HashMap<String, MetricsScope>> threadLocalScopes
+ = new ThreadLocal<HashMap<String,MetricsScope>>() {
+ @Override
+ protected HashMap<String,MetricsScope> initialValue() {
+ return new HashMap<String,MetricsScope>();
+ }
+ };
+
+ private boolean initialized = false;
+
+ public void init(HiveConf conf) throws Exception {
+ if (!initialized) {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ mbs.registerMBean(metrics, oname);
+ initialized = true;
+ }
+ }
+
+ public boolean isInitialized() {
+ return initialized;
+ }
+
+ public Long incrementCounter(String name) throws IOException{
+ if (!initialized) {
+ return null;
+ }
+ return incrementCounter(name,Long.valueOf(1));
+ }
+
+ public Long incrementCounter(String name, long increment) throws IOException{
+ if (!initialized) {
+ return null;
+ }
+ Long value;
+ synchronized(metrics) {
+ if (!metrics.hasKey(name)) {
+ value = Long.valueOf(increment);
+ set(name, value);
+ } else {
+ value = ((Long)get(name)) + increment;
+ set(name, value);
+ }
+ }
+ return value;
+ }
+
+ public void set(String name, Object value) throws IOException{
+ if (!initialized) {
+ return;
+ }
+ metrics.put(name,value);
+ }
+
+ public Object get(String name) throws IOException{
+ if (!initialized) {
+ return null;
+ }
+ return metrics.get(name);
+ }
+
+ public void startScope(String name) throws IOException{
+ if (!initialized) {
+ return;
+ }
+ if (threadLocalScopes.get().containsKey(name)) {
+ threadLocalScopes.get().get(name).open();
+ } else {
+ threadLocalScopes.get().put(name, new MetricsScope(name, this));
+ }
+ }
+
+ public MetricsScope getScope(String name) throws IOException {
+ if (!initialized) {
+ return null;
+ }
+ if (threadLocalScopes.get().containsKey(name)) {
+ return threadLocalScopes.get().get(name);
+ } else {
+ throw new IOException("No metrics scope named " + name);
+ }
+ }
+
+ public void endScope(String name) throws IOException{
+ if (!initialized) {
+ return;
+ }
+ if (threadLocalScopes.get().containsKey(name)) {
+ threadLocalScopes.get().get(name).close();
+ }
+ }
+
+ /**
+ * Resets the static context state to initial.
+ * Used primarily for testing purposes.
+ *
+ * Note that threadLocalScopes ThreadLocal is *not* cleared in this call.
+ */
+ public void deInit() throws Exception {
+ synchronized (metrics) {
+ if (initialized) {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (mbs.isRegistered(oname)) {
+ mbs.unregisterMBean(oname);
+ }
+ metrics.clear();
+ initialized = false;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/metrics/Metrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/Metrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/Metrics.java
deleted file mode 100644
index 01c9d1d..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/Metrics.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.metrics;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.HashMap;
-
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-
-/**
- * Metrics Subsystem - allows exposure of a number of named parameters/counters
- * via jmx, intended to be used as a static subsystem
- *
- * Has a couple of primary ways it can be used:
- * (i) Using the set and get methods to set and get named parameters
- * (ii) Using the incrementCounter method to increment and set named
- * parameters in one go, rather than having to make a get and then a set.
- * (iii) Using the startScope and endScope methods to start and end
- * named "scopes" that record the number of times they've been
- * instantiated and amount of time(in milliseconds) spent inside
- * the scopes.
- */
-public class Metrics {
-
- private Metrics() {
- // block
- }
-
- /**
- * MetricsScope : A class that encapsulates an idea of a metered scope.
- * Instantiating a named scope and then closing it exposes two counters:
- * (i) a "number of calls" counter ( <name>.n ), and
- * (ii) a "number of msecs spent between scope open and close" counter. ( <name>.t)
- */
- public static class MetricsScope {
-
- final String name;
- final String numCounter;
- final String timeCounter;
- final String avgTimeCounter;
-
- private boolean isOpen = false;
- private Long startTime = null;
-
- /**
- * Instantiates a named scope - intended to only be called by Metrics, so locally scoped.
- * @param name - name of the variable
- * @throws IOException
- */
- private MetricsScope(String name) throws IOException {
- this.name = name;
- this.numCounter = name + ".n";
- this.timeCounter = name + ".t";
- this.avgTimeCounter = name + ".avg_t";
- open();
- }
-
- public Long getNumCounter() throws IOException {
- return (Long)Metrics.get(numCounter);
- }
-
- public Long getTimeCounter() throws IOException {
- return (Long)Metrics.get(timeCounter);
- }
-
- /**
- * Opens scope, and makes note of the time started, increments run counter
- * @throws IOException
- *
- */
- public void open() throws IOException {
- if (!isOpen) {
- isOpen = true;
- startTime = System.currentTimeMillis();
- } else {
- throw new IOException("Scope named " + name + " is not closed, cannot be opened.");
- }
- }
-
- /**
- * Closes scope, and records the time taken
- * @throws IOException
- */
- public void close() throws IOException {
- if (isOpen) {
- Long endTime = System.currentTimeMillis();
- synchronized(metrics) {
- Long num = Metrics.incrementCounter(numCounter);
- Long time = Metrics.incrementCounter(timeCounter, endTime - startTime);
- if (num != null && time != null) {
- Metrics.set(avgTimeCounter, Double.valueOf(time.doubleValue() / num.doubleValue()));
- }
- }
- } else {
- throw new IOException("Scope named " + name + " is not open, cannot be closed.");
- }
- isOpen = false;
- }
-
-
- /**
- * Closes scope if open, and reopens it
- * @throws IOException
- */
- public void reopen() throws IOException {
- if(isOpen) {
- close();
- }
- open();
- }
-
- }
-
- private static final MetricsMBean metrics = new MetricsMBeanImpl();
-
- private static final ObjectName oname;
- static {
- try {
- oname = new ObjectName(
- "org.apache.hadoop.hive.common.metrics:type=MetricsMBean");
- } catch (MalformedObjectNameException mone) {
- throw new RuntimeException(mone);
- }
- }
-
-
- private static final ThreadLocal<HashMap<String, MetricsScope>> threadLocalScopes
- = new ThreadLocal<HashMap<String,MetricsScope>>() {
- @Override
- protected HashMap<String,MetricsScope> initialValue() {
- return new HashMap<String,MetricsScope>();
- }
- };
-
- private static boolean initialized = false;
-
- public static void init() throws Exception {
- synchronized (metrics) {
- if (!initialized) {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- mbs.registerMBean(metrics, oname);
- initialized = true;
- }
- }
- }
-
- public static Long incrementCounter(String name) throws IOException{
- if (!initialized) {
- return null;
- }
- return incrementCounter(name,Long.valueOf(1));
- }
-
- public static Long incrementCounter(String name, long increment) throws IOException{
- if (!initialized) {
- return null;
- }
- Long value;
- synchronized(metrics) {
- if (!metrics.hasKey(name)) {
- value = Long.valueOf(increment);
- set(name, value);
- } else {
- value = ((Long)get(name)) + increment;
- set(name, value);
- }
- }
- return value;
- }
-
- public static void set(String name, Object value) throws IOException{
- if (!initialized) {
- return;
- }
- metrics.put(name,value);
- }
-
- public static Object get(String name) throws IOException{
- if (!initialized) {
- return null;
- }
- return metrics.get(name);
- }
-
- public static MetricsScope startScope(String name) throws IOException{
- if (!initialized) {
- return null;
- }
- if (threadLocalScopes.get().containsKey(name)) {
- threadLocalScopes.get().get(name).open();
- } else {
- threadLocalScopes.get().put(name, new MetricsScope(name));
- }
- return threadLocalScopes.get().get(name);
- }
-
- public static MetricsScope getScope(String name) throws IOException {
- if (!initialized) {
- return null;
- }
- if (threadLocalScopes.get().containsKey(name)) {
- return threadLocalScopes.get().get(name);
- } else {
- throw new IOException("No metrics scope named " + name);
- }
- }
-
- public static void endScope(String name) throws IOException{
- if (!initialized) {
- return;
- }
- if (threadLocalScopes.get().containsKey(name)) {
- threadLocalScopes.get().get(name).close();
- }
- }
-
- /**
- * Resets the static context state to initial.
- * Used primarily for testing purposes.
- *
- * Note that threadLocalScopes ThreadLocal is *not* cleared in this call.
- */
- static void uninit() throws Exception {
- synchronized (metrics) {
- if (initialized) {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- if (mbs.isRegistered(oname)) {
- mbs.unregisterMBean(oname);
- }
- metrics.clear();
- initialized = false;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
new file mode 100644
index 0000000..13a5336
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import java.io.IOException;
+
+/**
+ * Generic Metics interface.
+ */
+public interface Metrics {
+
+ /**
+ * Initialize Metrics system with given Hive configuration.
+ * @param conf
+ */
+ public void init(HiveConf conf) throws Exception;
+
+ /**
+ * Deinitializes the Metrics system.
+ */
+ public void deInit() throws Exception;
+
+ /**
+ * @param name
+ * @throws IOException
+ */
+ public void startScope(String name) throws IOException;
+
+ public void endScope(String name) throws IOException;
+
+ //Counter-related methods
+
+ /**
+ * Increments a counter of the given name by 1.
+ * @param name
+ * @return
+ * @throws IOException
+ */
+ public Long incrementCounter(String name) throws IOException;
+
+ /**
+ * Increments a counter of the given name by "increment"
+ * @param name
+ * @param increment
+ * @return
+ * @throws IOException
+ */
+ public Long incrementCounter(String name, long increment) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
new file mode 100644
index 0000000..12a309d
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.common;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Class that manages a static Metric instance for this process.
+ */
+public class MetricsFactory {
+
+ private static Metrics metrics;
+ private static Object initLock = new Object();
+
+ public synchronized static void init(HiveConf conf) throws Exception {
+ if (metrics == null) {
+ metrics = (Metrics) ReflectionUtils.newInstance(conf.getClassByName(
+ conf.getVar(HiveConf.ConfVars.HIVE_METRICS_CLASS)), conf);
+ }
+ metrics.init(conf);
+ }
+
+ public synchronized static Metrics getMetricsInstance() {
+ return metrics;
+ }
+
+ public synchronized static void deInit() throws Exception {
+ if (metrics != null) {
+ metrics.deInit();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
new file mode 100644
index 0000000..e59da99
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
@@ -0,0 +1,366 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.ExponentiallyDecayingReservoir;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.json.MetricsModule;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimerTask;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Codahale-backed Metrics implementation.
+ */
+public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.common.Metrics {
+ public static final String API_PREFIX = "api_";
+ public static final Log LOGGER = LogFactory.getLog(CodahaleMetrics.class);
+
+ public final MetricRegistry metricRegistry = new MetricRegistry();
+ private final Lock timersLock = new ReentrantLock();
+ private final Lock countersLock = new ReentrantLock();
+
+ private LoadingCache<String, Timer> timers;
+ private LoadingCache<String, Counter> counters;
+
+ private boolean initialized = false;
+ private HiveConf conf;
+ private final Set<Closeable> reporters = new HashSet<Closeable>();
+
+ private final ThreadLocal<HashMap<String, MetricsScope>> threadLocalScopes
+ = new ThreadLocal<HashMap<String,MetricsScope>>() {
+ @Override
+ protected HashMap<String,MetricsScope> initialValue() {
+ return new HashMap<String,MetricsScope>();
+ }
+ };
+
+ public static class MetricsScope {
+
+ final String name;
+ final Timer timer;
+ Timer.Context timerContext;
+ CodahaleMetrics metrics;
+
+ private boolean isOpen = false;
+
+ /**
+ * Instantiates a named scope - intended to only be called by Metrics, so locally scoped.
+ * @param name - name of the variable
+ * @throws IOException
+ */
+ private MetricsScope(String name, CodahaleMetrics metrics) throws IOException {
+ this.name = name;
+ this.metrics = metrics;
+ this.timer = metrics.getTimer(name);
+ open();
+ }
+
+ /**
+ * Opens scope, and makes note of the time started, increments run counter
+ * @throws IOException
+ *
+ */
+ public void open() throws IOException {
+ if (!isOpen) {
+ isOpen = true;
+ this.timerContext = timer.time();
+ } else {
+ throw new IOException("Scope named " + name + " is not closed, cannot be opened.");
+ }
+ }
+
+ /**
+ * Closes scope, and records the time taken
+ * @throws IOException
+ */
+ public void close() throws IOException {
+ if (isOpen) {
+ timerContext.close();
+
+ } else {
+ throw new IOException("Scope named " + name + " is not open, cannot be closed.");
+ }
+ isOpen = false;
+ }
+ }
+
+ public synchronized void init(HiveConf conf) throws Exception {
+ if (initialized) {
+ return;
+ }
+
+ this.conf = conf;
+ //Codahale artifacts are lazily-created.
+ timers = CacheBuilder.newBuilder().build(
+ new CacheLoader<String, com.codahale.metrics.Timer>() {
+ @Override
+ public com.codahale.metrics.Timer load(String key) throws Exception {
+ Timer timer = new Timer(new ExponentiallyDecayingReservoir());
+ metricRegistry.register(key, timer);
+ return timer;
+ }
+ }
+ );
+ counters = CacheBuilder.newBuilder().build(
+ new CacheLoader<String, Counter>() {
+ @Override
+ public Counter load(String key) throws Exception {
+ Counter counter = new Counter();
+ metricRegistry.register(key, counter);
+ return counter;
+ }
+ }
+ );
+
+ //register JVM metrics
+ registerAll("gc", new GarbageCollectorMetricSet());
+ registerAll("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
+ registerAll("memory", new MemoryUsageGaugeSet());
+ registerAll("threads", new ThreadStatesGaugeSet());
+ registerAll("classLoading", new ClassLoadingGaugeSet());
+
+ //Metrics reporter
+ Set<MetricsReporting> finalReporterList = new HashSet<MetricsReporting>();
+ List<String> metricsReporterNames = Lists.newArrayList(
+ Splitter.on(",").trimResults().omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER)));
+
+ if(metricsReporterNames != null) {
+ for (String metricsReportingName : metricsReporterNames) {
+ try {
+ MetricsReporting reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase());
+ finalReporterList.add(reporter);
+ } catch (IllegalArgumentException e) {
+ LOGGER.warn("Metrics reporter skipped due to invalid configured reporter: " + metricsReportingName);
+ }
+ }
+ }
+ initReporting(finalReporterList);
+ initialized = true;
+ }
+
+
+ public synchronized void deInit() throws Exception {
+ if (initialized) {
+ if (reporters != null) {
+ for (Closeable reporter : reporters) {
+ reporter.close();
+ }
+ }
+ for (Map.Entry<String, Metric> metric : metricRegistry.getMetrics().entrySet()) {
+ metricRegistry.remove(metric.getKey());
+ }
+ timers.invalidateAll();
+ counters.invalidateAll();
+ initialized = false;
+ }
+ }
+
+ public void startScope(String name) throws IOException {
+ synchronized (this) {
+ if (!initialized) {
+ return;
+ }
+ }
+ name = API_PREFIX + name;
+ if (threadLocalScopes.get().containsKey(name)) {
+ threadLocalScopes.get().get(name).open();
+ } else {
+ threadLocalScopes.get().put(name, new MetricsScope(name, this));
+ }
+ }
+
+ public void endScope(String name) throws IOException{
+ synchronized (this) {
+ if (!initialized) {
+ return;
+ }
+ }
+ name = API_PREFIX + name;
+ if (threadLocalScopes.get().containsKey(name)) {
+ threadLocalScopes.get().get(name).close();
+ }
+ }
+
+ public Long incrementCounter(String name) throws IOException {
+ return incrementCounter(name, 1);
+ }
+
+ public Long incrementCounter(String name, long increment) throws IOException {
+ String key = name;
+ try {
+ countersLock.lock();
+ counters.get(key).inc(increment);
+ return counters.get(key).getCount();
+ } catch(ExecutionException ee) {
+ throw new RuntimeException(ee);
+ } finally {
+ countersLock.unlock();
+ }
+ }
+
+ // This method is necessary to synchronize lazy-creation to the timers.
+ private Timer getTimer(String name) throws IOException {
+ String key = name;
+ try {
+ timersLock.lock();
+ Timer timer = timers.get(key);
+ return timer;
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ } finally {
+ timersLock.unlock();
+ }
+ }
+
+ private void registerAll(String prefix, MetricSet metricSet) {
+ for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
+ if (entry.getValue() instanceof MetricSet) {
+ registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue());
+ } else {
+ metricRegistry.register(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public MetricRegistry getMetricRegistry() {
+ return metricRegistry;
+ }
+
+ /**
+ * Should be only called once to initialize the reporters
+ */
+ private void initReporting(Set<MetricsReporting> reportingSet) throws Exception {
+ for (MetricsReporting reporting : reportingSet) {
+ switch(reporting) {
+ case CONSOLE:
+ final ConsoleReporter consoleReporter = ConsoleReporter.forRegistry(metricRegistry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ consoleReporter.start(1, TimeUnit.SECONDS);
+ reporters.add(consoleReporter);
+ break;
+ case JMX:
+ final JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ jmxReporter.start();
+ reporters.add(jmxReporter);
+ break;
+ case JSON_FILE:
+ final JsonFileReporter jsonFileReporter = new JsonFileReporter();
+ jsonFileReporter.start();
+ reporters.add(jsonFileReporter);
+ break;
+ }
+ }
+ }
+
+ class JsonFileReporter implements Closeable {
+ private ObjectMapper jsonMapper = null;
+ private java.util.Timer timer = null;
+
+ public void start() {
+ this.jsonMapper = new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false));
+ this.timer = new java.util.Timer(true);
+
+ long time = conf.getTimeVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, TimeUnit.MILLISECONDS);
+ final String pathString = conf.getVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION);
+
+ timer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ BufferedWriter bw = null;
+ try {
+ String json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricRegistry);
+ Path tmpPath = new Path(pathString + ".tmp");
+ FileSystem fs = FileSystem.get(conf);
+ fs.delete(tmpPath, true);
+ bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true)));
+ bw.write(json);
+ bw.close();
+
+ Path path = new Path(pathString);
+ fs.rename(tmpPath, path);
+ fs.setPermission(path, FsPermission.createImmutable((short) 0644));
+ } catch (Exception e) {
+ LOGGER.warn("Error writing JSON Metrics to file", e);
+ } finally {
+ try {
+ if (bw != null) {
+ bw.close();
+ }
+ } catch (IOException e) {
+ //Ignore.
+ }
+ }
+
+
+ }
+ }, 0, time);
+ }
+
+ public void close() {
+ if (timer != null) {
+ this.timer.cancel();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
new file mode 100644
index 0000000..643246f
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+/**
+ * Reporting options for org.apache.hadoop.hive.common.metrics.metrics2.Metrics.
+ */
+public enum MetricsReporting {
+ JMX,
+ CONSOLE,
+ JSON_FILE
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d733d71..a724fd1 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -645,6 +645,7 @@ public class HiveConf extends Configuration {
"Maximum cache full % after which the cache cleaner thread kicks in."),
METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL("hive.metastore.aggregate.stats.cache.clean.until", (float) 0.8,
"The cleaner thread cleans until cache reaches this % full size."),
+ METASTORE_METRICS("hive.metastore.metrics.enabled", false, "Enable metrics on the metastore."),
// Parameters for exporting metadata on table drop (requires the use of the)
// org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener
@@ -1688,6 +1689,7 @@ public class HiveConf extends Configuration {
" EXECUTION: Log completion of tasks\n" +
" PERFORMANCE: Execution + Performance logs \n" +
" VERBOSE: All logs" ),
+ HIVE_SERVER2_METRICS_ENABLED("hive.server2.metrics.enabled", false, "Enable metrics on the HiveServer2."),
// logging configuration
HIVE_LOG4J_FILE("hive.log4j.file", "",
"Hive log4j configuration file.\n" +
@@ -1715,7 +1717,21 @@ public class HiveConf extends Configuration {
HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME(
"hive.autogen.columnalias.prefix.includefuncname", false,
"Whether to include function name in the column alias auto generated by Hive."),
-
+ HIVE_METRICS_CLASS("hive.service.metrics.class",
+ "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics",
+ new StringSet(
+ "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics",
+ "org.apache.hadoop.hive.common.metrics.LegacyMetrics"),
+ "Hive metrics subsystem implementation class."),
+ HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX",
+ "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, comma separated list of JMX, CONSOLE, JSON_FILE"),
+ HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "file:///tmp/my-logging.properties",
+ "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of JSON metrics file. " +
+ "This file will get overwritten at every interval."),
+ HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5s",
+ new TimeValidator(TimeUnit.MILLISECONDS),
+ "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, " +
+ "the frequency of updating JSON metrics file."),
HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger",
"The class responsible for logging client side performance metrics. \n" +
"Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger"),
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
new file mode 100644
index 0000000..c14c7ee
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
@@ -0,0 +1,295 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import javax.management.Attribute;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
+import javax.management.MBeanOperationInfo;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.common.metrics.LegacyMetrics.MetricsScope;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestLegacyMetrics {
+
+ private static final String scopeName = "foo";
+ private static final long periodMs = 50L;
+ private static LegacyMetrics metrics;
+
+ @Before
+ public void before() throws Exception {
+ MetricsFactory.deInit();
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, LegacyMetrics.class.getCanonicalName());
+ MetricsFactory.init(conf);
+ metrics = (LegacyMetrics) MetricsFactory.getMetricsInstance();
+ }
+
+ @After
+ public void after() throws Exception {
+ MetricsFactory.deInit();
+ }
+
+ @Test
+ public void testMetricsMBean() throws Exception {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ final ObjectName oname = new ObjectName(
+ "org.apache.hadoop.hive.common.metrics:type=MetricsMBean");
+ MBeanInfo mBeanInfo = mbs.getMBeanInfo(oname);
+ // check implementation class:
+ assertEquals(MetricsMBeanImpl.class.getName(), mBeanInfo.getClassName());
+
+ // check reset operation:
+ MBeanOperationInfo[] oops = mBeanInfo.getOperations();
+ boolean resetFound = false;
+ for (MBeanOperationInfo op : oops) {
+ if ("reset".equals(op.getName())) {
+ resetFound = true;
+ break;
+ }
+ }
+ assertTrue(resetFound);
+
+ // add metric with a non-null value:
+ Attribute attr = new Attribute("fooMetric", Long.valueOf(-77));
+ mbs.setAttribute(oname, attr);
+
+ mBeanInfo = mbs.getMBeanInfo(oname);
+ MBeanAttributeInfo[] attrinuteInfos = mBeanInfo.getAttributes();
+ assertEquals(1, attrinuteInfos.length);
+ boolean attrFound = false;
+ for (MBeanAttributeInfo info : attrinuteInfos) {
+ if ("fooMetric".equals(info.getName())) {
+ assertEquals("java.lang.Long", info.getType());
+ assertTrue(info.isReadable());
+ assertTrue(info.isWritable());
+ assertFalse(info.isIs());
+
+ attrFound = true;
+ break;
+ }
+ }
+ assertTrue(attrFound);
+
+ // check metric value:
+ Object v = mbs.getAttribute(oname, "fooMetric");
+ assertEquals(Long.valueOf(-77), v);
+
+ // reset the bean:
+ Object result = mbs.invoke(oname, "reset", new Object[0], new String[0]);
+ assertNull(result);
+
+ // the metric value must be zeroed:
+ v = mbs.getAttribute(oname, "fooMetric");
+ assertEquals(Long.valueOf(0), v);
+ }
+
+ private <T> void expectIOE(Callable<T> c) throws Exception {
+ try {
+ T t = c.call();
+ fail("IOE expected but ["+t+"] was returned.");
+ } catch (IOException ioe) {
+ // ok, expected
+ }
+ }
+
+ @Test
+ public void testScopeSingleThread() throws Exception {
+ metrics.startScope(scopeName);
+ final MetricsScope fooScope = metrics.getScope(scopeName);
+ // the time and number counters become available only after the 1st
+ // scope close:
+ expectIOE(new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ Long num = fooScope.getNumCounter();
+ return num;
+ }
+ });
+ expectIOE(new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ Long time = fooScope.getTimeCounter();
+ return time;
+ }
+ });
+ // cannot open scope that is already open:
+ expectIOE(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ fooScope.open();
+ return null;
+ }
+ });
+
+ assertSame(fooScope, metrics.getScope(scopeName));
+ Thread.sleep(periodMs+ 1);
+ // 1st close:
+ // closing of open scope should be ok:
+ metrics.endScope(scopeName);
+ expectIOE(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ metrics.endScope(scopeName); // closing of closed scope not allowed
+ return null;
+ }
+ });
+
+ assertEquals(Long.valueOf(1), fooScope.getNumCounter());
+ final long t1 = fooScope.getTimeCounter().longValue();
+ assertTrue(t1 > periodMs);
+
+ assertSame(fooScope, metrics.getScope(scopeName));
+
+ // opening allowed after closing:
+ metrics.startScope(scopeName);
+ // opening of already open scope not allowed:
+ expectIOE(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ metrics.startScope(scopeName);
+ return null;
+ }
+ });
+
+ assertEquals(Long.valueOf(1), fooScope.getNumCounter());
+ assertEquals(t1, fooScope.getTimeCounter().longValue());
+
+ assertSame(fooScope, metrics.getScope(scopeName));
+ Thread.sleep(periodMs + 1);
+ // Reopening (close + open) allowed in opened state:
+ fooScope.reopen();
+
+ assertEquals(Long.valueOf(2), fooScope.getNumCounter());
+ assertTrue(fooScope.getTimeCounter().longValue() > 2 * periodMs);
+
+ Thread.sleep(periodMs + 1);
+ // 3rd close:
+ fooScope.close();
+
+ assertEquals(Long.valueOf(3), fooScope.getNumCounter());
+ assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs);
+ Double avgT = (Double) metrics.get("foo.avg_t");
+ assertTrue(avgT.doubleValue() > periodMs);
+ }
+
+ @Test
+ public void testScopeConcurrency() throws Exception {
+ metrics.startScope(scopeName);
+ MetricsScope fooScope = metrics.getScope(scopeName);
+ final int threads = 10;
+ ExecutorService executorService = Executors.newFixedThreadPool(threads);
+ for (int i=0; i<threads; i++) {
+ final int n = i;
+ executorService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ testScopeImpl(n);
+ return null;
+ }
+ });
+ }
+ executorService.shutdown();
+ assertTrue(executorService.awaitTermination(periodMs * 3 * threads, TimeUnit.MILLISECONDS));
+
+ fooScope = metrics.getScope(scopeName);
+ assertEquals(Long.valueOf(3 * threads), fooScope.getNumCounter());
+ assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs * threads);
+ Double avgT = (Double) metrics.get("foo.avg_t");
+ assertTrue(avgT.doubleValue() > periodMs);
+ metrics.endScope(scopeName);
+ }
+
+ void testScopeImpl(int n) throws Exception {
+ metrics.startScope(scopeName);
+ final MetricsScope fooScope = metrics.getScope(scopeName);
+ // cannot open scope that is already open:
+ expectIOE(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ fooScope.open();
+ return null;
+ }
+ });
+
+ assertSame(fooScope, metrics.getScope(scopeName));
+ Thread.sleep(periodMs+ 1);
+ // 1st close:
+ metrics.endScope(scopeName); // closing of open scope should be ok.
+
+ assertTrue(fooScope.getNumCounter().longValue() >= 1);
+ final long t1 = fooScope.getTimeCounter().longValue();
+ assertTrue(t1 > periodMs);
+
+ expectIOE(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ metrics.endScope(scopeName); // closing of closed scope not allowed
+ return null;
+ }
+ });
+
+ assertSame(fooScope, metrics.getScope(scopeName));
+
+ // opening allowed after closing:
+ metrics.startScope(scopeName);
+
+ assertTrue(fooScope.getNumCounter().longValue() >= 1);
+ assertTrue(fooScope.getTimeCounter().longValue() >= t1);
+
+ // opening of already open scope not allowed:
+ expectIOE(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ metrics.startScope(scopeName);
+ return null;
+ }
+ });
+
+ assertSame(fooScope, metrics.getScope(scopeName));
+ Thread.sleep(periodMs + 1);
+ // Reopening (close + open) allowed in opened state:
+ fooScope.reopen();
+
+ assertTrue(fooScope.getNumCounter().longValue() >= 2);
+ assertTrue(fooScope.getTimeCounter().longValue() > 2 * periodMs);
+
+ Thread.sleep(periodMs + 1);
+ // 3rd close:
+ fooScope.close();
+
+ assertTrue(fooScope.getNumCounter().longValue() >= 3);
+ assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs);
+ Double avgT = (Double) metrics.get("foo.avg_t");
+ assertTrue(avgT.doubleValue() > periodMs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/test/org/apache/hadoop/hive/common/metrics/TestMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/TestMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/TestMetrics.java
deleted file mode 100644
index e85d3f8..0000000
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/TestMetrics.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.metrics;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import javax.management.Attribute;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanInfo;
-import javax.management.MBeanOperationInfo;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-import org.apache.hadoop.hive.common.metrics.Metrics.MetricsScope;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestMetrics {
-
- private static final String scopeName = "foo";
- private static final long periodMs = 50L;
-
- @Before
- public void before() throws Exception {
- Metrics.uninit();
- Metrics.init();
- }
-
- @After
- public void after() throws Exception {
- Metrics.uninit();
- }
-
- @Test
- public void testMetricsMBean() throws Exception {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- final ObjectName oname = new ObjectName(
- "org.apache.hadoop.hive.common.metrics:type=MetricsMBean");
- MBeanInfo mBeanInfo = mbs.getMBeanInfo(oname);
- // check implementation class:
- assertEquals(MetricsMBeanImpl.class.getName(), mBeanInfo.getClassName());
-
- // check reset operation:
- MBeanOperationInfo[] oops = mBeanInfo.getOperations();
- boolean resetFound = false;
- for (MBeanOperationInfo op : oops) {
- if ("reset".equals(op.getName())) {
- resetFound = true;
- break;
- }
- }
- assertTrue(resetFound);
-
- // add metric with a non-null value:
- Attribute attr = new Attribute("fooMetric", Long.valueOf(-77));
- mbs.setAttribute(oname, attr);
-
- mBeanInfo = mbs.getMBeanInfo(oname);
- MBeanAttributeInfo[] attrinuteInfos = mBeanInfo.getAttributes();
- assertEquals(1, attrinuteInfos.length);
- boolean attrFound = false;
- for (MBeanAttributeInfo info : attrinuteInfos) {
- if ("fooMetric".equals(info.getName())) {
- assertEquals("java.lang.Long", info.getType());
- assertTrue(info.isReadable());
- assertTrue(info.isWritable());
- assertFalse(info.isIs());
-
- attrFound = true;
- break;
- }
- }
- assertTrue(attrFound);
-
- // check metric value:
- Object v = mbs.getAttribute(oname, "fooMetric");
- assertEquals(Long.valueOf(-77), v);
-
- // reset the bean:
- Object result = mbs.invoke(oname, "reset", new Object[0], new String[0]);
- assertNull(result);
-
- // the metric value must be zeroed:
- v = mbs.getAttribute(oname, "fooMetric");
- assertEquals(Long.valueOf(0), v);
- }
-
- private <T> void expectIOE(Callable<T> c) throws Exception {
- try {
- T t = c.call();
- fail("IOE expected but ["+t+"] was returned.");
- } catch (IOException ioe) {
- // ok, expected
- }
- }
-
- @Test
- public void testScopeSingleThread() throws Exception {
- final MetricsScope fooScope = Metrics.startScope(scopeName);
- // the time and number counters become available only after the 1st
- // scope close:
- expectIOE(new Callable<Long>() {
- @Override
- public Long call() throws Exception {
- Long num = fooScope.getNumCounter();
- return num;
- }
- });
- expectIOE(new Callable<Long>() {
- @Override
- public Long call() throws Exception {
- Long time = fooScope.getTimeCounter();
- return time;
- }
- });
- // cannot open scope that is already open:
- expectIOE(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- fooScope.open();
- return null;
- }
- });
-
- assertSame(fooScope, Metrics.getScope(scopeName));
- Thread.sleep(periodMs+1);
- // 1st close:
- // closing of open scope should be ok:
- Metrics.endScope(scopeName);
- expectIOE(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Metrics.endScope(scopeName); // closing of closed scope not allowed
- return null;
- }
- });
-
- assertEquals(Long.valueOf(1), fooScope.getNumCounter());
- final long t1 = fooScope.getTimeCounter().longValue();
- assertTrue(t1 > periodMs);
-
- assertSame(fooScope, Metrics.getScope(scopeName));
-
- // opening allowed after closing:
- Metrics.startScope(scopeName);
- // opening of already open scope not allowed:
- expectIOE(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Metrics.startScope(scopeName);
- return null;
- }
- });
-
- assertEquals(Long.valueOf(1), fooScope.getNumCounter());
- assertEquals(t1, fooScope.getTimeCounter().longValue());
-
- assertSame(fooScope, Metrics.getScope(scopeName));
- Thread.sleep(periodMs + 1);
- // Reopening (close + open) allowed in opened state:
- fooScope.reopen();
-
- assertEquals(Long.valueOf(2), fooScope.getNumCounter());
- assertTrue(fooScope.getTimeCounter().longValue() > 2 * periodMs);
-
- Thread.sleep(periodMs + 1);
- // 3rd close:
- fooScope.close();
-
- assertEquals(Long.valueOf(3), fooScope.getNumCounter());
- assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs);
- Double avgT = (Double)Metrics.get("foo.avg_t");
- assertTrue(avgT.doubleValue() > periodMs);
- }
-
- @Test
- public void testScopeConcurrency() throws Exception {
- MetricsScope fooScope = Metrics.startScope(scopeName);
- final int threads = 10;
- ExecutorService executorService = Executors.newFixedThreadPool(threads);
- for (int i=0; i<threads; i++) {
- final int n = i;
- executorService.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- testScopeImpl(n);
- return null;
- }
- });
- }
- executorService.shutdown();
- assertTrue(executorService.awaitTermination(periodMs * 3 * threads, TimeUnit.MILLISECONDS));
-
- fooScope = Metrics.getScope(scopeName);
- assertEquals(Long.valueOf(3 * threads), fooScope.getNumCounter());
- assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs * threads);
- Double avgT = (Double)Metrics.get("foo.avg_t");
- assertTrue(avgT.doubleValue() > periodMs);
- Metrics.endScope(scopeName);
- }
-
- void testScopeImpl(int n) throws Exception {
- final MetricsScope fooScope = Metrics.startScope(scopeName);
- // cannot open scope that is already open:
- expectIOE(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- fooScope.open();
- return null;
- }
- });
-
- assertSame(fooScope, Metrics.getScope(scopeName));
- Thread.sleep(periodMs+1);
- // 1st close:
- Metrics.endScope(scopeName); // closing of open scope should be ok.
-
- assertTrue(fooScope.getNumCounter().longValue() >= 1);
- final long t1 = fooScope.getTimeCounter().longValue();
- assertTrue(t1 > periodMs);
-
- expectIOE(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Metrics.endScope(scopeName); // closing of closed scope not allowed
- return null;
- }
- });
-
- assertSame(fooScope, Metrics.getScope(scopeName));
-
- // opening allowed after closing:
- Metrics.startScope(scopeName);
-
- assertTrue(fooScope.getNumCounter().longValue() >= 1);
- assertTrue(fooScope.getTimeCounter().longValue() >= t1);
-
- // opening of already open scope not allowed:
- expectIOE(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Metrics.startScope(scopeName);
- return null;
- }
- });
-
- assertSame(fooScope, Metrics.getScope(scopeName));
- Thread.sleep(periodMs + 1);
- // Reopening (close + open) allowed in opened state:
- fooScope.reopen();
-
- assertTrue(fooScope.getNumCounter().longValue() >= 2);
- assertTrue(fooScope.getTimeCounter().longValue() > 2 * periodMs);
-
- Thread.sleep(periodMs + 1);
- // 3rd close:
- fooScope.close();
-
- assertTrue(fooScope.getNumCounter().longValue() >= 3);
- assertTrue(fooScope.getTimeCounter().longValue() > 3 * periodMs);
- Double avgT = (Double)Metrics.get("foo.avg_t");
- assertTrue(avgT.doubleValue() > periodMs);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
new file mode 100644
index 0000000..8749349
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.metrics.metrics2;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit test for new Metrics subsystem.
+ */
+public class TestCodahaleMetrics {
+
+ private static File workDir = new File(System.getProperty("test.tmp.dir"));
+ private static File jsonReportFile;
+ public static MetricRegistry metricRegistry;
+
+ @Before
+ public void before() throws Exception {
+ HiveConf conf = new HiveConf();
+
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+ String defaultFsName = ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS");
+ conf.set(defaultFsName, "local");
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+
+ MetricsFactory.init(conf);
+ metricRegistry = ((CodahaleMetrics) MetricsFactory.getMetricsInstance()).getMetricRegistry();
+ }
+
+ @After
+ public void after() throws Exception {
+ MetricsFactory.deInit();
+ }
+
+ @Test
+ public void testScope() throws Exception {
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ MetricsFactory.getMetricsInstance().startScope("method1");
+ MetricsFactory.getMetricsInstance().endScope("method1");
+ }
+
+ Timer timer = metricRegistry.getTimers().get("api_method1");
+ Assert.assertEquals(5, timer.getCount());
+ Assert.assertTrue(timer.getMeanRate() > 0);
+ }
+
+
+ @Test
+ public void testCount() throws Exception {
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ MetricsFactory.getMetricsInstance().incrementCounter("count1");
+ }
+ Counter counter = metricRegistry.getCounters().get("count1");
+ Assert.assertEquals(5L, counter.getCount());
+ }
+
+ @Test
+ public void testConcurrency() throws Exception {
+ int threads = 4;
+ ExecutorService executorService = Executors.newFixedThreadPool(threads);
+ for (int i=0; i< threads; i++) {
+ final int n = i;
+ executorService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ MetricsFactory.getMetricsInstance().startScope("method2");
+ MetricsFactory.getMetricsInstance().endScope("method2");
+ return null;
+ }
+ });
+ }
+ executorService.shutdown();
+ assertTrue(executorService.awaitTermination(10000, TimeUnit.MILLISECONDS));
+ Timer timer = metricRegistry.getTimers().get("api_method2");
+ Assert.assertEquals(4, timer.getCount());
+ Assert.assertTrue(timer.getMeanRate() > 0);
+ }
+
+ @Test
+ public void testFileReporting() throws Exception {
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ MetricsFactory.getMetricsInstance().incrementCounter("count2");
+ Thread.sleep(100);
+ }
+
+ Thread.sleep(2000);
+ byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ JsonNode rootNode = objectMapper.readTree(jsonData);
+ JsonNode countersNode = rootNode.path("counters");
+ JsonNode methodCounterNode = countersNode.path("count2");
+ JsonNode countNode = methodCounterNode.path("count");
+ Assert.assertEquals(countNode.asInt(), 5);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
new file mode 100644
index 0000000..25f34d1
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import junit.framework.TestCase;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * Tests Hive Metastore Metrics.
+ */
+public class TestMetaStoreMetrics {
+
+ private static File workDir = new File(System.getProperty("test.tmp.dir"));
+ private static File jsonReportFile;
+
+ private static HiveConf hiveConf;
+ private static Driver driver;
+
+
+ @Before
+ public void before() throws Exception {
+
+ int port = MetaStoreUtils.findFreePort();
+
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+
+ hiveConf = new HiveConf(TestMetaStoreMetrics.class);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+ hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+ hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
+
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ }
+
+ @Test
+ public void testMetricsFile() throws Exception {
+ driver.run("show databases");
+
+ //give timer thread a chance to print the metrics
+ Thread.sleep(2000);
+
+ //As the file is being written, try a few times.
+ //This can be replaced by CodahaleMetrics's JsonServlet reporter once it is exposed.
+ byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ JsonNode rootNode = objectMapper.readTree(jsonData);
+ JsonNode countersNode = rootNode.path("timers");
+ JsonNode methodCounterNode = countersNode.path("api_get_all_databases");
+ JsonNode countNode = methodCounterNode.path("count");
+ Assert.assertTrue(countNode.asInt() > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a96fbdee/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index d81c856..1688920 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -18,39 +18,14 @@
package org.apache.hadoop.hive.metastore;
-import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_COMMENT;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.validateName;
-
-import java.io.IOException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Formatter;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Set;
-import java.util.Timer;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Pattern;
-
-import javax.jdo.JDOException;
-
+import com.facebook.fb303.FacebookBase;
+import com.facebook.fb303.fb_status;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimaps;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -58,12 +33,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.JvmPauseMonitor;
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.common.cli.CommonCliOptions;
-import org.apache.hadoop.hive.common.metrics.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
@@ -221,14 +197,35 @@ import org.apache.thrift.transport.TServerTransport;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportFactory;
-import com.facebook.fb303.FacebookBase;
-import com.facebook.fb303.fb_status;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimaps;
+import javax.jdo.JDOException;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Formatter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Timer;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+import static org.apache.commons.lang.StringUtils.join;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.*;
/**
* TODO:pc remove application logic to a separate interface.
@@ -464,9 +461,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
- if (hiveConf.getBoolean("hive.metastore.metrics.enabled", false)) {
+ //Start Metrics for Embedded mode
+ if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
try {
- Metrics.init();
+ MetricsFactory.init(hiveConf);
} catch (Exception e) {
// log exception, but ignore inability to start
LOG.error("error in Metrics init: " + e.getClass().getName() + " "
@@ -750,11 +748,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
incrementCounter(function);
logInfo((getIpAddress() == null ? "" : "source:" + getIpAddress() + " ") +
function + extraLogInfo);
- try {
- Metrics.startScope(function);
- } catch (IOException e) {
- LOG.debug("Exception when starting metrics scope"
+ if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
+ try {
+ MetricsFactory.getMetricsInstance().startScope(function);
+ } catch (IOException e) {
+ LOG.debug("Exception when starting metrics scope"
+ e.getClass().getName() + " " + e.getMessage(), e);
+ }
}
return function;
}
@@ -792,10 +792,12 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
private void endFunction(String function, MetaStoreEndFunctionContext context) {
- try {
- Metrics.endScope(function);
- } catch (IOException e) {
- LOG.debug("Exception when closing metrics scope" + e);
+ if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
+ try {
+ MetricsFactory.getMetricsInstance().endScope(function);
+ } catch (IOException e) {
+ LOG.debug("Exception when closing metrics scope" + e);
+ }
}
for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
@@ -819,6 +821,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
threadLocalMS.remove();
}
}
+ if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
+ try {
+ MetricsFactory.deInit();
+ } catch (Exception e) {
+ LOG.error("error in Metrics deinit: " + e.getClass().getName() + " "
+ + e.getMessage(), e);
+ }
+ }
logInfo("Metastore shutdown complete.");
}
@@ -5901,6 +5911,16 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
});
+ //Start Metrics for Standalone (Remote) Mode
+ if (conf.getBoolVar(ConfVars.METASTORE_METRICS)) {
+ try {
+ MetricsFactory.init(conf);
+ } catch (Exception e) {
+ // log exception, but ignore inability to start
+ LOG.error("error in Metrics init: " + e.getClass().getName() + " "
+ + e.getMessage(), e);
+ }
+ }
Lock startLock = new ReentrantLock();
Condition startCondition = startLock.newCondition();
@@ -6091,7 +6111,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// Wrap the start of the threads in a catch Throwable loop so that any failures
// don't doom the rest of the metastore.
startLock.lock();
- ShimLoader.getHadoopShims().startPauseMonitor(conf);
+ try {
+ JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(conf);
+ pauseMonitor.start();
+ } catch (Throwable t) {
+ LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not be " +
+ "warned upon.", t);
+ }
try {
// Per the javadocs on Condition, do not depend on the condition alone as a start gate
[35/50] [abbrv] hive git commit: HIVE-10874: Fail in
TestMinimrCliDriver.testCliDriver_ql_rewrite_gbtoidx_cbo_2.q due to duplicate
column name (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Posted by xu...@apache.org.
HIVE-10874: Fail in TestMinimrCliDriver.testCliDriver_ql_rewrite_gbtoidx_cbo_2.q due to duplicate column name (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/17bedcc0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/17bedcc0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/17bedcc0
Branch: refs/heads/spark
Commit: 17bedcc0016b557228f7ae3be1596ae9cb93ecb8
Parents: 9c45f92
Author: jpullokk <jp...@apache.org>
Authored: Thu Jun 4 13:54:29 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Thu Jun 4 14:00:58 2015 -0700
----------------------------------------------------------------------
.../calcite/reloperators/HiveAggregate.java | 58 ++++++++++++++++++++
1 file changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/17bedcc0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
index bea5eec..903cc19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
import java.util.List;
+import java.util.Set;
+import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
@@ -29,10 +31,16 @@ import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.RelFactories.AggregateFactory;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.IntList;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
public class HiveAggregate extends Aggregate implements HiveRelNode {
@@ -81,6 +89,56 @@ public class HiveAggregate extends Aggregate implements HiveRelNode {
containsAll(groupSet.asList());
}
+ @Override
+ protected RelDataType deriveRowType() {
+ return deriveRowType(getCluster().getTypeFactory(), getInput().getRowType(),
+ indicator, groupSet, groupSets, aggCalls);
+ }
+
+ public static RelDataType deriveRowType(RelDataTypeFactory typeFactory,
+ final RelDataType inputRowType, boolean indicator,
+ ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets,
+ final List<AggregateCall> aggCalls) {
+ final IntList groupList = groupSet.toList();
+ assert groupList.size() == groupSet.cardinality();
+ final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
+ final List<RelDataTypeField> fieldList = inputRowType.getFieldList();
+ final Set<String> containedNames = Sets.newHashSet();
+ for (int groupKey : groupList) {
+ containedNames.add(fieldList.get(groupKey).getName());
+ builder.add(fieldList.get(groupKey));
+ }
+ if (indicator) {
+ for (int groupKey : groupList) {
+ final RelDataType booleanType =
+ typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.BOOLEAN), false);
+ String name = "i$" + fieldList.get(groupKey).getName();
+ int i = 0;
+ while (containedNames.contains(name)) {
+ name += "_" + i++;
+ }
+ containedNames.add(name);
+ builder.add(name, booleanType);
+ }
+ }
+ for (Ord<AggregateCall> aggCall : Ord.zip(aggCalls)) {
+ String name;
+ if (aggCall.e.name != null) {
+ name = aggCall.e.name;
+ } else {
+ name = "$f" + (groupList.size() + aggCall.i);
+ }
+ int i = 0;
+ while (containedNames.contains(name)) {
+ name += "_" + i++;
+ }
+ containedNames.add(name);
+ builder.add(name, aggCall.e.type);
+ }
+ return builder.build();
+ }
+
private static class HiveAggRelFactory implements AggregateFactory {
@Override
[40/50] [abbrv] hive git commit: HIVE-10736: HiveServer2 shutdown of
cached tez app-masters is not clean (Vikram Dixit K,
reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-10736: HiveServer2 shutdown of cached tez app-masters is not clean (Vikram Dixit K, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dbd89ac4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dbd89ac4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dbd89ac4
Branch: refs/heads/spark
Commit: dbd89ac4e001f64ef22aa0f955536ac28086b4e8
Parents: 29af41c
Author: vikram <vi...@hortonworks.com>
Authored: Sat Jun 6 10:46:54 2015 -0700
Committer: vikram <vi...@hortonworks.com>
Committed: Sat Jun 6 10:46:54 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 2 +-
.../hive/ql/exec/tez/TezSessionPoolManager.java | 21 ++++++++++++++++++--
.../hive/ql/exec/tez/TezSessionState.java | 10 ----------
3 files changed, 20 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dbd89ac4/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index aa84850..1a4decf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -127,7 +127,7 @@ public class TezJobMonitor {
TezJobMonitor.killRunningJobs();
}
try {
- for (TezSessionState s: TezSessionState.getOpenSessions()) {
+ for (TezSessionState s : TezSessionPoolManager.getInstance().getOpenSessions()) {
System.err.println("Shutting down tez session.");
TezSessionPoolManager.getInstance().close(s, false);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dbd89ac4/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index 1798201..dfa539f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
@@ -20,13 +20,16 @@ package org.apache.hadoop.hive.ql.exec.tez;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -49,6 +52,9 @@ public class TezSessionPoolManager {
private static TezSessionPoolManager sessionPool = null;
+ private static List<TezSessionState> openSessions = Collections
+ .synchronizedList(new LinkedList<TezSessionState>());
+
public static TezSessionPoolManager getInstance()
throws Exception {
if (sessionPool == null) {
@@ -68,6 +74,7 @@ public class TezSessionPoolManager {
TezSessionState sessionState = defaultQueuePool.take();
newConf.set("tez.queue.name", sessionState.getQueueName());
sessionState.open(newConf);
+ openSessions.add(sessionState);
defaultQueuePool.put(sessionState);
}
}
@@ -148,6 +155,7 @@ public class TezSessionPoolManager {
String what = "Created";
if (doOpen) {
retTezSessionState.open(conf);
+ openSessions.add(retTezSessionState);
what = "Started";
}
@@ -175,6 +183,7 @@ public class TezSessionPoolManager {
LOG.info("Closing tez session default? " + tezSessionState.isDefault());
if (!tezSessionState.isDefault()) {
tezSessionState.close(keepTmpDir);
+ openSessions.remove(tezSessionState);
}
}
@@ -184,9 +193,12 @@ public class TezSessionPoolManager {
}
// we can just stop all the sessions
- for (TezSessionState sessionState: TezSessionState.getOpenSessions()) {
+ Iterator<TezSessionState> iter = openSessions.iterator();
+ while (iter.hasNext()) {
+ TezSessionState sessionState = iter.next();
if (sessionState.isDefault()) {
sessionState.close(false);
+ iter.remove();
}
}
}
@@ -282,5 +294,10 @@ public class TezSessionPoolManager {
}
close(sessionState, keepTmpDir);
sessionState.open(conf, additionalFiles);
+ openSessions.add(sessionState);
+ }
+
+ public List<TezSessionState> getOpenSessions() {
+ return openSessions;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dbd89ac4/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index cef3303..8555c6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -79,9 +79,6 @@ public class TezSessionState {
private final Set<LocalResource> localizedResources = new HashSet<LocalResource>();
private boolean doAsEnabled;
- private static List<TezSessionState> openSessions
- = Collections.synchronizedList(new LinkedList<TezSessionState>());
-
/**
* Constructor. We do not automatically connect, because we only want to
* load tez classes when the user has tez installed.
@@ -110,10 +107,6 @@ public class TezSessionState {
* Get all open sessions. Only used to clean up at shutdown.
* @return List<TezSessionState>
*/
- public static List<TezSessionState> getOpenSessions() {
- return openSessions;
- }
-
public static String makeSessionId() {
return UUID.randomUUID().toString();
}
@@ -213,8 +206,6 @@ public class TezSessionState {
} catch(InterruptedException ie) {
//ignore
}
-
- openSessions.add(this);
}
public void refreshLocalResourcesFromConf(HiveConf conf)
@@ -264,7 +255,6 @@ public class TezSessionState {
LOG.info("Closing Tez Session");
try {
session.stop();
- openSessions.remove(this);
} catch (SessionNotRunning nr) {
// ignore
}
[45/50] [abbrv] hive git commit: HIVE-10939 : Make TestFileDump
robust (Ashutosh Chauhan via Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-10939 : Make TestFileDump robust (Ashutosh Chauhan via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbeee782
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbeee782
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbeee782
Branch: refs/heads/spark
Commit: cbeee7820945c322475b4d9a399cf0966877a56e
Parents: 4f8de20
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Sat Jun 6 14:29:18 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 6 14:29:18 2015 -0700
----------------------------------------------------------------------
.../test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cbeee782/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
index 5b200f1..e77e24d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
@@ -130,16 +130,19 @@ public class TestFileDump {
new BufferedReader(new FileReader(HiveTestUtils.getFileFromClasspath(expected)));
BufferedReader aStream =
new BufferedReader(new FileReader(actual));
- String expectedLine = eStream.readLine();
+ String expectedLine = eStream.readLine().trim();
while (expectedLine != null) {
- String actualLine = aStream.readLine();
+ String actualLine = aStream.readLine().trim();
System.out.println("actual: " + actualLine);
System.out.println("expected: " + expectedLine);
assertEquals(expectedLine, actualLine);
expectedLine = eStream.readLine();
+ expectedLine = expectedLine == null ? null : expectedLine.trim();
}
assertNull(eStream.readLine());
assertNull(aStream.readLine());
+ eStream.close();
+ aStream.close();
}
@Test