You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/09/18 22:35:40 UTC
[35/41] hive git commit: HIVE-11834: Lineage doesn't work with
dynamic partitioning query (Jimmy, reviewed by Szehon)
HIVE-11834: Lineage doesn't work with dynamic partitioning query (Jimmy, reviewed by Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2278548e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2278548e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2278548e
Branch: refs/heads/llap
Commit: 2278548e41bf7b51e7b604fe9c91905b1ca198f1
Parents: 3cf7bd9
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Wed Sep 16 08:09:41 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Fri Sep 18 07:19:23 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/hooks/LineageLogger.java | 93 +++++++++++---------
.../hive/ql/optimizer/lineage/LineageCtx.java | 8 +-
ql/src/test/queries/clientpositive/lineage3.q | 26 ++++++
.../test/results/clientpositive/lineage3.q.out | 68 +++++++++++++-
4 files changed, 153 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
index f615d81..9988c79 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
@@ -215,8 +215,7 @@ public class LineageLogger implements ExecuteWithHookContext {
private List<Edge> getEdges(QueryPlan plan, Index index) {
LinkedHashMap<String, ObjectPair<SelectOperator,
org.apache.hadoop.hive.ql.metadata.Table>> finalSelOps = index.getFinalSelectOps();
- Set<Vertex> allTargets = new LinkedHashSet<Vertex>();
- Map<String, Vertex> allSources = new LinkedHashMap<String, Vertex>();
+ Map<String, Vertex> vertexCache = new LinkedHashMap<String, Vertex>();
List<Edge> edges = new ArrayList<Edge>();
for (ObjectPair<SelectOperator,
org.apache.hadoop.hive.ql.metadata.Table> pair: finalSelOps.values()) {
@@ -244,41 +243,46 @@ public class LineageLogger implements ExecuteWithHookContext {
}
}
}
- int fields = fieldSchemas.size();
Map<ColumnInfo, Dependency> colMap = index.getDependencies(finalSelOp);
List<Dependency> dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null;
+ int fields = fieldSchemas.size();
+ if (t != null && colMap != null && fields < colMap.size()) {
+ // Dynamic partition keys should be added to field schemas.
+ List<FieldSchema> partitionKeys = t.getPartitionKeys();
+ int dynamicKeyCount = colMap.size() - fields;
+ int keyOffset = partitionKeys.size() - dynamicKeyCount;
+ if (keyOffset >= 0) {
+ fields += dynamicKeyCount;
+ for (int i = 0; i < dynamicKeyCount; i++) {
+ FieldSchema field = partitionKeys.get(keyOffset + i);
+ fieldSchemas.add(field);
+ if (colNames != null) {
+ colNames.add(field.getName());
+ }
+ }
+ }
+ }
if (dependencies == null || dependencies.size() != fields) {
log("Result schema has " + fields
+ " fields, but we don't get as many dependencies");
} else {
// Go through each target column, generate the lineage edges.
+ Set<Vertex> targets = new LinkedHashSet<Vertex>();
for (int i = 0; i < fields; i++) {
- Vertex target = new Vertex(
- getTargetFieldName(i, destTableName, colNames, fieldSchemas));
- allTargets.add(target);
+ Vertex target = getOrCreateVertex(vertexCache,
+ getTargetFieldName(i, destTableName, colNames, fieldSchemas),
+ Vertex.Type.COLUMN);
+ targets.add(target);
Dependency dep = dependencies.get(i);
- String expr = dep.getExpr();
- Set<Vertex> sources = createSourceVertices(allSources, dep.getBaseCols());
- Edge edge = findSimilarEdgeBySources(edges, sources, expr, Edge.Type.PROJECTION);
- if (edge == null) {
- Set<Vertex> targets = new LinkedHashSet<Vertex>();
- targets.add(target);
- edges.add(new Edge(sources, targets, expr, Edge.Type.PROJECTION));
- } else {
- edge.targets.add(target);
- }
+ addEdge(vertexCache, edges, dep.getBaseCols(), target,
+ dep.getExpr(), Edge.Type.PROJECTION);
}
Set<Predicate> conds = index.getPredicates(finalSelOp);
if (conds != null && !conds.isEmpty()) {
for (Predicate cond: conds) {
- String expr = cond.getExpr();
- Set<Vertex> sources = createSourceVertices(allSources, cond.getBaseCols());
- Edge edge = findSimilarEdgeByTargets(edges, allTargets, expr, Edge.Type.PREDICATE);
- if (edge == null) {
- edges.add(new Edge(sources, allTargets, expr, Edge.Type.PREDICATE));
- } else {
- edge.sources.addAll(sources);
- }
+ addEdge(vertexCache, edges, cond.getBaseCols(),
+ new LinkedHashSet<Vertex>(targets), cond.getExpr(),
+ Edge.Type.PREDICATE);
}
}
}
@@ -286,12 +290,35 @@ public class LineageLogger implements ExecuteWithHookContext {
return edges;
}
+ private void addEdge(Map<String, Vertex> vertexCache, List<Edge> edges,
+ Set<BaseColumnInfo> srcCols, Vertex target, String expr, Edge.Type type) {
+ Set<Vertex> targets = new LinkedHashSet<Vertex>();
+ targets.add(target);
+ addEdge(vertexCache, edges, srcCols, targets, expr, type);
+ }
+
+ /**
+ * Find an edge from all edges that has the same source vertices.
+ * If found, add the more targets to this edge's target vertex list.
+ * Otherwise, create a new edge and add to edge list.
+ */
+ private void addEdge(Map<String, Vertex> vertexCache, List<Edge> edges,
+ Set<BaseColumnInfo> srcCols, Set<Vertex> targets, String expr, Edge.Type type) {
+ Set<Vertex> sources = createSourceVertices(vertexCache, srcCols);
+ Edge edge = findSimilarEdgeBySources(edges, sources, expr, type);
+ if (edge == null) {
+ edges.add(new Edge(sources, targets, expr, type));
+ } else {
+ edge.targets.addAll(targets);
+ }
+ }
+
/**
* Convert a list of columns to a set of vertices.
* Use cached vertices if possible.
*/
private Set<Vertex> createSourceVertices(
- Map<String, Vertex> srcVertexCache, Collection<BaseColumnInfo> baseCols) {
+ Map<String, Vertex> vertexCache, Collection<BaseColumnInfo> baseCols) {
Set<Vertex> sources = new LinkedHashSet<Vertex>();
if (baseCols != null && !baseCols.isEmpty()) {
for(BaseColumnInfo col: baseCols) {
@@ -308,7 +335,7 @@ public class LineageLogger implements ExecuteWithHookContext {
type = Vertex.Type.COLUMN;
label = tableName + "." + fieldSchema.getName();
}
- sources.add(getOrCreateVertex(srcVertexCache, label, type));
+ sources.add(getOrCreateVertex(vertexCache, label, type));
}
}
return sources;
@@ -342,20 +369,6 @@ public class LineageLogger implements ExecuteWithHookContext {
}
/**
- * Find an edge that has the same type, expression, and targets.
- */
- private Edge findSimilarEdgeByTargets(
- List<Edge> edges, Set<Vertex> targets, String expr, Edge.Type type) {
- for (Edge edge: edges) {
- if (edge.type == type && StringUtils.equals(edge.expr, expr)
- && SetUtils.isEqualSet(edge.targets, targets)) {
- return edge;
- }
- }
- return null;
- }
-
- /**
* Generate normalized name for a given target column.
*/
private String getTargetFieldName(int fieldIndex,
http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
index c33d775..2d8b9e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.hive.ql.optimizer.lineage;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
@@ -171,6 +171,12 @@ public class LineageCtx implements NodeProcessorCtx {
conds = new LinkedHashSet<Predicate>();
condMap.put(op, conds);
}
+ for (Predicate p: conds) {
+ if (StringUtils.equals(cond.getExpr(), p.getExpr())) {
+ p.getBaseCols().addAll(cond.getBaseCols());
+ return;
+ }
+ }
conds.add(cond);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/test/queries/clientpositive/lineage3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/lineage3.q b/ql/src/test/queries/clientpositive/lineage3.q
index c24ff7d..70d4e57 100644
--- a/ql/src/test/queries/clientpositive/lineage3.q
+++ b/ql/src/test/queries/clientpositive/lineage3.q
@@ -176,3 +176,29 @@ alter view dest_v3 as
select * from dest_v3 limit 2;
+drop table if exists src_dp;
+create table src_dp (first string, word string, year int, month int, day int);
+drop table if exists dest_dp1;
+create table dest_dp1 (first string, word string) partitioned by (year int);
+drop table if exists dest_dp2;
+create table dest_dp2 (first string, word string) partitioned by (y int, m int);
+drop table if exists dest_dp3;
+create table dest_dp3 (first string, word string) partitioned by (y int, m int, d int);
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+insert into dest_dp1 partition (year) select first, word, year from src_dp;
+insert into dest_dp2 partition (y, m) select first, word, year, month from src_dp;
+insert into dest_dp2 partition (y=0, m) select first, word, month from src_dp where year=0;
+insert into dest_dp3 partition (y=0, m, d) select first, word, month m, day d from src_dp where year=0;
+
+drop table if exists src_dp1;
+create table src_dp1 (f string, w string, m int);
+
+from src_dp, src_dp1
+insert into dest_dp1 partition (year) select first, word, year
+insert into dest_dp2 partition (y, m) select first, word, year, month
+insert into dest_dp3 partition (y=2, m, d) select first, word, month m, day d where year=2
+insert into dest_dp2 partition (y=1, m) select f, w, m
+insert into dest_dp1 partition (year=0) select f, w;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/test/results/clientpositive/lineage3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out
index 708abee..ad965c8 100644
--- a/ql/src/test/results/clientpositive/lineage3.q.out
+++ b/ql/src/test/results/clientpositive/lineage3.q.out
@@ -25,7 +25,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: default@d1
PREHOOK: Output: default@d2
-{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[5],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.
cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
+{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[5],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint
"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
PREHOOK: query: drop table if exists t
PREHOOK: type: DROPTABLE
PREHOOK: query: create table t as
@@ -320,3 +320,69 @@ PREHOOK: Input: default@dest_v3
{"version":"1.0","engine":"mr","hash":"40bccc0722002f798d0548b59e369e83","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (. (tok_table_or_col $hdt$_0) csmallint)))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8],"targets":[0,1,2],"expression":"(a.cboolean2 = true)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = a.cint)","edgeType":"PREDICATE"},{"sources":[9],"targets":[0,1,2],"expression":"(a.cfloat > 0.0)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.c
int) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]}
38 216 false
38 229 true
+PREHOOK: query: drop table if exists src_dp
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table src_dp (first string, word string, year int, month int, day int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_dp
+PREHOOK: query: drop table if exists dest_dp1
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table dest_dp1 (first string, word string) partitioned by (year int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest_dp1
+PREHOOK: query: drop table if exists dest_dp2
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table dest_dp2 (first string, word string) partitioned by (y int, m int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest_dp2
+PREHOOK: query: drop table if exists dest_dp3
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table dest_dp3 (first string, word string) partitioned by (y int, m int, d int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest_dp3
+PREHOOK: query: insert into dest_dp1 partition (year) select first, word, year from src_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_dp
+PREHOOK: Output: default@dest_dp1
+{"version":"1.0","engine":"mr","hash":"b2d38401a3281e74a003d9650df97060","queryText":"insert into dest_dp1 partition (year) select first, word, year from src_dp","edges":[{"sources":[3],"targets":[0],"edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp1.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp1.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp1.year"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.year"}]}
+PREHOOK: query: insert into dest_dp2 partition (y, m) select first, word, year, month from src_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_dp
+PREHOOK: Output: default@dest_dp2
+{"version":"1.0","engine":"mr","hash":"237302d8ffd62b5b71d9544b22de7770","queryText":"insert into dest_dp2 partition (y, m) select first, word, year, month from src_dp","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp2.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp2.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp2.y"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_dp2.m"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src_dp.year"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src_dp.month"}]}
+PREHOOK: query: insert into dest_dp2 partition (y=0, m) select first, word, month from src_dp where year=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_dp
+PREHOOK: Output: default@dest_dp2@y=0
+{"version":"1.0","engine":"mr","hash":"63e990b47e7ab4eb6f2ea09dfb7453ff","queryText":"insert into dest_dp2 partition (y=0, m) select first, word, month from src_dp where year=0","edges":[{"sources":[3],"targets":[0],"edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[6],"targets":[0,1,2],"expression":"(src_dp.year = 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp2.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp2.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp2.m"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.month"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src_dp.year"}]}
+PREHOOK: query: insert into dest_dp3 partition (y=0, m, d) select first, word, month m, day d from src_dp where year=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_dp
+PREHOOK: Output: default@dest_dp3@y=0
+{"version":"1.0","engine":"mr","hash":"6bf71a9d02c0612c63b6f40b15c1e8b3","queryText":"insert into dest_dp3 partition (y=0, m, d) select first, word, month m, day d from src_dp where year=0","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[8],"targets":[0,1,2,3],"expression":"(src_dp.year = 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp3.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp3.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp3.m"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_dp3.d"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src_dp.month"},{"id":7,"vertexType":"CO
LUMN","vertexId":"default.src_dp.day"},{"id":8,"vertexType":"COLUMN","vertexId":"default.src_dp.year"}]}
+PREHOOK: query: drop table if exists src_dp1
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table src_dp1 (f string, w string, m int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_dp1
+Warning: Shuffle Join JOIN[4][tables = [src_dp, src_dp1]] in Stage 'Stage-5:MAPRED' is a cross product
+PREHOOK: query: from src_dp, src_dp1
+insert into dest_dp1 partition (year) select first, word, year
+insert into dest_dp2 partition (y, m) select first, word, year, month
+insert into dest_dp3 partition (y=2, m, d) select first, word, month m, day d where year=2
+insert into dest_dp2 partition (y=1, m) select f, w, m
+insert into dest_dp1 partition (year=0) select f, w
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_dp
+PREHOOK: Input: default@src_dp1
+PREHOOK: Output: default@dest_dp1
+PREHOOK: Output: default@dest_dp1@year=0
+PREHOOK: Output: default@dest_dp2
+PREHOOK: Output: default@dest_dp2@y=1
+PREHOOK: Output: default@dest_dp3@y=2
+{"version":"1.0","engine":"mr","hash":"44f16edbf35cfeaf3d4f7b0113a69b74","queryText":"from src_dp, src_dp1\ninsert into dest_dp1 partition (year) select first, word, year\ninsert into dest_dp2 partition (y, m) select first, word, year, month\ninsert into dest_dp3 partition (y=2, m, d) select first, word, month m, day d where year=2\ninsert into dest_dp2 partition (y=1, m) select f, w, m\ninsert into dest_dp1 partition (year=0) select f, w","edges":[{"sources":[11],"targets":[0,1,2],"edgeType":"PROJECTION"},{"sources":[12],"targets":[3,4,5],"edgeType":"PROJECTION"},{"sources":[13],"targets":[6,7],"edgeType":"PROJECTION"},{"sources":[14],"targets":[8,9],"edgeType":"PROJECTION"},{"sources":[15],"targets":[1,0],"edgeType":"PROJECTION"},{"sources":[16],"targets":[4,3],"edgeType":"PROJECTION"},{"sources":[17],"targets":[8],"edgeType":"PROJECTION"},{"sources":[18],"targets":[10],"edgeType":"PROJECTION"},{"sources":[13],"targets":[2,5,9,10],"expression":"(src_dp.year = 2)","edgeType":"PREDI
CATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp1.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp2.first"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp3.first"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_dp1.word"},{"id":4,"vertexType":"COLUMN","vertexId":"default.dest_dp2.word"},{"id":5,"vertexType":"COLUMN","vertexId":"default.dest_dp3.word"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dest_dp1.year"},{"id":7,"vertexType":"COLUMN","vertexId":"default.dest_dp2.y"},{"id":8,"vertexType":"COLUMN","vertexId":"default.dest_dp2.m"},{"id":9,"vertexType":"COLUMN","vertexId":"default.dest_dp3.m"},{"id":10,"vertexType":"COLUMN","vertexId":"default.dest_dp3.d"},{"id":11,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":12,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":13,"vertexType":"COLUMN","vertexId":"default.src_dp.year"},{"id":14,"vertexType":"COLUMN","vertexId":"default.src_dp.month
"},{"id":15,"vertexType":"COLUMN","vertexId":"default.src_dp1.f"},{"id":16,"vertexType":"COLUMN","vertexId":"default.src_dp1.w"},{"id":17,"vertexType":"COLUMN","vertexId":"default.src_dp1.m"},{"id":18,"vertexType":"COLUMN","vertexId":"default.src_dp.day"}]}