You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2017/08/15 16:18:37 UTC
hive git commit: HIVE-17305: New insert overwrite dynamic partitions
qtest need to have the golden file regenerated (Barna Zsombor Klara via Peter
Vary)
Repository: hive
Updated Branches:
refs/heads/master d3b820f64 -> 355c628f0
HIVE-17305: New insert overwrite dynamic partitions qtest need to have the golden file regenerated (Barna Zsombor Klara via Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/355c628f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/355c628f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/355c628f
Branch: refs/heads/master
Commit: 355c628f08de801aeac092809794e4ecdf1addf4
Parents: d3b820f
Author: Peter Vary <pv...@cloudera.com>
Authored: Tue Aug 15 18:17:23 2017 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Tue Aug 15 18:17:23 2017 +0200
----------------------------------------------------------------------
.../insert_overwrite_dynamic_partitions_move_only.q.out | 10 ++++++++++
.../org/apache/hadoop/hive/ql/session/LineageState.java | 9 ++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/355c628f/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out
index 530c036..cc1d018 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out
@@ -50,6 +50,16 @@ POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-10/product_id=1000
POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-10/product_id=10002
POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-12/product_id=10001
POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-12/product_id=10002
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10000).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10000).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10001).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10001).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10002).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10002).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10001).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10001).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10002).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ]
+POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10002).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ]
PREHOOK: query: select * from s3_table_move order by user_id
PREHOOK: type: QUERY
PREHOOK: Input: default@s3_table_move
http://git-wip-us.apache.org/repos/asf/hive/blob/355c628f/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
index 2a606a4..e2f2a68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
@@ -76,8 +76,15 @@ public class LineageState {
dirToFop.put(dir, fop);
}
+ /**
+ * Update the path of the captured lineage information in case the
+ * conditional input path and the linked MoveWork were merged into one MoveWork.
+ * This should only happen for Blobstore systems with optimization turned on.
+ * @param newPath conditional input path
+ * @param oldPath path of the old linked MoveWork
+ */
public void updateDirToOpMap(Path newPath, Path oldPath) {
- Operator op = dirToFop.remove(oldPath);
+ Operator op = dirToFop.get(oldPath);
if (op != null) {
dirToFop.put(newPath, op);
}