You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/01/30 23:48:03 UTC
hive git commit: HIVE-18237 : missing results for insert_only table
after DP insert (Steve Yeom, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 00145ee85 -> 20a34d329
HIVE-18237 : missing results for insert_only table after DP insert (Steve Yeom, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/20a34d32
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/20a34d32
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/20a34d32
Branch: refs/heads/master
Commit: 20a34d329d41c6b46eb576761fa55ae327d5208a
Parents: 00145ee
Author: sergey <se...@apache.org>
Authored: Tue Jan 30 15:41:38 2018 -0800
Committer: sergey <se...@apache.org>
Committed: Tue Jan 30 15:41:38 2018 -0800
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java | 7 +++++--
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 2 +-
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 9 +++++----
.../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 ++++++
.../java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java | 9 +++++++++
5 files changed, 26 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 0df30f1..114d455 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -503,8 +503,11 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
(tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(),
work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
!tbd.isMmTable(),
- work.getLoadTableWork().getTxnId(), tbd.getStmtId(), hasFollowingStatsTask(),
- work.getLoadTableWork().getWriteType());
+ work.getLoadTableWork().getTxnId(),
+ tbd.getStmtId(),
+ hasFollowingStatsTask(),
+ work.getLoadTableWork().getWriteType(),
+ tbd.isInsertOverwrite());
// publish DP columns to its subscribers
if (dps != null && dps.size() > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index d7b3e4b..7348faa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -4075,7 +4075,7 @@ public final class Utilities {
throws IOException {
int skipLevels = dpLevels + lbLevels;
if (filter == null) {
- filter = new JavaUtils.IdPathFilter(txnId, stmtId, true);
+ filter = new JavaUtils.IdPathFilter(txnId, stmtId, true, false, isBaseDir);
}
if (skipLevels == 0) {
return statusToPath(fs.listStatus(path, filter));
http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 23983d8..63bcedc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -2178,7 +2178,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @throws HiveException
*/
private Set<Path> getValidPartitionsInPath(
- int numDP, int numLB, Path loadPath, Long txnId, int stmtId, boolean isMmTable) throws HiveException {
+ int numDP, int numLB, Path loadPath, Long txnId, int stmtId,
+ boolean isMmTable, boolean isInsertOverwrite) throws HiveException {
Set<Path> validPartitions = new HashSet<Path>();
try {
FileSystem fs = loadPath.getFileSystem(conf);
@@ -2199,7 +2200,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
// where this is used; we always want to load everything; also the only case where
// we have multiple statements anyway is union.
Path[] leafStatus = Utilities.getMmDirectoryCandidates(
- fs, loadPath, numDP, numLB, null, txnId, -1, conf, false);
+ fs, loadPath, numDP, numLB, null, txnId, -1, conf, isInsertOverwrite);
for (Path p : leafStatus) {
Path dpPath = p.getParent(); // Skip the MM directory that we have found.
for (int i = 0; i < numLB; ++i) {
@@ -2247,7 +2248,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
public Map<Map<String, String>, Partition> loadDynamicPartitions(final Path loadPath,
final String tableName, final Map<String, String> partSpec, final LoadFileType loadFileType,
final int numDP, final int numLB, final boolean isAcid, final long txnId, final int stmtId,
- final boolean hasFollowingStatsTask, final AcidUtils.Operation operation)
+ final boolean hasFollowingStatsTask, final AcidUtils.Operation operation, boolean isInsertOverwrite)
throws HiveException {
final Map<Map<String, String>, Partition> partitionsMap =
@@ -2263,7 +2264,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
// Get all valid partition paths and existing partitions for them (if any)
final Table tbl = getTable(tableName);
final Set<Path> validPartitions = getValidPartitionsInPath(numDP, numLB, loadPath, txnId, stmtId,
- AcidUtils.isInsertOnlyTable(tbl.getParameters()));
+ AcidUtils.isInsertOnlyTable(tbl.getParameters()), isInsertOverwrite);
final int partsToLoad = validPartitions.size();
final AtomicInteger partitionsLoaded = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 69d4fa5..dbf9363 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6995,6 +6995,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
tableDesc.setWriter(fileSinkDesc);
}
+ if (fileSinkDesc.getInsertOverwrite()) {
+ if (ltd != null) {
+ ltd.setInsertOverwrite(true);
+ }
+ }
+
if (SessionState.get().isHiveServerQuery() &&
null != table_desc &&
table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
http://git-wip-us.apache.org/repos/asf/hive/blob/20a34d32/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index 1a9c11e..a40c486 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -40,6 +40,7 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
//table specs are to be used
private int stmtId;
private Long currentTransactionId;
+ private boolean isInsertOverwrite;
// TODO: the below seem like they should just be combined into partitionDesc
private org.apache.hadoop.hive.ql.plan.TableDesc table;
@@ -209,6 +210,14 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
this.inheritTableSpecs = inheritTableSpecs;
}
+ public boolean isInsertOverwrite() {
+ return this.isInsertOverwrite;
+ }
+
+ public void setInsertOverwrite(boolean v) {
+ this.isInsertOverwrite = v;
+ }
+
/**
* @return the lbCtx
*/