You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/11/01 22:20:17 UTC
[11/11] hive git commit: HIVE-14990 : run all tests for MM tables and
fix the issues that are found - many issues (Sergey Shelukhin)
HIVE-14990 : run all tests for MM tables and fix the issues that are found - many issues (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0119a8be
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0119a8be
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0119a8be
Branch: refs/heads/hive-14535
Commit: 0119a8be24d5ed99891070a9a3647542f3eb926c
Parents: c656b1b
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Nov 1 15:16:24 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Nov 1 15:16:24 2016 -0700
----------------------------------------------------------------------
.../hive/hcatalog/mapreduce/HCatOutputFormat.java | 5 +++++
ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java | 1 -
.../java/org/apache/hadoop/hive/ql/exec/MoveTask.java | 1 +
.../java/org/apache/hadoop/hive/ql/exec/Utilities.java | 12 ++++++++++--
.../hadoop/hive/ql/io/CombineHiveInputFormat.java | 2 +-
.../apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java | 5 +++++
.../apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +++++--
.../java/org/apache/hadoop/hive/ql/plan/MoveWork.java | 9 +++++++++
.../test/org/apache/hadoop/hive/ql/TestTxnCommands.java | 4 ++++
9 files changed, 40 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
index f9e71f0..cf0bb59 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Index;
@@ -113,6 +114,10 @@ public class HCatOutputFormat extends HCatBaseOutputFormat {
throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a partition with sorted column definition from Pig/Mapreduce is not supported");
}
+ if (MetaStoreUtils.isInsertOnlyTable(table.getParameters())) {
+ throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into an insert-only ACID table from Pig/Mapreduce is not supported");
+ }
+
// Set up a common id hash for this job, so that when we create any temporary directory
// later on, it is guaranteed to be unique.
String idHash;
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index b5ec3cb..ca6053d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3375,7 +3375,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
}
- // TODO# WRONG!! HERE
try {
if (allPartitions == null) {
db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), alterTbl.getEnvironmentContext());
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 76e399e..de2281b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -256,6 +256,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
@Override
public int execute(DriverContext driverContext) {
+ if (work.isNoop()) return 0; // TODO# temporary flag for HIVE-14990
Utilities.LOG14535.info("Executing MoveWork " + System.identityHashCode(work)
+ " with " + work.getLoadFileWork() + "; " + work.getLoadTableWork() + "; "
+ work.getLoadMultiFilesWork());
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 8e506aa..eed34ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -3819,13 +3819,21 @@ public final class Utilities {
if (skipLevels == 0) {
return statusToPath(fs.listStatus(path, filter));
}
- if (fs.getScheme().equalsIgnoreCase("s3a")
- && HiveConf.getBoolVar(conf, ConfVars.HIVE_MM_AVOID_GLOBSTATUS_ON_S3)) {
+ if (HiveConf.getBoolVar(conf, ConfVars.HIVE_MM_AVOID_GLOBSTATUS_ON_S3) && isS3(fs)) {
return getMmDirectoryCandidatesRecursive(fs, path, skipLevels, filter);
}
return getMmDirectoryCandidatesGlobStatus(fs, path, skipLevels, filter, mmWriteId);
}
+ private static boolean isS3(FileSystem fs) {
+ try {
+ return fs.getScheme().equalsIgnoreCase("s3a");
+ } catch (UnsupportedOperationException ex) {
+ // Some FS-es do not implement getScheme, e.g. ProxyLocalFileSystem.
+ return false;
+ }
+ }
+
private static Path[] statusToPath(FileStatus[] statuses) {
if (statuses == null) return null;
Path[] paths = new Path[statuses.length];
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index 15d6b9b..f0257ff 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -475,7 +475,7 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
ExecutorService executor = Executors.newFixedThreadPool(numThreads);
List<Future<Set<Integer>>> futureList = new ArrayList<Future<Set<Integer>>>(numThreads);
try {
- boolean isMerge = mrwork.isMergeFromResolver();
+ boolean isMerge = mrwork != null && mrwork.isMergeFromResolver();
for (int i = 0; i < numThreads; i++) {
int start = i * numPathPerThread;
int length = i != numThreads - 1 ? numPathPerThread : paths.length - start;
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index f0b5738..aa2ae35 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1364,6 +1364,11 @@ public final class GenMapRedUtils {
Utilities.LOG14535.info("creating dummy movetask for merge (with lfd)");
dummyMv = new MoveWork(null, null, null,
new LoadFileDesc(inputDirName, finalName, true, null, null), false);
+ } else {
+ // TODO# create the noop MoveWork to avoid q file changes for now. else should be removed.
+ dummyMv = new MoveWork(null, null, null,
+ new LoadFileDesc(inputDirName, finalName, true, null, null), false);
+ dummyMv.setNoop(true);
}
ConditionalTask cndTsk = GenMapRedUtils.createCondTask(conf, currTask, dummyMv, work,
fsInputDesc.getMergeInputDirName().toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index e8b2b84..a267b29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6961,8 +6961,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
fileSinkDesc.setHiveServerQuery(SessionState.get().isHiveServerQuery());
// If this is an insert, update, or delete on an ACID table then mark that so the
// FileSinkOperator knows how to properly write to it.
- if (destTableIsAcid && dest_part != null && dest_part.getTable() != null &&
- !MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
+ boolean isDestInsertOnly = (dest_part != null && dest_part.getTable() != null &&
+ MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters()))
+ || (table_desc != null && MetaStoreUtils.isInsertOnlyTable(table_desc.getProperties()));
+
+ if (destTableIsAcid && !isDestInsertOnly) {
AcidUtils.Operation wt = updating() ? AcidUtils.Operation.UPDATE :
(deleting() ? AcidUtils.Operation.DELETE : AcidUtils.Operation.INSERT);
fileSinkDesc.setWriteType(wt);
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index f0b2775..ff450db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -56,6 +56,7 @@ public class MoveWork implements Serializable {
* List of inserted partitions
*/
protected List<Partition> movedParts;
+ private boolean isNoop;
public MoveWork() {
}
@@ -142,4 +143,12 @@ public class MoveWork implements Serializable {
this.srcLocal = srcLocal;
}
+ // TODO# temporary test flag
+ public void setNoop(boolean b) {
+ this.isNoop = true;
+ }
+
+ public boolean isNoop() {
+ return this.isNoop;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0119a8be/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 64baa9f..8d79aee 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -43,6 +43,8 @@ import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
@@ -552,7 +554,9 @@ public class TestTxnCommands {
return sb.toString();
}
+ private static final Logger LOG = LoggerFactory.getLogger(TestTxnCommands.class);
private List<String> runStatementOnDriver(String stmt) throws Exception {
+ LOG.info("Running " + stmt);
CommandProcessorResponse cpr = d.run(stmt);
if(cpr.getResponseCode() != 0) {
throw new RuntimeException(stmt + " failed: " + cpr);