You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/02/28 03:37:14 UTC
[1/2] hive git commit: HIVE-14990 : run all tests for MM tables and
fix the issues that are found - addendum (Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/hive-14535 bf7cf328f -> 6cba3e7e4
HIVE-14990 : run all tests for MM tables and fix the issues that are found - addendum (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bd96c491
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bd96c491
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bd96c491
Branch: refs/heads/hive-14535
Commit: bd96c4917f77301ef932ce409996e0fc5718f379
Parents: bf7cf32
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Feb 27 17:36:12 2017 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Feb 27 17:36:12 2017 -0800
----------------------------------------------------------------------
.../src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java | 1 -
ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java | 1 -
2 files changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bd96c491/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index e24ab49..6259cda 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1947,7 +1947,6 @@ public class MetaStoreUtils {
public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- // TODO# for the test, all non-ACID tables are MM
return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/bd96c491/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index da78a99..4102d02 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@ -734,7 +734,6 @@ public class FetchOperator implements Serializable {
}
public RecordReader<WritableComparable, Writable> getRecordReader(JobConf job) throws IOException {
- LOG.error("TODO# calling origina getRr on " + inputFormat + "; " + getInputSplit());
return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL);
}
}
[2/2] hive git commit: HIVE-14990 : run all tests for MM tables and
fix the issues that are found - disable sort sampling (Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-14990 : run all tests for MM tables and fix the issues that are found - disable sort sampling (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6cba3e7e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6cba3e7e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6cba3e7e
Branch: refs/heads/hive-14535
Commit: 6cba3e7e464ca35751b7f7230f8b4cd23a8c89d0
Parents: bd96c49
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Feb 27 19:33:39 2017 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Feb 27 19:33:39 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/io/HiveInputFormat.java | 3 +++
.../hive/ql/optimizer/physical/SamplingOptimizer.java | 14 ++++++++++++++
2 files changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6cba3e7e/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 5ea3cec..921aa53 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -669,6 +669,9 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
throws IOException {
PartitionDesc partDesc = pathToPartitionInfo.get(dir);
if (partDesc == null) {
+ // TODO: HiveFileFormatUtils.getPartitionDescFromPathRecursively for MM tables?
+ // So far, the only case when this is called for a MM directory was in error.
+ // Keep it like this for now; may need replacement if we find a valid usage like this.
partDesc = pathToPartitionInfo.get(Path.getPathWithoutSchemeAndAuthority(dir));
}
if (partDesc == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6cba3e7e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
index 2def168..5b89059 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.hive.ql.optimizer.physical;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorUtils;
@@ -27,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
@@ -59,6 +61,18 @@ public class SamplingOptimizer implements PhysicalPlanResolver {
if (!(operator instanceof TableScanOperator)) {
continue;
}
+ TableScanOperator tsop = (TableScanOperator)operator;
+ Table tbl = tsop.getConf().getTableMetadata();
+ if (tbl == null) {
+ continue;
+ }
+ if (MetaStoreUtils.isInsertOnlyTable(tbl.getParameters())) {
+ // Not supported for MM tables - sampler breaks separate MM dirs into splits, resulting in
+ // mismatch when the downstream task looks at them again assuming they are MM table roots.
+ // We could somehow unset the MM flag for the main job when the sampler succeeds, since the
+ // sampler will limit the input to the the correct directories, but we don't care about MR.
+ continue;
+ }
ReduceSinkOperator child =
OperatorUtils.findSingleOperator(operator, ReduceSinkOperator.class);
if (child == null ||