You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/03/14 02:17:38 UTC
hive git commit: HIVE-16194 : MM tables: most of the the parquet
tests fail (w/o MM enabled) (Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/hive-14535 fecace165 -> b802d1b13
HIVE-16194 : MM tables: most of the the parquet tests fail (w/o MM enabled) (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b802d1b1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b802d1b1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b802d1b1
Branch: refs/heads/hive-14535
Commit: b802d1b1300f01782d0fa8632630469cd1bfc860
Parents: fecace1
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Mar 13 19:17:25 2017 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Mar 13 19:17:25 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java | 8 +++++++-
.../apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java | 4 ++--
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b802d1b1/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
index 1103170..cc77e4c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
@@ -357,6 +357,12 @@ public final class HiveFileFormatUtils {
public static <T> T getFromPathRecursively(Map<Path, T> pathToPartitionInfo, Path dir,
Map<Map<Path, T>, Map<Path, T>> cacheMap, boolean ignoreSchema) throws IOException {
+ return getFromPathRecursively(pathToPartitionInfo, dir, cacheMap, ignoreSchema, false);
+ }
+
+ public static <T> T getFromPathRecursively(Map<Path, T> pathToPartitionInfo, Path dir,
+ Map<Map<Path, T>, Map<Path, T>> cacheMap, boolean ignoreSchema, boolean ifPresent)
+ throws IOException {
T part = getFromPath(pathToPartitionInfo, dir);
if (part == null
@@ -378,7 +384,7 @@ public final class HiveFileFormatUtils {
}
part = getFromPath(newPathToPartitionInfo, dir);
}
- if (part != null) {
+ if (part != null || ifPresent) {
return part;
} else {
throw new IOException("cannot find dir = " + dir.toString()
http://git-wip-us.apache.org/repos/asf/hive/blob/b802d1b1/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java
index e608932..8c61568 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java
@@ -86,7 +86,7 @@ public class ProjectionPusher {
final Set<String> aliases = new HashSet<String>();
try {
ArrayList<String> a = HiveFileFormatUtils.getFromPathRecursively(
- mapWork.getPathToAliases(), new Path(splitPath), null);
+ mapWork.getPathToAliases(), new Path(splitPath), null, false, true);
if (a != null) {
aliases.addAll(a);
}
@@ -185,7 +185,7 @@ public class ProjectionPusher {
updateMrWork(jobConf); // TODO: refactor this in HIVE-6366
final JobConf cloneJobConf = new JobConf(jobConf);
final PartitionDesc part = HiveFileFormatUtils.getFromPathRecursively(
- pathToPartitionInfo, path, null);
+ pathToPartitionInfo, path, null, false, true);
if ((part != null) && (part.getTableDesc() != null)) {
Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);