You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/11/05 01:05:46 UTC
[3/3] hive git commit: HIVE-14990 : run all tests for MM tables and
fix the issues that are found - many issues (Sergey Shelukhin)
HIVE-14990 : run all tests for MM tables and fix the issues that are found - many issues (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f6c5ac4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f6c5ac4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f6c5ac4
Branch: refs/heads/hive-14535
Commit: 1f6c5ac4f814c60db123f14438bfc12055160749
Parents: 9aadce4
Author: Sergey Shelukhin <se...@apache.org>
Authored: Fri Nov 4 18:05:33 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Fri Nov 4 18:05:33 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/MetaStoreUtils.java | 2 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 21 +-
.../apache/hadoop/hive/ql/exec/MoveTask.java | 8 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 22 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 50 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 55 +-
.../hive/ql/stats/fs/FSStatsPublisher.java | 2 +-
ql/src/test/queries/clientpositive/mm_all.q | 17 +-
ql/src/test/queries/clientpositive/mm_current.q | 61 +-
ql/src/test/results/clientpositive/mm_all.q.out | 3195 ++++++++++++++++++
.../results/clientpositive/mm_current.q.out | 125 +
11 files changed, 3443 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 49e9345..53bdec0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -265,7 +265,7 @@ public class MetaStoreUtils {
numFiles += 1;
}
}
- LOG.info(s, new Exception());
+ LOG.info(s/*, new Exception()*/);
params.put(StatsSetupConst.NUM_FILES, Integer.toString(numFiles));
params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(tableSize));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 26e9cd0..2864af4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -134,6 +134,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
protected transient long cntr = 1;
protected transient long logEveryNRows = 0;
protected transient int rowIndex = 0;
+ private transient boolean inheritPerms = false;
/**
* Counters.
*/
@@ -242,7 +243,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
if ((bDynParts || isSkewedStoredAsSubDirectories)
&& !fs.exists(finalPaths[idx].getParent())) {
Utilities.LOG14535.info("commit making path for dyn/skew: " + finalPaths[idx].getParent());
- fs.mkdirs(finalPaths[idx].getParent());
+ FileUtils.mkdir(fs, finalPaths[idx].getParent(), inheritPerms, hconf);
}
// If we're updating or deleting there may be no file to close. This can happen
// because the where clause strained out all of the records for a given bucket. So
@@ -461,6 +462,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
serializer = (Serializer) conf.getTableInfo().getDeserializerClass().newInstance();
serializer.initialize(hconf, conf.getTableInfo().getProperties());
outputClass = serializer.getSerializedClass();
+ inheritPerms = HiveConf.getBoolVar(hconf, ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
if (isLogInfoEnabled) {
LOG.info("Using serializer : " + serializer + " and formatter : " + hiveOutputFormat +
@@ -696,8 +698,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
// buckets of dynamic partitions will be created for each newly created partition
if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID ||
conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) {
+ Path outPath = fsp.outPaths[filesIdx];
+ mkDirIfPermsAreNeeded(outPath); // Make sure we inherit permissions.
fsp.outWriters[filesIdx] = HiveFileFormatUtils.getHiveRecordWriter(jc, conf.getTableInfo(),
- outputClass, conf, fsp.outPaths[filesIdx], reporter);
+ outputClass, conf, outPath, reporter);
// If the record writer provides stats, get it from there instead of the serde
statsFromRecordWriter[filesIdx] = fsp.outWriters[filesIdx] instanceof
StatsProvidingRecordWriter;
@@ -720,6 +724,12 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
}
+ private void mkDirIfPermsAreNeeded(Path outPath) throws IOException {
+ if (inheritPerms && !FileUtils.mkdir(fs, outPath.getParent(), inheritPerms, hconf)) {
+ LOG.warn("Unable to create directory with inheritPerms: " + outPath);
+ }
+ }
+
/**
* Report status to JT so that JT won't kill this task if closing takes too long
* due to too many files to close and the NN is overloaded.
@@ -1273,6 +1283,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
private void publishStats() throws HiveException {
+ Utilities.LOG14535.error("FSOP publishStats called.");
boolean isStatsReliable = conf.isStatsReliable();
// Initializing a stats publisher
@@ -1303,6 +1314,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
for (Map.Entry<String, FSPaths> entry : valToPaths.entrySet()) {
String fspKey = entry.getKey(); // DP/LB
FSPaths fspValue = entry.getValue();
+ Utilities.LOG14535.info("Observing entry for stats " + fspKey
+ + " => FSP with tmpPath " + fspValue.getTmpPath());
// for bucketed tables, hive.optimize.sort.dynamic.partition optimization
// adds the taskId to the fspKey.
if (conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) {
@@ -1315,6 +1328,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
// not be retrieved from staging table and hence not aggregated. To avoid this issue
// we will remove the taskId from the key which is redundant anyway.
fspKey = fspKey.split(taskID)[0];
+ Utilities.LOG14535.info("Adjusting fspKey for stats to " + fspKey);
}
// split[0] = DP, split[1] = LB
@@ -1326,12 +1340,14 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
String prefix = conf.getTableInfo().getTableName().toLowerCase();
prefix = Utilities.join(prefix, spSpec, dpSpec);
prefix = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR;
+ Utilities.LOG14535.info("Prefix for stats " + prefix + " (from " + spSpec + ", " + dpSpec + ")");
Map<String, String> statsToPublish = new HashMap<String, String>();
for (String statType : fspValue.getStoredStats()) {
statsToPublish.put(statType, Long.toString(fspValue.stat.getStat(statType)));
}
if (!statsPublisher.publishStat(prefix, statsToPublish)) {
+ Utilities.LOG14535.error("Failed to publish stats");
// The original exception is lost.
// Not changing the interface to maintain backward compatibility
if (isStatsReliable) {
@@ -1341,6 +1357,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
sContext.setIndexForTezUnion(this.getIndexForTezUnion());
if (!statsPublisher.closeConnection(sContext)) {
+ Utilities.LOG14535.error("Failed to close stats");
// The original exception is lost.
// Not changing the interface to maintain backward compatibility
if (isStatsReliable) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index de2281b..721d22a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -273,8 +273,12 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
if (lfd != null) {
Path targetPath = lfd.getTargetDir();
Path sourcePath = lfd.getSourcePath();
- Utilities.LOG14535.info("MoveTask moving LFD " + sourcePath + " to " + targetPath);
- moveFile(sourcePath, targetPath, lfd.getIsDfsDir());
+ if (targetPath.equals(sourcePath)) {
+ Utilities.LOG14535.info("MoveTask not moving LFD " + sourcePath);
+ } else {
+ Utilities.LOG14535.info("MoveTask moving LFD " + sourcePath + " to " + targetPath);
+ moveFile(sourcePath, targetPath, lfd.getIsDfsDir());
+ }
}
// Multi-file load is for dynamic partitions when some partitions do not
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index d8edeb1..8c9f622 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -3052,12 +3052,23 @@ public final class Utilities {
Set<Path> pathsProcessed = new HashSet<Path>();
List<Path> pathsToAdd = new LinkedList<Path>();
// AliasToWork contains all the aliases
- for (String alias : work.getAliasToWork().keySet()) {
+ Collection<String> aliasToWork = work.getAliasToWork().keySet();
+ if (!skipDummy) {
+ // ConcurrentModification otherwise if adding dummy.
+ aliasToWork = new ArrayList<>(aliasToWork);
+ }
+ for (String alias : aliasToWork) {
LOG.info("Processing alias " + alias);
// The alias may not have any path
+ Collection<Map.Entry<Path, ArrayList<String>>> pathToAliases =
+ work.getPathToAliases().entrySet();
+ if (!skipDummy) {
+ // ConcurrentModification otherwise if adding dummy.
+ pathToAliases = new ArrayList<>(pathToAliases);
+ }
Path path = null;
- for (Map.Entry<Path, ArrayList<String>> e : work.getPathToAliases().entrySet()) {
+ for (Map.Entry<Path, ArrayList<String>> e : pathToAliases) {
Path file = e.getKey();
List<String> aliases = e.getValue();
if (aliases.contains(alias)) {
@@ -3072,11 +3083,8 @@ public final class Utilities {
pathsProcessed.add(path);
LOG.info("Adding input file " + path);
- if (!skipDummy
- && isEmptyPath(job, path, ctx)) {
- path = createDummyFileForEmptyPartition(path, job, work,
- hiveScratchDir);
-
+ if (!skipDummy && isEmptyPath(job, path, ctx)) {
+ path = createDummyFileForEmptyPartition(path, job, work, hiveScratchDir);
}
pathsToAdd.add(path);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index f0a3a6c..a5de511 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -1642,7 +1643,6 @@ public class Hive {
org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
/* Construct list bucketing location mappings from sub-directory name. */
- // TODO# HERE probably broken
Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(
newPartPath, skewedInfo);
/* Add list bucketing location mappings. */
@@ -1768,8 +1768,11 @@ public class Hive {
private void walkDirTree(FileStatus fSta, FileSystem fSys,
Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
throws IOException {
+ // TODO# HERE broken
+
/* Base Case. It's leaf. */
if (!fSta.isDir()) {
+ Utilities.LOG14535.info("Processing LB leaf " + fSta.getPath());
/* construct one location map if not exists. */
constructOneLBLocationMap(fSta, skewedColValueLocationMaps, newPartPath, skewedInfo);
return;
@@ -1778,6 +1781,7 @@ private void walkDirTree(FileStatus fSta, FileSystem fSys,
/* dfs. */
FileStatus[] children = fSys.listStatus(fSta.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
if (children != null) {
+ Utilities.LOG14535.info("Processing LB dir " + fSta.getPath());
for (FileStatus child : children) {
walkDirTree(child, fSys, skewedColValueLocationMaps, newPartPath, skewedInfo);
}
@@ -1798,22 +1802,37 @@ private void constructOneLBLocationMap(FileStatus fSta,
List<String> skewedValue = new ArrayList<String>();
String lbDirName = FileUtils.unescapePathName(lbdPath.toString());
String partDirName = FileUtils.unescapePathName(newPartPath.toString());
- String lbDirSuffix = lbDirName.replace(partDirName, "");
+ String lbDirSuffix = lbDirName.replace(partDirName, ""); // TODO: wtf?
+ if (lbDirSuffix.startsWith(Path.SEPARATOR)) {
+ lbDirSuffix = lbDirSuffix.substring(1);
+ }
String[] dirNames = lbDirSuffix.split(Path.SEPARATOR);
- for (String dirName : dirNames) {
- if ((dirName != null) && (dirName.length() > 0)) {
- // Construct skewed-value to location map except default directory.
- // why? query logic knows default-dir structure and don't need to get from map
- if (!dirName
- .equalsIgnoreCase(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)) {
- String[] kv = dirName.split("=");
- if (kv.length == 2) {
- skewedValue.add(kv[1]);
- }
+ int keysFound = 0, dirsToTake = 0;
+ int colCount = skewedInfo.getSkewedColNames().size();
+ while (dirsToTake < dirNames.length && keysFound < colCount) {
+ String dirName = dirNames[dirsToTake++];
+ // Construct skewed-value to location map except default directory.
+ // why? query logic knows default-dir structure and don't need to get from map
+ if (dirName.equalsIgnoreCase(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)) {
+ ++keysFound;
+ } else {
+ String[] kv = dirName.split("=");
+ if (kv.length == 2) {
+ skewedValue.add(kv[1]);
+ ++keysFound;
+ } else {
+ // TODO: we should really probably throw. Keep the existing logic for now.
+ LOG.warn("Skipping unknown directory " + dirName
+ + " when expecting LB keys or default directory (from " + lbDirName + ")");
}
}
}
- if ((skewedValue.size() > 0) && (skewedValue.size() == skewedInfo.getSkewedColNames().size())
+ for (int i = 0; i < (dirNames.length - dirsToTake); ++i) {
+ lbdPath = lbdPath.getParent();
+ }
+ Utilities.LOG14535.info("Saving LB location " + lbdPath + " based on "
+ + colCount + " keys and " + fSta.getPath());
+ if ((skewedValue.size() > 0) && (skewedValue.size() == colCount)
&& !skewedColValueLocationMaps.containsKey(skewedValue)) {
skewedColValueLocationMaps.put(skewedValue, lbdPath.toString());
}
@@ -1830,10 +1849,11 @@ private void constructOneLBLocationMap(FileStatus fSta,
*/
private Map<List<String>, String> constructListBucketingLocationMap(Path newPartPath,
SkewedInfo skewedInfo) throws IOException, FileNotFoundException {
+ Utilities.LOG14535.info("Constructing list bucketing map for " + newPartPath);
Map<List<String>, String> skewedColValueLocationMaps = new HashMap<List<String>, String>();
FileSystem fSys = newPartPath.getFileSystem(conf);
- walkDirTree(fSys.getFileStatus(newPartPath), fSys, skewedColValueLocationMaps, newPartPath,
- skewedInfo);
+ walkDirTree(fSys.getFileStatus(newPartPath),
+ fSys, skewedColValueLocationMaps, newPartPath, skewedInfo);
return skewedColValueLocationMaps;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index aa2ae35..c3228d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1386,28 +1386,14 @@ public final class GenMapRedUtils {
// MM directory, the original MoveTask still commits based on the parent. Note that this path
// can only be triggered for a merge that's part of insert for now; MM tables do not support
// concatenate. Keeping the old logic for non-MM tables with temp directories and stuff.
+ // TODO# is this correct?
Path fsopPath = srcMmWriteId != null ? fsInputDesc.getFinalDirName() : finalName;
- linkMoveTask(fsopPath, cndTsk, mvTasks, conf, dependencyTask);
- }
-
- /**
- * Make the move task in the GenMRProcContext following the FileSinkOperator a dependent of all
- * possible subtrees branching from the ConditionalTask.
- *
- * @param newOutput
- * @param cndTsk
- * @param mvTasks
- * @param hconf
- * @param dependencyTask
- */
- private static void linkMoveTask(Path fsopPath,
- ConditionalTask cndTsk, List<Task<MoveWork>> mvTasks, HiveConf hconf,
- DependencyCollectionTask dependencyTask) {
-
- Task<MoveWork> mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(mvTasks, fsopPath);
+ Utilities.LOG14535.info("Looking for MoveTask to make it dependant on the conditional tasks");
+ Task<MoveWork> mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
+ mvTasks, fsopPath, fsInputDesc.isMmTable());
for (Task<? extends Serializable> tsk : cndTsk.getListTasks()) {
- linkMoveTask(mvTask, tsk, hconf, dependencyTask);
+ linkMoveTask(mvTask, tsk, conf, dependencyTask);
}
}
@@ -1726,18 +1712,25 @@ public final class GenMapRedUtils {
}
public static Task<MoveWork> findMoveTaskForFsopOutput(
- List<Task<MoveWork>> mvTasks, Path fsopFinalDir) {
+ List<Task<MoveWork>> mvTasks, Path fsopFinalDir, boolean isMmFsop) {
// find the move task
for (Task<MoveWork> mvTsk : mvTasks) {
MoveWork mvWork = mvTsk.getWork();
+ if (mvWork.isNoop()) continue;
Path srcDir = null;
+ boolean isLfd = false;
if (mvWork.getLoadFileWork() != null) {
srcDir = mvWork.getLoadFileWork().getSourcePath();
+ isLfd = true;
+ if (isMmFsop) {
+ srcDir = srcDir.getParent();
+ }
} else if (mvWork.getLoadTableWork() != null) {
- srcDir = mvWork.getLoadTableWork().getSourcePath();
+ srcDir = mvWork.getLoadTableWork().getSourcePath(); // TODO# THIS
}
Utilities.LOG14535.info("Observing MoveWork " + System.identityHashCode(mvWork)
- + " with " + srcDir + " while looking for " + fsopFinalDir);
+ + " with " + srcDir + "(from " + (isLfd ? "LFD" : "LTD") + ") while looking for "
+ + fsopFinalDir + "(mm = " + isMmFsop + ")");
if ((srcDir != null) && srcDir.equals(fsopFinalDir)) {
return mvTsk;
@@ -1748,23 +1741,16 @@ public final class GenMapRedUtils {
/**
* Returns true iff the fsOp requires a merge
- * @param mvTasks
- * @param hconf
- * @param fsOp
- * @param currTask
- * @param isInsertTable
- * @return
*/
- public static boolean isMergeRequired(List<Task<MoveWork>> mvTasks, HiveConf hconf, FileSinkOperator fsOp,
- Task<? extends Serializable> currTask, boolean isInsertTable) {
-
+ public static boolean isMergeRequired(List<Task<MoveWork>> mvTasks, HiveConf hconf,
+ FileSinkOperator fsOp, Task<? extends Serializable> currTask, boolean isInsertTable) {
// Has the user enabled merging of files for map-only jobs or for all jobs
if (mvTasks == null || mvTasks.isEmpty()) return false;
// no need of merging if the move is to a local file system
// We are looking based on the original FSOP, so use the original path as is.
MoveTask mvTask = (MoveTask) GenMapRedUtils.findMoveTaskForFsopOutput(
- mvTasks, fsOp.getConf().getFinalDirName());
+ mvTasks, fsOp.getConf().getFinalDirName(), fsOp.getConf().isMmTable());
// TODO: wtf? wtf?!! why is this in this method?
if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)
@@ -1864,7 +1850,10 @@ public final class GenMapRedUtils {
Task<MoveWork> mvTask = null;
if (!chDir) {
- mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(mvTasks, fsOp.getConf().getFinalDirName());
+ // TODO# is it correct to always use MM dir in MM case here? Where does MoveTask point?
+ Utilities.LOG14535.info("Looking for MoveTask from createMoveTask");
+ mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
+ mvTasks, fsOp.getConf().getFinalDirName(), fsOp.getConf().isMmTable());
}
// Set the move task to be dependent on the current task
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
index 8077721..a92465f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
@@ -77,7 +77,7 @@ public class FSStatsPublisher implements StatsPublisher {
@Override
public boolean publishStat(String partKV, Map<String, String> stats) {
- LOG.debug("Putting in map : " + partKV + "\t" + stats);
+ Utilities.LOG14535.info("Putting in map : " + partKV + "\t" + stats);
// we need to do new hashmap, since stats object is reused across calls.
Map<String,String> cpy = new HashMap<String, String>(stats);
Map<String,String> statMap = statsMap.get(partKV);
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q
index 7e0330c..f9a829c 100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@ -18,12 +18,13 @@ insert into table intermediate partition(p='457') select distinct key from src w
drop table part_mm;
create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
-explain insert into table part_mm partition(key_mm='455') select key from intermediate;
-insert into table part_mm partition(key_mm='455') select key from intermediate;
-insert into table part_mm partition(key_mm='456') select key from intermediate;
-insert into table part_mm partition(key_mm='455') select key from intermediate;
+explain insert into table part_mm partition(key_mm=455) select key from intermediate;
+insert into table part_mm partition(key_mm=455) select key from intermediate;
+insert into table part_mm partition(key_mm=456) select key from intermediate;
+insert into table part_mm partition(key_mm=455) select key from intermediate;
select * from part_mm order by key, key_mm;
-truncate table part_mm partition(key_mm='455');
+
+-- TODO: doesn't work truncate table part_mm partition(key_mm=455);
select * from part_mm order by key, key_mm;
truncate table part_mm;
select * from part_mm order by key, key_mm;
@@ -430,6 +431,12 @@ insert into table stats_mm select key from intermediate;
desc formatted stats_mm;
drop table stats_mm;
+drop table stats2_mm;
+create table stats2_mm tblproperties("transactional"="true", "transactional_properties"="insert_only") as select array(key, value) from src;
+desc formatted stats2_mm;
+drop table stats2_mm;
+
+
drop table intermediate;
http://git-wip-us.apache.org/repos/asf/hive/blob/1f6c5ac4/ql/src/test/queries/clientpositive/mm_current.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_current.q b/ql/src/test/queries/clientpositive/mm_current.q
index 93103d3..291a81a 100644
--- a/ql/src/test/queries/clientpositive/mm_current.q
+++ b/ql/src/test/queries/clientpositive/mm_current.q
@@ -14,59 +14,22 @@ insert into table intermediate partition(p='455') select distinct key from src w
insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2;
-drop table multi0_1_mm;
-drop table multi0_2_mm;
-create table multi0_1_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
-create table multi0_2_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+drop table intermmediate_nonpart;
+create table intermmediate_nonpart(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+insert into intermmediate_nonpart select * from intermediate;
-from intermediate
-insert overwrite table multi0_1_mm select key, p
-insert overwrite table multi0_2_mm select p, key;
+set hive.exim.test.mode=true;
-select * from multi0_1_mm order by key, key2;
-select * from multi0_2_mm order by key, key2;
-
-set hive.merge.mapredfiles=true;
-set hive.merge.sparkfiles=true;
-set hive.merge.tezfiles=true;
-
-from intermediate
-insert into table multi0_1_mm select p, key
-insert overwrite table multi0_2_mm select key, p;
-select * from multi0_1_mm order by key, key2;
-select * from multi0_2_mm order by key, key2;
-
-set hive.merge.mapredfiles=false;
-set hive.merge.sparkfiles=false;
-set hive.merge.tezfiles=false;
-
-drop table multi0_1_mm;
-drop table multi0_2_mm;
-
-
-drop table multi1_mm;
-create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
-from intermediate
-insert into table multi1_mm partition(p=1) select p, key
-insert into table multi1_mm partition(p=2) select key, p;
-select * from multi1_mm order by key, key2, p;
-from intermediate
-insert into table multi1_mm partition(p=2) select p, key
-insert overwrite table multi1_mm partition(p=1) select key, p;
-select * from multi1_mm order by key, key2, p;
-
-from intermediate
-insert into table multi1_mm partition(p) select p, key, p
-insert into table multi1_mm partition(p=1) select key, p;
-select key, key2, p from multi1_mm order by key, key2, p;
-
-from intermediate
-insert into table multi1_mm partition(p) select p, key, 1
-insert into table multi1_mm partition(p=1) select key, p;
-select key, key2, p from multi1_mm order by key, key2, p;
-drop table multi1_mm;
+export table intermmediate_nonpart to 'ql/test/data/exports/intermmediate_nonpart';
+drop table intermmediate_nonpart;
+-- MM export into new MM table, non-part and part
+drop table import2_mm;
+import table import2_mm from 'ql/test/data/exports/intermmediate_nonpart';
+desc import2_mm;
+select * from import2_mm order by key, p;
+drop table import2_mm;
drop table intermediate;