You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/10/13 00:16:05 UTC
[26/50] [abbrv] hive git commit: HIVE-15212 : merge branch into
master - some initial cleanup and fixes (Sergey Shelukhin)
HIVE-15212 : merge branch into master - some initial cleanup and fixes (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/543e0b01
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/543e0b01
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/543e0b01
Branch: refs/heads/master
Commit: 543e0b01a27fa5a51ac3adcfb6c878c403c6be7b
Parents: 9a9f254
Author: sergey <se...@apache.org>
Authored: Fri Sep 15 12:51:35 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Sep 15 12:51:35 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/MetaStoreUtils.java | 3 +-
.../hadoop/hive/metastore/TestObjectStore.java | 3 +-
.../hive/ql/exec/HashTableSinkOperator.java | 3 +-
.../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +-
.../hadoop/hive/ql/exec/ReplCopyTask.java | 2 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 32 ++++++++++++--------
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../BucketingSortingReduceSinkOptimizer.java | 1 -
.../hive/ql/optimizer/GenMapRedUtils.java | 7 ++---
.../hive/ql/optimizer/StatsOptimizer.java | 6 ----
.../hive/ql/parse/ImportSemanticAnalyzer.java | 20 ++++++------
.../hive/ql/plan/BucketMapJoinContext.java | 1 -
.../apache/hadoop/hive/ql/plan/MoveWork.java | 1 -
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 17 ++++++-----
14 files changed, 49 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 49005b9..faec3a8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1977,7 +1977,6 @@ public class MetaStoreUtils {
* @param params table properties
* @return true if table is an INSERT_ONLY table, false otherwise
*/
- // TODO# also check that transactional is true
public static boolean isInsertOnlyTable(Map<String, String> params) {
return isInsertOnlyTable(params, false);
}
@@ -1998,7 +1997,7 @@ public class MetaStoreUtils {
/** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
public static Boolean isToInsertOnlyTable(Map<String, String> props) {
- // TODO# Setting these separately is a very hairy issue in certain combinations, since we
+ // TODO: Setting these separately is a very hairy issue in certain combinations, since we
// cannot decide what type of table this becomes without taking both into account, and
// in many cases the conversion might be illegal.
// The only thing we allow is tx = true w/o tx-props, for backward compat.
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 176fc4c..e6661f9 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore;
import com.codahale.metrics.Counter;
+import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
@@ -57,8 +58,6 @@ import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
import javax.jdo.Query;
import java.util.Arrays;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
index f4f20f0..0b8045e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
@@ -301,9 +301,10 @@ public class HashTableSinkOperator extends TerminalOperator<HashTableSinkDesc> i
String bigBucketFileName = getExecContext().getCurrentBigBucketFile();
String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName);
// get the tmp URI path; it will be a hdfs path if not local mode
- // TODO# this doesn't work... the path for writer and reader mismatch
+ // TODO: this doesn't work... the path for writer and reader mismatch
// Dump the side-table for tag ... -local-10004/HashTable-Stage-1/MapJoin-a-00-(ds%3D2008-04-08)mm_2.hashtable
// Load back 1 hashtable file -local-10004/HashTable-Stage-1/MapJoin-a-00-srcsortbucket3outof4.txt.hashtable
+ // Hive3 probably won't support MR so do we really care?
String dumpFilePrefix = conf.getDumpFilePrefix();
Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName);
console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag +
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 34d0598..739ce18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -268,7 +268,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
@Override
public int execute(DriverContext driverContext) {
- if (work.isNoop()) return 0; // TODO# temporary flag for HIVE-14990
+ if (work.isNoop()) return 0;
Utilities.LOG14535.info("Executing MoveWork " + System.identityHashCode(work)
+ " with " + work.getLoadFileWork() + "; " + work.getLoadTableWork() + "; "
+ work.getLoadMultiFilesWork());
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
index 86e21cb..1a9b39f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
@@ -61,7 +61,7 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
FileSystem dstFs = null;
Path toPath = null;
try {
- // TODO# merge with real CopyTask logic
+ // TODO: merge with real CopyTask logic?
Path fromPath = work.getFromPaths()[0];
toPath = work.getToPaths()[0];
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index c6d4616..3f6fb36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1124,9 +1124,6 @@ public final class Utilities {
if (orig.getName().indexOf(tmpPrefix) == 0) {
return orig;
}
- if (orig.getName().contains("=1")) {
- LOG.error("TODO# creating tmp path from " + orig, new Exception());
- }
return new Path(orig.getParent(), tmpPrefix + orig.getName());
}
@@ -1466,7 +1463,8 @@ public final class Utilities {
Set<Path> filesKept = new HashSet<Path>();
perfLogger.PerfLogBegin("FileSinkOperator", "RemoveTempOrDuplicateFiles");
// remove any tmp file or double-committed output files
- List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, statuses, dpCtx, conf, hconf, filesKept);
+ List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(
+ fs, statuses, dpCtx, conf, hconf, filesKept);
perfLogger.PerfLogEnd("FileSinkOperator", "RemoveTempOrDuplicateFiles");
// create empty buckets if necessary
if (emptyBuckets.size() > 0) {
@@ -1578,11 +1576,12 @@ public final class Utilities {
* @return a list of path names corresponding to should-be-created empty buckets.
*/
public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
- DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<Path> filesKept) throws IOException {
+ DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<Path> filesKept)
+ throws IOException {
int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
- numBuckets = (conf != null && conf.getTable() != null)
- ? conf.getTable().getNumBuckets() : 0;
- return removeTempOrDuplicateFiles(fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false);
+ numBuckets = (conf != null && conf.getTable() != null) ? conf.getTable().getNumBuckets() : 0;
+ return removeTempOrDuplicateFiles(
+ fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false, filesKept);
}
private static boolean removeEmptyDpDirectory(FileSystem fs, Path path) throws IOException {
@@ -1598,7 +1597,8 @@ public final class Utilities {
}
public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
- int dpLevels, int numBuckets, Configuration hconf, Long txnId, int stmtId, boolean isMmTable) throws IOException {
+ int dpLevels, int numBuckets, Configuration hconf, Long txnId, int stmtId, boolean isMmTable,
+ Set<Path> filesKept) throws IOException {
if (fileStats == null) {
return null;
}
@@ -1625,6 +1625,9 @@ public final class Utilities {
Utilities.LOG14535.info("removeTempOrDuplicateFiles processing files in MM directory " + mmDir);
}
taskIDToFile = removeTempOrDuplicateFilesNonMm(items, fs);
+ if (filesKept != null && taskIDToFile != null) {
+ addFilesToPathSet(taskIDToFile.values(), filesKept);
+ }
// TODO: not clear why two if conditions are different. Preserve the existing logic for now.
addBucketFileToResults(taskIDToFile, numBuckets, hconf, result);
@@ -1636,6 +1639,9 @@ public final class Utilities {
}
if (!isMmTable) {
taskIDToFile = removeTempOrDuplicateFilesNonMm(items, fs);
+ if (filesKept != null && taskIDToFile != null) {
+ addFilesToPathSet(taskIDToFile.values(), filesKept);
+ }
} else {
if (items.length > 1) {
throw new IOException("Unexpected directories for non-DP MM: " + Arrays.toString(items));
@@ -1647,6 +1653,9 @@ public final class Utilities {
Utilities.LOG14535.info(
"removeTempOrDuplicateFiles processing files in MM directory " + mmDir);
taskIDToFile = removeTempOrDuplicateFilesNonMm(fs.listStatus(mmDir), fs);
+ if (filesKept != null && taskIDToFile != null) {
+ addFilesToPathSet(taskIDToFile.values(), filesKept);
+ }
}
// TODO: not clear why two if conditions are different. Preserve the existing logic for now.
addBucketFileToResults2(taskIDToFile, numBuckets, hconf, result);
@@ -4194,7 +4203,6 @@ public final class Utilities {
}
Utilities.LOG14535.info("Looking for manifests in: " + manifestDir + " (" + txnId + ")");
- // TODO# may be wrong if there are no splits (empty insert/CTAS)
List<Path> manifests = new ArrayList<>();
if (fs.exists(manifestDir)) {
FileStatus[] manifestFiles = fs.listStatus(manifestDir);
@@ -4275,8 +4283,8 @@ public final class Utilities {
for (int i = 0; i < mmDirectories.size(); ++i) {
finalResults[i] = new PathOnlyFileStatus(mmDirectories.get(i));
}
- List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(
- fs, finalResults, dpLevels, mbc == null ? 0 : mbc.numBuckets, hconf, txnId, stmtId, isMmTable);
+ List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, finalResults, dpLevels,
+ mbc == null ? 0 : mbc.numBuckets, hconf, txnId, stmtId, isMmTable, null);
// create empty buckets if necessary
if (emptyBuckets.size() > 0) {
assert mbc != null;
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ff5ffe9..e51e648 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1911,7 +1911,7 @@ public class Hive {
private void walkDirTree(FileStatus fSta, FileSystem fSys,
Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
throws IOException {
- // TODO# HERE broken
+ // TODO: may be broken? no LB bugs for now but if any are found.
/* Base Case. It's leaf. */
if (!fSta.isDir()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
index 09c3079..85dcdd7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
@@ -237,7 +237,6 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
Map<String, Integer> bucketFileNameMapping = new HashMap<String, Integer>();
for (int pos = 0; pos < srcs.length; pos++) {
if (ShimLoader.getHadoopShims().isDirectory(srcs[pos])) {
- // TODO# HERE
throw new RuntimeException("Was expecting '" + srcs[pos].getPath() + "' to be bucket file.");
}
bucketFileNameMapping.put(srcs[pos].getPath().getName(), pos);
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 02e6512..6dfa607 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1371,7 +1371,7 @@ public final class GenMapRedUtils {
dummyMv = new MoveWork(null, null, null,
new LoadFileDesc(inputDirName, finalName, true, null, null, false), false);
} else {
- // TODO# create the noop MoveWork to avoid q file changes for now. Should be removed.
+ // TODO# create the noop MoveWork to avoid q file changes for now. Should be removed w/the flag just before merge
dummyMv = new MoveWork(null, null, null,
new LoadFileDesc(inputDirName, finalName, true, null, null, false), false);
dummyMv.setNoop(true);
@@ -1380,13 +1380,11 @@ public final class GenMapRedUtils {
// MM directory, the original MoveTask still commits based on the parent. Note that this path
// can only be triggered for a merge that's part of insert for now; MM tables do not support
// concatenate. Keeping the old logic for non-MM tables with temp directories and stuff.
- // TODO# is this correct?
Path fsopPath = srcMmWriteId != null ? fsInputDesc.getFinalDirName() : finalName;
Utilities.LOG14535.info("Looking for MoveTask to make it dependant on the conditional tasks");
Task<MoveWork> mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
mvTasks, fsopPath, fsInputDesc.isMmTable());
- // TODO# questionable master merge here
ConditionalTask cndTsk = GenMapRedUtils.createCondTask(conf, currTask, dummyMv, work,
fsInputDesc.getMergeInputDirName(), finalName, mvTask, dependencyTask);
@@ -1847,7 +1845,7 @@ public final class GenMapRedUtils {
srcDir = srcDir.getParent();
}
} else if (mvWork.getLoadTableWork() != null) {
- srcDir = mvWork.getLoadTableWork().getSourcePath(); // TODO# THIS
+ srcDir = mvWork.getLoadTableWork().getSourcePath();
}
Utilities.LOG14535.info("Observing MoveWork " + System.identityHashCode(mvWork)
+ " with " + srcDir + "(from " + (isLfd ? "LFD" : "LTD") + ") while looking for "
@@ -1970,7 +1968,6 @@ public final class GenMapRedUtils {
Task<MoveWork> mvTask = null;
if (!chDir) {
- // TODO# is it correct to always use MM dir in MM case here? Where does MoveTask point?
Utilities.LOG14535.info("Looking for MoveTask from createMoveTask");
mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
mvTasks, fsOp.getConf().getFinalDirName(), fsOp.getConf().isMmTable());
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index 0fe1ea3..7b2a1a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -342,12 +342,6 @@ public class StatsOptimizer extends Transform {
return null; // todo we can collapse this part of tree into single TS
}
- /* TODO# seems to be removed in master?
- Table tbl = tsOp.getConf().getTableMetadata();
- if (AcidUtils.isFullAcidTable(tbl)) {
- Logger.info("Table " + tbl.getTableName() + " is ACID table. Skip StatsOptimizer.");
- return null;
- }*/
List<Object> oneRow = new ArrayList<Object>();
Hive hive = Hive.get(pctx.getConf());
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index a41cc8b..814910d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -314,9 +314,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
Long txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
int stmtId = 0;
- //todo due to the master merge, tblDesc is no longer CreateTableDesc, but ImportTableDesc
+ // TODO: due to the master merge, tblDesc is no longer CreateTableDesc, but ImportTableDesc
/*
- if (txnId != null) {
+ if (isAcid(txnId)) {
tblDesc.setInitialMmWriteId(txnId);
}
*/
@@ -374,10 +374,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
Task<?> copyTask = null;
if (replicationSpec.isInReplicationScope()) {
- if (isSourceMm || txnId != null) {
+ if (isSourceMm || isAcid(txnId)) {
// TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
- throw new RuntimeException(
- "Not supported right now because Replication is completely screwed");
+ throw new RuntimeException("Replicating MM and ACID tables is not supported");
}
copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf());
} else {
@@ -397,6 +396,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
return loadTableTask;
}
+ private static boolean isAcid(Long txnId) {
+ return (txnId != null) && (txnId != 0);
+ }
+
private static Task<?> createTableTask(ImportTableDesc tableDesc, EximUtil.SemanticAnalyzerWrapperContext x){
return tableDesc.getCreateTableTask(x.getInputs(), x.getOutputs(), x.getConf());
}
@@ -458,10 +461,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
Task<?> copyTask = null;
if (replicationSpec.isInReplicationScope()) {
- if (isSourceMm || txnId != null) {
+ if (isSourceMm || isAcid(txnId)) {
// TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
- throw new RuntimeException(
- "Not supported right now because Replication is completely screwed");
+ throw new RuntimeException("Replicating MM and ACID tables is not supported");
}
copyTask = ReplCopyTask.getLoadCopyTask(
replicationSpec, new Path(srcLocation), destPath, x.getConf());
@@ -480,7 +482,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
loadTableWork.setInheritTableSpecs(false);
// Do not commit the write ID from each task; need to commit once.
// TODO: we should just change the import to use a single MoveTask, like dynparts.
- loadTableWork.setIntermediateInMmWrite(txnId != null);
+ loadTableWork.setIntermediateInMmWrite(isAcid(txnId));
Task<?> loadPartTask = TaskFactory.get(new MoveWork(
x.getInputs(), x.getOutputs(), loadTableWork, null, false), x.getConf());
copyTask.addDependentTask(loadPartTask);
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
index 96f722c..4587c32 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
@@ -39,7 +39,6 @@ public class BucketMapJoinContext implements Serializable {
private static final long serialVersionUID = 1L;
- // TODO# this is completely broken, esp. w/load into bucketed tables (should perhaps be forbidden for MM tables)
// table alias (small) --> input file name (big) --> target file names (small)
private Map<String, Map<String, List<String>>> aliasBucketFileNameMapping;
private String mapJoinBigTableAlias;
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index 50adc42..8594edf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -153,7 +153,6 @@ public class MoveWork implements Serializable {
this.srcLocal = srcLocal;
}
- // TODO# temporary test flag
public void setNoop(boolean b) {
this.isNoop = true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 9171e60..a8a1fd6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -368,18 +368,19 @@ public class TestTxnCommands2 {
*/
String[][] expected = {
{"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t0\t13", "bucket_00000"},
- {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000"},
- {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000"},
- {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000"},
+ {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000"},
+ {"{\"transactionid\":22,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000"},
+ {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000"},
{"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t1\t2", "bucket_00001"},
{"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":3}\t1\t4", "bucket_00001"},
{"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":2}\t1\t5", "bucket_00001"},
{"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":4}\t1\t6", "bucket_00001"},
- {"{\"transactionid\":18,\"bucketid\":536936448,\"rowid\":0}\t1\t16", "bucket_00001"}
+ {"{\"transactionid\":20,\"bucketid\":536936448,\"rowid\":0}\t1\t16", "bucket_00001"}
};
Assert.assertEquals("Unexpected row count before compaction", expected.length, rs.size());
for(int i = 0; i < expected.length; i++) {
- Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i) + "; expected " + expected[i][0],
+ rs.get(i).startsWith(expected[i][0]));
}
//run Compaction
runStatementOnDriver("alter table "+ TestTxnCommands2.Table.NONACIDORCTBL +" compact 'major'");
@@ -763,7 +764,7 @@ public class TestTxnCommands2 {
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
} else if (numDelta == 2) {
- Assert.assertEquals("delta_0000023_0000023_0000", status[i].getPath().getName());
+ Assert.assertEquals("delta_0000025_0000025_0000", status[i].getPath().getName());
Assert.assertEquals(1, buckets.length);
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
}
@@ -772,7 +773,7 @@ public class TestTxnCommands2 {
FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
Arrays.sort(buckets);
if (numDeleteDelta == 1) {
- Assert.assertEquals("delete_delta_0000022_0000022_0000", status[i].getPath().getName());
+ Assert.assertEquals("delete_delta_0000024_0000024_0000", status[i].getPath().getName());
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
}
@@ -819,7 +820,7 @@ public class TestTxnCommands2 {
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
} else if (numBase == 2) {
// The new base dir now has two bucket files, since the delta dir has two bucket files
- Assert.assertEquals("base_0000023", status[i].getPath().getName());
+ Assert.assertEquals("base_0000025", status[i].getPath().getName());
Assert.assertEquals(1, buckets.length);
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
}