You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/19 18:18:07 UTC
[40/40] hive git commit: HIVE-14996 : handle load for MM tables
(Sergey Shelukhin)
HIVE-14996 : handle load for MM tables (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a057e12c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a057e12c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a057e12c
Branch: refs/heads/hive-14535
Commit: a057e12cfeff59a6955d58e3bc4107a204b88ea2
Parents: 4de0d96
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Oct 19 11:13:02 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Oct 19 11:13:02 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +-
.../hive/ql/exec/OrcFileMergeOperator.java | 2 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 73 ++--
.../hive/ql/parse/ImportSemanticAnalyzer.java | 7 +-
.../hive/ql/parse/LoadSemanticAnalyzer.java | 15 +-
.../hadoop/hive/ql/plan/FileSinkDesc.java | 16 -
.../hadoop/hive/ql/plan/LoadFileDesc.java | 2 +-
.../hadoop/hive/ql/plan/LoadTableDesc.java | 6 +-
ql/src/test/queries/clientpositive/mm_all.q | 86 +++--
ql/src/test/queries/clientpositive/mm_current.q | 51 ++-
.../results/clientpositive/llap/mm_all.q.out | 354 ++++++++++++++++---
.../clientpositive/llap/mm_current.q.out | 313 ++++++++++++++--
12 files changed, 753 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 02059fb..eea4357 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -325,7 +325,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
DataContainer dc = null;
if (tbd.getPartitionSpec().size() == 0) {
dc = new DataContainer(table.getTTable());
- Utilities.LOG14535.info("loadTable called from " + tbd.getSourcePath() + " into " + tbd.getTable().getTableName(), new Exception());
+ Utilities.LOG14535.info("loadTable called from " + tbd.getSourcePath() + " into " + tbd.getTable().getTableName());
db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getReplace(),
work.isSrcLocal(), isSkewedStoredAsDirs(tbd), isAcid, hasFollowingStatsTask(),
tbd.getMmWriteId());
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index 835791b..a845b50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@ -94,7 +94,7 @@ public class OrcFileMergeOperator extends
filePath = k.getInputPath().toUri().getPath();
- Utilities.LOG14535.info("OrcFileMergeOperator processing " + filePath, new Exception());
+ Utilities.LOG14535.info("OrcFileMergeOperator processing " + filePath);
fixTmpPath(k.getInputPath().getParent());
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 4b93350..d6dc2d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1588,22 +1588,31 @@ public class Hive {
List<Path> newFiles = null;
PerfLogger perfLogger = SessionState.getPerfLogger();
perfLogger.PerfLogBegin("MoveTask", "FileMoves");
- if (mmWriteId != null) {
- Utilities.LOG14535.info("not moving " + loadPath + " to " + newPartPath);
+ if (mmWriteId != null && loadPath.equals(newPartPath)) {
+ // MM insert query, move itself is a no-op.
+ Utilities.LOG14535.info("not moving " + loadPath + " to " + newPartPath + " (MM)");
assert !isAcid;
if (areEventsForDmlNeeded(tbl, oldPart)) {
newFiles = listFilesCreatedByQuery(loadPath, mmWriteId);
}
Utilities.LOG14535.info("maybe deleting stuff from " + oldPartPath + " (new " + newPartPath + ") for replace");
- if (replace && oldPartPath != null) { // TODO# is this correct? ignore until iow jira
- deleteOldPathForReplace(newPartPath, oldPartPath,
- getConf(), new ValidWriteIds.IdPathFilter(mmWriteId, false));
+ if (replace && oldPartPath != null) {
+ deleteOldPathForReplace(newPartPath, oldPartPath, getConf(),
+ new ValidWriteIds.IdPathFilter(mmWriteId, false), mmWriteId != null);
}
} else {
- Utilities.LOG14535.info("moving " + loadPath + " to " + newPartPath);
+ // Either a non-MM query, or a load into MM table from an external source.
+ PathFilter filter = FileUtils.HIDDEN_FILES_PATH_FILTER;
+ Path destPath = newPartPath;
+ if (mmWriteId != null) {
+ // We will load into MM directory, and delete from the parent if needed.
+ destPath = new Path(destPath, ValidWriteIds.getMmFilePrefix(mmWriteId));
+ filter = replace ? new ValidWriteIds.IdPathFilter(mmWriteId, false) : filter;
+ }
+ Utilities.LOG14535.info("moving " + loadPath + " to " + destPath);
if (replace || (oldPart == null && !isAcid)) {
- replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(),
- isSrcLocal);
+ replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(),
+ isSrcLocal, filter, mmWriteId != null);
} else {
if (areEventsForDmlNeeded(tbl, oldPart)) {
newFiles = Collections.synchronizedList(new ArrayList<Path>());
@@ -2038,28 +2047,35 @@ private void constructOneLBLocationMap(FileStatus fSta,
if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
newFiles = Collections.synchronizedList(new ArrayList<Path>());
}
- if (mmWriteId == null) {
- Utilities.LOG14535.info("moving " + loadPath + " to " + tbl.getPath());
+ if (mmWriteId != null && loadPath.equals(tbl.getPath())) {
+ Utilities.LOG14535.info("not moving " + loadPath + " to " + tbl.getPath());
if (replace) {
Path tableDest = tbl.getPath();
- replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal);
+ deleteOldPathForReplace(tableDest, tableDest, sessionConf,
+ new ValidWriteIds.IdPathFilter(mmWriteId, false), mmWriteId != null);
+ }
+ newFiles = listFilesCreatedByQuery(loadPath, mmWriteId);
+ } else {
+ // Either a non-MM query, or a load into MM table from an external source.
+ Path tblPath = tbl.getPath(), destPath = tblPath;
+ PathFilter filter = FileUtils.HIDDEN_FILES_PATH_FILTER;
+ if (mmWriteId != null) {
+ // We will load into MM directory, and delete from the parent if needed.
+ destPath = new Path(destPath, ValidWriteIds.getMmFilePrefix(mmWriteId));
+ filter = replace ? new ValidWriteIds.IdPathFilter(mmWriteId, false) : filter;
+ }
+ Utilities.LOG14535.info("moving " + loadPath + " to " + tblPath);
+ if (replace) {
+ replaceFiles(tblPath, loadPath, destPath, tblPath,
+ sessionConf, isSrcLocal, filter, mmWriteId != null);
} else {
- FileSystem fs;
try {
- fs = tbl.getDataLocation().getFileSystem(sessionConf);
- copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid, newFiles);
+ FileSystem fs = tbl.getDataLocation().getFileSystem(sessionConf);
+ copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcid, newFiles);
} catch (IOException e) {
throw new HiveException("addFiles: filesystem error in check phase", e);
}
}
- } else {
- Utilities.LOG14535.info("not moving " + loadPath + " to " + tbl.getPath());
- if (replace) {
- Path tableDest = tbl.getPath();
- deleteOldPathForReplace(tableDest, tableDest, sessionConf,
- new ValidWriteIds.IdPathFilter(mmWriteId, false));
- }
- newFiles = listFilesCreatedByQuery(loadPath, mmWriteId);
}
if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE);
@@ -3423,7 +3439,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
* If the source directory is LOCAL
*/
protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf conf,
- boolean isSrcLocal) throws HiveException {
+ boolean isSrcLocal, PathFilter deletePathFilter, boolean isMmTable) throws HiveException {
try {
FileSystem destFs = destf.getFileSystem(conf);
@@ -3442,10 +3458,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
if (oldPath != null) {
- deleteOldPathForReplace(destf, oldPath, conf, FileUtils.HIDDEN_FILES_PATH_FILTER);
+ deleteOldPathForReplace(destf, oldPath, conf, deletePathFilter, isMmTable);
}
- // TODO# what are the paths that use this? MM tables will need to do this beforehand
// first call FileUtils.mkdir to make sure that destf directory exists, if not, it creates
// destf with inherited permissions
boolean inheritPerms = HiveConf.getBoolVar(conf, HiveConf.ConfVars
@@ -3478,9 +3493,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
-
private void deleteOldPathForReplace(Path destPath, Path oldPath, HiveConf conf,
- PathFilter pathFilter) throws HiveException {
+ PathFilter pathFilter, boolean isMmTable) throws HiveException {
+ Utilities.LOG14535.info("Deleting old paths for replace in " + destPath + " and old path " + oldPath);
boolean isOldPathUnderDestf = false;
try {
FileSystem oldFs = oldPath.getFileSystem(conf);
@@ -3490,7 +3505,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
// But not sure why we changed not to delete the oldPath in HIVE-8750 if it is
// not the destf or its subdir?
isOldPathUnderDestf = isSubDir(oldPath, destPath, oldFs, destFs, false);
- if (isOldPathUnderDestf) {
+ if (isOldPathUnderDestf || isMmTable) {
FileStatus[] statuses = oldFs.listStatus(oldPath, pathFilter);
if (statuses == null || statuses.length == 0) return;
String s = "Deleting files under " + oldPath + " for replace: ";
@@ -3504,7 +3519,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
} catch (IOException e) {
- if (isOldPathUnderDestf) {
+ if (isOldPathUnderDestf || isMmTable) {
// if oldPath is a subdir of destf but it could not be cleaned
throw new HiveException("Directory " + oldPath.toString()
+ " could not be cleaned up.", e);
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index a360378..043de2f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -323,10 +323,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
Path tmpPath = ctx.getExternalTmpPath(tgtPath);
Task<?> copyTask = TaskFactory.get(new CopyWork(dataPath,
tmpPath, false), conf);
+ // TODO# we assume mm=false here
LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
Utilities.getTableDesc(table), new TreeMap<String, String>(),
- replace);
- // TODO# movetask is created here; handle MM tables
+ replace, null);
Task<?> loadTableTask = TaskFactory.get(new MoveWork(getInputs(),
getOutputs(), loadTableWork, null, false), conf);
copyTask.addDependentTask(loadTableTask);
@@ -397,9 +397,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
tmpPath, false), conf);
Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
getOutputs(), addPartitionDesc), conf);
+ // TODO# we assume mm=false here
LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
Utilities.getTableDesc(table),
- partSpec.getPartSpec(), true);
+ partSpec.getPartSpec(), true, null);
loadTableWork.setInheritTableSpecs(false);
// TODO# movetask is created here; handle MM tables
Task<?> loadPartTask = TaskFactory.get(new MoveWork(
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 96311c1..e38b0f7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -46,6 +47,7 @@ import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.StatsWork;
@@ -259,10 +261,20 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
+ // TODO# movetask is created here; handle MM tables
+ Long mmWriteId = null;
+ Table tbl = ts.tableHandle;
+ if (MetaStoreUtils.isMmTable(tbl.getParameters())) {
+ try {
+ mmWriteId = db.getNextTableWriteId(tbl.getDbName(), tbl.getTableName());
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ }
LoadTableDesc loadTableWork;
loadTableWork = new LoadTableDesc(new Path(fromURI),
- Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+ Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite, mmWriteId);
if (preservePartitionSpecs){
// Note : preservePartitionSpecs=true implies inheritTableSpecs=false but
// but preservePartitionSpecs=false(default) here is not sufficient enough
@@ -270,7 +282,6 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
loadTableWork.setInheritTableSpecs(false);
}
- // TODO# movetask is created here; handle MM tables
Task<? extends Serializable> childTask = TaskFactory.get(new MoveWork(getInputs(),
getOutputs(), loadTableWork, null, true, isLocal), conf);
if (rTask != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 83f2c92..1f84531 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -493,22 +493,6 @@ public class FileSinkDesc extends AbstractOperatorDesc {
this.mmWriteId = mmWriteId;
}
- public class FileSinkOperatorExplainVectorization extends OperatorExplainVectorization {
-
- public FileSinkOperatorExplainVectorization(VectorDesc vectorDesc) {
- // Native vectorization not supported.
- super(vectorDesc, false);
- }
- }
-
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "File Sink Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public FileSinkOperatorExplainVectorization getFileSinkVectorization() {
- if (vectorDesc == null) {
- return null;
- }
- return new FileSinkOperatorExplainVectorization(vectorDesc);
- }
-
public void setIsMerge(boolean b) {
this.isMerge = b;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
index 7670ef2..072148c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
@@ -60,7 +60,7 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
final boolean isDfsDir, final String columns, final String columnTypes) {
super(sourcePath);
- Utilities.LOG14535.info("creating LFD from " + sourcePath + " to " + targetDir, new Exception());
+ Utilities.LOG14535.info("creating LFD from " + sourcePath + " to " + targetDir);
this.targetDir = targetDir;
this.isDfsDir = isDfsDir;
this.columns = columns;
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index fc8726c..bf858b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -68,9 +68,9 @@ public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
public LoadTableDesc(final Path sourcePath,
final TableDesc table,
final Map<String, String> partitionSpec,
- final boolean replace) {
- // TODO# we assume mm=false here
- this(sourcePath, table, partitionSpec, replace, AcidUtils.Operation.NOT_ACID, null);
+ final boolean replace,
+ final Long mmWriteId) {
+ this(sourcePath, table, partitionSpec, replace, AcidUtils.Operation.NOT_ACID, mmWriteId);
}
public LoadTableDesc(final Path sourcePath,
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q
index 90523d3..5377568 100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@ -177,6 +177,10 @@ select * from merge1_mm;
drop table merge1_mm;
+set hive.merge.tezfiles=false;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+
-- TODO: need to include merge+union+DP, but it's broken for now
@@ -192,9 +196,6 @@ select * from ctas1_mm;
drop table ctas1_mm;
-set hive.merge.tezfiles=false;
-set hive.merge.mapfiles=false;
-set hive.merge.mapredfiles=false;
drop table iow0_mm;
create table iow0_mm(key int) tblproperties('hivecommit'='true');
@@ -223,40 +224,65 @@ drop table iow1_mm;
+
+drop table load0_mm;
+create table load0_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true');
+load data local inpath '../../data/files/kv1.txt' into table load0_mm;
+select count(1) from load0_mm;
+load data local inpath '../../data/files/kv2.txt' into table load0_mm;
+select count(1) from load0_mm;
+load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm;
+select count(1) from load0_mm;
+drop table load0_mm;
+
+
+drop table intermediate2;
+create table intermediate2 (key string, value string) stored as textfile
+location 'file:${system:test.tmp.dir}/intermediate2';
+load data local inpath '../../data/files/kv1.txt' into table intermediate2;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data local inpath '../../data/files/kv3.txt' into table intermediate2;
+
+drop table load1_mm;
+create table load1_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true');
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' into table load1_mm;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv1.txt' into table load1_mm;
+select count(1) from load1_mm;
+load data local inpath '../../data/files/kv1.txt' into table intermediate2;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data local inpath '../../data/files/kv3.txt' into table intermediate2;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' overwrite into table load1_mm;
+select count(1) from load1_mm;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' overwrite into table load1_mm;
+select count(1) from load1_mm;
+drop table load1_mm;
+
+drop table load2_mm;
+create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties('hivecommit'='true');
+load data local inpath '../../data/files/kv1.txt' into table intermediate2;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data local inpath '../../data/files/kv3.txt' into table intermediate2;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' into table load2_mm partition(k=5, l=5);
+select count(1) from load2_mm;
+drop table load2_mm;
+drop table intermediate2;
+
+
+-- IMPORT
+
+
+
-- TODO# future
--
---create table load_overwrite (key string, value string) stored as textfile location 'file:${system:test.tmp.dir}/load_overwrite';
---create table load_overwrite2 (key string, value string) stored as textfile location 'file:${system:test.tmp.dir}/load2_overwrite2';
---
---load data local inpath '../../data/files/kv1.txt' into table load_overwrite;
---load data local inpath '../../data/files/kv2.txt' into table load_overwrite;
---load data local inpath '../../data/files/kv3.txt' into table load_overwrite;
---
---show table extended like load_overwrite;
---desc extended load_overwrite;
---select count(*) from load_overwrite;
---
---load data inpath '${system:test.tmp.dir}/load_overwrite/kv*.txt' overwrite into table load_overwrite2;
---
---
---load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
---
---
---
---
---
--create table exim_department ( dep_id int) stored as textfile;
---load data local inpath "../../data/files/test.dat" into table exim_department;
---dfs ${system:test.dfs.mkdir} target/tmp/ql/test/data/exports/exim_department/temp;
--dfs -rmr target/tmp/ql/test/data/exports/exim_department;
--export table exim_department to 'ql/test/data/exports/exim_department';
--drop table exim_department;
---
--create database importer;
--use importer;
---
--create table exim_department ( dep_id int) stored as textfile;
---set hive.security.authorization.enabled=true;
--import from 'ql/test/data/exports/exim_department';
--
--
@@ -275,8 +301,8 @@ drop table iow1_mm;
--- TODO multi-insert
+-- TODO multi-insert, truncate
-drop table intermediate;
\ No newline at end of file
+drop table intermediate;
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/test/queries/clientpositive/mm_current.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_current.q b/ql/src/test/queries/clientpositive/mm_current.q
index 6884928..391017b 100644
--- a/ql/src/test/queries/clientpositive/mm_current.q
+++ b/ql/src/test/queries/clientpositive/mm_current.q
@@ -12,17 +12,50 @@ insert into table intermediate partition(p='455') select distinct key from src w
insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2;
-set hive.merge.tezfiles=true;
-set hive.merge.mapfiles=true;
-set hive.merge.mapredfiles=true;
-set hive.merge.orcfile.stripe.level=true;
+drop table load0_mm;
+create table load0_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true');
+load data local inpath '../../data/files/kv1.txt' into table load0_mm;
+select count(1) from load0_mm;
+load data local inpath '../../data/files/kv2.txt' into table load0_mm;
+select count(1) from load0_mm;
+load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm;
+select count(1) from load0_mm;
+drop table load0_mm;
-drop table merge2_mm;
-create table merge2_mm(key int) tblproperties('hivecommit'='true');
-insert overwrite table merge2_mm select key from intermediate;
-select * from merge2_mm;
-drop table merge2_mm;
+
+drop table intermediate2;
+create table intermediate2 (key string, value string) stored as textfile
+location 'file:${system:test.tmp.dir}/intermediate2';
+load data local inpath '../../data/files/kv1.txt' into table intermediate2;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data local inpath '../../data/files/kv3.txt' into table intermediate2;
+
+drop table load1_mm;
+create table load1_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true');
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' into table load1_mm;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv1.txt' into table load1_mm;
+select count(1) from load1_mm;
+load data local inpath '../../data/files/kv1.txt' into table intermediate2;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data local inpath '../../data/files/kv3.txt' into table intermediate2;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' overwrite into table load1_mm;
+select count(1) from load1_mm;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' overwrite into table load1_mm;
+select count(1) from load1_mm;
+drop table load1_mm;
+
+drop table load2_mm;
+create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties('hivecommit'='true');
+load data local inpath '../../data/files/kv1.txt' into table intermediate2;
+load data local inpath '../../data/files/kv2.txt' into table intermediate2;
+load data local inpath '../../data/files/kv3.txt' into table intermediate2;
+load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' into table load2_mm partition(k=5, l=5);
+select count(1) from load2_mm;
+drop table load2_mm;
+drop table intermediate2;
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/test/results/clientpositive/llap/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out
index 92f32e5..6569369 100644
--- a/ql/src/test/results/clientpositive/llap/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out
@@ -1628,38 +1628,315 @@ POSTHOOK: query: drop table iow1_mm
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@iow1_mm
POSTHOOK: Output: default@iow1_mm
-PREHOOK: query: -- TODO# future
---
+PREHOOK: query: drop table load0_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load0_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
+PREHOOK: type: LOAD
#### A masked pattern was here ####
---
---load data local inpath '../../data/files/kv1.txt' into table load_overwrite;
---load data local inpath '../../data/files/kv2.txt' into table load_overwrite;
---load data local inpath '../../data/files/kv3.txt' into table load_overwrite;
---
---show table extended like load_overwrite;
---desc extended load_overwrite;
---select count(*) from load_overwrite;
---
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
+POSTHOOK: type: LOAD
#### A masked pattern was here ####
---
---
---load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
---
---
---
---
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: drop table load0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load0_mm
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: drop table load0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load0_mm
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: drop table intermediate2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table intermediate2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: drop table load1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load1_mm
+POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+1050
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: drop table load1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load1_mm
+PREHOOK: Output: default@load1_mm
+POSTHOOK: query: drop table load1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load1_mm
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: drop table load2_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load2_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load2_mm
+POSTHOOK: query: create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load2_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load2_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load2_mm
+POSTHOOK: Output: default@load2_mm@k=5/l=5
+PREHOOK: query: select count(1) from load2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load2_mm
+PREHOOK: Input: default@load2_mm@k=5/l=5
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load2_mm
+POSTHOOK: Input: default@load2_mm@k=5/l=5
+#### A masked pattern was here ####
+1025
+PREHOOK: query: drop table load2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load2_mm
+PREHOOK: Output: default@load2_mm
+POSTHOOK: query: drop table load2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load2_mm
+POSTHOOK: Output: default@load2_mm
+PREHOOK: query: drop table intermediate2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@intermediate2
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: drop table intermediate2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@intermediate2
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: -- IMPORT
+
+
+
+-- TODO# future
--
--create table exim_department ( dep_id int) stored as textfile;
---load data local inpath "../../data/files/test.dat" into table exim_department;
#### A masked pattern was here ####
--export table exim_department to 'ql/test/data/exports/exim_department';
--drop table exim_department;
---
--create database importer;
--use importer;
---
--create table exim_department ( dep_id int) stored as textfile;
---set hive.security.authorization.enabled=true;
--import from 'ql/test/data/exports/exim_department';
--
--
@@ -1677,7 +1954,7 @@ PREHOOK: query: -- TODO# future
--- TODO multi-insert
+-- TODO multi-insert, truncate
@@ -1685,38 +1962,19 @@ drop table intermediate
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@intermediate
PREHOOK: Output: default@intermediate
-POSTHOOK: query: -- TODO# future
---
-#### A masked pattern was here ####
---
---load data local inpath '../../data/files/kv1.txt' into table load_overwrite;
---load data local inpath '../../data/files/kv2.txt' into table load_overwrite;
---load data local inpath '../../data/files/kv3.txt' into table load_overwrite;
---
---show table extended like load_overwrite;
---desc extended load_overwrite;
---select count(*) from load_overwrite;
---
-#### A masked pattern was here ####
---
---
---load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
---
---
---
---
+POSTHOOK: query: -- IMPORT
+
+
+
+-- TODO# future
--
--create table exim_department ( dep_id int) stored as textfile;
---load data local inpath "../../data/files/test.dat" into table exim_department;
#### A masked pattern was here ####
--export table exim_department to 'ql/test/data/exports/exim_department';
--drop table exim_department;
---
--create database importer;
--use importer;
---
--create table exim_department ( dep_id int) stored as textfile;
---set hive.security.authorization.enabled=true;
--import from 'ql/test/data/exports/exim_department';
--
--
@@ -1734,7 +1992,7 @@ POSTHOOK: query: -- TODO# future
--- TODO multi-insert
+-- TODO multi-insert, truncate
http://git-wip-us.apache.org/repos/asf/hive/blob/a057e12c/ql/src/test/results/clientpositive/llap/mm_current.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_current.q.out b/ql/src/test/results/clientpositive/llap/mm_current.q.out
index 934a5fb..7ccc2ee 100644
--- a/ql/src/test/results/clientpositive/llap/mm_current.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_current.q.out
@@ -28,51 +28,302 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@intermediate@p=456
POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: drop table merge2_mm
+PREHOOK: query: drop table load0_mm
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table merge2_mm
+POSTHOOK: query: drop table load0_mm
POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table merge2_mm(key int) tblproperties('hivecommit'='true')
+PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: create table merge2_mm(key int) tblproperties('hivecommit'='true')
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
-POSTHOOK: Output: default@merge2_mm
-PREHOOK: query: insert overwrite table merge2_mm select key from intermediate
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: insert overwrite table merge2_mm select key from intermediate
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Output: default@merge2_mm
-POSTHOOK: Lineage: merge2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge2_mm
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: drop table load0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load0_mm
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: drop table load0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load0_mm
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: drop table intermediate2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table intermediate2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: drop table load1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load1_mm
+POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+1050
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
PREHOOK: type: QUERY
-PREHOOK: Input: default@merge2_mm
+PREHOOK: Input: default@load1_mm
#### A masked pattern was here ####
-POSTHOOK: query: select * from merge2_mm
+POSTHOOK: query: select count(1) from load1_mm
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge2_mm
+POSTHOOK: Input: default@load1_mm
#### A masked pattern was here ####
-98
-97
-0
-10
-PREHOOK: query: drop table merge2_mm
+500
+PREHOOK: query: drop table load1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load1_mm
+PREHOOK: Output: default@load1_mm
+POSTHOOK: query: drop table load1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load1_mm
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: drop table load2_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load2_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load2_mm
+POSTHOOK: query: create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load2_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load2_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load2_mm
+POSTHOOK: Output: default@load2_mm@k=5/l=5
+PREHOOK: query: select count(1) from load2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load2_mm
+PREHOOK: Input: default@load2_mm@k=5/l=5
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load2_mm
+POSTHOOK: Input: default@load2_mm@k=5/l=5
+#### A masked pattern was here ####
+1025
+PREHOOK: query: drop table load2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load2_mm
+PREHOOK: Output: default@load2_mm
+POSTHOOK: query: drop table load2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load2_mm
+POSTHOOK: Output: default@load2_mm
+PREHOOK: query: drop table intermediate2
PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@merge2_mm
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: drop table merge2_mm
+PREHOOK: Input: default@intermediate2
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: drop table intermediate2
POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@merge2_mm
-POSTHOOK: Output: default@merge2_mm
+POSTHOOK: Input: default@intermediate2
+POSTHOOK: Output: default@intermediate2
PREHOOK: query: drop table intermediate
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@intermediate