You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dj...@apache.org on 2018/02/02 21:27:16 UTC
hive git commit: HIVE-18516 : load data should rename files
consistent with insert statements for ACID Tables (Deepak Jaiswal,
reviewed by Eugene Koifman and Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master c2e335fc0 -> 1223031b9
HIVE-18516 : load data should rename files consistent with insert statements for ACID Tables (Deepak Jaiswal, reviewed by Eugene Koifman and Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1223031b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1223031b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1223031b
Branch: refs/heads/master
Commit: 1223031b9afdb7e22f9da09f6c8bb47037e447d0
Parents: c2e335f
Author: djaiswal <dj...@apache.org>
Authored: Fri Feb 2 13:25:53 2018 -0800
Committer: djaiswal <dj...@apache.org>
Committed: Fri Feb 2 13:27:08 2018 -0800
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 75 +++++++++++++-------
.../hive/ql/parse/LoadSemanticAnalyzer.java | 8 ---
.../apache/hadoop/hive/ql/TestTxnLoadData.java | 12 ++--
.../hive/ql/metadata/TestHiveCopyFiles.java | 12 ++--
.../clientnegative/load_data_into_acid.q | 20 ------
.../test/queries/clientpositive/smb_mapjoin_7.q | 4 +-
.../clientnegative/load_data_into_acid.q.out | 33 ---------
.../clientpositive/beeline/smb_mapjoin_7.q.out | 8 +--
.../results/clientpositive/smb_mapjoin_7.q.out | 8 +--
.../clientpositive/spark/smb_mapjoin_7.q.out | 8 +--
11 files changed, 75 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index d86ff58..70d0749 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -570,6 +570,7 @@ minillaplocal.query.files=\
list_bucket_dml_10.q,\
llap_partitioned.q,\
llap_vector_nohybridgrace.q,\
+ load_data_acid_rename.q,\
load_dyn_part5.q,\
lvj_mapjoin.q,\
materialized_view_create_rewrite_dummy.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 3b97dac..b1e05df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -31,18 +31,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -1881,11 +1871,12 @@ public class Hive {
// base_x. (there is Insert Overwrite and Load Data Overwrite)
boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(),
- isSrcLocal, isAutoPurge, newFiles, filter, isMmTableWrite?true:false, !tbl.isTemporary());
+ isSrcLocal, isAutoPurge, newFiles, filter, isMmTableWrite, !tbl.isTemporary());
} else {
FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation,
- (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles);
+ (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles, tbl.getNumBuckets() > 0,
+ isFullAcidTable);
}
}
perfLogger.PerfLogEnd("MoveTask", "FileMoves");
@@ -2432,7 +2423,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
try {
FileSystem fs = tbl.getDataLocation().getFileSystem(sessionConf);
copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation,
- loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles);
+ loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles,
+ tbl.getNumBuckets() > 0 ? true : false, isFullAcidTable);
} catch (IOException e) {
throw new HiveException("addFiles: filesystem error in check phase", e);
}
@@ -3302,8 +3294,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
private static void copyFiles(final HiveConf conf, final FileSystem destFs,
- FileStatus[] srcs, final FileSystem srcFs, final Path destf, final boolean isSrcLocal,
- boolean isOverwrite, final List<Path> newFiles) throws HiveException {
+ FileStatus[] srcs, final FileSystem srcFs, final Path destf,
+ final boolean isSrcLocal, boolean isOverwrite,
+ final List<Path> newFiles, boolean acidRename) throws HiveException {
final HdfsUtils.HadoopFileStatus fullDestStatus;
try {
@@ -3319,6 +3312,12 @@ private void constructOneLBLocationMap(FileStatus fSta,
final ExecutorService pool = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25) > 0 ?
Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Move-Thread-%d").build()) : null;
+ // For ACID non-bucketed case, the filenames have to be in the format consistent with INSERT/UPDATE/DELETE Ops,
+ // i.e, like 000000_0, 000001_0_copy_1, 000002_0.gz etc.
+ // The extension is only maintained for files which are compressed.
+ int taskId = 0;
+ // Sort the files
+ Arrays.sort(srcs);
for (FileStatus src : srcs) {
FileStatus[] files;
if (src.isDirectory()) {
@@ -3333,6 +3332,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
final SessionState parentSession = SessionState.get();
+ // Sort the files
+ Arrays.sort(files);
for (final FileStatus srcFile : files) {
final Path srcP = srcFile.getPath();
final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs);
@@ -3346,7 +3347,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
// copy from source to destination, we will inherit the destination's parent group ownership.
if (null == pool) {
try {
- Path destPath = mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed);
+ Path destPath = mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed,
+ acidRename ? taskId++ : -1);
if (null != newFiles) {
newFiles.add(destPath);
@@ -3355,6 +3357,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
throw getHiveException(e, msg, "Failed to move: {}");
}
} else {
+ // future only takes final or seemingly final values. Make a final copy of taskId
+ final int finalTaskId = acidRename ? taskId++ : -1;
futures.add(pool.submit(new Callable<ObjectPair<Path, Path>>() {
@Override
public ObjectPair<Path, Path> call() throws HiveException {
@@ -3362,7 +3366,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
try {
Path destPath =
- mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed);
+ mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed, finalTaskId);
if (null != newFiles) {
newFiles.add(destPath);
@@ -3432,6 +3436,10 @@ private void constructOneLBLocationMap(FileStatus fSta,
return ShimLoader.getHadoopShims().getPathWithoutSchemeAndAuthority(path);
}
+ private static String getPathName(int taskId) {
+ return Utilities.replaceTaskId("000000", taskId) + "_0";
+ }
+
/**
* <p>
* Moves a file from one {@link Path} to another. If {@code isRenameAllowed} is true then the
@@ -3459,15 +3467,22 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @throws IOException if there was an issue moving the file
*/
private static Path mvFile(HiveConf conf, FileSystem sourceFs, Path sourcePath, FileSystem destFs, Path destDirPath,
- boolean isSrcLocal, boolean isOverwrite, boolean isRenameAllowed) throws IOException {
+ boolean isSrcLocal, boolean isOverwrite, boolean isRenameAllowed,
+ int taskId) throws IOException {
// Strip off the file type, if any so we don't make:
// 000000_0.gz -> 000000_0.gz_copy_1
final String fullname = sourcePath.getName();
- final String name = FilenameUtils.getBaseName(sourcePath.getName());
+ final String name;
+ if (taskId == -1) { // non-acid
+ name = FilenameUtils.getBaseName(sourcePath.getName());
+ } else { // acid
+ name = getPathName(taskId);
+ }
final String type = FilenameUtils.getExtension(sourcePath.getName());
- Path destFilePath = new Path(destDirPath, fullname);
+ // Incase of ACID, the file is ORC so the extension is not relevant and should not be inherited.
+ Path destFilePath = new Path(destDirPath, taskId == -1 ? fullname : name);
/*
* The below loop may perform bad when the destination file already exists and it has too many _copy_
@@ -3482,7 +3497,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
destFs.delete(destFilePath, false);
break;
}
- destFilePath = new Path(destDirPath, name + (Utilities.COPY_KEYWORD + counter) + (!type.isEmpty() ? "." + type : ""));
+ destFilePath = new Path(destDirPath, name + (Utilities.COPY_KEYWORD + counter) +
+ ((taskId == -1 && !type.isEmpty()) ? "." + type : ""));
}
if (isRenameAllowed) {
@@ -3769,15 +3785,16 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @param destf directory to move files into
* @param fs Filesystem
* @param isSrcLocal true if source is on local file system
- * @param isAcid true if this is an ACID based write
+ * @param isAcidIUD true if this is an ACID based Insert/Update/Delete
* @param isOverwrite if true, then overwrite if destination file exist, else add a duplicate copy
* @param newFiles if this is non-null, a list of files that were created as a result of this
* move will be returned.
* @throws HiveException
*/
static protected void copyFiles(HiveConf conf, Path srcf, Path destf, FileSystem fs,
- boolean isSrcLocal, boolean isAcid,
- boolean isOverwrite, List<Path> newFiles) throws HiveException {
+ boolean isSrcLocal, boolean isAcidIUD,
+ boolean isOverwrite, List<Path> newFiles, boolean isBucketed,
+ boolean isFullAcidTable) throws HiveException {
try {
// create the destination if it does not exist
if (!fs.exists(destf)) {
@@ -3806,10 +3823,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
// If we're moving files around for an ACID write then the rules and paths are all different.
// You can blame this on Owen.
- if (isAcid) {
+ if (isAcidIUD) {
moveAcidFiles(srcFs, srcs, destf, newFiles);
} else {
- copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, isOverwrite, newFiles);
+ // For ACID non-bucketed case, the filenames have to be in the format consistent with INSERT/UPDATE/DELETE Ops,
+ // i.e, like 000000_0, 000001_0_copy_1, 000002_0.gz etc.
+ // The extension is only maintained for files which are compressed.
+ copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, isOverwrite,
+ newFiles, isFullAcidTable && !isBucketed);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 5868d4d..54f5bab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -159,14 +159,6 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast,
"source contains directory: " + oneSrc.getPath().toString()));
}
- if(AcidUtils.isAcidTable(table)) {
- if(!AcidUtils.originalBucketFilter.accept(oneSrc.getPath())) {
- //acid files (e.g. bucket_0000) have ROW_ID embedded in them and so can't be simply
- //copied to a table so only allow non-acid files for now
- throw new SemanticException(ErrorMsg.ACID_LOAD_DATA_INVALID_FILE_NAME,
- oneSrc.getPath().getName(), table.getFullyQualifiedName());
- }
- }
}
} catch (IOException e) {
// Has to use full name to make sure it does not conflict with
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index a9cba45..3a3272f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -264,10 +264,10 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
//from Load Data into acid converted table
{"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/delta_0000024_0000024_0000/000000_0"},
{"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000024_0000024_0000/000000_0"},
- {"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":2}\t2\t2", "t/delta_0000024_0000024_0000/000000_0_copy_1"},
- {"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":3}\t3\t3", "t/delta_0000024_0000024_0000/000000_0_copy_1"},
- {"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":4}\t4\t4", "t/delta_0000024_0000024_0000/000000_0_copy_2"},
- {"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":5}\t5\t5", "t/delta_0000024_0000024_0000/000000_0_copy_2"},
+ {"{\"transactionid\":24,\"bucketid\":536936448,\"rowid\":0}\t2\t2", "t/delta_0000024_0000024_0000/000001_0"},
+ {"{\"transactionid\":24,\"bucketid\":536936448,\"rowid\":1}\t3\t3", "t/delta_0000024_0000024_0000/000001_0"},
+ {"{\"transactionid\":24,\"bucketid\":537001984,\"rowid\":0}\t4\t4", "t/delta_0000024_0000024_0000/000002_0"},
+ {"{\"transactionid\":24,\"bucketid\":537001984,\"rowid\":1}\t5\t5", "t/delta_0000024_0000024_0000/000002_0"},
};
checkResult(expected, testQuery, isVectorized, "load data inpath");
@@ -281,7 +281,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
String[][] expected2 = new String[][] {
{"{\"transactionid\":30,\"bucketid\":536870912,\"rowid\":0}\t5\t6", "t/base_0000030/000000_0"},
{"{\"transactionid\":30,\"bucketid\":536870912,\"rowid\":1}\t7\t8", "t/base_0000030/000000_0"},
- {"{\"transactionid\":30,\"bucketid\":536870912,\"rowid\":2}\t8\t8", "t/base_0000030/000000_0_copy_1"}
+ {"{\"transactionid\":30,\"bucketid\":536936448,\"rowid\":0}\t8\t8", "t/base_0000030/000001_0"}
};
checkResult(expected2, testQuery, isVectorized, "load data inpath overwrite");
@@ -293,7 +293,7 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
String[][] expected3 = new String[][] {
{"{\"transactionid\":30,\"bucketid\":536870912,\"rowid\":0}\t5\t6", "t/base_0000033/bucket_00000"},
{"{\"transactionid\":30,\"bucketid\":536870912,\"rowid\":1}\t7\t8", "t/base_0000033/bucket_00000"},
- {"{\"transactionid\":30,\"bucketid\":536870912,\"rowid\":2}\t8\t8", "t/base_0000033/bucket_00000"},
+ {"{\"transactionid\":30,\"bucketid\":536936448,\"rowid\":0}\t8\t8", "t/base_0000033/bucket_00001"},
{"{\"transactionid\":33,\"bucketid\":536870912,\"rowid\":0}\t9\t9", "t/base_0000033/bucket_00000"}
};
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java
index c6a4a89..a20a2ae 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java
@@ -83,7 +83,7 @@ public class TestHiveCopyFiles {
FileSystem targetFs = targetPath.getFileSystem(hiveConf);
try {
- Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false,null);
+ Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false,null, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
@@ -107,7 +107,7 @@ public class TestHiveCopyFiles {
FileSystem targetFs = targetPath.getFileSystem(hiveConf);
try {
- Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null);
+ Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
@@ -127,7 +127,7 @@ public class TestHiveCopyFiles {
sourceFolder.newFile("000001_0.gz");
try {
- Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null);
+ Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
@@ -158,7 +158,7 @@ public class TestHiveCopyFiles {
Mockito.when(spyTargetFs.getUri()).thenReturn(URI.create("hdfs://" + targetPath.toUri().getPath()));
try {
- Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null);
+ Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
@@ -185,7 +185,7 @@ public class TestHiveCopyFiles {
Mockito.when(spyTargetFs.getUri()).thenReturn(URI.create("hdfs://" + targetPath.toUri().getPath()));
try {
- Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null);
+ Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
@@ -205,7 +205,7 @@ public class TestHiveCopyFiles {
sourceFolder.newFile("000001_0.gz");
try {
- Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null);
+ Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false, false);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Hive.copyFiles() threw an unexpected exception.", false);
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/queries/clientnegative/load_data_into_acid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/load_data_into_acid.q b/ql/src/test/queries/clientnegative/load_data_into_acid.q
deleted file mode 100644
index 2ac5b56..0000000
--- a/ql/src/test/queries/clientnegative/load_data_into_acid.q
+++ /dev/null
@@ -1,20 +0,0 @@
-set hive.support.concurrency=true;
-set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
-
-create table acid_ivot(
- ctinyint TINYINT,
- csmallint SMALLINT,
- cint INT,
- cbigint BIGINT,
- cfloat FLOAT,
- cdouble DOUBLE,
- cstring1 STRING,
- cstring2 STRING,
- ctimestamp1 TIMESTAMP,
- ctimestamp2 TIMESTAMP,
- cboolean1 BOOLEAN,
- cboolean2 BOOLEAN) stored as orc TBLPROPERTIES ('transactional'='true');
-
-LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot;
-
-
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/queries/clientpositive/smb_mapjoin_7.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_7.q b/ql/src/test/queries/clientpositive/smb_mapjoin_7.q
index 4a6afb0..fed931c 100644
--- a/ql/src/test/queries/clientpositive/smb_mapjoin_7.q
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_7.q
@@ -16,8 +16,8 @@ create table smb_join_results(k1 int, v1 string, k2 int, v2 string);
create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string);
create table normal_join_results(k1 int, v1 string, k2 int, v2 string);
-load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1;
-load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1;
+load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1;
+load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1;
insert overwrite table smb_bucket4_2
select * from src;
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/results/clientnegative/load_data_into_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/load_data_into_acid.q.out b/ql/src/test/results/clientnegative/load_data_into_acid.q.out
deleted file mode 100644
index 46b5cdd..0000000
--- a/ql/src/test/results/clientnegative/load_data_into_acid.q.out
+++ /dev/null
@@ -1,33 +0,0 @@
-PREHOOK: query: create table acid_ivot(
- ctinyint TINYINT,
- csmallint SMALLINT,
- cint INT,
- cbigint BIGINT,
- cfloat FLOAT,
- cdouble DOUBLE,
- cstring1 STRING,
- cstring2 STRING,
- ctimestamp1 TIMESTAMP,
- ctimestamp2 TIMESTAMP,
- cboolean1 BOOLEAN,
- cboolean2 BOOLEAN) stored as orc TBLPROPERTIES ('transactional'='true')
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@acid_ivot
-POSTHOOK: query: create table acid_ivot(
- ctinyint TINYINT,
- csmallint SMALLINT,
- cint INT,
- cbigint BIGINT,
- cfloat FLOAT,
- cdouble DOUBLE,
- cstring1 STRING,
- cstring2 STRING,
- ctimestamp1 TIMESTAMP,
- ctimestamp2 TIMESTAMP,
- cboolean1 BOOLEAN,
- cboolean2 BOOLEAN) stored as orc TBLPROPERTIES ('transactional'='true')
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@acid_ivot
-FAILED: SemanticException [Error 30023]: alltypesorc file name is not valid in Load Data into Acid table default.acid_ivot. Examples of valid names are: 00000_0, 00000_0_copy_1
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out
index 7a6f8c5..4b1313d 100644
--- a/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out
+++ b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out
@@ -38,19 +38,19 @@ POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@normal_join_results
-PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1
+PREHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@smb_bucket4_1
-POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1
+POSTHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket4_1
-PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1
+PREHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@smb_bucket4_1
-POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1
+POSTHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket4_1
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out
index b71c5b8..83033b0 100644
--- a/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out
+++ b/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out
@@ -38,19 +38,19 @@ POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@normal_join_results
-PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1
+PREHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@smb_bucket4_1
-POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1
+POSTHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket4_1
-PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1
+PREHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@smb_bucket4_1
-POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1
+POSTHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket4_1
http://git-wip-us.apache.org/repos/asf/hive/blob/1223031b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
index ac49c02..610abab 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
@@ -38,19 +38,19 @@ POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@normal_join_results
-PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1
+PREHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@smb_bucket4_1
-POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1
+POSTHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket4_1
-PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1
+PREHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1
PREHOOK: type: LOAD
#### A masked pattern was here ####
PREHOOK: Output: default@smb_bucket4_1
-POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1
+POSTHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket4_1