You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/09/20 01:41:25 UTC
[18/22] hive git commit: HIVE-17196: CM: ReplCopyTask should retain
the original file names even if copied from CM path (Daniel Dai,
reviewed by Thejas Nair)
HIVE-17196: CM: ReplCopyTask should retain the original file names even if copied from CM path (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d45d462f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d45d462f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d45d462f
Branch: refs/heads/hive-14535
Commit: d45d462f5447a55b1f3cbec1b9ba71ec47781bb1
Parents: 9a14984
Author: Daniel Dai <da...@hortonworks.com>
Authored: Tue Sep 19 13:53:37 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Tue Sep 19 13:53:37 2017 -0700
----------------------------------------------------------------------
.../hive/metastore/TestReplChangeManager.java | 44 ++++++++++----------
.../hive/ql/parse/TestReplicationScenarios.java | 25 +++++++++++
.../hive/metastore/ReplChangeManager.java | 11 ++---
3 files changed, 53 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d45d462f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
index d9e206c..6ade76d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
@@ -175,19 +175,19 @@ public class TestReplChangeManager {
// verify cm.recycle(db, table, part) api moves file to cmroot dir
int ret = cm.recycle(part1Path, RecycleType.MOVE, false);
Assert.assertEquals(ret, 1);
- Path cmPart1Path = ReplChangeManager.getCMPath(hiveConf, path1Chksum);
+ Path cmPart1Path = ReplChangeManager.getCMPath(hiveConf, part1Path.getName(), path1Chksum);
assertTrue(cmPart1Path.getFileSystem(hiveConf).exists(cmPart1Path));
// Verify dropPartition recycle part files
client.dropPartition(dbName, tblName, Arrays.asList("20160102"));
assertFalse(part2Path.getFileSystem(hiveConf).exists(part2Path));
- Path cmPart2Path = ReplChangeManager.getCMPath(hiveConf, path2Chksum);
+ Path cmPart2Path = ReplChangeManager.getCMPath(hiveConf, part2Path.getName(), path2Chksum);
assertTrue(cmPart2Path.getFileSystem(hiveConf).exists(cmPart2Path));
// Verify dropTable recycle partition files
client.dropTable(dbName, tblName);
assertFalse(part3Path.getFileSystem(hiveConf).exists(part3Path));
- Path cmPart3Path = ReplChangeManager.getCMPath(hiveConf, path3Chksum);
+ Path cmPart3Path = ReplChangeManager.getCMPath(hiveConf, part3Path.getName(), path3Chksum);
assertTrue(cmPart3Path.getFileSystem(hiveConf).exists(cmPart3Path));
client.dropDatabase(dbName, true, true);
@@ -246,17 +246,17 @@ public class TestReplChangeManager {
cm.recycle(filePath1, RecycleType.MOVE, false);
assertFalse(filePath1.getFileSystem(hiveConf).exists(filePath1));
- Path cmPath1 = ReplChangeManager.getCMPath(hiveConf, fileChksum1);
+ Path cmPath1 = ReplChangeManager.getCMPath(hiveConf, filePath1.getName(), fileChksum1);
assertTrue(cmPath1.getFileSystem(hiveConf).exists(cmPath1));
// Verify dropTable recycle table files
client.dropTable(dbName, tblName);
- Path cmPath2 = ReplChangeManager.getCMPath(hiveConf, fileChksum2);
+ Path cmPath2 = ReplChangeManager.getCMPath(hiveConf, filePath2.getName(), fileChksum2);
assertFalse(filePath2.getFileSystem(hiveConf).exists(filePath2));
assertTrue(cmPath2.getFileSystem(hiveConf).exists(cmPath2));
- Path cmPath3 = ReplChangeManager.getCMPath(hiveConf, fileChksum3);
+ Path cmPath3 = ReplChangeManager.getCMPath(hiveConf, filePath3.getName(), fileChksum3);
assertFalse(filePath3.getFileSystem(hiveConf).exists(filePath3));
assertTrue(cmPath3.getFileSystem(hiveConf).exists(cmPath3));
@@ -298,17 +298,17 @@ public class TestReplChangeManager {
ReplChangeManager.getInstance(hiveConf).recycle(dirTbl2, RecycleType.MOVE, false);
ReplChangeManager.getInstance(hiveConf).recycle(dirTbl3, RecycleType.MOVE, true);
- assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum11)));
- assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum12)));
- assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum21)));
- assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum22)));
- assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31)));
- assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum32)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part12.getName(), fileChksum12)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part22.getName(), fileChksum22)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32)));
- fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum11), now - 86400*1000*2, now - 86400*1000*2);
- fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum21), now - 86400*1000*2, now - 86400*1000*2);
- fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum31), now - 86400*1000*2, now - 86400*1000*2);
- fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum32), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32), now - 86400*1000*2, now - 86400*1000*2);
ReplChangeManager.scheduleCMClearer(hiveConf);
@@ -321,12 +321,12 @@ public class TestReplChangeManager {
if (end - start > 5000) {
Assert.fail("timeout, cmroot has not been cleared");
}
- if (!fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum11)) &&
- fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum12)) &&
- !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum21)) &&
- fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum22)) &&
- !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31)) &&
- !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31))) {
+ if (!fs.exists(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11)) &&
+ fs.exists(ReplChangeManager.getCMPath(hiveConf, part12.getName(), fileChksum12)) &&
+ !fs.exists(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21)) &&
+ fs.exists(ReplChangeManager.getCMPath(hiveConf, part22.getName(), fileChksum22)) &&
+ !fs.exists(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31)) &&
+ !fs.exists(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32))) {
cleared = true;
}
} while (!cleared);
http://git-wip-us.apache.org/repos/asf/hive/blob/d45d462f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 3800e6a..2e880c7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -3102,6 +3102,31 @@ public class TestReplicationScenarios {
}
}
+ @Test
+ public void testCMConflict() throws IOException {
+ String testName = "cmConflict";
+ String dbName = createDB(testName, driver);
+
+ // Create table and insert two file of the same content
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver);
+
+ // Bootstrap test
+ advanceDumpDir();
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0, 0,driver);
+ String replDumpId = getResult(0, 1, true, driver);
+
+ // Drop two files so they are moved to CM
+ run("TRUNCATE TABLE " + dbName + ".unptned", driver);
+
+ LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+
+ verifyRun("SELECT count(*) from " + dbName + "_dupe.unptned", new String[]{"2"}, driverMirror);
+ }
+
private static String createDB(String name, Driver myDriver) {
LOG.info("Testing " + name);
String dbName = name + "_" + tid;
http://git-wip-us.apache.org/repos/asf/hive/blob/d45d462f/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
index 88d6a7a..6cb5fa8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
@@ -132,7 +132,7 @@ public class ReplChangeManager {
}
} else {
String fileCheckSum = checksumFor(path, fs);
- Path cmPath = getCMPath(hiveConf, fileCheckSum);
+ Path cmPath = getCMPath(hiveConf, path.getName(), fileCheckSum);
// set timestamp before moving to cmroot, so we can
// avoid race condition CM remove the file before setting
@@ -237,11 +237,12 @@ public class ReplChangeManager {
* to a deterministic location of cmroot. So user can retrieve the file back
* with the original location plus checksum.
* @param conf
+ * @param name original filename
* @param checkSum checksum of the file, can be retrieved by {@link #checksumFor(Path, FileSystem)}
* @return Path
*/
- static Path getCMPath(Configuration conf, String checkSum) throws IOException, MetaException {
- String newFileName = checkSum;
+ static Path getCMPath(Configuration conf, String name, String checkSum) throws IOException, MetaException {
+ String newFileName = name + "_" + checkSum;
int maxLength = conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
@@ -269,14 +270,14 @@ public class ReplChangeManager {
}
if (!srcFs.exists(src)) {
- return srcFs.getFileStatus(getCMPath(hiveConf, checksumString));
+ return srcFs.getFileStatus(getCMPath(hiveConf, src.getName(), checksumString));
}
String currentChecksumString = checksumFor(src, srcFs);
if (currentChecksumString == null || checksumString.equals(currentChecksumString)) {
return srcFs.getFileStatus(src);
} else {
- return srcFs.getFileStatus(getCMPath(hiveConf, checksumString));
+ return srcFs.getFileStatus(getCMPath(hiveConf, src.getName(), checksumString));
}
} catch (IOException e) {
throw new MetaException(StringUtils.stringifyException(e));