You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2017/12/21 20:44:14 UTC
hive git commit: HIVE-17897: Repl load in bootstrap phase fails when
partitions have whitespace (Thejas Nair, reviewed by Sankar Hariappan)
Repository: hive
Updated Branches:
refs/heads/master ad5bcb150 -> b4b06ac15
HIVE-17897: Repl load in bootstrap phase fails when partitions have whitespace (Thejas Nair, reviewed by Sankar Hariappan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4b06ac1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4b06ac1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4b06ac1
Branch: refs/heads/master
Commit: b4b06ac1568490200f817cd7f1855b9a81df6bf9
Parents: ad5bcb1
Author: Sankar Hariappan <sa...@apache.org>
Authored: Fri Dec 22 02:13:26 2017 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Fri Dec 22 02:13:45 2017 +0530
----------------------------------------------------------------------
.../hive/ql/parse/TestReplicationScenarios.java | 36 ++++++++++++++++++--
.../ql/parse/repl/dump/io/FileOperations.java | 2 +-
2 files changed, 35 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b06ac1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 9062d43..0e08007 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -1953,13 +1953,16 @@ public class TestReplicationScenarios {
String testName = "dropPartitionEventWithPartitionOnTimestampColumn";
String dbName = createDB(testName, driver);
run("CREATE TABLE " + dbName + ".ptned(a string) PARTITIONED BY (b timestamp)", driver);
+ String[] ptn_data = new String[] { "fourteen" };
+ String ptnVal = "2017-10-01 01:00:10.1";
+ run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=\"" + ptnVal +"\") values('" + ptn_data[0] + "')", driver);
// Bootstrap dump/load
String replDbName = dbName + "_dupe";
Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
- String[] ptn_data = new String[] { "fifteen" };
- String ptnVal = "2017-10-24 00:00:00.0";
+ ptn_data = new String[] { "fifteen" };
+ ptnVal = "2017-10-24 00:00:00.0";
run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=\"" + ptnVal +"\") values('" + ptn_data[0] + "')", driver);
// Replicate insert event and verify
@@ -1973,6 +1976,35 @@ public class TestReplicationScenarios {
verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal)), metaStoreClientMirror);
}
+ /**
+ * Verify replication when string partition column value has special chars
+ * @throws IOException
+ */
+ @Test
+ public void testWithStringPartitionSpecialChars() throws IOException {
+ String testName = "testWithStringPartitionSpecialChars";
+ String dbName = createDB(testName, driver);
+ run("CREATE TABLE " + dbName + ".ptned(v string) PARTITIONED BY (p string)", driver);
+ String[] ptn_data = new String[] { "fourteen", "fifteen" };
+ String[] ptnVal = new String [] {"has a space, /, and \t tab", "another set of '#@ chars" };
+ run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(p=\"" + ptnVal[0] +"\") values('" + ptn_data[0] + "')", driver);
+
+ // Bootstrap dump/load
+ String replDbName = dbName + "_dupe";
+ Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
+
+ run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(p=\"" + ptnVal[1] +"\") values('" + ptn_data[1] + "')", driver);
+ // Replicate insert event and verify
+ Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName);
+ verifyRun("SELECT p from " + replDbName + ".ptned ORDER BY p desc", ptnVal, driverMirror);
+
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION(p=\"" + ptnVal[0] + "\")", driver);
+
+ // Replicate drop partition event and verify
+ incrementalLoadAndVerify(dbName, incrDump.lastReplId, replDbName);
+ verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal[0])), metaStoreClientMirror);
+ }
+
@Test
public void testRenameTableWithCM() throws IOException {
String testName = "renameTableWithCM";
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b06ac1/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
index 2f636b6..4642012 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
@@ -108,6 +108,6 @@ public class FileOperations {
private String encodedUri(FileStatus fileStatus) throws IOException {
Path currentDataFilePath = fileStatus.getPath();
String checkSum = ReplChangeManager.checksumFor(currentDataFilePath, dataFileSystem);
- return ReplChangeManager.encodeFileUri(currentDataFilePath.toUri().toString(), checkSum);
+ return ReplChangeManager.encodeFileUri(currentDataFilePath.toString(), checkSum);
}
}