You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2019/07/18 07:35:44 UTC
[hive] branch master updated: HIVE-21992: REPL DUMP throws NPE when
dumping Create Function event (Sankar Hariappan,
reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8412b37 HIVE-21992: REPL DUMP throws NPE when dumping Create Function event (Sankar Hariappan, reviewed by Mahesh Kumar Behera)
8412b37 is described below
commit 8412b3748b7b384689a39b8747c7f8fd41e58f28
Author: Sankar Hariappan <sa...@apache.org>
AuthorDate: Thu Jul 18 13:05:17 2019 +0530
HIVE-21992: REPL DUMP throws NPE when dumping Create Function event (Sankar Hariappan, reviewed by Mahesh Kumar Behera)
Signed-off-by: Sankar Hariappan <sa...@apache.org>
---
.../TestReplicationScenariosAcrossInstances.java | 10 ++++++---
.../ql/parse/repl/dump/io/FunctionSerializer.java | 26 ++++++++++++----------
.../repl/load/message/CreateFunctionHandler.java | 6 ++---
3 files changed, 24 insertions(+), 18 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 3ee8490..af5746f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -84,7 +84,9 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro
primary.run("CREATE FUNCTION " + primaryDbName
+ ".testFunctionOne as 'hivemall.tools.string.StopwordUDF' "
- + "using jar 'ivy://io.github.myui:hivemall:0.4.0-2'");
+ + "using jar 'ivy://io.github.myui:hivemall:0.4.0-2'")
+ .run("CREATE FUNCTION " + primaryDbName
+ + ".testFunctionTwo as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax'");
WarehouseInstance.Tuple incrementalDump =
primary.dump(primaryDbName, bootStrapDump.lastReplicationId);
@@ -92,14 +94,16 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro
.run("REPL STATUS " + replicatedDbName)
.verifyResult(incrementalDump.lastReplicationId)
.run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'")
- .verifyResult(replicatedDbName + ".testFunctionOne");
+ .verifyResults(new String[] { replicatedDbName + ".testFunctionOne",
+ replicatedDbName + ".testFunctionTwo" });
// Test the idempotent behavior of CREATE FUNCTION
replica.load(replicatedDbName, incrementalDump.dumpLocation)
.run("REPL STATUS " + replicatedDbName)
.verifyResult(incrementalDump.lastReplicationId)
.run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'")
- .verifyResult(replicatedDbName + ".testFunctionOne");
+ .verifyResults(new String[] { replicatedDbName + ".testFunctionOne",
+ replicatedDbName + ".testFunctionTwo" });
}
@Test
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
index 6258c9e..576eb06 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
@@ -51,18 +51,20 @@ public class FunctionSerializer implements JsonWriter.Serializer {
throws SemanticException, IOException, MetaException {
TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
List<ResourceUri> resourceUris = new ArrayList<>();
- for (ResourceUri uri : function.getResourceUris()) {
- Path inputPath = new Path(uri.getUri());
- if ("hdfs".equals(inputPath.toUri().getScheme())) {
- FileSystem fileSystem = inputPath.getFileSystem(hiveConf);
- Path qualifiedUri = PathBuilder.fullyQualifiedHDFSUri(inputPath, fileSystem);
- // Initialize ReplChangeManager instance since we will require it to encode file URI.
- ReplChangeManager.getInstance(hiveConf);
- String checkSum = ReplChangeManager.checksumFor(qualifiedUri, fileSystem);
- String newFileUri = ReplChangeManager.encodeFileUri(qualifiedUri.toString(), checkSum, null);
- resourceUris.add(new ResourceUri(uri.getResourceType(), newFileUri));
- } else {
- resourceUris.add(uri);
+ if (function.getResourceUris() != null) {
+ for (ResourceUri uri : function.getResourceUris()) {
+ Path inputPath = new Path(uri.getUri());
+ if ("hdfs".equals(inputPath.toUri().getScheme())) {
+ FileSystem fileSystem = inputPath.getFileSystem(hiveConf);
+ Path qualifiedUri = PathBuilder.fullyQualifiedHDFSUri(inputPath, fileSystem);
+ // Initialize ReplChangeManager instance since we will require it to encode file URI.
+ ReplChangeManager.getInstance(hiveConf);
+ String checkSum = ReplChangeManager.checksumFor(qualifiedUri, fileSystem);
+ String newFileUri = ReplChangeManager.encodeFileUri(qualifiedUri.toString(), checkSum, null);
+ resourceUris.add(new ResourceUri(uri.getResourceType(), newFileUri));
+ } else {
+ resourceUris.add(uri);
+ }
}
}
Function copyObj = new Function(this.function);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
index c5a0519..bc891f7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
@@ -129,9 +129,9 @@ public class CreateFunctionHandler extends AbstractMessageHandler {
// and not do them lazily. The reason being the function class used for transformations additionally
// also creates the corresponding replCopyTasks, which cannot be evaluated lazily. since the query
// plan needs to be complete before we execute and not modify it while execution in the driver.
- List<ResourceUri> transformedUris = ImmutableList.copyOf(
- Lists.transform(metadata.function.getResourceUris(), conversionFunction)
- );
+ List<ResourceUri> transformedUris = (metadata.function.getResourceUris() == null)
+ ? null
+ : ImmutableList.copyOf(Lists.transform(metadata.function.getResourceUris(), conversionFunction));
replCopyTasks.addAll(conversionFunction.replCopyTasks);
String fullQualifiedFunctionName = FunctionUtils.qualifyFunctionName(
metadata.function.getFunctionName(), destinationDbName