You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/05/14 21:05:27 UTC
[GitHub] merlimat closed pull request #1769: refactor source and sink
classname to be pulsar source and sink when not set
merlimat closed pull request #1769: refactor source and sink classname to be pulsar source and sink when not set
URL: https://github.com/apache/incubator-pulsar/pull/1769
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 9f2ce10fc3..b704ec7361 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -651,9 +651,6 @@ protected FunctionDetails convert(FunctionConfig functionConfig)
Map<String, String> topicToSerDeClassNameMap = new HashMap<>();
topicToSerDeClassNameMap.putAll(functionConfig.getCustomSerdeInputs());
SourceSpec.Builder sourceSpecBuilder = SourceSpec.newBuilder();
- if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
- sourceSpecBuilder.setClassName(PulsarSource.class.getName());
- }
functionConfig.getInputs().forEach(v -> topicToSerDeClassNameMap.put(v, ""));
sourceSpecBuilder.putAllTopicsToSerDeClassName(topicToSerDeClassNameMap);
if (functionConfig.getSubscriptionType() != null) {
@@ -667,9 +664,6 @@ protected FunctionDetails convert(FunctionConfig functionConfig)
// Setup sink
SinkSpec.Builder sinkSpecBuilder = SinkSpec.newBuilder();
- if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
- sinkSpecBuilder.setClassName(PulsarSink.class.getName());
- }
if (functionConfig.getOutput() != null) {
sinkSpecBuilder.setTopic(functionConfig.getOutput());
}
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
index bac98d9331..a64ff17373 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
@@ -289,8 +289,8 @@ protected FunctionDetails createSinkConfig(SinkConfig sinkConfig) {
}
// set source spec
+ // source spec classname should be empty so that the default pulsar source will be used
SourceSpec.Builder sourceSpecBuilder = SourceSpec.newBuilder();
- sourceSpecBuilder.setClassName(PulsarSource.class.getName());
sourceSpecBuilder.setSubscriptionType(Function.SubscriptionType.SHARED);
sourceSpecBuilder.putAllTopicsToSerDeClassName(sinkConfig.getTopicToSerdeClassName());
sourceSpecBuilder.setTypeClassName(typeArg.getName());
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
index 267673c9b7..b78ebde7df 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
@@ -279,9 +279,9 @@ protected FunctionDetails createSourceConfig(SourceConfig sourceConfig) {
sourceSpecBuilder.setTypeClassName(typeArg.getName());
functionDetailsBuilder.setSource(sourceSpecBuilder);
- // set up sink spec
+ // set up sink spec.
+ // Sink spec classname should be empty so that the default pulsar sink will be used
SinkSpec.Builder sinkSpecBuilder = SinkSpec.newBuilder();
- sinkSpecBuilder.setClassName(PulsarSink.class.getName());
if (sourceConfig.getSerdeClassName() != null && !sourceConfig.getSerdeClassName().isEmpty()) {
sinkSpecBuilder.setSerDeClassName(sourceConfig.getSerdeClassName());
}
diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
index aec1e8e159..2ee8a0e462 100644
--- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
+++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
@@ -444,8 +444,8 @@ public void setupInput() throws Exception {
SourceSpec sourceSpec = this.instanceConfig.getFunctionDetails().getSource();
Object object;
- if (sourceSpec.getClassName().equals(PulsarSource.class.getName())) {
-
+ // If source classname is not set, we default pulsar source
+ if (sourceSpec.getClassName().isEmpty()) {
PulsarSourceConfig pulsarSourceConfig = new PulsarSourceConfig();
pulsarSourceConfig.setTopicSerdeClassNameMap(sourceSpec.getTopicsToSerDeClassNameMap());
pulsarSourceConfig.setSubscriptionName(
@@ -461,7 +461,7 @@ public void setupInput() throws Exception {
Class[] paramTypes = {PulsarClient.class, PulsarSourceConfig.class};
object = Reflections.createInstance(
- sourceSpec.getClassName(),
+ PulsarSource.class.getName(),
PulsarSource.class.getClassLoader(), params, paramTypes);
} else {
@@ -487,7 +487,8 @@ public void setupOutput() throws Exception {
SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
Object object;
- if (sinkSpec.getClassName().equals(PulsarSink.class.getName())) {
+ // If sink classname is not set, we default pulsar sink
+ if (sinkSpec.getClassName().isEmpty()) {
PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(
this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
@@ -499,7 +500,7 @@ public void setupOutput() throws Exception {
Class[] paramTypes = {PulsarClient.class, PulsarSinkConfig.class};
object = Reflections.createInstance(
- sinkSpec.getClassName(),
+ PulsarSink.class.getName(),
PulsarSink.class.getClassLoader(), params, paramTypes);
} else {
object = Reflections.createInstance(
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
index 1f34b8b7e9..4e23038325 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
@@ -92,7 +92,7 @@
@Parameter(names = "--auto_ack", description = "Enable Auto Acking?\n")
protected String autoAck = "true";
- @Parameter(names = "--source_classname", description = "The source classname", required = true)
+ @Parameter(names = "--source_classname", description = "The source classname")
protected String sourceClassname;
@Parameter(names = "--source_configs", description = "The source configs")
@@ -113,7 +113,7 @@
@Parameter(names = "--sink_configs", description = "The sink configs\n")
protected String sinkConfigs;
- @Parameter(names = "--sink_classname", description = "The sink classname\n", required = true)
+ @Parameter(names = "--sink_classname", description = "The sink classname\n")
protected String sinkClassname;
@Parameter(names = "--sink_topic", description = "The sink Topic Name\n")
@@ -154,7 +154,9 @@ public void start() throws Exception {
// Setup source
SourceSpec.Builder sourceDetailsBuilder = SourceSpec.newBuilder();
- sourceDetailsBuilder.setClassName(sourceClassname);
+ if (sourceClassname != null) {
+ sourceDetailsBuilder.setClassName(sourceClassname);
+ }
if (sourceConfigs != null && !sourceConfigs.isEmpty()) {;
sourceDetailsBuilder.setConfigs(sourceConfigs);
}
@@ -165,7 +167,9 @@ public void start() throws Exception {
// Setup sink
SinkSpec.Builder sinkSpecBuilder = SinkSpec.newBuilder();
- sinkSpecBuilder.setClassName(sinkClassname);
+ if (sinkClassname != null) {
+ sinkSpecBuilder.setClassName(sinkClassname);
+ }
if (sinkConfigs != null) {
sinkSpecBuilder.setConfigs(sinkConfigs);
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services