You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ac...@apache.org on 2025/06/04 16:54:26 UTC
(kudu) branch master updated: KUDU-3662 [2/n] Add job config parsing
This is an automated email from the ASF dual-hosted git repository.
achennaka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new d1e316417 KUDU-3662 [2/n] Add job config parsing
d1e316417 is described below
commit d1e316417c079af6e35140f44a83292080b6e63f
Author: Zoltan Chovan <zc...@cloudera.com>
AuthorDate: Wed May 14 17:57:10 2025 +0200
KUDU-3662 [2/n] Add job config parsing
This patch introduces a configuration model and CLI parser for job-level
replication settings.
- Introduced ReplicationJobConfig to encapsulate parameters like
source/sink Kudu master addresses, table name, etc.
- Added ReplicationConfigParser with a static parseJobConfig method
for extracting job.* parameters from ParameterTool.
- CLI argument parsing follows a naming convention with job., reader.,
and writer.* prefixes. This patch adds support for job.* only.
- The parsed job config is passed to ReplicationEnvProvider and will be
used in follow-up patches.
- Added unit tests for config parsing in TestReplicationConfigParser.
- Added AssertJ as a test dependency for more expressive assertions.
Note: added a temporary exclusion rule to suppress "URF_UNREAD_FIELD"
SpotBugs error, since the jobConfig field will be used in following
commits, this exclusion should be removed when no longer relevant.
Change-Id: Ic7229e11baa6a03c8986f206f456725acda00774
Reviewed-on: http://gerrit.cloudera.org:8080/22902
Tested-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Abhishek Chennaka <ac...@cloudera.com>
---
java/config/spotbugs/excludeFilter.xml | 6 +
java/gradle/dependencies.gradle | 2 +
java/kudu-replication/build.gradle | 3 +
.../kudu/replication/ReplicationConfigParser.java | 80 ++++++++++++
.../kudu/replication/ReplicationEnvProvider.java | 5 +
.../apache/kudu/replication/ReplicationJob.java | 8 +-
.../kudu/replication/ReplicationJobConfig.java | 144 +++++++++++++++++++++
.../replication/TestReplicationConfigParser.java | 87 +++++++++++++
8 files changed, 334 insertions(+), 1 deletion(-)
diff --git a/java/config/spotbugs/excludeFilter.xml b/java/config/spotbugs/excludeFilter.xml
index 11da3f997..90080e9cb 100644
--- a/java/config/spotbugs/excludeFilter.xml
+++ b/java/config/spotbugs/excludeFilter.xml
@@ -375,4 +375,10 @@
<Class name="org.apache.kudu.test.cluster.FakeDNS"/>
<Bug pattern="DP_DO_INSIDE_DO_PRIVILEGED" />
</Match>
+
+ <Match>
+ <!-- TODO: Remove this in follow-up commit. -->
+ <Class name="org.apache.kudu.replication.ReplicationEnvProvider" />
+ <Bug pattern="URF_UNREAD_FIELD" />
+ </Match>
</FindBugsFilter>
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index 0b48627a4..988ae7ea3 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -25,6 +25,7 @@ ext {
}
versions += [
+ assertj : "3.27.3",
async : "1.4.1",
checkstyle : "8.36.1",
clojure : "1.12.0",
@@ -85,6 +86,7 @@ versions["scalaBase"] = versions.scala.substring(0, versions.scala.lastIndexOf("
versions["sparkBase"] = versions.spark.substring(0, versions.spark.indexOf("."))
libs += [
+ assertj : "org.assertj:assertj-core:$versions.assertj",
async : "com.stumbleupon:async:$versions.async",
clojure : "org.clojure:clojure:$versions.clojure",
clojureToolsCli : "org.clojure:tools.cli:$versions.clojureToolsCli",
diff --git a/java/kudu-replication/build.gradle b/java/kudu-replication/build.gradle
index 9d86c24a9..6c1c775b6 100644
--- a/java/kudu-replication/build.gradle
+++ b/java/kudu-replication/build.gradle
@@ -20,6 +20,9 @@ apply from: "$rootDir/gradle/shadow.gradle"
dependencies {
// todo(zchovan) confirm if any dependencies should be shaded to avoid classpath issues
implementation libs.flinkStreamingJava
+
+ testImplementation libs.junit
+ testImplementation libs.assertj
}
// kudu-replication has no public Javadoc.
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationConfigParser.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationConfigParser.java
new file mode 100644
index 000000000..030a8fa37
--- /dev/null
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationConfigParser.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.kudu.replication;
+
+import org.apache.flink.api.java.utils.ParameterTool;
+
+/**
+ * Utility class for parsing replication-related configuration parameters from
+ * {@link ParameterTool}. It expects parameters to follow a prefix-based naming convention:
+ * <ul>
+ * <li><b>job.*</b> – general replication job configuration (e.g., source/sink master addresses,
+ * table name)</li>
+ * <li><b>reader.*</b> – reader-specific options for the source Kudu table</li>
+ * <li><b>writer.*</b> – writer-specific options for the sink Kudu table</li>
+ * </ul>
+ */
+public class ReplicationConfigParser {
+
+ /**
+ * Parses replication job-related parameters and constructs a
+ * {@link ReplicationJobConfig} instance.
+ * <p>
+ * The following parameters are expected under the {@code job.} prefix:
+ * <ul>
+ * <li>{@code job.sourceMasterAddresses} (required) – comma-separated list of source Kudu master
+ * addresses</li>
+ * <li>{@code job.sinkMasterAddresses} (required) – comma-separated list of sink Kudu master
+ * addresses</li>
+ * <li>{@code job.tableName} (required) – name of the source table to replicate</li>
+ * <li>{@code job.restoreOwner} (optional) – whether to restore table ownership on
+ * the sink (default: false)</li>
+ * <li>{@code job.tableSuffix} (optional) – suffix to append to the sink table name
+ * (default: empty string)</li>
+ * <li>{@code job.discoveryIntervalSeconds} (optional) – interval in seconds at which the source
+ * tries to perform a diff scan to get new or changed data (default: 300)</li>
+ * </ul>
+ *
+ * @param params the Flink {@code ParameterTool} containing command-line parameters
+ * @return a fully constructed {@link ReplicationJobConfig} instance
+ * @throws RuntimeException if any required parameter is missing
+ */
+ public static ReplicationJobConfig parseJobConfig(ParameterTool params) {
+ ReplicationJobConfig.Builder builder = ReplicationJobConfig.builder()
+ .setSourceMasterAddresses(params.getRequired("job.sourceMasterAddresses"))
+ .setSinkMasterAddresses(params.getRequired("job.sinkMasterAddresses"))
+ .setTableName(params.getRequired("job.tableName"));
+
+ if (params.has("job.restoreOwner")) {
+ builder.setRestoreOwner(params.getBoolean("job.restoreOwner"));
+ }
+
+ if (params.has("job.tableSuffix")) {
+ builder.setTableSuffix(params.get("job.tableSuffix"));
+ }
+
+ if (params.has("job.discoveryIntervalSeconds")) {
+ builder.setDiscoveryIntervalSeconds(params.getInt("job.discoveryIntervalSeconds"));
+ }
+
+ return builder.build();
+ }
+
+ //TODO(mgreber): add parseReaderConfig()
+
+ //TODO(mgreber): add parseWriterConfig()
+
+}
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java
index de7154eea..118526bcd 100644
--- a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java
@@ -22,6 +22,11 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
* This class does not execute the environment; it only prepares it.
*/
public class ReplicationEnvProvider {
+ private final ReplicationJobConfig jobConfig;
+
+ ReplicationEnvProvider(ReplicationJobConfig jobConfig) {
+ this.jobConfig = jobConfig;
+ }
/**
* Builds and configures the {@link StreamExecutionEnvironment} for the replication job.
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java
index 48a2c50fc..adb0f78de 100644
--- a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java
@@ -15,6 +15,8 @@
package org.apache.kudu.replication;
+import org.apache.flink.api.java.utils.ParameterTool;
+
/**
* This class is used to submit Kudu Replication Jobs into a Flink Cluster.
* This is a wrapper for convenience.
@@ -28,7 +30,11 @@ public class ReplicationJob {
* @throws Exception
*/
public static void main(String[] args) throws Exception {
- ReplicationEnvProvider provider = new ReplicationEnvProvider();
+ ParameterTool parameters = ParameterTool.fromArgs(args);
+
+ ReplicationJobConfig jobConfig = ReplicationConfigParser.parseJobConfig(parameters);
+
+ ReplicationEnvProvider provider = new ReplicationEnvProvider(jobConfig);
provider.getEnv().execute();
}
}
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobConfig.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobConfig.java
new file mode 100644
index 000000000..f65ea20a5
--- /dev/null
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobConfig.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.kudu.replication;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.io.Serializable;
+
+/**
+ * A configuration object for ReplicationJob used for the Kudu Flink based replication.
+ * <p>
+ * This object encapsulates the necessary parameters for setting up and running
+ * a replication job, including source and sink Kudu master addresses, table name,
+ * optional table suffix, replication discovery interval, and an option to restore
+ * table ownership.
+ * <p>
+ */
+public class ReplicationJobConfig implements Serializable {
+ private static final long serialVersionUID = 7667951599874025860L;
+
+ private final String sourceMasterAddresses;
+ private final String sinkMasterAddresses;
+ private final String tableName;
+ private final boolean restoreOwner;
+ private final String tableSuffix;
+ private final long discoveryIntervalSeconds;
+
+ private ReplicationJobConfig(
+ String sourceMasterAddresses,
+ String sinkMasterAddresses,
+ String tableName,
+ boolean restoreOwner,
+ String tableSuffix,
+ long discoveryIntervalSeconds) {
+ this.sourceMasterAddresses = checkNotNull(sourceMasterAddresses,
+ "sourceMasterAddresses cannot be null");
+ this.sinkMasterAddresses = checkNotNull(sinkMasterAddresses,
+ "sinkMasterAddresses cannot be null");
+ this.tableName = checkNotNull(tableName, "tableName cannot be null");
+ this.restoreOwner = restoreOwner;
+ this.tableSuffix = tableSuffix != null ? tableSuffix : "";
+ this.discoveryIntervalSeconds = discoveryIntervalSeconds;
+ }
+
+ public String getSourceMasterAddresses() {
+ return sourceMasterAddresses;
+ }
+
+ public String getSinkMasterAddresses() {
+ return sinkMasterAddresses;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public boolean getRestoreOwner() {
+ return restoreOwner;
+ }
+
+ public String getTableSuffix() {
+ return tableSuffix;
+ }
+
+ public long getDiscoveryIntervalSeconds() {
+ return discoveryIntervalSeconds;
+ }
+
+ public String getSinkTableName() {
+ return tableName + tableSuffix;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /** Builder for {@link ReplicationJobConfig}. */
+ public static class Builder {
+ private String sourceMasterAddresses;
+ private String sinkMasterAddresses;
+ private String tableName;
+ // By default, we don't restore the owner.
+ private boolean restoreOwner = false;
+ // By default, there is no tableSuffix for the replicated table.
+ private String tableSuffix = "";
+ // The default discover interval is 5 minutes.
+ private long discoveryIntervalSeconds = 5 * 60;
+
+ public Builder setSourceMasterAddresses(String sourceMasterAddresses) {
+ this.sourceMasterAddresses = sourceMasterAddresses;
+ return this;
+ }
+
+ public Builder setSinkMasterAddresses(String sinkMasterAddresses) {
+ this.sinkMasterAddresses = sinkMasterAddresses;
+ return this;
+ }
+
+ public Builder setTableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public Builder setRestoreOwner(boolean restoreOwner) {
+ this.restoreOwner = restoreOwner;
+ return this;
+ }
+
+ public Builder setTableSuffix(String tableSuffix) {
+ this.tableSuffix = tableSuffix;
+ return this;
+ }
+
+ public Builder setDiscoveryIntervalSeconds(long discoveryIntervalSeconds) {
+ this.discoveryIntervalSeconds = discoveryIntervalSeconds;
+ return this;
+ }
+
+ public ReplicationJobConfig build() {
+ return new ReplicationJobConfig(
+ sourceMasterAddresses,
+ sinkMasterAddresses,
+ tableName,
+ restoreOwner,
+ tableSuffix,
+ discoveryIntervalSeconds);
+ }
+ }
+
+
+}
diff --git a/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplicationConfigParser.java b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplicationConfigParser.java
new file mode 100644
index 000000000..b493084ef
--- /dev/null
+++ b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplicationConfigParser.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.kudu.replication;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.junit.Test;
+
+public class TestReplicationConfigParser {
+
+ @Test
+ public void testJobConfigMissingSourceMasterAddressesThrows() {
+ String[] args = {
+ "--job.sinkMasterAddresses", "sink1:7051",
+ "--job.tableName", "my_table"
+ };
+ ParameterTool params = ParameterTool.fromArgs(args);
+
+ assertThatThrownBy(() -> ReplicationConfigParser.parseJobConfig(params))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("No data for required key 'job.sourceMasterAddresses'");
+ }
+
+ @Test
+ public void testJobConfigMissingSinkMasterAddressesThrows() {
+ String[] args = {
+ "--job.sourceMasterAddresses", "source1:7051",
+ "--job.tableName", "my_table"
+ };
+ ParameterTool params = ParameterTool.fromArgs(args);
+
+ assertThatThrownBy(() -> ReplicationConfigParser.parseJobConfig(params))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("No data for required key 'job.sinkMasterAddresses'");
+ }
+
+ @Test
+ public void testJobConfigMissingTableNameThrows() {
+ String[] args = {
+ "--job.sourceMasterAddresses", "source1:7051",
+ "--job.sinkMasterAddresses", "sink1:7051"
+ };
+ ParameterTool params = ParameterTool.fromArgs(args);
+
+ assertThatThrownBy(() -> ReplicationConfigParser.parseJobConfig(params))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("No data for required key 'job.tableName'");
+ }
+
+ @Test
+ public void testJobConfigAllRequiredParamsPresent() {
+ String[] args = {
+ "--job.sourceMasterAddresses", "source1:7051",
+ "--job.sinkMasterAddresses", "sink1:7051",
+ "--job.tableName", "my_table"
+ };
+
+ ParameterTool params = ParameterTool.fromArgs(args);
+ ReplicationJobConfig config = ReplicationConfigParser.parseJobConfig(params);
+
+ assertEquals("source1:7051", config.getSourceMasterAddresses());
+ assertEquals("sink1:7051", config.getSinkMasterAddresses());
+ assertEquals("my_table", config.getTableName());
+ // The default value for restoreOwner is False.
+ assertFalse(config.getRestoreOwner());
+ assertEquals("", config.getTableSuffix());
+ // The default value for the discovery interval is 300.
+ assertEquals(300, config.getDiscoveryIntervalSeconds());
+ }
+
+}