You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ac...@apache.org on 2025/06/04 16:54:26 UTC

(kudu) branch master updated: KUDU-3662 [2/n] Add job config parsing

This is an automated email from the ASF dual-hosted git repository.

achennaka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new d1e316417 KUDU-3662 [2/n] Add job config parsing
d1e316417 is described below

commit d1e316417c079af6e35140f44a83292080b6e63f
Author: Zoltan Chovan <zc...@cloudera.com>
AuthorDate: Wed May 14 17:57:10 2025 +0200

    KUDU-3662 [2/n] Add job config parsing
    
    This patch introduces a configuration model and CLI parser for job-level
    replication settings.
     - Introduced ReplicationJobConfig to encapsulate parameters like
    source/sink Kudu master addresses, table name, etc.
    - Added ReplicationConfigParser with a static parseJobConfig method
    for extracting job.* parameters from ParameterTool.
    - CLI argument parsing follows a naming convention with job., reader.,
    and writer.* prefixes. This patch adds support for job.* only.
    - The parsed job config is passed to ReplicationEnvProvider and will be
    used in follow-up patches.
    - Added unit tests for config parsing in TestReplicationConfigParser.
    - Added AssertJ as a test dependency for more expressive assertions.
    
    Note: added a temporary exclusion rule to suppress "URF_UNREAD_FIELD"
    SpotBugs error, since the jobConfig field will be used in following
    commits, this exclusion should be removed when no longer relevant.
    
    Change-Id: Ic7229e11baa6a03c8986f206f456725acda00774
    Reviewed-on: http://gerrit.cloudera.org:8080/22902
    Tested-by: Marton Greber <gr...@gmail.com>
    Reviewed-by: Marton Greber <gr...@gmail.com>
    Reviewed-by: Abhishek Chennaka <ac...@cloudera.com>
---
 java/config/spotbugs/excludeFilter.xml             |   6 +
 java/gradle/dependencies.gradle                    |   2 +
 java/kudu-replication/build.gradle                 |   3 +
 .../kudu/replication/ReplicationConfigParser.java  |  80 ++++++++++++
 .../kudu/replication/ReplicationEnvProvider.java   |   5 +
 .../apache/kudu/replication/ReplicationJob.java    |   8 +-
 .../kudu/replication/ReplicationJobConfig.java     | 144 +++++++++++++++++++++
 .../replication/TestReplicationConfigParser.java   |  87 +++++++++++++
 8 files changed, 334 insertions(+), 1 deletion(-)

diff --git a/java/config/spotbugs/excludeFilter.xml b/java/config/spotbugs/excludeFilter.xml
index 11da3f997..90080e9cb 100644
--- a/java/config/spotbugs/excludeFilter.xml
+++ b/java/config/spotbugs/excludeFilter.xml
@@ -375,4 +375,10 @@
         <Class name="org.apache.kudu.test.cluster.FakeDNS"/>
         <Bug pattern="DP_DO_INSIDE_DO_PRIVILEGED" />
     </Match>
+
+    <Match>
+        <!-- TODO: Remove this in follow-up commit. -->
+        <Class name="org.apache.kudu.replication.ReplicationEnvProvider" />
+        <Bug pattern="URF_UNREAD_FIELD" />
+    </Match>
 </FindBugsFilter>
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index 0b48627a4..988ae7ea3 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -25,6 +25,7 @@ ext {
 }
 
 versions += [
+    assertj        : "3.27.3",
     async          : "1.4.1",
     checkstyle     : "8.36.1",
     clojure        : "1.12.0",
@@ -85,6 +86,7 @@ versions["scalaBase"] = versions.scala.substring(0, versions.scala.lastIndexOf("
 versions["sparkBase"] = versions.spark.substring(0, versions.spark.indexOf("."))
 
 libs += [
+    assertj              : "org.assertj:assertj-core:$versions.assertj",
     async                : "com.stumbleupon:async:$versions.async",
     clojure              : "org.clojure:clojure:$versions.clojure",
     clojureToolsCli      : "org.clojure:tools.cli:$versions.clojureToolsCli",
diff --git a/java/kudu-replication/build.gradle b/java/kudu-replication/build.gradle
index 9d86c24a9..6c1c775b6 100644
--- a/java/kudu-replication/build.gradle
+++ b/java/kudu-replication/build.gradle
@@ -20,6 +20,9 @@ apply from: "$rootDir/gradle/shadow.gradle"
 dependencies {
     // todo(zchovan) confirm if any dependencies should be shaded to avoid classpath issues
     implementation libs.flinkStreamingJava
+
+    testImplementation libs.junit
+    testImplementation libs.assertj
 }
 
 // kudu-replication has no public Javadoc.
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationConfigParser.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationConfigParser.java
new file mode 100644
index 000000000..030a8fa37
--- /dev/null
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationConfigParser.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.kudu.replication;
+
+import org.apache.flink.api.java.utils.ParameterTool;
+
+/**
+ * Utility class for parsing replication-related configuration parameters from
+ * {@link ParameterTool}. It expects parameters to follow a prefix-based naming convention:
+ * <ul>
+ *   <li><b>job.*</b> – general replication job configuration (e.g., source/sink master addresses,
+ *   table name)</li>
+ *   <li><b>reader.*</b> – reader-specific options for the source Kudu table</li>
+ *   <li><b>writer.*</b> – writer-specific options for the sink Kudu table</li>
+ * </ul>
+ */
+public class ReplicationConfigParser {
+
+  /**
+   * Parses replication job-related parameters and constructs a
+   * {@link ReplicationJobConfig} instance.
+   * <p>
+   * The following parameters are expected under the {@code job.} prefix:
+   * <ul>
+   *   <li>{@code job.sourceMasterAddresses} (required) – comma-separated list of source Kudu master
+   *   addresses</li>
+   *   <li>{@code job.sinkMasterAddresses} (required) – comma-separated list of sink Kudu master
+   *   addresses</li>
+   *   <li>{@code job.tableName} (required) – name of the source table to replicate</li>
+   *   <li>{@code job.restoreOwner} (optional) – whether to restore table ownership on
+   *   the sink (default: false)</li>
+   *   <li>{@code job.tableSuffix} (optional) – suffix to append to the sink table name
+   *   (default: empty string)</li>
+   *   <li>{@code job.discoveryIntervalSeconds} (optional) – interval in seconds at which the source
+   *   tries to perform a diff scan to get new or changed data (default: 300)</li>
+   * </ul>
+   *
+   * @param params the Flink {@code ParameterTool} containing command-line parameters
+   * @return a fully constructed {@link ReplicationJobConfig} instance
+   * @throws RuntimeException if any required parameter is missing
+   */
+  public static ReplicationJobConfig parseJobConfig(ParameterTool params) {
+    ReplicationJobConfig.Builder builder = ReplicationJobConfig.builder()
+            .setSourceMasterAddresses(params.getRequired("job.sourceMasterAddresses"))
+            .setSinkMasterAddresses(params.getRequired("job.sinkMasterAddresses"))
+            .setTableName(params.getRequired("job.tableName"));
+
+    if (params.has("job.restoreOwner")) {
+      builder.setRestoreOwner(params.getBoolean("job.restoreOwner"));
+    }
+
+    if (params.has("job.tableSuffix")) {
+      builder.setTableSuffix(params.get("job.tableSuffix"));
+    }
+
+    if (params.has("job.discoveryIntervalSeconds")) {
+      builder.setDiscoveryIntervalSeconds(params.getInt("job.discoveryIntervalSeconds"));
+    }
+
+    return builder.build();
+  }
+
+  //TODO(mgreber): add parseReaderConfig()
+
+  //TODO(mgreber): add parseWriterConfig()
+
+}
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java
index de7154eea..118526bcd 100644
--- a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationEnvProvider.java
@@ -22,6 +22,11 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
  * This class does not execute the environment; it only prepares it.
  */
 public class ReplicationEnvProvider {
+  private final ReplicationJobConfig jobConfig;
+
+  ReplicationEnvProvider(ReplicationJobConfig jobConfig) {
+    this.jobConfig = jobConfig;
+  }
 
   /**
    * Builds and configures the {@link StreamExecutionEnvironment} for the replication job.
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java
index 48a2c50fc..adb0f78de 100644
--- a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJob.java
@@ -15,6 +15,8 @@
 
 package org.apache.kudu.replication;
 
+import org.apache.flink.api.java.utils.ParameterTool;
+
 /**
  * This class is used to submit Kudu Replication Jobs into a Flink Cluster.
  * This is a wrapper for convenience.
@@ -28,7 +30,11 @@ public class ReplicationJob {
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
-    ReplicationEnvProvider provider = new ReplicationEnvProvider();
+    ParameterTool parameters = ParameterTool.fromArgs(args);
+
+    ReplicationJobConfig jobConfig = ReplicationConfigParser.parseJobConfig(parameters);
+
+    ReplicationEnvProvider provider = new ReplicationEnvProvider(jobConfig);
     provider.getEnv().execute();
   }
 }
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobConfig.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobConfig.java
new file mode 100644
index 000000000..f65ea20a5
--- /dev/null
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobConfig.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.kudu.replication;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+import java.io.Serializable;
+
+/**
+ * A configuration object for ReplicationJob used for the Kudu Flink based replication.
+ * <p>
+ * This object encapsulates the necessary parameters for setting up and running
+ * a replication job, including source and sink Kudu master addresses, table name,
+ * optional table suffix, replication discovery interval, and an option to restore
+ * table ownership.
+ * <p>
+ */
+public class ReplicationJobConfig implements Serializable {
+  private static final long serialVersionUID = 7667951599874025860L;
+
+  private final String sourceMasterAddresses;
+  private final String sinkMasterAddresses;
+  private final String tableName;
+  private final boolean restoreOwner;
+  private final String tableSuffix;
+  private final long discoveryIntervalSeconds;
+
+  private ReplicationJobConfig(
+          String sourceMasterAddresses,
+          String sinkMasterAddresses,
+          String tableName,
+          boolean restoreOwner,
+          String tableSuffix,
+          long discoveryIntervalSeconds) {
+    this.sourceMasterAddresses = checkNotNull(sourceMasterAddresses,
+      "sourceMasterAddresses cannot be null");
+    this.sinkMasterAddresses = checkNotNull(sinkMasterAddresses,
+      "sinkMasterAddresses cannot be null");
+    this.tableName = checkNotNull(tableName, "tableName cannot be null");
+    this.restoreOwner = restoreOwner;
+    this.tableSuffix = tableSuffix != null ? tableSuffix : "";
+    this.discoveryIntervalSeconds = discoveryIntervalSeconds;
+  }
+
+  public String getSourceMasterAddresses() {
+    return sourceMasterAddresses;
+  }
+
+  public String getSinkMasterAddresses() {
+    return sinkMasterAddresses;
+  }
+
+  public String getTableName() {
+    return tableName;
+  }
+
+  public boolean getRestoreOwner() {
+    return restoreOwner;
+  }
+
+  public String getTableSuffix() {
+    return tableSuffix;
+  }
+
+  public long getDiscoveryIntervalSeconds() {
+    return discoveryIntervalSeconds;
+  }
+
+  public String getSinkTableName() {
+    return tableName + tableSuffix;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  /** Builder for {@link ReplicationJobConfig}. */
+  public static class Builder {
+    private String sourceMasterAddresses;
+    private String sinkMasterAddresses;
+    private String tableName;
+    // By default, we don't restore the owner.
+    private boolean restoreOwner = false;
+    // By default, there is no tableSuffix for the replicated table.
+    private String tableSuffix = "";
+    // The default discover interval is 5 minutes.
+    private long discoveryIntervalSeconds = 5 * 60;
+
+    public Builder setSourceMasterAddresses(String sourceMasterAddresses) {
+      this.sourceMasterAddresses = sourceMasterAddresses;
+      return this;
+    }
+
+    public Builder setSinkMasterAddresses(String sinkMasterAddresses) {
+      this.sinkMasterAddresses = sinkMasterAddresses;
+      return this;
+    }
+
+    public Builder setTableName(String tableName) {
+      this.tableName = tableName;
+      return this;
+    }
+
+    public Builder setRestoreOwner(boolean restoreOwner) {
+      this.restoreOwner = restoreOwner;
+      return this;
+    }
+
+    public Builder setTableSuffix(String tableSuffix) {
+      this.tableSuffix = tableSuffix;
+      return this;
+    }
+
+    public Builder setDiscoveryIntervalSeconds(long discoveryIntervalSeconds) {
+      this.discoveryIntervalSeconds = discoveryIntervalSeconds;
+      return this;
+    }
+
+    public ReplicationJobConfig build() {
+      return new ReplicationJobConfig(
+              sourceMasterAddresses,
+              sinkMasterAddresses,
+              tableName,
+              restoreOwner,
+              tableSuffix,
+              discoveryIntervalSeconds);
+    }
+  }
+
+
+}
diff --git a/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplicationConfigParser.java b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplicationConfigParser.java
new file mode 100644
index 000000000..b493084ef
--- /dev/null
+++ b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplicationConfigParser.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.kudu.replication;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.junit.Test;
+
+public class TestReplicationConfigParser {
+
+  @Test
+  public void testJobConfigMissingSourceMasterAddressesThrows() {
+    String[] args = {
+        "--job.sinkMasterAddresses", "sink1:7051",
+        "--job.tableName", "my_table"
+    };
+    ParameterTool params = ParameterTool.fromArgs(args);
+
+    assertThatThrownBy(() -> ReplicationConfigParser.parseJobConfig(params))
+            .isInstanceOf(RuntimeException.class)
+            .hasMessageContaining("No data for required key 'job.sourceMasterAddresses'");
+  }
+
+  @Test
+  public void testJobConfigMissingSinkMasterAddressesThrows() {
+    String[] args = {
+        "--job.sourceMasterAddresses", "source1:7051",
+        "--job.tableName", "my_table"
+    };
+    ParameterTool params = ParameterTool.fromArgs(args);
+
+    assertThatThrownBy(() -> ReplicationConfigParser.parseJobConfig(params))
+            .isInstanceOf(RuntimeException.class)
+            .hasMessageContaining("No data for required key 'job.sinkMasterAddresses'");
+  }
+
+  @Test
+  public void testJobConfigMissingTableNameThrows() {
+    String[] args = {
+        "--job.sourceMasterAddresses", "source1:7051",
+        "--job.sinkMasterAddresses", "sink1:7051"
+    };
+    ParameterTool params = ParameterTool.fromArgs(args);
+
+    assertThatThrownBy(() -> ReplicationConfigParser.parseJobConfig(params))
+            .isInstanceOf(RuntimeException.class)
+            .hasMessageContaining("No data for required key 'job.tableName'");
+  }
+
+  @Test
+  public void testJobConfigAllRequiredParamsPresent() {
+    String[] args = {
+        "--job.sourceMasterAddresses", "source1:7051",
+        "--job.sinkMasterAddresses", "sink1:7051",
+        "--job.tableName", "my_table"
+    };
+
+    ParameterTool params = ParameterTool.fromArgs(args);
+    ReplicationJobConfig config = ReplicationConfigParser.parseJobConfig(params);
+
+    assertEquals("source1:7051", config.getSourceMasterAddresses());
+    assertEquals("sink1:7051", config.getSinkMasterAddresses());
+    assertEquals("my_table", config.getTableName());
+    // The default value for restoreOwner is False.
+    assertFalse(config.getRestoreOwner());
+    assertEquals("", config.getTableSuffix());
+    // The default value for the discovery interval is 300.
+    assertEquals(300, config.getDiscoveryIntervalSeconds());
+  }
+
+}