You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2025/04/14 16:58:25 UTC
(kudu) 04/14: organisation of replication job, added test stubs
This is an automated email from the ASF dual-hosted git repository.
mgreber pushed a commit to branch flink-replication
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit e6218646af7004b6898fd8357e0841410dd6fbff
Author: zchovan <zc...@cloudera.com>
AuthorDate: Tue Feb 11 14:49:32 2025 +0100
organisation of replication job, added test stubs
Change-Id: Iad71eaedd7f136458719d52d2fb14047d61fa227
---
.../kudu/replication/ReplicationJobExecutor.java | 161 ++++++---------------
.../apache/kudu/replication/TestReplication.java | 58 +++++---
2 files changed, 83 insertions(+), 136 deletions(-)
diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java
index c5d568ace..ce5029cb4 100644
--- a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java
@@ -17,8 +17,13 @@
package org.apache.kudu.replication;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.connector.sink2.Sink;
import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceSplit;
+import org.apache.flink.api.connector.source.SplitEnumerator;
import org.apache.flink.connector.kudu.connector.ColumnSchemasFactory;
import org.apache.flink.connector.kudu.connector.KuduTableInfo;
import org.apache.flink.connector.kudu.connector.converter.RowResultRowConverter;
@@ -28,21 +33,23 @@ import org.apache.flink.connector.kudu.connector.reader.KuduReaderConfig;
import org.apache.flink.connector.kudu.connector.reader.KuduReaderIterator;
import org.apache.flink.connector.kudu.connector.writer.AbstractSingleOperationMapper;
import org.apache.flink.connector.kudu.connector.writer.KuduWriterConfig;
+import org.apache.flink.connector.kudu.connector.writer.RowDataUpsertOperationMapper;
import org.apache.flink.connector.kudu.connector.writer.RowOperationMapper;
import org.apache.flink.connector.kudu.sink.KuduSink;
import org.apache.flink.connector.kudu.sink.KuduSinkBuilder;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
import org.apache.flink.types.Row;
import org.apache.kudu.ColumnSchema;
import org.apache.kudu.Type;
import org.apache.kudu.client.CreateTableOptions;
import org.apache.kudu.shaded.com.google.common.collect.Lists;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
import java.util.stream.Collectors;
/**
@@ -50,40 +57,9 @@ import java.util.stream.Collectors;
*/
class ReplicationJobExecutor {
private ReplicationJobConfig config;
- private final String[] columns = new String[] {"id", "title", "author", "price", "quantity"};
- private final Object[][] booksTableData = {
- {1001, "Java for dummies", "Tan Ah Teck", 11.11, 11},
- {1002, "More Java for dummies", "Tan Ah Teck", 22.22, 22},
- {1003, "More Java for more dummies", "Mohammad Ali", 33.33, 33},
- {1004, "A Cup of Java", "Kumar", 44.44, 44},
- {1005, "A Teaspoon of Java", "Kevin Jones", 55.55, 55}
- };
-
- private List<Row> booksDataRow;
ReplicationJobExecutor(ReplicationJobConfig config) {
this.config = config;
- this.booksDataRow = Arrays.stream(booksTableData)
- .map(
- row -> {
- Integer rowId = (Integer) row[0];
- if (rowId % 2 == 1) {
- Row values = new Row(5);
- values.setField(0, row[0]);
- values.setField(1, row[1]);
- values.setField(2, row[2]);
- values.setField(3, row[3]);
- values.setField(4, row[4]);
- return values;
- } else {
- Row values = new Row(5);
- values.setField(0, row[0]);
- values.setField(1, row[1]);
- values.setField(2, row[2]);
- return values;
- }
- })
- .collect(Collectors.toList());
}
/**
@@ -91,8 +67,18 @@ class ReplicationJobExecutor {
*/
public void runJob() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-// DataStream<String> kuduSourceStream = env.addSource(
-// new KuduSource(config.getSourceMasterAddresses(), config.getTableName()));
+ KuduTableInfo tableInfo = KuduTableInfo.forTable(config.getTableName());
+
+
+
+
+ KuduReaderConfig readerConfig =
+ KuduReaderConfig.Builder
+ .setMasters(String.join(",", config.getSinkMasterAddresses()))
+ .build();
+
+ // TODO(zchovan): replace this with new implementation, when available
+ KuduSource source = new KuduSource(config.getSourceMasterAddresses(), config.getTableName());
KuduWriterConfig writerConfig =
KuduWriterConfig.Builder
@@ -100,99 +86,42 @@ class ReplicationJobExecutor {
.setStrongConsistency()
.build();
- KuduTableInfo tableInfo = booksTableInfo(UUID.randomUUID().toString(), true);
+
+ RowOperationMapper operationMapper = new RowOperationMapper(
+ tableInfo.getSchema()
+ .getColumns()
+ .stream()
+ .map(c -> c.getName())
+ .toArray(size -> new String[size]), AbstractSingleOperationMapper.KuduOperation.UPSERT);
+
KuduSink<Row> sink =
KuduSink.<Row>builder()
- .setWriterConfig(writerConfig)
.setTableInfo(tableInfo)
- .setOperationMapper(initOperationMapper(columns))
+ .setOperationMapper(operationMapper)
+ .setWriterConfig(writerConfig)
.build();
+ env.fromSource(source)
+ .sinkTo(sink)
+ .execute();
+ // 1. Establish connection to both source and sink clusters. Fail the replication
+ // job early if there are any problems (e.g. connectivity, auth).
+ // Check that the table(s) to be replicated do exist on the source and sink clusters.
+ // If either is missing the job fails.
- try {
- SinkWriter<Row> writer = sink.createWriter((Sink.InitContext) null);
+ // 3. Check if the table(s) have been bootstrapped (the target table is not empty).
- for (Row kuduRow : booksDataRow) {
- writer.write(kuduRow, null);
- }
- writer.close();
+ // 4a. If bootstrapping is needed, start a full replication.
- List<Row> rows = readRows(tableInfo, String.join(",", config.getSinkMasterAddresses()));
-
- env.execute("Kudu Test Job");
- } catch (Exception e) {
- System.out.println(e.getMessage());
- }
- }
+ // 4b. If bootstrapping is not needed, start from the latest replicated timestamp.
+ // 5. If the replication was successful, update and persist the latest replicated timestamp.
- // everything below this is just temp helper stuff
- private RowOperationMapper initOperationMapper(String[] cols) {
- return new RowOperationMapper(cols, AbstractSingleOperationMapper.KuduOperation.INSERT);
- }
- public static KuduTableInfo booksTableInfo(String tableName, boolean createIfNotExist) {
-
- KuduTableInfo tableInfo = KuduTableInfo.forTable(tableName);
-
- if (createIfNotExist) {
- ColumnSchemasFactory schemasFactory =
- () -> {
- List<ColumnSchema> schemas = new ArrayList<>();
- schemas.add(
- new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32)
- .key(true)
- .build());
- schemas.add(
- new ColumnSchema.ColumnSchemaBuilder("title", Type.STRING).build());
- schemas.add(
- new ColumnSchema.ColumnSchemaBuilder("author", Type.STRING)
- .build());
- schemas.add(
- new ColumnSchema.ColumnSchemaBuilder("price", Type.DOUBLE)
- .nullable(true)
- .build());
- schemas.add(
- new ColumnSchema.ColumnSchemaBuilder("quantity", Type.INT32)
- .nullable(true)
- .build());
- return schemas;
- };
-
- tableInfo.createTableIfNotExists(
- schemasFactory,
- () ->
- new CreateTableOptions()
- .setNumReplicas(1)
- .addHashPartitions(Lists.newArrayList("id"), 2));
- }
-
- return tableInfo;
- }
- protected List<Row> readRows(KuduTableInfo tableInfo, String masterAddresses) throws Exception {
- KuduReaderConfig readerConfig =
- KuduReaderConfig.Builder.setMasters(masterAddresses).build();
- KuduReader<Row> reader =
- new KuduReader<>(tableInfo, readerConfig, new RowResultRowConverter());
-
- KuduInputSplit[] splits = reader.createInputSplits(1);
- List<Row> rows = new ArrayList<>();
- for (KuduInputSplit split : splits) {
- KuduReaderIterator<Row> resultIterator = reader.scanner(split.getScanToken());
- while (resultIterator.hasNext()) {
- Row row = resultIterator.next();
- if (row != null) {
- rows.add(row);
- }
- }
- }
- reader.close();
-
- return rows;
- }
+ }
}
diff --git a/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java
index 869f38e2e..4030bd43d 100644
--- a/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java
+++ b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java
@@ -22,8 +22,8 @@ public class TestReplication {
@Rule
public final KuduTestHarness sourceHarness = new KuduTestHarness();
- @Rule
- public final KuduTestHarness sinkHarness = new KuduTestHarness();
+// @Rule
+// public final KuduTestHarness sinkHarness = new KuduTestHarness();
private KuduClient sourceClient;
private KuduClient sinkClient;
@@ -31,7 +31,7 @@ public class TestReplication {
@Before
public void setup() {
this.sourceClient = sourceHarness.getClient();
- this.sinkClient = sinkHarness.getClient();
+// this.sinkClient = sinkHarness.getClient();
}
@Test
@@ -49,7 +49,7 @@ public class TestReplication {
ReplicationJobConfig config = new ReplicationJobConfig();
config.setSourceMasterAddresses(Arrays.asList(sourceHarness.getMasterAddressesAsString().split(",")));
- config.setSinkMasterAddresses(Arrays.asList(sinkHarness.getMasterAddressesAsString().split(",")));
+// config.setSinkMasterAddresses(Arrays.asList(sinkHarness.getMasterAddressesAsString().split(",")));
config.setTableName(TABLE_NAME);
ReplicationJobExecutor executor = new ReplicationJobExecutor(config);
@@ -59,26 +59,44 @@ public class TestReplication {
}
@Test
- @KuduTestHarness.EnableKerberos
- public void TestReplicationWithKerberos() {
- try {
- createTableWithOneThousandRows(
- this.sourceHarness.getAsyncClient(), TABLE_NAME, 32 * 1024, DEFAULT_SLEEP);
- } catch (Exception e) {
- e.printStackTrace();
- LOG.error(e.getMessage());
- fail(e.getMessage());
- }
+ public void TestSourceTableMissing() {
+ assertTrue(true);
+ }
- ReplicationJobConfig config = new ReplicationJobConfig();
+ @Test
+ public void TestSinkTableMissing() {
+ assertTrue(true);
+ }
- config.setSourceMasterAddresses(Arrays.asList(sourceHarness.getMasterAddressesAsString().split(",")));
- config.setSinkMasterAddresses(Arrays.asList(sinkHarness.getMasterAddressesAsString().split(",")));
- config.setTableName(TABLE_NAME);
+ @Test
+ public void TestTableSchemaDoNotMatch() {
+ assertTrue(true);
+ }
- ReplicationJobExecutor executor = new ReplicationJobExecutor(config);
- executor.runJob();
+ @Test
+ public void TestSourceClusterUnavailable() {
+ assertTrue(true);
+ }
+ @Test
+ public void TestSinkClusterUnavailable() {
+ assertTrue(true);
+ }
+
+ @Test
+ public void TestRestartFailedJob() {
assertTrue(true);
}
+
+ @Test
+ public void TestReplicatingAlreadyBootstrappedTable() {
+ assertTrue(true);
+ }
+
+ @Test
+ public void TestReplicatingMultipleTables() {
+ assertTrue(true);
+ }
+
+
}