You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2025/04/14 16:58:25 UTC

(kudu) 04/14: organisation of replication job, added test stubs

This is an automated email from the ASF dual-hosted git repository.

mgreber pushed a commit to branch flink-replication
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit e6218646af7004b6898fd8357e0841410dd6fbff
Author: zchovan <zc...@cloudera.com>
AuthorDate: Tue Feb 11 14:49:32 2025 +0100

    organisation of replication job, added test stubs
    
    Change-Id: Iad71eaedd7f136458719d52d2fb14047d61fa227
---
 .../kudu/replication/ReplicationJobExecutor.java   | 161 ++++++---------------
 .../apache/kudu/replication/TestReplication.java   |  58 +++++---
 2 files changed, 83 insertions(+), 136 deletions(-)

diff --git a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java
index c5d568ace..ce5029cb4 100644
--- a/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java
+++ b/java/kudu-replication/src/main/java/org/apache/kudu/replication/ReplicationJobExecutor.java
@@ -17,8 +17,13 @@
 
 package org.apache.kudu.replication;
 
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.connector.sink2.Sink;
 import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceSplit;
+import org.apache.flink.api.connector.source.SplitEnumerator;
 import org.apache.flink.connector.kudu.connector.ColumnSchemasFactory;
 import org.apache.flink.connector.kudu.connector.KuduTableInfo;
 import org.apache.flink.connector.kudu.connector.converter.RowResultRowConverter;
@@ -28,21 +33,23 @@ import org.apache.flink.connector.kudu.connector.reader.KuduReaderConfig;
 import org.apache.flink.connector.kudu.connector.reader.KuduReaderIterator;
 import org.apache.flink.connector.kudu.connector.writer.AbstractSingleOperationMapper;
 import org.apache.flink.connector.kudu.connector.writer.KuduWriterConfig;
+import org.apache.flink.connector.kudu.connector.writer.RowDataUpsertOperationMapper;
 import org.apache.flink.connector.kudu.connector.writer.RowOperationMapper;
 import org.apache.flink.connector.kudu.sink.KuduSink;
 import org.apache.flink.connector.kudu.sink.KuduSinkBuilder;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
 import org.apache.flink.types.Row;
 import org.apache.kudu.ColumnSchema;
 import org.apache.kudu.Type;
 import org.apache.kudu.client.CreateTableOptions;
 import org.apache.kudu.shaded.com.google.common.collect.Lists;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -50,40 +57,9 @@ import java.util.stream.Collectors;
  */
 class ReplicationJobExecutor {
     private ReplicationJobConfig config;
-    private final String[] columns = new String[] {"id", "title", "author", "price", "quantity"};
-    private final Object[][] booksTableData = {
-            {1001, "Java for dummies", "Tan Ah Teck", 11.11, 11},
-            {1002, "More Java for dummies", "Tan Ah Teck", 22.22, 22},
-            {1003, "More Java for more dummies", "Mohammad Ali", 33.33, 33},
-            {1004, "A Cup of Java", "Kumar", 44.44, 44},
-            {1005, "A Teaspoon of Java", "Kevin Jones", 55.55, 55}
-    };
-
-    private List<Row> booksDataRow;
 
     ReplicationJobExecutor(ReplicationJobConfig config) {
         this.config = config;
-        this.booksDataRow =  Arrays.stream(booksTableData)
-            .map(
-                row -> {
-                    Integer rowId = (Integer) row[0];
-                    if (rowId % 2 == 1) {
-                        Row values = new Row(5);
-                        values.setField(0, row[0]);
-                        values.setField(1, row[1]);
-                        values.setField(2, row[2]);
-                        values.setField(3, row[3]);
-                        values.setField(4, row[4]);
-                        return values;
-                    } else {
-                        Row values = new Row(5);
-                        values.setField(0, row[0]);
-                        values.setField(1, row[1]);
-                        values.setField(2, row[2]);
-                        return values;
-                    }
-                })
-            .collect(Collectors.toList());
     }
 
     /**
@@ -91,8 +67,18 @@ class ReplicationJobExecutor {
      */
     public void runJob() {
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-//        DataStream<String> kuduSourceStream = env.addSource(
-//                new KuduSource(config.getSourceMasterAddresses(), config.getTableName()));
+        KuduTableInfo tableInfo = KuduTableInfo.forTable(config.getTableName());
+
+
+
+
+        KuduReaderConfig readerConfig =
+                KuduReaderConfig.Builder
+                        .setMasters(String.join(",", config.getSinkMasterAddresses()))
+                        .build();
+
+        // TODO(zchovan): replace this with new implementation, when available
+        KuduSource source = new KuduSource(config.getSourceMasterAddresses(), config.getTableName());
 
         KuduWriterConfig writerConfig =
                 KuduWriterConfig.Builder
@@ -100,99 +86,42 @@ class ReplicationJobExecutor {
                         .setStrongConsistency()
                         .build();
 
-        KuduTableInfo tableInfo = booksTableInfo(UUID.randomUUID().toString(), true);
+
+        RowOperationMapper operationMapper = new RowOperationMapper(
+                tableInfo.getSchema()
+                        .getColumns()
+                        .stream()
+                        .map(c -> c.getName())
+                        .toArray(size -> new String[size]), AbstractSingleOperationMapper.KuduOperation.UPSERT);
+
 
         KuduSink<Row> sink =
                 KuduSink.<Row>builder()
-                        .setWriterConfig(writerConfig)
                         .setTableInfo(tableInfo)
-                        .setOperationMapper(initOperationMapper(columns))
+                        .setOperationMapper(operationMapper)
+                        .setWriterConfig(writerConfig)
                         .build();
 
+        env.fromSource(source)
+                .sinkTo(sink)
+                .execute();
+        // 1. Establish connection to both source and sink clusters. Fail the replication
+        // job early if there are any problems (e.g. connectivity, auth).
+        // Check that the table(s) to be replicated do exist on the source and sink clusters.
+        // If either is missing the job fails.
 
-        try {
-            SinkWriter<Row> writer = sink.createWriter((Sink.InitContext) null);
+        // 3. Check if the table(s) have been bootstrapped (the target table is not empty).
 
-            for (Row kuduRow : booksDataRow) {
-                writer.write(kuduRow, null);
-            }
-            writer.close();
+        // 4a. If bootstrapping is needed, start a full replication.
 
-            List<Row> rows = readRows(tableInfo, String.join(",", config.getSinkMasterAddresses()));
-
-            env.execute("Kudu Test Job");
-        } catch (Exception e) {
-            System.out.println(e.getMessage());
-        }
-    }
+        // 4b. If bootstrapping is not needed, start from the latest replicated timestamp.
 
+        // 5. If the replication was successful, update and persist the latest replicated timestamp.
 
-    // everything below this is just temp helper stuff
-    private RowOperationMapper initOperationMapper(String[] cols) {
-        return new RowOperationMapper(cols, AbstractSingleOperationMapper.KuduOperation.INSERT);
-    }
 
-    public static KuduTableInfo booksTableInfo(String tableName, boolean createIfNotExist) {
-
-        KuduTableInfo tableInfo = KuduTableInfo.forTable(tableName);
-
-        if (createIfNotExist) {
-            ColumnSchemasFactory schemasFactory =
-                    () -> {
-                        List<ColumnSchema> schemas = new ArrayList<>();
-                        schemas.add(
-                                new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32)
-                                        .key(true)
-                                        .build());
-                        schemas.add(
-                                new ColumnSchema.ColumnSchemaBuilder("title", Type.STRING).build());
-                        schemas.add(
-                                new ColumnSchema.ColumnSchemaBuilder("author", Type.STRING)
-                                        .build());
-                        schemas.add(
-                                new ColumnSchema.ColumnSchemaBuilder("price", Type.DOUBLE)
-                                        .nullable(true)
-                                        .build());
-                        schemas.add(
-                                new ColumnSchema.ColumnSchemaBuilder("quantity", Type.INT32)
-                                        .nullable(true)
-                                        .build());
-                        return schemas;
-                    };
-
-            tableInfo.createTableIfNotExists(
-                    schemasFactory,
-                    () ->
-                            new CreateTableOptions()
-                                    .setNumReplicas(1)
-                                    .addHashPartitions(Lists.newArrayList("id"), 2));
-        }
-
-        return tableInfo;
-    }
 
-    protected List<Row> readRows(KuduTableInfo tableInfo, String masterAddresses) throws Exception {
-        KuduReaderConfig readerConfig =
-                KuduReaderConfig.Builder.setMasters(masterAddresses).build();
-        KuduReader<Row> reader =
-                new KuduReader<>(tableInfo, readerConfig, new RowResultRowConverter());
-
-        KuduInputSplit[] splits = reader.createInputSplits(1);
-        List<Row> rows = new ArrayList<>();
-        for (KuduInputSplit split : splits) {
-            KuduReaderIterator<Row> resultIterator = reader.scanner(split.getScanToken());
-            while (resultIterator.hasNext()) {
-                Row row = resultIterator.next();
-                if (row != null) {
-                    rows.add(row);
-                }
-            }
-        }
-        reader.close();
-
-        return rows;
-    }
 
 
+    }
 
 }
diff --git a/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java
index 869f38e2e..4030bd43d 100644
--- a/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java
+++ b/java/kudu-replication/src/test/java/org/apache/kudu/replication/TestReplication.java
@@ -22,8 +22,8 @@ public class TestReplication {
 
     @Rule
     public final KuduTestHarness sourceHarness = new KuduTestHarness();
-    @Rule
-    public final KuduTestHarness sinkHarness = new KuduTestHarness();
+//    @Rule
+//    public final KuduTestHarness sinkHarness = new KuduTestHarness();
 
     private KuduClient sourceClient;
     private KuduClient sinkClient;
@@ -31,7 +31,7 @@ public class TestReplication {
     @Before
     public void setup()  {
         this.sourceClient = sourceHarness.getClient();
-        this.sinkClient = sinkHarness.getClient();
+//        this.sinkClient = sinkHarness.getClient();
     }
 
     @Test
@@ -49,7 +49,7 @@ public class TestReplication {
         ReplicationJobConfig config = new ReplicationJobConfig();
 
         config.setSourceMasterAddresses(Arrays.asList(sourceHarness.getMasterAddressesAsString().split(",")));
-        config.setSinkMasterAddresses(Arrays.asList(sinkHarness.getMasterAddressesAsString().split(",")));
+//        config.setSinkMasterAddresses(Arrays.asList(sinkHarness.getMasterAddressesAsString().split(",")));
         config.setTableName(TABLE_NAME);
 
         ReplicationJobExecutor executor = new ReplicationJobExecutor(config);
@@ -59,26 +59,44 @@ public class TestReplication {
     }
 
     @Test
-    @KuduTestHarness.EnableKerberos
-    public void TestReplicationWithKerberos() {
-        try {
-            createTableWithOneThousandRows(
-                    this.sourceHarness.getAsyncClient(), TABLE_NAME, 32 * 1024, DEFAULT_SLEEP);
-        } catch (Exception e) {
-            e.printStackTrace();
-            LOG.error(e.getMessage());
-            fail(e.getMessage());
-        }
+    public void TestSourceTableMissing() {
+        assertTrue(true);
+    }
 
-        ReplicationJobConfig config = new ReplicationJobConfig();
+    @Test
+    public void TestSinkTableMissing() {
+        assertTrue(true);
+    }
 
-        config.setSourceMasterAddresses(Arrays.asList(sourceHarness.getMasterAddressesAsString().split(",")));
-        config.setSinkMasterAddresses(Arrays.asList(sinkHarness.getMasterAddressesAsString().split(",")));
-        config.setTableName(TABLE_NAME);
+    @Test
+    public void TestTableSchemaDoNotMatch() {
+        assertTrue(true);
+    }
 
-        ReplicationJobExecutor executor = new ReplicationJobExecutor(config);
-        executor.runJob();
+    @Test
+    public void TestSourceClusterUnavailable() {
+        assertTrue(true);
+    }
 
+    @Test
+    public void TestSinkClusterUnavailable() {
+        assertTrue(true);
+    }
+
+    @Test
+    public void TestRestartFailedJob() {
         assertTrue(true);
     }
+
+    @Test
+    public void TestReplicatingAlreadyBootstrappedTable() {
+        assertTrue(true);
+    }
+
+    @Test
+    public void TestReplicatingMultipleTables() {
+        assertTrue(true);
+    }
+    
+
 }