You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ar...@apache.org on 2021/12/11 15:34:18 UTC

[flink] 01/02: [FLINK-24077][HBase/IT] Add check of row count after insert and wait explicitly for job to finish.

This is an automated email from the ASF dual-hosted git repository.

arvid pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 4e962d9a7980dc143a2e4ace1211887314648d7d
Author: Jing Ge <ge...@gmail.com>
AuthorDate: Tue Nov 23 11:49:57 2021 +0100

    [FLINK-24077][HBase/IT] Add check of row count after insert and wait explicitly for job to finish.
    
    (cherry picked from commit 7976be0f8675a8753a5bb7e7a44dda6b4a347247)
---
 .../connector/hbase2/HBaseConnectorITCase.java     | 99 +++++++++++++++-------
 1 file changed, 67 insertions(+), 32 deletions(-)

diff --git a/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java b/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java
index ee98c61..a512058 100644
--- a/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java
+++ b/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java
@@ -29,10 +29,12 @@ import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.TableResult;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.functions.ScalarFunction;
 import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
+import org.apache.flink.types.RowKind;
 import org.apache.flink.util.CollectionUtil;
 
 import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
@@ -250,13 +252,33 @@ public class HBaseConnectorITCase extends HBaseTestBase {
                         + " FROM "
                         + TEST_TABLE_1;
 
+        TableResult tableResult = tEnv.executeSql(query);
+
         // wait to finish
-        tEnv.executeSql(query).await();
+        tableResult.await();
+
+        assertEquals(
+                "Expected INSERT rowKind", RowKind.INSERT, tableResult.collect().next().getKind());
 
         // start a batch scan job to verify contents in HBase table
         TableEnvironment batchEnv = TableEnvironment.create(batchSettings);
         batchEnv.executeSql(table2DDL);
 
+        List<String> expected = new ArrayList<>();
+        expected.add("+I[1, 10, Hello-1, 100, 1.01, false, Welt-1]\n");
+        expected.add("+I[2, 20, Hello-2, 200, 2.02, true, Welt-2]\n");
+        expected.add("+I[3, 30, Hello-3, 300, 3.03, false, Welt-3]\n");
+        expected.add("+I[4, 40, null, 400, 4.04, true, Welt-4]\n");
+        expected.add("+I[5, 50, Hello-5, 500, 5.05, false, Welt-5]\n");
+        expected.add("+I[6, 60, Hello-6, 600, 6.06, true, Welt-6]\n");
+        expected.add("+I[7, 70, Hello-7, 700, 7.07, false, Welt-7]\n");
+        expected.add("+I[8, 80, null, 800, 8.08, true, Welt-8]\n");
+
+        Table countTable =
+                batchEnv.sqlQuery("SELECT COUNT(h.rowkey) FROM " + TEST_TABLE_2 + " AS h");
+
+        assertEquals(new Long(expected.size()), countTable.execute().collect().next().getField(0));
+
         Table table =
                 batchEnv.sqlQuery(
                         "SELECT "
@@ -270,18 +292,14 @@ public class HBaseConnectorITCase extends HBaseTestBase {
                                 + "FROM "
                                 + TEST_TABLE_2
                                 + " AS h");
-        List<Row> results = CollectionUtil.iteratorToList(table.execute().collect());
-        String expected =
-                "+I[1, 10, Hello-1, 100, 1.01, false, Welt-1]\n"
-                        + "+I[2, 20, Hello-2, 200, 2.02, true, Welt-2]\n"
-                        + "+I[3, 30, Hello-3, 300, 3.03, false, Welt-3]\n"
-                        + "+I[4, 40, null, 400, 4.04, true, Welt-4]\n"
-                        + "+I[5, 50, Hello-5, 500, 5.05, false, Welt-5]\n"
-                        + "+I[6, 60, Hello-6, 600, 6.06, true, Welt-6]\n"
-                        + "+I[7, 70, Hello-7, 700, 7.07, false, Welt-7]\n"
-                        + "+I[8, 80, null, 800, 8.08, true, Welt-8]\n";
 
-        TestBaseUtils.compareResultAsText(results, expected);
+        TableResult tableResult2 = table.execute();
+        // wait to finish
+        tableResult2.getJobClient().get().getJobExecutionResult().get();
+
+        List<Row> results = CollectionUtil.iteratorToList(tableResult2.collect());
+
+        TestBaseUtils.compareResultAsText(results, String.join("", expected));
     }
 
     @Test
@@ -307,12 +325,42 @@ public class HBaseConnectorITCase extends HBaseTestBase {
                         + " family4"
                         + " from "
                         + TEST_TABLE_1;
+
+        TableResult tableResult = tEnv.executeSql(insertStatement);
+
         // wait to finish
-        tEnv.executeSql(insertStatement).await();
+        tableResult.await();
+
+        assertEquals(
+                "Expected INSERT rowKind", RowKind.INSERT, tableResult.collect().next().getKind());
 
         // start a batch scan job to verify contents in HBase table
         TableEnvironment batchEnv = TableEnvironment.create(batchSettings);
         batchEnv.executeSql(table3DDL);
+
+        List<String> expected = new ArrayList<>();
+        expected.add(
+                "+I[1, 10, Hello-1, 100, 1.01, false, Welt-1, 2019-08-18T19:00, 2019-08-18, 19:00, 12345678.0001]");
+        expected.add(
+                "+I[2, 20, Hello-2, 200, 2.02, true, Welt-2, 2019-08-18T19:01, 2019-08-18, 19:01, 12345678.0002]");
+        expected.add(
+                "+I[3, 30, Hello-3, 300, 3.03, false, Welt-3, 2019-08-18T19:02, 2019-08-18, 19:02, 12345678.0003]");
+        expected.add(
+                "+I[4, 40, null, 400, 4.04, true, Welt-4, 2019-08-18T19:03, 2019-08-18, 19:03, 12345678.0004]");
+        expected.add(
+                "+I[5, 50, Hello-5, 500, 5.05, false, Welt-5, 2019-08-19T19:10, 2019-08-19, 19:10, 12345678.0005]");
+        expected.add(
+                "+I[6, 60, Hello-6, 600, 6.06, true, Welt-6, 2019-08-19T19:20, 2019-08-19, 19:20, 12345678.0006]");
+        expected.add(
+                "+I[7, 70, Hello-7, 700, 7.07, false, Welt-7, 2019-08-19T19:30, 2019-08-19, 19:30, 12345678.0007]");
+        expected.add(
+                "+I[8, 80, null, 800, 8.08, true, Welt-8, 2019-08-19T19:40, 2019-08-19, 19:40, 12345678.0008]");
+
+        Table countTable =
+                batchEnv.sqlQuery("SELECT COUNT(h.rowkey) FROM " + TEST_TABLE_3 + " AS h");
+
+        assertEquals(new Long(expected.size()), countTable.execute().collect().next().getField(0));
+
         String query =
                 "SELECT "
                         + "  h.rowkey, "
@@ -329,30 +377,17 @@ public class HBaseConnectorITCase extends HBaseTestBase {
                         + " FROM "
                         + TEST_TABLE_3
                         + " AS h";
-        Iterator<Row> collected = tEnv.executeSql(query).collect();
+
+        TableResult tableResult3 = batchEnv.executeSql(query);
+        // wait to finish
+        tableResult3.getJobClient().get().getJobExecutionResult().get();
+
         List<String> result =
-                Lists.newArrayList(collected).stream()
+                Lists.newArrayList(tableResult3.collect()).stream()
                         .map(Row::toString)
                         .sorted()
                         .collect(Collectors.toList());
 
-        List<String> expected = new ArrayList<>();
-        expected.add(
-                "+I[1, 10, Hello-1, 100, 1.01, false, Welt-1, 2019-08-18T19:00, 2019-08-18, 19:00, 12345678.0001]");
-        expected.add(
-                "+I[2, 20, Hello-2, 200, 2.02, true, Welt-2, 2019-08-18T19:01, 2019-08-18, 19:01, 12345678.0002]");
-        expected.add(
-                "+I[3, 30, Hello-3, 300, 3.03, false, Welt-3, 2019-08-18T19:02, 2019-08-18, 19:02, 12345678.0003]");
-        expected.add(
-                "+I[4, 40, null, 400, 4.04, true, Welt-4, 2019-08-18T19:03, 2019-08-18, 19:03, 12345678.0004]");
-        expected.add(
-                "+I[5, 50, Hello-5, 500, 5.05, false, Welt-5, 2019-08-19T19:10, 2019-08-19, 19:10, 12345678.0005]");
-        expected.add(
-                "+I[6, 60, Hello-6, 600, 6.06, true, Welt-6, 2019-08-19T19:20, 2019-08-19, 19:20, 12345678.0006]");
-        expected.add(
-                "+I[7, 70, Hello-7, 700, 7.07, false, Welt-7, 2019-08-19T19:30, 2019-08-19, 19:30, 12345678.0007]");
-        expected.add(
-                "+I[8, 80, null, 800, 8.08, true, Welt-8, 2019-08-19T19:40, 2019-08-19, 19:40, 12345678.0008]");
         assertEquals(expected, result);
     }