You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ar...@apache.org on 2021/12/11 15:34:18 UTC
[flink] 01/02: [FLINK-24077][HBase/IT] Add check of row count after insert and wait explicitly for job to finish.
This is an automated email from the ASF dual-hosted git repository.
arvid pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4e962d9a7980dc143a2e4ace1211887314648d7d
Author: Jing Ge <ge...@gmail.com>
AuthorDate: Tue Nov 23 11:49:57 2021 +0100
[FLINK-24077][HBase/IT] Add check of row count after insert and wait explicitly for job to finish.
(cherry picked from commit 7976be0f8675a8753a5bb7e7a44dda6b4a347247)
---
.../connector/hbase2/HBaseConnectorITCase.java | 99 +++++++++++++++-------
1 file changed, 67 insertions(+), 32 deletions(-)
diff --git a/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java b/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java
index ee98c61..a512058 100644
--- a/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java
+++ b/flink-connectors/flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseConnectorITCase.java
@@ -29,10 +29,12 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.functions.ScalarFunction;
import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
+import org.apache.flink.types.RowKind;
import org.apache.flink.util.CollectionUtil;
import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
@@ -250,13 +252,33 @@ public class HBaseConnectorITCase extends HBaseTestBase {
+ " FROM "
+ TEST_TABLE_1;
+ TableResult tableResult = tEnv.executeSql(query);
+
// wait to finish
- tEnv.executeSql(query).await();
+ tableResult.await();
+
+ assertEquals(
+ "Expected INSERT rowKind", RowKind.INSERT, tableResult.collect().next().getKind());
// start a batch scan job to verify contents in HBase table
TableEnvironment batchEnv = TableEnvironment.create(batchSettings);
batchEnv.executeSql(table2DDL);
+ List<String> expected = new ArrayList<>();
+ expected.add("+I[1, 10, Hello-1, 100, 1.01, false, Welt-1]\n");
+ expected.add("+I[2, 20, Hello-2, 200, 2.02, true, Welt-2]\n");
+ expected.add("+I[3, 30, Hello-3, 300, 3.03, false, Welt-3]\n");
+ expected.add("+I[4, 40, null, 400, 4.04, true, Welt-4]\n");
+ expected.add("+I[5, 50, Hello-5, 500, 5.05, false, Welt-5]\n");
+ expected.add("+I[6, 60, Hello-6, 600, 6.06, true, Welt-6]\n");
+ expected.add("+I[7, 70, Hello-7, 700, 7.07, false, Welt-7]\n");
+ expected.add("+I[8, 80, null, 800, 8.08, true, Welt-8]\n");
+
+ Table countTable =
+ batchEnv.sqlQuery("SELECT COUNT(h.rowkey) FROM " + TEST_TABLE_2 + " AS h");
+
+ assertEquals(new Long(expected.size()), countTable.execute().collect().next().getField(0));
+
Table table =
batchEnv.sqlQuery(
"SELECT "
@@ -270,18 +292,14 @@ public class HBaseConnectorITCase extends HBaseTestBase {
+ "FROM "
+ TEST_TABLE_2
+ " AS h");
- List<Row> results = CollectionUtil.iteratorToList(table.execute().collect());
- String expected =
- "+I[1, 10, Hello-1, 100, 1.01, false, Welt-1]\n"
- + "+I[2, 20, Hello-2, 200, 2.02, true, Welt-2]\n"
- + "+I[3, 30, Hello-3, 300, 3.03, false, Welt-3]\n"
- + "+I[4, 40, null, 400, 4.04, true, Welt-4]\n"
- + "+I[5, 50, Hello-5, 500, 5.05, false, Welt-5]\n"
- + "+I[6, 60, Hello-6, 600, 6.06, true, Welt-6]\n"
- + "+I[7, 70, Hello-7, 700, 7.07, false, Welt-7]\n"
- + "+I[8, 80, null, 800, 8.08, true, Welt-8]\n";
- TestBaseUtils.compareResultAsText(results, expected);
+ TableResult tableResult2 = table.execute();
+ // wait to finish
+ tableResult2.getJobClient().get().getJobExecutionResult().get();
+
+ List<Row> results = CollectionUtil.iteratorToList(tableResult2.collect());
+
+ TestBaseUtils.compareResultAsText(results, String.join("", expected));
}
@Test
@@ -307,12 +325,42 @@ public class HBaseConnectorITCase extends HBaseTestBase {
+ " family4"
+ " from "
+ TEST_TABLE_1;
+
+ TableResult tableResult = tEnv.executeSql(insertStatement);
+
// wait to finish
- tEnv.executeSql(insertStatement).await();
+ tableResult.await();
+
+ assertEquals(
+ "Expected INSERT rowKind", RowKind.INSERT, tableResult.collect().next().getKind());
// start a batch scan job to verify contents in HBase table
TableEnvironment batchEnv = TableEnvironment.create(batchSettings);
batchEnv.executeSql(table3DDL);
+
+ List<String> expected = new ArrayList<>();
+ expected.add(
+ "+I[1, 10, Hello-1, 100, 1.01, false, Welt-1, 2019-08-18T19:00, 2019-08-18, 19:00, 12345678.0001]");
+ expected.add(
+ "+I[2, 20, Hello-2, 200, 2.02, true, Welt-2, 2019-08-18T19:01, 2019-08-18, 19:01, 12345678.0002]");
+ expected.add(
+ "+I[3, 30, Hello-3, 300, 3.03, false, Welt-3, 2019-08-18T19:02, 2019-08-18, 19:02, 12345678.0003]");
+ expected.add(
+ "+I[4, 40, null, 400, 4.04, true, Welt-4, 2019-08-18T19:03, 2019-08-18, 19:03, 12345678.0004]");
+ expected.add(
+ "+I[5, 50, Hello-5, 500, 5.05, false, Welt-5, 2019-08-19T19:10, 2019-08-19, 19:10, 12345678.0005]");
+ expected.add(
+ "+I[6, 60, Hello-6, 600, 6.06, true, Welt-6, 2019-08-19T19:20, 2019-08-19, 19:20, 12345678.0006]");
+ expected.add(
+ "+I[7, 70, Hello-7, 700, 7.07, false, Welt-7, 2019-08-19T19:30, 2019-08-19, 19:30, 12345678.0007]");
+ expected.add(
+ "+I[8, 80, null, 800, 8.08, true, Welt-8, 2019-08-19T19:40, 2019-08-19, 19:40, 12345678.0008]");
+
+ Table countTable =
+ batchEnv.sqlQuery("SELECT COUNT(h.rowkey) FROM " + TEST_TABLE_3 + " AS h");
+
+ assertEquals(new Long(expected.size()), countTable.execute().collect().next().getField(0));
+
String query =
"SELECT "
+ " h.rowkey, "
@@ -329,30 +377,17 @@ public class HBaseConnectorITCase extends HBaseTestBase {
+ " FROM "
+ TEST_TABLE_3
+ " AS h";
- Iterator<Row> collected = tEnv.executeSql(query).collect();
+
+ TableResult tableResult3 = batchEnv.executeSql(query);
+ // wait to finish
+ tableResult3.getJobClient().get().getJobExecutionResult().get();
+
List<String> result =
- Lists.newArrayList(collected).stream()
+ Lists.newArrayList(tableResult3.collect()).stream()
.map(Row::toString)
.sorted()
.collect(Collectors.toList());
- List<String> expected = new ArrayList<>();
- expected.add(
- "+I[1, 10, Hello-1, 100, 1.01, false, Welt-1, 2019-08-18T19:00, 2019-08-18, 19:00, 12345678.0001]");
- expected.add(
- "+I[2, 20, Hello-2, 200, 2.02, true, Welt-2, 2019-08-18T19:01, 2019-08-18, 19:01, 12345678.0002]");
- expected.add(
- "+I[3, 30, Hello-3, 300, 3.03, false, Welt-3, 2019-08-18T19:02, 2019-08-18, 19:02, 12345678.0003]");
- expected.add(
- "+I[4, 40, null, 400, 4.04, true, Welt-4, 2019-08-18T19:03, 2019-08-18, 19:03, 12345678.0004]");
- expected.add(
- "+I[5, 50, Hello-5, 500, 5.05, false, Welt-5, 2019-08-19T19:10, 2019-08-19, 19:10, 12345678.0005]");
- expected.add(
- "+I[6, 60, Hello-6, 600, 6.06, true, Welt-6, 2019-08-19T19:20, 2019-08-19, 19:20, 12345678.0006]");
- expected.add(
- "+I[7, 70, Hello-7, 700, 7.07, false, Welt-7, 2019-08-19T19:30, 2019-08-19, 19:30, 12345678.0007]");
- expected.add(
- "+I[8, 80, null, 800, 8.08, true, Welt-8, 2019-08-19T19:40, 2019-08-19, 19:40, 12345678.0008]");
assertEquals(expected, result);
}