You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2022/01/05 15:14:08 UTC
[GitHub] [flink] fapaul commented on a change in pull request #17937: [FLINK-25044][testing][Pulsar Connector] Add More Unit Test For Pulsar Source
fapaul commented on a change in pull request #17937:
URL: https://github.com/apache/flink/pull/17937#discussion_r778886160
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ @RegisterExtension
+ PulsarSourceReaderInvocationContextProvider provider =
+ new PulsarSourceReaderInvocationContextProvider();
+
+ @BeforeEach
+ void beforeEach(String topicName) {
+ Random random = new Random(System.currentTimeMillis());
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(String topicName) {
+ operator().deleteTopic(topicName, true);
+ }
+
+ @TestTemplate
+ void assignZeroSplitsCreatesZeroSubscription(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @TestTemplate
+ void assigningEmptySplits(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
+ // We sleep for 1 sec to avoid entering false negative case:
+ // The fetcher could become idle upon entry because :
+ // 1. the AddSplitsTask is dequeued from taskQueue,
+ // 2. The dequeued splits is not assigned to assignedSplits.
+ // And if both assignedSplits is null and taskQueue is empty, the fetcher could become idle,
+ // and return NOTHING_AVAILABLE upon entry. This happened multiple times during local test.
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ assertThat(status).isEqualTo(InputStatus.NOTHING_AVAILABLE);
+ reader.close();
+ }
+
+ protected void setupSourceReader(
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ @RegisterExtension
+ PulsarSourceReaderInvocationContextProvider provider =
+ new PulsarSourceReaderInvocationContextProvider();
+
+ @BeforeEach
+ void beforeEach(String topicName) {
+ Random random = new Random(System.currentTimeMillis());
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(String topicName) {
+ operator().deleteTopic(topicName, true);
+ }
+
+ @TestTemplate
+ void assignZeroSplitsCreatesZeroSubscription(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @TestTemplate
+ void assigningEmptySplits(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
Review comment:
Nit: why the split?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ @RegisterExtension
+ PulsarSourceReaderInvocationContextProvider provider =
+ new PulsarSourceReaderInvocationContextProvider();
+
+ @BeforeEach
+ void beforeEach(String topicName) {
+ Random random = new Random(System.currentTimeMillis());
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(String topicName) {
+ operator().deleteTopic(topicName, true);
+ }
+
+ @TestTemplate
+ void assignZeroSplitsCreatesZeroSubscription(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @TestTemplate
+ void assigningEmptySplits(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
+ // We sleep for 1 sec to avoid entering false negative case:
+ // The fetcher could become idle upon entry because :
+ // 1. the AddSplitsTask is dequeued from taskQueue,
+ // 2. The dequeued splits is not assigned to assignedSplits.
+ // And if both assignedSplits is null and taskQueue is empty, the fetcher could become idle,
+ // and return NOTHING_AVAILABLE upon entry. This happened multiple times during local test.
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ assertThat(status).isEqualTo(InputStatus.NOTHING_AVAILABLE);
+ reader.close();
+ }
+
+ protected void setupSourceReader(
+ PulsarSourceReaderBase<Integer> reader,
+ String topicName,
+ int partitionId,
+ Boundedness boundedness) {
+ PulsarPartitionSplit split = createPartitionSplit(topicName, partitionId, boundedness);
+ reader.addSplits(Collections.singletonList(split));
+ reader.notifyNoMoreSplits();
+ }
+
+ protected void pollUtilReadExpectedNumberOfRecordsAndValidate(
+ PulsarSourceReaderBase<Integer> reader,
+ TestingReaderOutput<Integer> output,
+ int expectedRecords,
+ String topicNameWithPartition)
+ throws Exception {
+ pollUntil(
+ reader,
+ output,
+ () -> output.getEmittedRecords().size() == expectedRecords,
+ "The output didn't poll enough records before timeout.");
+ reader.close();
+ verifyAllMessageAcknowledged(expectedRecords, topicNameWithPartition);
+ assertThat(output.getEmittedRecords()).hasSize(expectedRecords);
+ }
+
+ protected void pollUntil(
Review comment:
This can be a static util I do not an instance reference
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ @RegisterExtension
+ PulsarSourceReaderInvocationContextProvider provider =
+ new PulsarSourceReaderInvocationContextProvider();
+
+ @BeforeEach
+ void beforeEach(String topicName) {
+ Random random = new Random(System.currentTimeMillis());
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(String topicName) {
+ operator().deleteTopic(topicName, true);
+ }
+
+ @TestTemplate
+ void assignZeroSplitsCreatesZeroSubscription(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @TestTemplate
+ void assigningEmptySplits(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
+ // We sleep for 1 sec to avoid entering false negative case:
+ // The fetcher could become idle upon entry because :
+ // 1. the AddSplitsTask is dequeued from taskQueue,
+ // 2. The dequeued splits is not assigned to assignedSplits.
+ // And if both assignedSplits is null and taskQueue is empty, the fetcher could become idle,
+ // and return NOTHING_AVAILABLE upon entry. This happened multiple times during local test.
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ assertThat(status).isEqualTo(InputStatus.NOTHING_AVAILABLE);
+ reader.close();
+ }
+
+ protected void setupSourceReader(
+ PulsarSourceReaderBase<Integer> reader,
+ String topicName,
+ int partitionId,
+ Boundedness boundedness) {
+ PulsarPartitionSplit split = createPartitionSplit(topicName, partitionId, boundedness);
+ reader.addSplits(Collections.singletonList(split));
+ reader.notifyNoMoreSplits();
+ }
+
+ protected void pollUtilReadExpectedNumberOfRecordsAndValidate(
+ PulsarSourceReaderBase<Integer> reader,
+ TestingReaderOutput<Integer> output,
+ int expectedRecords,
+ String topicNameWithPartition)
+ throws Exception {
+ pollUntil(
+ reader,
+ output,
+ () -> output.getEmittedRecords().size() == expectedRecords,
+ "The output didn't poll enough records before timeout.");
+ reader.close();
+ verifyAllMessageAcknowledged(expectedRecords, topicNameWithPartition);
+ assertThat(output.getEmittedRecords()).hasSize(expectedRecords);
+ }
+
+ protected void pollUntil(
+ PulsarSourceReaderBase<Integer> reader,
+ ReaderOutput<Integer> output,
+ Supplier<Boolean> condition,
+ String errorMessage)
+ throws InterruptedException, TimeoutException {
+ CommonTestUtils.waitUtil(
+ () -> {
+ try {
+ reader.pollNext(output);
+ } catch (Exception exception) {
+ throw new RuntimeException(
+ "Caught unexpected exception when polling from the reader",
+ exception);
+ }
+ return condition.get();
+ },
+ Duration.ofSeconds(DEFAULT_POLLING_TIMEOUT_SECONDS),
+ errorMessage);
+ }
+
+ protected void verifyAllMessageAcknowledged(int expectedMessages, String partitionName)
Review comment:
I wonder whether we can introduce an assertion helper class that is constructed with the `operator`. So we do not need to use inheritance to share utils
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarOrderedSourceReaderTest.java
##########
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.testutils.extension.SubType;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.core.io.InputStatus;
+
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.junit.jupiter.api.TestTemplate;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplits;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
+class PulsarOrderedSourceReaderTest extends PulsarSourceReaderTestBase {
+
+ private static final int MAX_EMPTY_POLLING_TIMES = 10;
+
+ @SubType SubscriptionType subscriptionType = SubscriptionType.Failover;
+
+ @TestTemplate
+ void pollMessageWithIntervalLongerThanTimeout(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ setupSourceReader(reader, topicName, 0, boundedness);
+ sleepUninterruptibly(15, TimeUnit.SECONDS);
Review comment:
Can you make the test faster than 15 seconds?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ @RegisterExtension
+ PulsarSourceReaderInvocationContextProvider provider =
+ new PulsarSourceReaderInvocationContextProvider();
+
+ @BeforeEach
+ void beforeEach(String topicName) {
+ Random random = new Random(System.currentTimeMillis());
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(String topicName) {
+ operator().deleteTopic(topicName, true);
+ }
+
+ @TestTemplate
+ void assignZeroSplitsCreatesZeroSubscription(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @TestTemplate
+ void assigningEmptySplits(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
+ // We sleep for 1 sec to avoid entering false negative case:
+ // The fetcher could become idle upon entry because :
+ // 1. the AddSplitsTask is dequeued from taskQueue,
+ // 2. The dequeued splits is not assigned to assignedSplits.
+ // And if both assignedSplits is null and taskQueue is empty, the fetcher could become idle,
+ // and return NOTHING_AVAILABLE upon entry. This happened multiple times during local test.
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ assertThat(status).isEqualTo(InputStatus.NOTHING_AVAILABLE);
+ reader.close();
+ }
+
+ protected void setupSourceReader(
+ PulsarSourceReaderBase<Integer> reader,
+ String topicName,
+ int partitionId,
+ Boundedness boundedness) {
+ PulsarPartitionSplit split = createPartitionSplit(topicName, partitionId, boundedness);
+ reader.addSplits(Collections.singletonList(split));
+ reader.notifyNoMoreSplits();
+ }
+
+ protected void pollUtilReadExpectedNumberOfRecordsAndValidate(
Review comment:
Move to PulsarOrderedSourceReaderTest?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
@RegisterExtension
PulsarSplitReaderInvocationContextProvider provider =
new PulsarSplitReaderInvocationContextProvider();
+ /** Default reader config: max message 1, fetch timeout 1s. */
protected Configuration readerConfig() {
Configuration config = operator().config();
config.set(PULSAR_MAX_FETCH_RECORDS, 1);
config.set(PULSAR_MAX_FETCH_TIME, 1000L);
config.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
config.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
-
return config;
}
protected SourceConfiguration sourceConfig() {
return new SourceConfiguration(readerConfig());
}
- protected SplitsAddition<PulsarPartitionSplit> createSplit(String topicName, int partitionId) {
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ handleSplit(reader, topicName, partitionId, null);
+ }
+
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader,
+ String topicName,
+ int partitionId,
+ MessageId startPosition) {
+ TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
+ PulsarPartitionSplit split =
+ new PulsarPartitionSplit(partition, StopCursor.never(), startPosition, null);
+ SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
+ reader.handleSplitsChanges(addition);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ seekStartPositionAndHandleSplit(reader, topicName, partitionId, MessageId.latest);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
@RegisterExtension
PulsarSplitReaderInvocationContextProvider provider =
new PulsarSplitReaderInvocationContextProvider();
+ /** Default reader config: max message 1, fetch timeout 1s. */
protected Configuration readerConfig() {
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
@RegisterExtension
PulsarSplitReaderInvocationContextProvider provider =
new PulsarSplitReaderInvocationContextProvider();
+ /** Default reader config: max message 1, fetch timeout 1s. */
protected Configuration readerConfig() {
Configuration config = operator().config();
config.set(PULSAR_MAX_FETCH_RECORDS, 1);
config.set(PULSAR_MAX_FETCH_TIME, 1000L);
config.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
config.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
-
return config;
}
protected SourceConfiguration sourceConfig() {
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ @RegisterExtension
+ PulsarSourceReaderInvocationContextProvider provider =
+ new PulsarSourceReaderInvocationContextProvider();
+
+ @BeforeEach
+ void beforeEach(String topicName) {
+ Random random = new Random(System.currentTimeMillis());
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(String topicName) {
+ operator().deleteTopic(topicName, true);
+ }
+
+ @TestTemplate
+ void assignZeroSplitsCreatesZeroSubscription(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @TestTemplate
+ void assigningEmptySplits(
+ PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName)
+ throws Exception {
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
+ // We sleep for 1 sec to avoid entering false negative case:
+ // The fetcher could become idle upon entry because :
+ // 1. the AddSplitsTask is dequeued from taskQueue,
+ // 2. The dequeued splits is not assigned to assignedSplits.
+ // And if both assignedSplits is null and taskQueue is empty, the fetcher could become idle,
+ // and return NOTHING_AVAILABLE upon entry. This happened multiple times during local test.
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ assertThat(status).isEqualTo(InputStatus.NOTHING_AVAILABLE);
+ reader.close();
+ }
+
+ protected void setupSourceReader(
+ PulsarSourceReaderBase<Integer> reader,
+ String topicName,
+ int partitionId,
+ Boundedness boundedness) {
+ PulsarPartitionSplit split = createPartitionSplit(topicName, partitionId, boundedness);
+ reader.addSplits(Collections.singletonList(split));
+ reader.notifyNoMoreSplits();
+ }
+
+ protected void pollUtilReadExpectedNumberOfRecordsAndValidate(
+ PulsarSourceReaderBase<Integer> reader,
+ TestingReaderOutput<Integer> output,
+ int expectedRecords,
+ String topicNameWithPartition)
+ throws Exception {
+ pollUntil(
+ reader,
+ output,
+ () -> output.getEmittedRecords().size() == expectedRecords,
+ "The output didn't poll enough records before timeout.");
+ reader.close();
+ verifyAllMessageAcknowledged(expectedRecords, topicNameWithPartition);
+ assertThat(output.getEmittedRecords()).hasSize(expectedRecords);
+ }
+
+ protected void pollUntil(
+ PulsarSourceReaderBase<Integer> reader,
+ ReaderOutput<Integer> output,
+ Supplier<Boolean> condition,
+ String errorMessage)
+ throws InterruptedException, TimeoutException {
+ CommonTestUtils.waitUtil(
+ () -> {
+ try {
+ reader.pollNext(output);
+ } catch (Exception exception) {
+ throw new RuntimeException(
+ "Caught unexpected exception when polling from the reader",
+ exception);
+ }
+ return condition.get();
+ },
+ Duration.ofSeconds(DEFAULT_POLLING_TIMEOUT_SECONDS),
+ errorMessage);
+ }
+
+ protected void verifyAllMessageAcknowledged(int expectedMessages, String partitionName)
+ throws PulsarAdminException {
+ TopicStats topicStats = operator().admin().topics().getStats(partitionName, true, true);
+ // verify if the messages has been consumed
+ Map<String, ? extends SubscriptionStats> subscriptionStats = topicStats.getSubscriptions();
+ assertThat(subscriptionStats).hasSizeGreaterThan(0);
+ subscriptionStats.forEach(
+ (subscription, stats) -> {
+ assertThat(stats.getUnackedMessages()).isZero();
+ assertThat(stats.getMsgOutCounter()).isEqualTo(expectedMessages);
+ });
+ }
+
+ protected void verifyNoSubscriptionCreated(String partitionName) throws PulsarAdminException {
Review comment:
private
##########
File path: flink-test-utils-parent/flink-test-utils-junit/pom.xml
##########
@@ -21,6 +21,10 @@ under the License.
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
Review comment:
Why is this needed? I am not sure whether we want to do this because usually all modules should work with java 8 and java 11
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
+
+ /** Expected the add or override config entries in inherited classes. */
+ protected abstract void customConfig(Configuration config);
+
+ @BeforeEach
+ void beforeEach(TestInfo testInfo) {
+ Random random = new Random(System.currentTimeMillis());
+ String topicName = uniqueTestName(testInfo);
+ operator().setupTopic(topicName, Schema.INT32, () -> random.nextInt(20));
+ }
+
+ @AfterEach
+ void afterEach(TestInfo testInfo) {
+ String topicName = uniqueTestName(testInfo);
+ operator().deleteTopic(topicName, true);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void assignZeroSplitsCreatesZeroSubscription(
+ boolean autoAcknowledgementEnabled, TestInfo testInfo) throws Exception {
+ PulsarSourceReaderBase<Integer> reader = sourceReader(autoAcknowledgementEnabled);
+ String topicName = uniqueTestName(testInfo);
+ reader.snapshotState(100L);
+ reader.notifyCheckpointComplete(100L);
+ // Verify the committed offsets.
+ reader.close();
+ for (int i = 0; i < PulsarRuntimeOperator.DEFAULT_PARTITIONS; i++) {
+ verifyNoSubscriptionCreated(TopicNameUtils.topicNameWithPartition(topicName, i));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void assigningEmptySplits(boolean autoAcknowledgementEnabled, TestInfo testInfo)
+ throws Exception {
+ PulsarSourceReaderBase<Integer> reader = sourceReader(autoAcknowledgementEnabled);
+ String topicName = uniqueTestName(testInfo);
+ final PulsarPartitionSplit emptySplit =
+ createPartitionSplit(
+ topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
+
+ reader.addSplits(Collections.singletonList(emptySplit));
+
+ TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
+ InputStatus status;
+ status = reader.pollNext(output);
+ // We sleep for 1 sec to avoid entering false negative case:
+ // we perform assertion when add splits task
+ // is dequeued but not executed.
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
Review comment:
I still do not understand this point if `pollNext` returns we already have the inputStatus and it is not requested again. So why the sleep?
In general, the situation you described cannot be reliably solved with a sleep because it may take longer than one second until the fetcher it idle.
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
@RegisterExtension
PulsarSplitReaderInvocationContextProvider provider =
new PulsarSplitReaderInvocationContextProvider();
+ /** Default reader config: max message 1, fetch timeout 1s. */
protected Configuration readerConfig() {
Configuration config = operator().config();
config.set(PULSAR_MAX_FETCH_RECORDS, 1);
config.set(PULSAR_MAX_FETCH_TIME, 1000L);
config.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
config.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
-
return config;
}
protected SourceConfiguration sourceConfig() {
return new SourceConfiguration(readerConfig());
}
- protected SplitsAddition<PulsarPartitionSplit> createSplit(String topicName, int partitionId) {
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ handleSplit(reader, topicName, partitionId, null);
+ }
+
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader,
+ String topicName,
+ int partitionId,
+ MessageId startPosition) {
+ TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
+ PulsarPartitionSplit split =
+ new PulsarPartitionSplit(partition, StopCursor.never(), startPosition, null);
+ SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
+ reader.handleSplitsChanges(addition);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ seekStartPositionAndHandleSplit(reader, topicName, partitionId, MessageId.latest);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
+ PulsarPartitionSplitReaderBase<String> reader,
+ String topicName,
+ int partitionId,
+ MessageId startPosition) {
TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
- PulsarPartitionSplit split = new PulsarPartitionSplit(partition, never());
- return new SplitsAddition<>(singletonList(split));
+ PulsarPartitionSplit split =
+ new PulsarPartitionSplit(partition, StopCursor.never(), null, null);
+ SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
+
+ // create consumer and seek before split changes
+ try (Consumer<byte[]> consumer = reader.createPulsarConsumer(partition)) {
+ // inclusive messageId
+ StartCursor startCursor = StartCursor.fromMessageId(startPosition);
+ startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
+ } catch (PulsarClientException e) {
+ sneakyThrow(e);
+ }
+
+ reader.handleSplitsChanges(addition);
}
protected <T> PulsarMessage<T> fetchedMessage(PulsarPartitionSplitReaderBase<T> splitReader) {
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/MessageIdStartCursor.java
##########
@@ -53,9 +53,13 @@ public MessageIdStartCursor(MessageId messageId, boolean inclusive) {
messageId instanceof MessageIdImpl,
"We only support normal message id and batch message id.");
MessageIdImpl id = (MessageIdImpl) messageId;
- this.messageId =
- new MessageIdImpl(
- id.getLedgerId(), id.getEntryId() + 1, id.getPartitionIndex());
+ if (MessageId.earliest.equals(messageId) || MessageId.latest.equals(messageId)) {
+ this.messageId = messageId;
+ } else {
+ this.messageId =
+ new MessageIdImpl(
+ id.getLedgerId(), id.getEntryId() + 1, id.getPartitionIndex());
+ }
Review comment:
Maybe I am missing something on this commit but I see you have fixed the production but I would also expect to see a test case on this commit to verify the fix.
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
@RegisterExtension
PulsarSplitReaderInvocationContextProvider provider =
new PulsarSplitReaderInvocationContextProvider();
+ /** Default reader config: max message 1, fetch timeout 1s. */
protected Configuration readerConfig() {
Configuration config = operator().config();
config.set(PULSAR_MAX_FETCH_RECORDS, 1);
config.set(PULSAR_MAX_FETCH_TIME, 1000L);
config.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
config.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
-
return config;
}
protected SourceConfiguration sourceConfig() {
return new SourceConfiguration(readerConfig());
}
- protected SplitsAddition<PulsarPartitionSplit> createSplit(String topicName, int partitionId) {
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ handleSplit(reader, topicName, partitionId, null);
+ }
+
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader,
+ String topicName,
+ int partitionId,
+ MessageId startPosition) {
+ TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
+ PulsarPartitionSplit split =
+ new PulsarPartitionSplit(partition, StopCursor.never(), startPosition, null);
+ SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
+ reader.handleSplitsChanges(addition);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/extension/PulsarSourceOrderlinessExtension.java
##########
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.testutils.extension;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.platform.commons.support.AnnotationSupport;
+
+import java.lang.annotation.Annotation;
+import java.util.Collection;
+import java.util.List;
+
+/** An extension for subclasses to specify {@link org.apache.pulsar.client.api.SubscriptionType}. */
+public class PulsarSourceOrderlinessExtension implements BeforeAllCallback {
Review comment:
Is orderliness the same as ordered here?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/source/PulsarSourceReaderTestBase.java
##########
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.source.reader.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
+import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
+import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
+import org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderContext;
+import org.apache.flink.connector.testutils.source.reader.TestingReaderOutput;
+import org.apache.flink.connectors.test.common.junit.extensions.TestLoggerExtension;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.policies.data.SubscriptionStats;
+import org.apache.pulsar.common.policies.data.TopicStats;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.pulsarSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.createPartitionSplit;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
+abstract class PulsarSourceReaderTestBase extends PulsarTestSuiteBase {
+
+ protected static final int DEFAULT_POLLING_TIMEOUT_SECONDS = 120;
Review comment:
private
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
@RegisterExtension
PulsarSplitReaderInvocationContextProvider provider =
new PulsarSplitReaderInvocationContextProvider();
+ /** Default reader config: max message 1, fetch timeout 1s. */
protected Configuration readerConfig() {
Configuration config = operator().config();
config.set(PULSAR_MAX_FETCH_RECORDS, 1);
config.set(PULSAR_MAX_FETCH_TIME, 1000L);
config.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
config.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
-
return config;
}
protected SourceConfiguration sourceConfig() {
return new SourceConfiguration(readerConfig());
}
- protected SplitsAddition<PulsarPartitionSplit> createSplit(String topicName, int partitionId) {
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ handleSplit(reader, topicName, partitionId, null);
+ }
+
+ protected void handleSplit(
+ PulsarPartitionSplitReaderBase<String> reader,
+ String topicName,
+ int partitionId,
+ MessageId startPosition) {
+ TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
+ PulsarPartitionSplit split =
+ new PulsarPartitionSplit(partition, StopCursor.never(), startPosition, null);
+ SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
+ reader.handleSplitsChanges(addition);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
+ PulsarPartitionSplitReaderBase<String> reader, String topicName, int partitionId) {
+ seekStartPositionAndHandleSplit(reader, topicName, partitionId, MessageId.latest);
+ }
+
+ protected void seekStartPositionAndHandleSplit(
+ PulsarPartitionSplitReaderBase<String> reader,
+ String topicName,
+ int partitionId,
+ MessageId startPosition) {
TopicPartition partition = new TopicPartition(topicName, partitionId, createFullRange());
- PulsarPartitionSplit split = new PulsarPartitionSplit(partition, never());
- return new SplitsAddition<>(singletonList(split));
+ PulsarPartitionSplit split =
+ new PulsarPartitionSplit(partition, StopCursor.never(), null, null);
+ SplitsAddition<PulsarPartitionSplit> addition = new SplitsAddition<>(singletonList(split));
+
+ // create consumer and seek before split changes
+ try (Consumer<byte[]> consumer = reader.createPulsarConsumer(partition)) {
+ // inclusive messageId
+ StartCursor startCursor = StartCursor.fromMessageId(startPosition);
+ startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
+ } catch (PulsarClientException e) {
+ sneakyThrow(e);
+ }
+
+ reader.handleSplitsChanges(addition);
}
protected <T> PulsarMessage<T> fetchedMessage(PulsarPartitionSplitReaderBase<T> splitReader) {
- try {
- RecordsWithSplitIds<PulsarMessage<T>> records = splitReader.fetch();
- if (records.nextSplit() != null) {
- return records.nextRecordFromSplit();
+ return fetchedMessages(splitReader, 1, false).stream().findFirst().orElse(null);
+ }
+
+ protected <T> List<PulsarMessage<T>> fetchedMessages(
+ PulsarPartitionSplitReaderBase<T> splitReader, int expectedCount, boolean verify) {
+ return fetchedMessages(
+ splitReader, expectedCount, verify, Boundedness.CONTINUOUS_UNBOUNDED);
+ }
+
+ protected <T> List<PulsarMessage<T>> fetchedMessages(
Review comment:
private?
##########
File path: flink-connectors/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/split/PulsarPartitionSplitReaderTestBase.java
##########
@@ -39,78 +50,164 @@
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import static java.time.Duration.ofSeconds;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
+import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_RECORDS;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_MAX_FETCH_TIME;
import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
-import static org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor.never;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange.createFullRange;
+import static org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema.flinkSchema;
+import static org.apache.flink.connector.pulsar.testutils.PulsarTestCommonUtils.isAssignableFromParameterContext;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_SOURCE_READER_SUBSCRIPTION_TYPE_STORE_KEY;
+import static org.apache.flink.connector.pulsar.testutils.extension.PulsarSourceOrderlinessExtension.PULSAR_TEST_RESOURCE_NAMESPACE;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.DEFAULT_PARTITIONS;
+import static org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntimeOperator.NUM_RECORDS_PER_PARTITION;
import static org.apache.flink.core.testutils.CommonTestUtils.waitUtil;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.pulsar.client.api.Schema.STRING;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Test utils for split readers. */
+@ExtendWith({
+ PulsarSourceOrderlinessExtension.class,
+ TestLoggerExtension.class,
+})
public abstract class PulsarPartitionSplitReaderTestBase extends PulsarTestSuiteBase {
Review comment:
A lot of the methods of this class seem to unnecessarily be protected either they can be private or even better provide util outside if they do not work with an instance reference.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org