You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dm...@apache.org on 2023/05/05 19:03:18 UTC
[flink] 04/04: [FLINK-32010][kubernetes] Properly handle KubernetesLeaderRetrievalDriver.ConfigMapCallbackHandlerImpl#onAdded events in case the leader is already known.
This is an automated email from the ASF dual-hosted git repository.
dmvk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 026d7ccfe1d6f4cfa26c9038dd05403c889d2e0d
Author: David Moravek <dm...@apache.org>
AuthorDate: Fri May 5 12:32:55 2023 +0200
[FLINK-32010][kubernetes] Properly handle KubernetesLeaderRetrievalDriver.ConfigMapCallbackHandlerImpl#onAdded events in case the leader is already known.
---
.../KubernetesLeaderRetrievalDriver.java | 11 ++-
...KubernetesLeaderElectionAndRetrievalITCase.java | 102 +++++++++++++--------
2 files changed, 74 insertions(+), 39 deletions(-)
diff --git a/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderRetrievalDriver.java b/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderRetrievalDriver.java
index f08e4622165..035057c7441 100644
--- a/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderRetrievalDriver.java
+++ b/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderRetrievalDriver.java
@@ -98,9 +98,14 @@ public class KubernetesLeaderRetrievalDriver implements LeaderRetrievalDriver {
@Override
public void onAdded(List<KubernetesConfigMap> configMaps) {
- // The ConfigMap is created by KubernetesLeaderElectionDriver with empty data. We do not
- // process this
- // useless event.
+ // The ConfigMap is created by KubernetesLeaderElectionDriver with empty data. We don't
+ // really need to process anything unless the retriever was started after the leader
+ // election has already succeeded.
+ final KubernetesConfigMap configMap = getOnlyConfigMap(configMaps, configMapName);
+ final LeaderInformation leaderInformation = leaderInformationExtractor.apply(configMap);
+ if (!leaderInformation.isEmpty()) {
+ leaderRetrievalEventHandler.notifyLeaderAddress(leaderInformation);
+ }
}
@Override
diff --git a/flink-kubernetes/src/test/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderElectionAndRetrievalITCase.java b/flink-kubernetes/src/test/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderElectionAndRetrievalITCase.java
index 4d922cbda06..d101d19b805 100644
--- a/flink-kubernetes/src/test/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderElectionAndRetrievalITCase.java
+++ b/flink-kubernetes/src/test/java/org/apache/flink/kubernetes/highavailability/KubernetesLeaderElectionAndRetrievalITCase.java
@@ -21,6 +21,7 @@ package org.apache.flink.kubernetes.highavailability;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.KubernetesExtension;
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
+import org.apache.flink.kubernetes.configuration.KubernetesHighAvailabilityOptions;
import org.apache.flink.kubernetes.configuration.KubernetesLeaderElectionConfiguration;
import org.apache.flink.kubernetes.kubeclient.FlinkKubeClient;
import org.apache.flink.kubernetes.kubeclient.KubernetesConfigMapSharedWatcher;
@@ -28,15 +29,18 @@ import org.apache.flink.kubernetes.utils.KubernetesUtils;
import org.apache.flink.runtime.leaderelection.LeaderInformation;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionEventHandler;
import org.apache.flink.runtime.leaderretrieval.TestingLeaderRetrievalEventHandler;
-import org.apache.flink.util.ExecutorUtils;
+import org.apache.flink.testutils.executor.TestExecutorExtension;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import static org.apache.flink.kubernetes.utils.Constants.LABEL_CONFIGMAP_TYPE_HIGH_AVAILABILITY;
import static org.assertj.core.api.Assertions.assertThat;
@@ -54,73 +58,99 @@ class KubernetesLeaderElectionAndRetrievalITCase {
"akka.tcp://flink@172.20.1.21:6123/user/rpc/dispatcher";
@RegisterExtension
- private static final KubernetesExtension kubernetesExtension = new KubernetesExtension();
+ private static final KubernetesExtension KUBERNETES_EXTENSION = new KubernetesExtension();
+
+ @RegisterExtension
+ private static final TestExecutorExtension<ExecutorService> EXECUTOR_EXTENSION =
+ new TestExecutorExtension<>(Executors::newCachedThreadPool);
@Test
void testLeaderElectionAndRetrieval() throws Exception {
- final String configMapName = LEADER_CONFIGMAP_NAME + System.currentTimeMillis();
- KubernetesLeaderElectionDriver leaderElectionDriver = null;
- KubernetesLeaderRetrievalDriver leaderRetrievalDriver = null;
-
- final FlinkKubeClient flinkKubeClient = kubernetesExtension.getFlinkKubeClient();
- final Configuration configuration = kubernetesExtension.getConfiguration();
+ final String configMapName = LEADER_CONFIGMAP_NAME + UUID.randomUUID();
+ final FlinkKubeClient flinkKubeClient = KUBERNETES_EXTENSION.getFlinkKubeClient();
+ final Configuration configuration = KUBERNETES_EXTENSION.getConfiguration();
final String clusterId = configuration.getString(KubernetesConfigOptions.CLUSTER_ID);
+
+ // This will make the leader election retrieval time out if we won't process already
+ // existing leader information when starting it up.
+ configuration.set(
+ KubernetesHighAvailabilityOptions.KUBERNETES_LEASE_DURATION, Duration.ofHours(1));
+ configuration.set(
+ KubernetesHighAvailabilityOptions.KUBERNETES_RETRY_PERIOD, Duration.ofHours(1));
+ configuration.set(
+ KubernetesHighAvailabilityOptions.KUBERNETES_RENEW_DEADLINE, Duration.ofHours(1));
+
+ final List<AutoCloseable> closeables = new ArrayList<>();
+
final KubernetesConfigMapSharedWatcher configMapSharedWatcher =
flinkKubeClient.createConfigMapSharedWatcher(
KubernetesUtils.getConfigMapLabels(
clusterId, LABEL_CONFIGMAP_TYPE_HIGH_AVAILABILITY));
- final ExecutorService watchExecutorService = Executors.newCachedThreadPool();
+ closeables.add(configMapSharedWatcher);
final TestingLeaderElectionEventHandler electionEventHandler =
new TestingLeaderElectionEventHandler(LEADER_ADDRESS);
+ closeables.add(electionEventHandler);
try {
- leaderElectionDriver =
+ final KubernetesLeaderElectionDriver leaderElectionDriver =
new KubernetesLeaderElectionDriver(
flinkKubeClient,
configMapSharedWatcher,
- watchExecutorService,
+ EXECUTOR_EXTENSION.getExecutor(),
new KubernetesLeaderElectionConfiguration(
configMapName, UUID.randomUUID().toString(), configuration),
electionEventHandler,
electionEventHandler::handleError);
+ closeables.add(leaderElectionDriver);
+
electionEventHandler.init(leaderElectionDriver);
- final TestingLeaderRetrievalEventHandler retrievalEventHandler =
+ final Function<TestingLeaderRetrievalEventHandler, AutoCloseable>
+ leaderRetrievalDriverFactory =
+ leaderRetrievalEventHandler ->
+ new KubernetesLeaderRetrievalDriver(
+ configMapSharedWatcher,
+ EXECUTOR_EXTENSION.getExecutor(),
+ configMapName,
+ leaderRetrievalEventHandler,
+ KubernetesUtils::getLeaderInformationFromConfigMap,
+ leaderRetrievalEventHandler::handleError);
+
+ final TestingLeaderRetrievalEventHandler firstLeaderRetrievalEventHandler =
new TestingLeaderRetrievalEventHandler();
- leaderRetrievalDriver =
- new KubernetesLeaderRetrievalDriver(
- configMapSharedWatcher,
- watchExecutorService,
- configMapName,
- retrievalEventHandler,
- KubernetesUtils::getLeaderInformationFromConfigMap,
- retrievalEventHandler::handleError);
+ closeables.add(leaderRetrievalDriverFactory.apply(firstLeaderRetrievalEventHandler));
+ // Wait for the driver to obtain leadership.
electionEventHandler.waitForLeader();
- // Check the new leader is confirmed
final LeaderInformation confirmedLeaderInformation =
electionEventHandler.getConfirmedLeaderInformation();
assertThat(confirmedLeaderInformation.getLeaderAddress()).isEqualTo(LEADER_ADDRESS);
- // Check the leader retrieval driver should be notified the leader address
- retrievalEventHandler.waitForNewLeader();
- assertThat(retrievalEventHandler.getLeaderSessionID())
- .isEqualByComparingTo(confirmedLeaderInformation.getLeaderSessionID());
- assertThat(retrievalEventHandler.getAddress())
- .isEqualTo(confirmedLeaderInformation.getLeaderAddress());
+ // Check if the leader retrieval driver is notified about the leader address
+ awaitLeadership(firstLeaderRetrievalEventHandler, confirmedLeaderInformation);
+
+ // Start a second leader retrieval that should be notified immediately because we
+ // already know who the leader is.
+ final TestingLeaderRetrievalEventHandler secondRetrievalEventHandler =
+ new TestingLeaderRetrievalEventHandler();
+ closeables.add(leaderRetrievalDriverFactory.apply(secondRetrievalEventHandler));
+ awaitLeadership(secondRetrievalEventHandler, confirmedLeaderInformation);
} finally {
- electionEventHandler.close();
- if (leaderElectionDriver != null) {
- leaderElectionDriver.close();
- }
- if (leaderRetrievalDriver != null) {
- leaderRetrievalDriver.close();
+ for (AutoCloseable closeable : closeables) {
+ closeable.close();
}
flinkKubeClient.deleteConfigMap(configMapName).get();
- configMapSharedWatcher.close();
- ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, watchExecutorService);
}
}
+
+ private static void awaitLeadership(
+ TestingLeaderRetrievalEventHandler handler, LeaderInformation leaderInformation)
+ throws Exception {
+ handler.waitForNewLeader();
+ assertThat(handler.getLeaderSessionID())
+ .isEqualByComparingTo(leaderInformation.getLeaderSessionID());
+ assertThat(handler.getAddress()).isEqualTo(leaderInformation.getLeaderAddress());
+ }
}