You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2022/03/24 10:10:24 UTC

[flink-kubernetes-operator] branch main updated: [FLINK-26370] Use configurable ThreadPool for Flink cluster communication

This is an automated email from the ASF dual-hosted git repository.

gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git


The following commit(s) were added to refs/heads/main by this push:
     new 14fd310  [FLINK-26370] Use configurable ThreadPool for Flink cluster communication
14fd310 is described below

commit 14fd31020a54c5613ff2a6fd3f0d624f9c53d31f
Author: Sandor Kelemen <ke...@efox.hu>
AuthorDate: Tue Mar 22 23:57:34 2022 +0000

    [FLINK-26370] Use configurable ThreadPool for Flink cluster communication
    
    Closes #44
---
 .../flink/kubernetes/operator/FlinkOperator.java   | 61 ++++++++++++++----
 .../config/FlinkOperatorConfiguration.java         |  6 ++
 .../operator/config/OperatorConfigOptions.java     |  9 +++
 .../kubernetes/operator/service/FlinkService.java  |  7 +--
 .../kubernetes/operator/FlinkOperatorTest.java     | 73 ++++++++++++++++++++++
 .../controller/FlinkDeploymentControllerTest.java  |  1 +
 .../operator/observer/SessionObserverTest.java     |  3 +
 .../conf/flink-operator-config/flink-conf.yaml     |  1 +
 8 files changed, 145 insertions(+), 16 deletions(-)

diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java
index d674388..2ceeafb 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.kubernetes.operator;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.kubernetes.operator.config.DefaultConfig;
 import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration;
 import org.apache.flink.kubernetes.operator.controller.FlinkControllerConfig;
@@ -30,6 +31,7 @@ import org.apache.flink.kubernetes.operator.validation.DefaultDeploymentValidato
 import org.apache.flink.kubernetes.operator.validation.FlinkDeploymentValidator;
 
 import io.fabric8.kubernetes.client.DefaultKubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClient;
 import io.javaoperatorsdk.operator.Operator;
 import io.javaoperatorsdk.operator.api.config.ConfigurationService;
 import io.javaoperatorsdk.operator.api.config.ConfigurationServiceOverrider;
@@ -37,32 +39,38 @@ import io.javaoperatorsdk.operator.config.runtime.DefaultConfigurationService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.concurrent.Executors;
+
 /** Main Class for Flink native k8s operator. */
 public class FlinkOperator {
     private static final Logger LOG = LoggerFactory.getLogger(FlinkOperator.class);
 
-    public static void main(String... args) {
+    private final Operator operator;
+    private final FlinkDeploymentController controller;
+    private final FlinkControllerConfig controllerConfig;
+
+    public FlinkOperator() {
+        this(FlinkUtils.loadDefaultConfig());
+    }
+
+    public FlinkOperator(DefaultConfig defaultConfig) {
 
         LOG.info("Starting Flink Kubernetes Operator");
-        DefaultConfig defaultConfig = FlinkUtils.loadDefaultConfig();
         OperatorMetricUtils.initOperatorMetrics(defaultConfig.getOperatorConfig());
 
-        DefaultKubernetesClient client = new DefaultKubernetesClient();
+        KubernetesClient client = new DefaultKubernetesClient();
         String namespace = client.getNamespace();
         if (namespace == null) {
             namespace = "default";
         }
 
-        ConfigurationService configurationService =
-                new ConfigurationServiceOverrider(DefaultConfigurationService.instance())
-                        .checkingCRDAndValidateLocalModel(false)
-                        .build();
-
-        Operator operator = new Operator(client, configurationService);
-
         FlinkOperatorConfiguration operatorConfiguration =
                 FlinkOperatorConfiguration.fromConfiguration(defaultConfig.getOperatorConfig());
 
+        ConfigurationService configurationService = getConfigurationService(operatorConfiguration);
+
+        operator = new Operator(client, configurationService);
+
         FlinkService flinkService = new FlinkService(client, operatorConfiguration);
 
         FlinkDeploymentValidator validator = new DefaultDeploymentValidator();
@@ -70,7 +78,7 @@ public class FlinkOperator {
                 new ReconcilerFactory(client, flinkService, operatorConfiguration);
         ObserverFactory observerFactory = new ObserverFactory(flinkService, operatorConfiguration);
 
-        FlinkDeploymentController controller =
+        controller =
                 new FlinkDeploymentController(
                         defaultConfig,
                         operatorConfiguration,
@@ -80,13 +88,42 @@ public class FlinkOperator {
                         reconcilerFactory,
                         observerFactory);
 
-        FlinkControllerConfig controllerConfig =
+        controllerConfig =
                 new FlinkControllerConfig(controller, operatorConfiguration.getWatchedNamespaces());
         controller.setControllerConfig(controllerConfig);
         controllerConfig.setConfigurationService(configurationService);
+    }
 
+    private ConfigurationService getConfigurationService(
+            FlinkOperatorConfiguration operatorConfiguration) {
+
+        ConfigurationServiceOverrider configOverrider =
+                new ConfigurationServiceOverrider(DefaultConfigurationService.instance())
+                        .checkingCRDAndValidateLocalModel(false);
+
+        int parallelism = operatorConfiguration.getReconcilerMaxParallelism();
+        if (parallelism == -1) {
+            LOG.info("Configuring operator with unbounded reconciliation thread pool.");
+            configOverrider = configOverrider.withExecutorService(Executors.newCachedThreadPool());
+        } else {
+            LOG.info("Configuring operator with {} reconciliation threads.", parallelism);
+            configOverrider = configOverrider.withConcurrentReconciliationThreads(parallelism);
+        }
+        return configOverrider.build();
+    }
+
+    public void run() {
         operator.register(controller, controllerConfig);
         operator.installShutdownHook();
         operator.start();
     }
+
+    @VisibleForTesting
+    protected Operator getOperator() {
+        return operator;
+    }
+
+    public static void main(String... args) {
+        new FlinkOperator().run();
+    }
 }
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
index 4b273f1..e933b11 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
@@ -32,6 +32,7 @@ import java.util.Set;
 public class FlinkOperatorConfiguration {
 
     Duration reconcileInterval;
+    int reconcilerMaxParallelism;
     Duration progressCheckInterval;
     Duration restApiReadyDelay;
     Duration savepointTriggerGracePeriod;
@@ -43,6 +44,10 @@ public class FlinkOperatorConfiguration {
         Duration reconcileInterval =
                 operatorConfig.get(OperatorConfigOptions.OPERATOR_RECONCILER_RESCHEDULE_INTERVAL);
 
+        int reconcilerMaxParallelism =
+                operatorConfig.getInteger(
+                        OperatorConfigOptions.OPERATOR_RECONCILER_MAX_PARALLELISM);
+
         Duration restApiReadyDelay =
                 operatorConfig.get(OperatorConfigOptions.OPERATOR_OBSERVER_REST_READY_DELAY);
 
@@ -66,6 +71,7 @@ public class FlinkOperatorConfiguration {
 
         return new FlinkOperatorConfiguration(
                 reconcileInterval,
+                reconcilerMaxParallelism,
                 progressCheckInterval,
                 restApiReadyDelay,
                 savepointTriggerGracePeriod,
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/OperatorConfigOptions.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/OperatorConfigOptions.java
index 9fbaca3..923bab6 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/OperatorConfigOptions.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/OperatorConfigOptions.java
@@ -21,6 +21,8 @@ package org.apache.flink.kubernetes.operator.config;
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ConfigOptions;
 
+import io.javaoperatorsdk.operator.api.config.ConfigurationService;
+
 import java.time.Duration;
 
 /** This class holds configuration constants used by flink operator. */
@@ -40,6 +42,13 @@ public class OperatorConfigOptions {
                     .withDescription(
                             "Final delay before deployment is marked ready after port becomes accessible.");
 
+    public static final ConfigOption<Integer> OPERATOR_RECONCILER_MAX_PARALLELISM =
+            ConfigOptions.key("operator.reconciler.max.parallelism")
+                    .intType()
+                    .defaultValue(ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER)
+                    .withDescription(
+                            "The maximum number of threads running the reconciliation loop. Use -1 for infinite.");
+
     public static final ConfigOption<Duration> OPERATOR_OBSERVER_PROGRESS_CHECK_INTERVAL =
             ConfigOptions.key("operator.observer.progress-check.interval")
                     .durationType()
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
index 1551f1e..b74106b 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
@@ -53,7 +53,7 @@ import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMes
 import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerRequestBody;
 
 import io.fabric8.kubernetes.api.model.PodList;
-import io.fabric8.kubernetes.client.NamespacedKubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClient;
 import org.apache.commons.lang3.ObjectUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -74,12 +74,11 @@ public class FlinkService {
 
     private static final Logger LOG = LoggerFactory.getLogger(FlinkService.class);
 
-    private final NamespacedKubernetesClient kubernetesClient;
+    private final KubernetesClient kubernetesClient;
     private final FlinkOperatorConfiguration operatorConfiguration;
 
     public FlinkService(
-            NamespacedKubernetesClient kubernetesClient,
-            FlinkOperatorConfiguration operatorConfiguration) {
+            KubernetesClient kubernetesClient, FlinkOperatorConfiguration operatorConfiguration) {
         this.kubernetesClient = kubernetesClient;
         this.operatorConfiguration = operatorConfiguration;
     }
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/FlinkOperatorTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/FlinkOperatorTest.java
new file mode 100644
index 0000000..281a357
--- /dev/null
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/FlinkOperatorTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.kubernetes.operator.config.DefaultConfig;
+
+import io.javaoperatorsdk.operator.api.config.ConfigurationService;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import static java.util.Optional.empty;
+
+/** @link FlinkOperator unit tests. */
+public class FlinkOperatorTest {
+
+    @Test
+    public void testExecutorServiceDefaultsToMaxParallelism() {
+        checkExecutorServiceThreadCount(
+                empty(), ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER);
+    }
+
+    @Test
+    public void testExecutorServiceUsesReconciliationMaxParallelismFromConfig() {
+        checkExecutorServiceThreadCount(Optional.of(42), 42);
+    }
+
+    @Test
+    public void testExecutorServiceUsesMaxParallelismForMinusOneReconciliationMaxParallelism() {
+        checkExecutorServiceThreadCount(Optional.of(-1), Integer.MAX_VALUE);
+    }
+
+    private void checkExecutorServiceThreadCount(
+            Optional<Integer> parallelism, int expectedThreadCount) {
+        var es = getExecutorForParallelismConfig(parallelism);
+
+        Assertions.assertInstanceOf(ThreadPoolExecutor.class, es);
+
+        ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) es;
+        Assertions.assertEquals(expectedThreadCount, threadPoolExecutor.getMaximumPoolSize());
+    }
+
+    private ExecutorService getExecutorForParallelismConfig(Optional<Integer> parallelism) {
+        var operatorConfig = new Configuration();
+        parallelism.ifPresent(
+                p -> operatorConfig.setInteger("operator.reconciler.max.parallelism", p));
+
+        var flinkConfig = new Configuration();
+        var config = new DefaultConfig(operatorConfig, flinkConfig);
+
+        FlinkOperator flinkOperator = new FlinkOperator(config);
+        return flinkOperator.getOperator().getConfigurationService().getExecutorService();
+    }
+}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkDeploymentControllerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkDeploymentControllerTest.java
index e984bf6..cc0bace 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkDeploymentControllerTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkDeploymentControllerTest.java
@@ -76,6 +76,7 @@ public class FlinkDeploymentControllerTest {
     private final FlinkOperatorConfiguration operatorConfiguration =
             new FlinkOperatorConfiguration(
                     Duration.ofSeconds(1),
+                    -1,
                     Duration.ofSeconds(2),
                     Duration.ofSeconds(3),
                     Duration.ofSeconds(4),
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/SessionObserverTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/SessionObserverTest.java
index 2719438..87694dd 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/SessionObserverTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/SessionObserverTest.java
@@ -109,6 +109,7 @@ public class SessionObserverTest {
         FlinkOperatorConfiguration allNsConfig =
                 new FlinkOperatorConfiguration(
                         Duration.ofSeconds(1),
+                        -1,
                         Duration.ofSeconds(2),
                         Duration.ofSeconds(3),
                         Duration.ofSeconds(4),
@@ -118,6 +119,7 @@ public class SessionObserverTest {
         FlinkOperatorConfiguration specificNsConfig =
                 new FlinkOperatorConfiguration(
                         Duration.ofSeconds(1),
+                        -1,
                         Duration.ofSeconds(2),
                         Duration.ofSeconds(3),
                         Duration.ofSeconds(4),
@@ -127,6 +129,7 @@ public class SessionObserverTest {
         FlinkOperatorConfiguration multipleNsConfig =
                 new FlinkOperatorConfiguration(
                         Duration.ofSeconds(1),
+                        -1,
                         Duration.ofSeconds(2),
                         Duration.ofSeconds(3),
                         Duration.ofSeconds(4),
diff --git a/helm/flink-operator/conf/flink-operator-config/flink-conf.yaml b/helm/flink-operator/conf/flink-operator-config/flink-conf.yaml
index ae2fab8..6e6e104 100644
--- a/helm/flink-operator/conf/flink-operator-config/flink-conf.yaml
+++ b/helm/flink-operator/conf/flink-operator-config/flink-conf.yaml
@@ -17,6 +17,7 @@
 ################################################################################
 
 # operator.reconciler.reschedule.interval: 60 s
+# operator.reconciler.max.parallelism: 5
 # operator.observer.rest-ready.delay: 10 s
 # operator.observer.progress-check.interval: 10 s
 # operator.observer.savepoint.trigger.grace-period: 10 s