You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2023/03/30 11:26:30 UTC

[flink-kubernetes-operator] branch main updated: [FLINK-31592] Improve Flink dependent resource deletion

This is an automated email from the ASF dual-hosted git repository.

gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git


The following commit(s) were added to refs/heads/main by this push:
     new 6620466d [FLINK-31592] Improve Flink dependent resource deletion
6620466d is described below

commit 6620466d31df7b22e4cae4ed0bd2eab70280c8c8
Author: Gyula Fora <g_...@apple.com>
AuthorDate: Thu Mar 30 13:26:23 2023 +0200

    [FLINK-31592] Improve Flink dependent resource deletion
---
 .../kubernetes_operator_config_configuration.html  |  6 ++++
 .../generated/system_advanced_section.html         |  6 ++++
 e2e-tests/test_application_operations.sh           |  2 +-
 .../config/FlinkOperatorConfiguration.java         |  8 +++++-
 .../config/KubernetesOperatorConfigOptions.java    |  8 ++++++
 .../AbstractFlinkResourceReconciler.java           |  2 +-
 .../operator/service/AbstractFlinkService.java     | 16 ++++++++---
 .../operator/service/NativeFlinkService.java       |  7 ++++-
 .../operator/service/StandaloneFlinkService.java   |  8 +++++-
 .../flink/kubernetes/operator/TestUtils.java       |  2 +-
 .../kubernetes/operator/TestingFlinkService.java   |  6 +++-
 .../operator/service/NativeFlinkServiceTest.java   | 32 ++++++++++++++++++++++
 12 files changed, 92 insertions(+), 11 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html b/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
index 2d18bbdd..6b335b8d 100644
--- a/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
+++ b/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
@@ -242,6 +242,12 @@
             <td>Duration</td>
             <td>The timeout for the resource clean up to wait for flink to shutdown cluster.</td>
         </tr>
+        <tr>
+            <td><h5>kubernetes.operator.resource.deletion.propagation</h5></td>
+            <td style="word-wrap: break-word;">Foreground</td>
+            <td><p>Enum</p></td>
+            <td>JM/TM Deployment deletion propagation.<br /><br />Possible values:<ul><li>"Orphan"</li><li>"Background"</li><li>"Foreground"</li></ul></td>
+        </tr>
         <tr>
             <td><h5>kubernetes.operator.retry.initial.interval</h5></td>
             <td style="word-wrap: break-word;">5 s</td>
diff --git a/docs/layouts/shortcodes/generated/system_advanced_section.html b/docs/layouts/shortcodes/generated/system_advanced_section.html
index 1d250b93..487f6fc7 100644
--- a/docs/layouts/shortcodes/generated/system_advanced_section.html
+++ b/docs/layouts/shortcodes/generated/system_advanced_section.html
@@ -68,6 +68,12 @@
             <td>Duration</td>
             <td>Final delay before deployment is marked ready after port becomes accessible.</td>
         </tr>
+        <tr>
+            <td><h5>kubernetes.operator.resource.deletion.propagation</h5></td>
+            <td style="word-wrap: break-word;">Foreground</td>
+            <td><p>Enum</p></td>
+            <td>JM/TM Deployment deletion propagation.<br /><br />Possible values:<ul><li>"Orphan"</li><li>"Background"</li><li>"Foreground"</li></ul></td>
+        </tr>
         <tr>
             <td><h5>kubernetes.operator.savepoint.history.max.age.threshold</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
diff --git a/e2e-tests/test_application_operations.sh b/e2e-tests/test_application_operations.sh
index f6d1ace3..d69da980 100755
--- a/e2e-tests/test_application_operations.sh
+++ b/e2e-tests/test_application_operations.sh
@@ -56,7 +56,7 @@ fi
 # Testing last-state mode upgrade
 # Update the FlinkDeployment and trigger the last state upgrade
 kubectl patch flinkdep ${CLUSTER_ID} --type merge --patch '{"spec":{"job": {"parallelism": 1 } } }'
-kubectl wait --for=delete pod --timeout=${TIMEOUT}s --selector="app=${CLUSTER_ID}"
+kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${CLUSTER_ID}"
 wait_for_jobmanager_running $CLUSTER_ID $TIMEOUT
 jm_pod_name=$(get_jm_pod_name $CLUSTER_ID)
 
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
index ce6b97c7..b29077aa 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.IllegalConfigurationException;
 import org.apache.flink.kubernetes.operator.metrics.KubernetesOperatorMetricOptions;
 import org.apache.flink.kubernetes.operator.utils.EnvUtils;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
 import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration;
 import io.javaoperatorsdk.operator.api.config.RetryConfiguration;
 import lombok.Value;
@@ -67,6 +68,7 @@ public class FlinkOperatorConfiguration {
     int exceptionThrowableCountThreshold;
     String labelSelector;
     LeaderElectionConfiguration leaderElectionConfiguration;
+    DeletionPropagation deletionPropagation;
 
     public static FlinkOperatorConfiguration fromConfiguration(Configuration operatorConfig) {
         Duration reconcileInterval =
@@ -172,6 +174,9 @@ public class FlinkOperatorConfiguration {
         String labelSelector =
                 operatorConfig.getString(KubernetesOperatorConfigOptions.OPERATOR_LABEL_SELECTOR);
 
+        DeletionPropagation deletionPropagation =
+                operatorConfig.get(KubernetesOperatorConfigOptions.RESOURCE_DELETION_PROPAGATION);
+
         return new FlinkOperatorConfiguration(
                 reconcileInterval,
                 reconcilerMaxParallelism,
@@ -196,7 +201,8 @@ public class FlinkOperatorConfiguration {
                 exceptionFieldLengthThreshold,
                 exceptionThrowableCountThreshold,
                 labelSelector,
-                getLeaderElectionConfig(operatorConfig));
+                getLeaderElectionConfig(operatorConfig),
+                deletionPropagation);
     }
 
     private static LeaderElectionConfiguration getLeaderElectionConfig(Configuration conf) {
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java
index a9f28619..f361310b 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ConfigOptions;
 import org.apache.flink.core.execution.SavepointFormatType;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
 import io.javaoperatorsdk.operator.api.config.ConfigurationService;
 import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration;
 import io.javaoperatorsdk.operator.api.reconciler.Constants;
@@ -474,4 +475,11 @@ public class KubernetesOperatorConfigOptions {
                     .defaultValue(false)
                     .withDescription(
                             "Configure the array merge behaviour during pod merging. Arrays can be either merged by position or name matching.");
+
+    @Documentation.Section(SECTION_ADVANCED)
+    public static final ConfigOption<DeletionPropagation> RESOURCE_DELETION_PROPAGATION =
+            operatorConfig("resource.deletion.propagation")
+                    .enumType(DeletionPropagation.class)
+                    .defaultValue(DeletionPropagation.FOREGROUND)
+                    .withDescription("JM/TM Deployment deletion propagation.");
 }
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
index c74545c9..251b8547 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
@@ -465,7 +465,7 @@ public abstract class AbstractFlinkResourceReconciler<
                         "kind", owner.getKind(),
                         "name", owner.getMetadata().getName(),
                         "uid", owner.getMetadata().getUid(),
-                        "blockOwnerDeletion", "false",
+                        "blockOwnerDeletion", "true",
                         "controller", "false");
         deployConfig.set(
                 KubernetesConfigOptions.JOB_MANAGER_OWNER_REFERENCE, List.of(ownerReference));
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java
index 27578c04..d71996af 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java
@@ -87,6 +87,7 @@ import org.apache.flink.util.FlinkRuntimeException;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.concurrent.ExecutorThreadFactory;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.api.model.Service;
@@ -784,6 +785,8 @@ public abstract class AbstractFlinkService implements FlinkService {
     /** Wait until the FLink cluster has completely shut down. */
     @VisibleForTesting
     void waitForClusterShutdown(String namespace, String clusterId, long shutdownTimeout) {
+        LOG.info("Waiting for cluster shutdown...");
+
         boolean jobManagerRunning = true;
         boolean serviceRunning = true;
 
@@ -918,7 +921,10 @@ public abstract class AbstractFlinkService implements FlinkService {
             FlinkDeploymentStatus status,
             Configuration conf,
             boolean deleteHaData) {
-        deleteClusterInternal(meta, conf, deleteHaData);
+
+        var deletionPropagation = configManager.getOperatorConfiguration().getDeletionPropagation();
+        LOG.info("Deleting cluster with {} propagation", deletionPropagation);
+        deleteClusterInternal(meta, conf, deleteHaData, deletionPropagation);
         updateStatusAfterClusterDeletion(status);
     }
 
@@ -930,9 +936,13 @@ public abstract class AbstractFlinkService implements FlinkService {
      * @param conf Configuration of the Flink application
      * @param deleteHaData Flag to indicate whether k8s or Zookeeper HA metadata should be removed
      *     as well
+     * @param deletionPropagation Resource deletion propagation policy
      */
     protected abstract void deleteClusterInternal(
-            ObjectMeta meta, Configuration conf, boolean deleteHaData);
+            ObjectMeta meta,
+            Configuration conf,
+            boolean deleteHaData,
+            DeletionPropagation deletionPropagation);
 
     protected void deleteHAData(String namespace, String clusterId, Configuration conf) {
         // We need to wait for cluster shutdown otherwise HA data might be recreated
@@ -950,8 +960,6 @@ public abstract class AbstractFlinkService implements FlinkService {
         } else if (FlinkUtils.isZookeeperHAActivated(conf)) {
             LOG.info("Deleting Zookeeper HA metadata");
             FlinkUtils.deleteZookeeperHAMetadata(conf);
-        } else {
-            LOG.warn("Can't delete HA metadata since HA is not enabled");
         }
     }
 
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java
index 9c752f7f..912a19e1 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java
@@ -32,6 +32,7 @@ import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode;
 import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
 import org.apache.flink.kubernetes.utils.KubernetesUtils;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.client.KubernetesClient;
@@ -108,7 +109,10 @@ public class NativeFlinkService extends AbstractFlinkService {
 
     @Override
     protected void deleteClusterInternal(
-            ObjectMeta meta, Configuration conf, boolean deleteHaData) {
+            ObjectMeta meta,
+            Configuration conf,
+            boolean deleteHaData,
+            DeletionPropagation deletionPropagation) {
 
         String namespace = meta.getNamespace();
         String clusterId = meta.getName();
@@ -121,6 +125,7 @@ public class NativeFlinkService extends AbstractFlinkService {
                 .deployments()
                 .inNamespace(namespace)
                 .withName(KubernetesUtils.getDeploymentName(clusterId))
+                .withPropagationPolicy(deletionPropagation)
                 .delete();
 
         if (deleteHaData) {
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java
index 6c7d7052..bdc6b65f 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java
@@ -38,6 +38,7 @@ import org.apache.flink.kubernetes.operator.standalone.StandaloneKubernetesConfi
 import org.apache.flink.kubernetes.operator.utils.StandaloneKubernetesUtils;
 import org.apache.flink.util.concurrent.ExecutorThreadFactory;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.client.KubernetesClient;
@@ -129,7 +130,10 @@ public class StandaloneFlinkService extends AbstractFlinkService {
 
     @Override
     protected void deleteClusterInternal(
-            ObjectMeta meta, Configuration conf, boolean deleteHaData) {
+            ObjectMeta meta,
+            Configuration conf,
+            boolean deleteHaData,
+            DeletionPropagation deletionPropagation) {
         final String clusterId = meta.getName();
         final String namespace = meta.getNamespace();
 
@@ -139,6 +143,7 @@ public class StandaloneFlinkService extends AbstractFlinkService {
                 .deployments()
                 .inNamespace(namespace)
                 .withName(StandaloneKubernetesUtils.getJobManagerDeploymentName(clusterId))
+                .withPropagationPolicy(deletionPropagation)
                 .delete();
 
         LOG.info("Deleting Flink Standalone cluster TM resources");
@@ -147,6 +152,7 @@ public class StandaloneFlinkService extends AbstractFlinkService {
                 .deployments()
                 .inNamespace(namespace)
                 .withName(StandaloneKubernetesUtils.getTaskManagerDeploymentName(clusterId))
+                .withPropagationPolicy(deletionPropagation)
                 .delete();
         if (deleteHaData) {
             deleteHAData(namespace, clusterId, conf);
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java
index 448fb85d..a39d09a4 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java
@@ -134,7 +134,7 @@ public class TestUtils extends BaseTestUtils {
                 "kind", owner.getKind(),
                 "name", owner.getMetadata().getName(),
                 "uid", owner.getMetadata().getUid(),
-                "blockOwnerDeletion", "false",
+                "blockOwnerDeletion", "true",
                 "controller", "false");
     }
 
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
index 83f75738..3aea5daf 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
@@ -60,6 +60,7 @@ import org.apache.flink.runtime.rest.messages.EmptyResponseBody;
 import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders;
 import org.apache.flink.util.SerializedThrowable;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
 import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.PodList;
@@ -429,7 +430,10 @@ public class TestingFlinkService extends AbstractFlinkService {
 
     @Override
     protected void deleteClusterInternal(
-            ObjectMeta meta, Configuration conf, boolean deleteHaMeta) {
+            ObjectMeta meta,
+            Configuration conf,
+            boolean deleteHaMeta,
+            DeletionPropagation deletionPropagation) {
         jobs.clear();
         sessions.remove(meta.getName());
     }
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java
index 59bf99a7..d7239018 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java
@@ -32,10 +32,12 @@ import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
 import org.apache.flink.kubernetes.operator.api.spec.FlinkVersion;
 import org.apache.flink.kubernetes.operator.api.spec.JobSpec;
 import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode;
+import org.apache.flink.kubernetes.operator.api.status.FlinkDeploymentStatus;
 import org.apache.flink.kubernetes.operator.api.status.JobManagerDeploymentStatus;
 import org.apache.flink.kubernetes.operator.api.status.JobStatus;
 import org.apache.flink.kubernetes.operator.api.status.SavepointTriggerType;
 import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
+import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
 import org.apache.flink.kubernetes.operator.exception.RecoveryFailureException;
 import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
 import org.apache.flink.runtime.execution.ExecutionState;
@@ -50,6 +52,8 @@ import org.apache.flink.runtime.rest.util.RestMapperUtils;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
 
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
+import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
 import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
 import io.fabric8.kubernetes.client.KubernetesClient;
@@ -457,6 +461,34 @@ public class NativeFlinkServiceTest {
         assertEquals(savepointPath, stopWithSavepointFuture.get().f2);
     }
 
+    @Test
+    public void testDeletionPropagation() {
+        var propagation = new ArrayList<DeletionPropagation>();
+        var flinkService =
+                new NativeFlinkService(client, configManager) {
+                    @Override
+                    protected void deleteClusterInternal(
+                            ObjectMeta meta,
+                            Configuration conf,
+                            boolean deleteHaData,
+                            DeletionPropagation deletionPropagation) {
+                        propagation.add(deletionPropagation);
+                    }
+                };
+
+        flinkService.deleteClusterDeployment(
+                new ObjectMeta(), new FlinkDeploymentStatus(), configuration, true);
+        assertEquals(DeletionPropagation.FOREGROUND, propagation.get(0));
+
+        configuration.set(
+                KubernetesOperatorConfigOptions.RESOURCE_DELETION_PROPAGATION,
+                DeletionPropagation.BACKGROUND);
+        configManager.updateDefaultConfig(configuration);
+        flinkService.deleteClusterDeployment(
+                new ObjectMeta(), new FlinkDeploymentStatus(), configuration, true);
+        assertEquals(DeletionPropagation.BACKGROUND, propagation.get(1));
+    }
+
     class TestingNativeFlinkService extends NativeFlinkService {
         private Configuration runtimeConfig;