You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2023/03/30 11:26:30 UTC
[flink-kubernetes-operator] branch main updated: [FLINK-31592] Improve Flink dependent resource deletion
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 6620466d [FLINK-31592] Improve Flink dependent resource deletion
6620466d is described below
commit 6620466d31df7b22e4cae4ed0bd2eab70280c8c8
Author: Gyula Fora <g_...@apple.com>
AuthorDate: Thu Mar 30 13:26:23 2023 +0200
[FLINK-31592] Improve Flink dependent resource deletion
---
.../kubernetes_operator_config_configuration.html | 6 ++++
.../generated/system_advanced_section.html | 6 ++++
e2e-tests/test_application_operations.sh | 2 +-
.../config/FlinkOperatorConfiguration.java | 8 +++++-
.../config/KubernetesOperatorConfigOptions.java | 8 ++++++
.../AbstractFlinkResourceReconciler.java | 2 +-
.../operator/service/AbstractFlinkService.java | 16 ++++++++---
.../operator/service/NativeFlinkService.java | 7 ++++-
.../operator/service/StandaloneFlinkService.java | 8 +++++-
.../flink/kubernetes/operator/TestUtils.java | 2 +-
.../kubernetes/operator/TestingFlinkService.java | 6 +++-
.../operator/service/NativeFlinkServiceTest.java | 32 ++++++++++++++++++++++
12 files changed, 92 insertions(+), 11 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html b/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
index 2d18bbdd..6b335b8d 100644
--- a/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
+++ b/docs/layouts/shortcodes/generated/kubernetes_operator_config_configuration.html
@@ -242,6 +242,12 @@
<td>Duration</td>
<td>The timeout for the resource clean up to wait for flink to shutdown cluster.</td>
</tr>
+ <tr>
+ <td><h5>kubernetes.operator.resource.deletion.propagation</h5></td>
+ <td style="word-wrap: break-word;">Foreground</td>
+ <td><p>Enum</p></td>
+ <td>JM/TM Deployment deletion propagation.<br /><br />Possible values:<ul><li>"Orphan"</li><li>"Background"</li><li>"Foreground"</li></ul></td>
+ </tr>
<tr>
<td><h5>kubernetes.operator.retry.initial.interval</h5></td>
<td style="word-wrap: break-word;">5 s</td>
diff --git a/docs/layouts/shortcodes/generated/system_advanced_section.html b/docs/layouts/shortcodes/generated/system_advanced_section.html
index 1d250b93..487f6fc7 100644
--- a/docs/layouts/shortcodes/generated/system_advanced_section.html
+++ b/docs/layouts/shortcodes/generated/system_advanced_section.html
@@ -68,6 +68,12 @@
<td>Duration</td>
<td>Final delay before deployment is marked ready after port becomes accessible.</td>
</tr>
+ <tr>
+ <td><h5>kubernetes.operator.resource.deletion.propagation</h5></td>
+ <td style="word-wrap: break-word;">Foreground</td>
+ <td><p>Enum</p></td>
+ <td>JM/TM Deployment deletion propagation.<br /><br />Possible values:<ul><li>"Orphan"</li><li>"Background"</li><li>"Foreground"</li></ul></td>
+ </tr>
<tr>
<td><h5>kubernetes.operator.savepoint.history.max.age.threshold</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/e2e-tests/test_application_operations.sh b/e2e-tests/test_application_operations.sh
index f6d1ace3..d69da980 100755
--- a/e2e-tests/test_application_operations.sh
+++ b/e2e-tests/test_application_operations.sh
@@ -56,7 +56,7 @@ fi
# Testing last-state mode upgrade
# Update the FlinkDeployment and trigger the last state upgrade
kubectl patch flinkdep ${CLUSTER_ID} --type merge --patch '{"spec":{"job": {"parallelism": 1 } } }'
-kubectl wait --for=delete pod --timeout=${TIMEOUT}s --selector="app=${CLUSTER_ID}"
+kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${CLUSTER_ID}"
wait_for_jobmanager_running $CLUSTER_ID $TIMEOUT
jm_pod_name=$(get_jm_pod_name $CLUSTER_ID)
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
index ce6b97c7..b29077aa 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/FlinkOperatorConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.kubernetes.operator.metrics.KubernetesOperatorMetricOptions;
import org.apache.flink.kubernetes.operator.utils.EnvUtils;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration;
import io.javaoperatorsdk.operator.api.config.RetryConfiguration;
import lombok.Value;
@@ -67,6 +68,7 @@ public class FlinkOperatorConfiguration {
int exceptionThrowableCountThreshold;
String labelSelector;
LeaderElectionConfiguration leaderElectionConfiguration;
+ DeletionPropagation deletionPropagation;
public static FlinkOperatorConfiguration fromConfiguration(Configuration operatorConfig) {
Duration reconcileInterval =
@@ -172,6 +174,9 @@ public class FlinkOperatorConfiguration {
String labelSelector =
operatorConfig.getString(KubernetesOperatorConfigOptions.OPERATOR_LABEL_SELECTOR);
+ DeletionPropagation deletionPropagation =
+ operatorConfig.get(KubernetesOperatorConfigOptions.RESOURCE_DELETION_PROPAGATION);
+
return new FlinkOperatorConfiguration(
reconcileInterval,
reconcilerMaxParallelism,
@@ -196,7 +201,8 @@ public class FlinkOperatorConfiguration {
exceptionFieldLengthThreshold,
exceptionThrowableCountThreshold,
labelSelector,
- getLeaderElectionConfig(operatorConfig));
+ getLeaderElectionConfig(operatorConfig),
+ deletionPropagation);
}
private static LeaderElectionConfiguration getLeaderElectionConfig(Configuration conf) {
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java
index a9f28619..f361310b 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/config/KubernetesOperatorConfigOptions.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.core.execution.SavepointFormatType;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.javaoperatorsdk.operator.api.config.ConfigurationService;
import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration;
import io.javaoperatorsdk.operator.api.reconciler.Constants;
@@ -474,4 +475,11 @@ public class KubernetesOperatorConfigOptions {
.defaultValue(false)
.withDescription(
"Configure the array merge behaviour during pod merging. Arrays can be either merged by position or name matching.");
+
+ @Documentation.Section(SECTION_ADVANCED)
+ public static final ConfigOption<DeletionPropagation> RESOURCE_DELETION_PROPAGATION =
+ operatorConfig("resource.deletion.propagation")
+ .enumType(DeletionPropagation.class)
+ .defaultValue(DeletionPropagation.FOREGROUND)
+ .withDescription("JM/TM Deployment deletion propagation.");
}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
index c74545c9..251b8547 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
@@ -465,7 +465,7 @@ public abstract class AbstractFlinkResourceReconciler<
"kind", owner.getKind(),
"name", owner.getMetadata().getName(),
"uid", owner.getMetadata().getUid(),
- "blockOwnerDeletion", "false",
+ "blockOwnerDeletion", "true",
"controller", "false");
deployConfig.set(
KubernetesConfigOptions.JOB_MANAGER_OWNER_REFERENCE, List.of(ownerReference));
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java
index 27578c04..d71996af 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/AbstractFlinkService.java
@@ -87,6 +87,7 @@ import org.apache.flink.util.FlinkRuntimeException;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.PodList;
import io.fabric8.kubernetes.api.model.Service;
@@ -784,6 +785,8 @@ public abstract class AbstractFlinkService implements FlinkService {
/** Wait until the FLink cluster has completely shut down. */
@VisibleForTesting
void waitForClusterShutdown(String namespace, String clusterId, long shutdownTimeout) {
+ LOG.info("Waiting for cluster shutdown...");
+
boolean jobManagerRunning = true;
boolean serviceRunning = true;
@@ -918,7 +921,10 @@ public abstract class AbstractFlinkService implements FlinkService {
FlinkDeploymentStatus status,
Configuration conf,
boolean deleteHaData) {
- deleteClusterInternal(meta, conf, deleteHaData);
+
+ var deletionPropagation = configManager.getOperatorConfiguration().getDeletionPropagation();
+ LOG.info("Deleting cluster with {} propagation", deletionPropagation);
+ deleteClusterInternal(meta, conf, deleteHaData, deletionPropagation);
updateStatusAfterClusterDeletion(status);
}
@@ -930,9 +936,13 @@ public abstract class AbstractFlinkService implements FlinkService {
* @param conf Configuration of the Flink application
* @param deleteHaData Flag to indicate whether k8s or Zookeeper HA metadata should be removed
* as well
+ * @param deletionPropagation Resource deletion propagation policy
*/
protected abstract void deleteClusterInternal(
- ObjectMeta meta, Configuration conf, boolean deleteHaData);
+ ObjectMeta meta,
+ Configuration conf,
+ boolean deleteHaData,
+ DeletionPropagation deletionPropagation);
protected void deleteHAData(String namespace, String clusterId, Configuration conf) {
// We need to wait for cluster shutdown otherwise HA data might be recreated
@@ -950,8 +960,6 @@ public abstract class AbstractFlinkService implements FlinkService {
} else if (FlinkUtils.isZookeeperHAActivated(conf)) {
LOG.info("Deleting Zookeeper HA metadata");
FlinkUtils.deleteZookeeperHAMetadata(conf);
- } else {
- LOG.warn("Can't delete HA metadata since HA is not enabled");
}
}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java
index 9c752f7f..912a19e1 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/NativeFlinkService.java
@@ -32,6 +32,7 @@ import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.utils.KubernetesUtils;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.PodList;
import io.fabric8.kubernetes.client.KubernetesClient;
@@ -108,7 +109,10 @@ public class NativeFlinkService extends AbstractFlinkService {
@Override
protected void deleteClusterInternal(
- ObjectMeta meta, Configuration conf, boolean deleteHaData) {
+ ObjectMeta meta,
+ Configuration conf,
+ boolean deleteHaData,
+ DeletionPropagation deletionPropagation) {
String namespace = meta.getNamespace();
String clusterId = meta.getName();
@@ -121,6 +125,7 @@ public class NativeFlinkService extends AbstractFlinkService {
.deployments()
.inNamespace(namespace)
.withName(KubernetesUtils.getDeploymentName(clusterId))
+ .withPropagationPolicy(deletionPropagation)
.delete();
if (deleteHaData) {
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java
index 6c7d7052..bdc6b65f 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/StandaloneFlinkService.java
@@ -38,6 +38,7 @@ import org.apache.flink.kubernetes.operator.standalone.StandaloneKubernetesConfi
import org.apache.flink.kubernetes.operator.utils.StandaloneKubernetesUtils;
import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.PodList;
import io.fabric8.kubernetes.client.KubernetesClient;
@@ -129,7 +130,10 @@ public class StandaloneFlinkService extends AbstractFlinkService {
@Override
protected void deleteClusterInternal(
- ObjectMeta meta, Configuration conf, boolean deleteHaData) {
+ ObjectMeta meta,
+ Configuration conf,
+ boolean deleteHaData,
+ DeletionPropagation deletionPropagation) {
final String clusterId = meta.getName();
final String namespace = meta.getNamespace();
@@ -139,6 +143,7 @@ public class StandaloneFlinkService extends AbstractFlinkService {
.deployments()
.inNamespace(namespace)
.withName(StandaloneKubernetesUtils.getJobManagerDeploymentName(clusterId))
+ .withPropagationPolicy(deletionPropagation)
.delete();
LOG.info("Deleting Flink Standalone cluster TM resources");
@@ -147,6 +152,7 @@ public class StandaloneFlinkService extends AbstractFlinkService {
.deployments()
.inNamespace(namespace)
.withName(StandaloneKubernetesUtils.getTaskManagerDeploymentName(clusterId))
+ .withPropagationPolicy(deletionPropagation)
.delete();
if (deleteHaData) {
deleteHAData(namespace, clusterId, conf);
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java
index 448fb85d..a39d09a4 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestUtils.java
@@ -134,7 +134,7 @@ public class TestUtils extends BaseTestUtils {
"kind", owner.getKind(),
"name", owner.getMetadata().getName(),
"uid", owner.getMetadata().getUid(),
- "blockOwnerDeletion", "false",
+ "blockOwnerDeletion", "true",
"controller", "false");
}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
index 83f75738..3aea5daf 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
@@ -60,6 +60,7 @@ import org.apache.flink.runtime.rest.messages.EmptyResponseBody;
import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders;
import org.apache.flink.util.SerializedThrowable;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.PodList;
@@ -429,7 +430,10 @@ public class TestingFlinkService extends AbstractFlinkService {
@Override
protected void deleteClusterInternal(
- ObjectMeta meta, Configuration conf, boolean deleteHaMeta) {
+ ObjectMeta meta,
+ Configuration conf,
+ boolean deleteHaMeta,
+ DeletionPropagation deletionPropagation) {
jobs.clear();
sessions.remove(meta.getName());
}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java
index 59bf99a7..d7239018 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/service/NativeFlinkServiceTest.java
@@ -32,10 +32,12 @@ import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
import org.apache.flink.kubernetes.operator.api.spec.FlinkVersion;
import org.apache.flink.kubernetes.operator.api.spec.JobSpec;
import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode;
+import org.apache.flink.kubernetes.operator.api.status.FlinkDeploymentStatus;
import org.apache.flink.kubernetes.operator.api.status.JobManagerDeploymentStatus;
import org.apache.flink.kubernetes.operator.api.status.JobStatus;
import org.apache.flink.kubernetes.operator.api.status.SavepointTriggerType;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
+import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
import org.apache.flink.kubernetes.operator.exception.RecoveryFailureException;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
import org.apache.flink.runtime.execution.ExecutionState;
@@ -50,6 +52,8 @@ import org.apache.flink.runtime.rest.util.RestMapperUtils;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import io.fabric8.kubernetes.api.model.DeletionPropagation;
+import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
@@ -457,6 +461,34 @@ public class NativeFlinkServiceTest {
assertEquals(savepointPath, stopWithSavepointFuture.get().f2);
}
+ @Test
+ public void testDeletionPropagation() {
+ var propagation = new ArrayList<DeletionPropagation>();
+ var flinkService =
+ new NativeFlinkService(client, configManager) {
+ @Override
+ protected void deleteClusterInternal(
+ ObjectMeta meta,
+ Configuration conf,
+ boolean deleteHaData,
+ DeletionPropagation deletionPropagation) {
+ propagation.add(deletionPropagation);
+ }
+ };
+
+ flinkService.deleteClusterDeployment(
+ new ObjectMeta(), new FlinkDeploymentStatus(), configuration, true);
+ assertEquals(DeletionPropagation.FOREGROUND, propagation.get(0));
+
+ configuration.set(
+ KubernetesOperatorConfigOptions.RESOURCE_DELETION_PROPAGATION,
+ DeletionPropagation.BACKGROUND);
+ configManager.updateDefaultConfig(configuration);
+ flinkService.deleteClusterDeployment(
+ new ObjectMeta(), new FlinkDeploymentStatus(), configuration, true);
+ assertEquals(DeletionPropagation.BACKGROUND, propagation.get(1));
+ }
+
class TestingNativeFlinkService extends NativeFlinkService {
private Configuration runtimeConfig;