You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2019/02/15 17:22:13 UTC

[GitHub] jerrypeng commented on a change in pull request #3603: fix: add retry and wait for kubernetes creation/deletion

jerrypeng commented on a change in pull request #3603: fix: add retry and wait for kubernetes creation/deletion
URL: https://github.com/apache/pulsar/pull/3603#discussion_r257322596
 
 

 ##########
 File path: pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/KubernetesRuntime.java
 ##########
 @@ -397,75 +419,298 @@ private void submitStatefulSet() throws Exception {
 
         log.info("Submitting the following spec to k8 {}", appsClient.getApiClient().getJSON().serialize(statefulSet));
 
-        final Response response =
-                appsClient.createNamespacedStatefulSetCall(jobNamespace, statefulSet, null,
-                        null, null).execute();
-        if (!response.isSuccessful()) {
-            if (response.code() == HTTP_CONFLICT) {
-                log.warn("Statefulset already present for function {}/{}/{}",
-                        instanceConfig.getFunctionDetails().getTenant(),
-                        instanceConfig.getFunctionDetails().getNamespace(),
-                        instanceConfig.getFunctionDetails().getName());
-            } else {
-                log.error("Error creating statefulset for function {}/{}/{}:- {}",
-                        instanceConfig.getFunctionDetails().getTenant(),
-                        instanceConfig.getFunctionDetails().getNamespace(),
-                        instanceConfig.getFunctionDetails().getName(),
-                        response.message());
-                // construct a message based on the k8s api server response
-                throw new IllegalStateException(response.message());
-            }
-        } else {
-            log.info("Successfully created statefulset for function {}/{}/{}",
-                    instanceConfig.getFunctionDetails().getTenant(),
-                    instanceConfig.getFunctionDetails().getNamespace(),
-                    instanceConfig.getFunctionDetails().getName());
+        String fqfn = FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails());
+
+        RuntimeUtils.Actions.Action createStatefulSet = RuntimeUtils.Actions.Action.builder()
+                .actionName(String.format("Submitting statefulset for function %s", fqfn))
+                .numRetries(NUM_RETRIES)
+                .sleepBetweenInvocationsMs(SLEEP_BETWEEN_RETRIES_MS)
+                .supplier(() -> {
+                    final V1StatefulSet response;
+                    try {
+                        response = appsClient.createNamespacedStatefulSet(jobNamespace, statefulSet, null);
+                    } catch (ApiException e) {
+                        // already exists
+                        if (e.getCode() == HTTP_CONFLICT) {
+                            log.warn("Statefulset already present for function {}", fqfn);
+                            return RuntimeUtils.Actions.ActionResult.builder().success(true).build();
+                        }
+
+                        String errorMsg = e.getResponseBody() != null ? e.getResponseBody() : e.getMessage();
+                        return RuntimeUtils.Actions.ActionResult.builder()
+                                .success(false)
+                                .errorMsg(errorMsg)
+                                .build();
+                    }
+
+                    return RuntimeUtils.Actions.ActionResult.builder().success(true).build();
+                })
+                .build();
+
+
+        AtomicBoolean success = new AtomicBoolean(false);
+        RuntimeUtils.Actions.newBuilder()
+                .addAction(createStatefulSet.toBuilder()
+                        .onSuccess(() -> success.set(true))
+                        .build())
+                .run();
+
+        if (!success.get()) {
+            throw new RuntimeException(String.format("Failed to create statefulset for function %s", fqfn));
         }
     }
 
-    public void deleteStatefulSet() throws Exception {
+
+    public void deleteStatefulSet() throws InterruptedException {
+        String statefulSetName = createJobName(instanceConfig.getFunctionDetails());
         final V1DeleteOptions options = new V1DeleteOptions();
         options.setGracePeriodSeconds(0L);
         options.setPropagationPolicy("Foreground");
-        final Response response = appsClient.deleteNamespacedStatefulSetCall(
-                createJobName(instanceConfig.getFunctionDetails()),
-                jobNamespace, options, null, null, null, null, null, null)
-                .execute();
 
-        if (!response.isSuccessful()) {
-            throw new RuntimeException(String.format("Error deleting statefulset for function {}/{}/{} :- {} ",
-                    instanceConfig.getFunctionDetails().getTenant(),
-                    instanceConfig.getFunctionDetails().getNamespace(),
-                    instanceConfig.getFunctionDetails().getName(),
-                    response.message()));
+        String fqfn = FunctionDetailsUtils.getFullyQualifiedName(instanceConfig.getFunctionDetails());
+        RuntimeUtils.Actions.Action deleteStatefulSet = RuntimeUtils.Actions.Action.builder()
+                .actionName(String.format("Deleting statefulset for function %s", fqfn))
+                .numRetries(NUM_RETRIES)
+                .sleepBetweenInvocationsMs(SLEEP_BETWEEN_RETRIES_MS)
+                .supplier(() -> {
+                    Response response;
+                    try {
+                        // cannot use deleteNamespacedStatefulSet because of bug in kuberenetes
+                        // https://github.com/kubernetes-client/java/issues/86
+                        response = appsClient.deleteNamespacedStatefulSetCall(
+                                statefulSetName,
+                                jobNamespace, options, null,
+                                null, null, null,
+                                null, null)
+                                .execute();
+                    } catch (ApiException e) {
+                        // if already deleted
+                        if (e.getCode() == HTTP_NOT_FOUND) {
+                            log.warn("Statefulset for function {} does not exist", fqfn);
+                            return RuntimeUtils.Actions.ActionResult.builder().success(true).build();
+                        }
+
+                        String errorMsg = e.getResponseBody() != null ? e.getResponseBody() : e.getMessage();
+                        return RuntimeUtils.Actions.ActionResult.builder()
+                                .success(false)
+                                .errorMsg(errorMsg)
+                                .build();
+                    } catch (IOException e) {
+                        return RuntimeUtils.Actions.ActionResult.builder()
+                                .success(false)
+                                .errorMsg(e.getMessage())
+                                .build();
+                    }
+
+                    // if already deleted
+                    if (response.code() == HTTP_NOT_FOUND) {
+                        log.warn("Statefulset for function {} does not exist", fqfn);
+                        return RuntimeUtils.Actions.ActionResult.builder().success(true).build();
+                    } else {
+                        return RuntimeUtils.Actions.ActionResult.builder()
+                                .success(response.isSuccessful())
+                                .errorMsg(response.message())
+                                .build();
+                    }
+                })
+                .build();
+
+
+        RuntimeUtils.Actions.Action waitForStatefulSetDeletion = RuntimeUtils.Actions.Action.builder()
+                .actionName(String.format("Waiting for statefulset for function %s to complete deletion", fqfn))
+                .numRetries(NUM_RETRIES)
+                .sleepBetweenInvocationsMs(SLEEP_BETWEEN_RETRIES_MS)
+                .supplier(() -> {
+                    V1StatefulSet response;
+                    try {
+                        response = appsClient.readNamespacedStatefulSet(jobNamespace, statefulSetName,
+                                null, null, null);
+                    } catch (ApiException e) {
+                        // statefulset is gone
+                        if (e.getCode() == HTTP_NOT_FOUND) {
+                            return RuntimeUtils.Actions.ActionResult.builder().success(true).build();
+                        }
+
+                        String errorMsg = e.getResponseBody() != null ? e.getResponseBody() : e.getMessage();
+                        return RuntimeUtils.Actions.ActionResult.builder()
+                                .success(false)
+                                .errorMsg(errorMsg)
+                                .build();
+                    }
+                    return RuntimeUtils.Actions.ActionResult.builder()
+                            .success(false)
+                            .errorMsg(response.getStatus().toString())
+                            .build();
+                })
+                .build();
+
+        // Need to wait for all pods to die so we can cleanup subscriptions.
+        RuntimeUtils.Actions.Action waitForStatefulPodsToTerminate = RuntimeUtils.Actions.Action.builder()
+                .actionName(String.format("Waiting for pods for function %s to terminate", fqfn))
+                .numRetries(NUM_RETRIES * 2)
+                .sleepBetweenInvocationsMs(SLEEP_BETWEEN_RETRIES_MS * 2)
+                .supplier(() -> {
+                    String labels = String.format("tenant=%s,namespace=%s,name=%s",
+                            instanceConfig.getFunctionDetails().getTenant(),
+                            instanceConfig.getFunctionDetails().getNamespace(),
+                            instanceConfig.getFunctionDetails().getName());
+
+                    V1PodList response;
+                    try {
+                        response = coreClient.listNamespacedPod("default", null, null,
 
 Review comment:
   good catch

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services