You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2022/03/18 07:59:23 UTC

[pulsar] 01/16: [Functions] Pass configured metricsPort to k8s runtime (#14502)

This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit f9b020305a2d058793b351a41085276d4a110434
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Wed Mar 2 15:00:42 2022 +0200

    [Functions] Pass configured metricsPort to k8s runtime (#14502)
    
    (cherry picked from commit daed6a0b3ac94e77a3d7d4212ee297b9046317a2)
---
 conf/functions_worker.yml                          |  1 +
 .../runtime/kubernetes/KubernetesRuntime.java      | 18 +++++---
 .../kubernetes/KubernetesRuntimeFactory.java       |  7 ++-
 .../runtime/kubernetes/KubernetesRuntimeTest.java  | 52 ++++++++++++++++++++++
 .../functions/worker/FunctionsStatsGenerator.java  |  5 ++-
 5 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index 9ca5f7b..b235d60 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -206,6 +206,7 @@ functionRuntimeFactoryConfigs:
 #    # The port inside the function pod which is used by the worker to communicate with the pod
 #    grpcPort: 9093
 #    # The port inside the function pod on which prometheus metrics are exposed
+#    # An empty value disables prometheus metrics.
 #    metricsPort: 9094
 #    # The directory inside the function pod where nar packages will be extracted
 #    narExtractionDirectory:
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
index 70839fc..c443c4b 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java
@@ -427,7 +427,11 @@ public class KubernetesRuntime implements Runtime {
 
     @Override
     public String getPrometheusMetrics() throws IOException {
-        return RuntimeUtils.getPrometheusMetrics(metricsPort);
+        if (metricsPort != null) {
+            return RuntimeUtils.getPrometheusMetrics(metricsPort);
+        } else {
+            return null;
+        }
     }
 
     @Override
@@ -975,10 +979,14 @@ public class KubernetesRuntime implements Runtime {
     }
 
     private Map<String, String> getPrometheusAnnotations() {
-        final Map<String, String> annotations = new HashMap<>();
-        annotations.put("prometheus.io/scrape", "true");
-        annotations.put("prometheus.io/port", String.valueOf(metricsPort));
-        return annotations;
+        if (metricsPort != null) {
+            final Map<String, String> annotations = new HashMap<>();
+            annotations.put("prometheus.io/scrape", "true");
+            annotations.put("prometheus.io/port", String.valueOf(metricsPort));
+            return annotations;
+        } else {
+            return Collections.emptyMap();
+        }
     }
 
     private Map<String, String> getLabels(Function.FunctionDetails functionDetails) {
diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
index d98a161..4b2c7e7 100644
--- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
+++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java
@@ -290,6 +290,11 @@ public class KubernetesRuntimeFactory implements RuntimeFactory {
         String overriddenNamespace = manifestCustomizer.map((customizer) -> customizer.customizeNamespace(instanceConfig.getFunctionDetails(), jobNamespace)).orElse(jobNamespace);
         String overriddenName = manifestCustomizer.map((customizer) -> customizer.customizeName(instanceConfig.getFunctionDetails(), jobName)).orElse(jobName);
 
+        // pass metricsPort configured in functionRuntimeFactoryConfigs.metricsPort in functions_worker.yml
+        if (metricsPort != null) {
+            instanceConfig.setMetricsPort(metricsPort);
+        }
+
         return new KubernetesRuntime(
             appsClient,
             coreClient,
@@ -351,7 +356,7 @@ public class KubernetesRuntimeFactory implements RuntimeFactory {
             if (k8Uri == null) {
                 log.info("k8Uri is null thus going by defaults");
                 ApiClient cli;
-                if (submittingInsidePod) {
+                if (submittingInsidePod != null && submittingInsidePod) {
                     log.info("Looks like we are inside a k8 pod ourselves. Initializing as cluster");
                     cli = Config.fromCluster();
                 } else {
diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
index b295cf8..be9c52c 100644
--- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
+++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java
@@ -29,12 +29,14 @@ import io.kubernetes.client.openapi.apis.CoreV1Api;
 import io.kubernetes.client.custom.Quantity;
 import io.kubernetes.client.openapi.models.V1Container;
 import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.openapi.models.V1PodTemplateSpec;
 import io.kubernetes.client.openapi.models.V1ResourceRequirements;
 import io.kubernetes.client.openapi.models.V1Service;
 import io.kubernetes.client.openapi.models.V1StatefulSet;
 import io.kubernetes.client.openapi.models.V1Toleration;
 import org.apache.commons.lang.StringUtils;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.functions.instance.AuthenticationConfig;
 import org.apache.pulsar.functions.instance.InstanceConfig;
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.Function.ConsumerSpec;
@@ -42,6 +44,7 @@ import org.apache.pulsar.functions.proto.Function.FunctionDetails;
 import org.apache.pulsar.functions.runtime.RuntimeCustomizer;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntime;
 import org.apache.pulsar.functions.secretsprovider.ClearTextSecretsProvider;
+import org.apache.pulsar.functions.secretsproviderconfigurator.DefaultSecretsProviderConfigurator;
 import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator;
 import org.apache.pulsar.functions.utils.FunctionCommon;
 import org.apache.pulsar.functions.worker.ConnectorsManager;
@@ -66,6 +69,7 @@ import static org.apache.pulsar.functions.utils.FunctionCommon.roundDecimal;
 import static org.powermock.api.mockito.PowerMockito.doNothing;
 import static org.powermock.api.mockito.PowerMockito.spy;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
@@ -1145,4 +1149,52 @@ public class KubernetesRuntimeTest {
         String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2);
         assertTrue(containerCommand.contains(expectedDownloadCommand));
     }
+
+    @Test
+    public void shouldUseConfiguredMetricsPort() throws Exception {
+        assertMetricsPortConfigured(Collections.singletonMap("metricsPort", 12345), 12345);
+    }
+
+    @Test
+    public void shouldUseDefaultMetricsPortWhenMetricsPortIsntSet() throws Exception {
+        assertMetricsPortConfigured(Collections.emptyMap(), 9094);
+    }
+
+    @Test
+    public void shouldNotAddPrometheusAnnotationIfMetricsPortIsSetToEmpty() throws Exception {
+        assertMetricsPortConfigured(Collections.singletonMap("metricsPort", ""), -1);
+    }
+
+    private void assertMetricsPortConfigured(Map<String, Object> functionRuntimeFactoryConfigs,
+                                             int expectedPort) throws Exception {
+        KubernetesRuntimeFactory kubernetesRuntimeFactory = new KubernetesRuntimeFactory();
+        WorkerConfig workerConfig = new WorkerConfig();
+        workerConfig.setFunctionRuntimeFactoryClassName(KubernetesRuntimeFactory.class.getName());
+        workerConfig.setFunctionRuntimeFactoryConfigs(functionRuntimeFactoryConfigs);
+        AuthenticationConfig authenticationConfig = AuthenticationConfig.builder().build();
+        kubernetesRuntimeFactory.initialize(workerConfig, authenticationConfig, new DefaultSecretsProviderConfigurator(), Mockito.mock(ConnectorsManager.class), Optional.empty(), Optional.empty());
+        InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true);
+        KubernetesRuntime container = kubernetesRuntimeFactory.createContainer(config, userJarFile, userJarFile, 30l);
+        V1PodTemplateSpec template = container.createStatefulSet().getSpec().getTemplate();
+        Map<String, String> annotations =
+                template.getMetadata().getAnnotations();
+        if (expectedPort != -1) {
+            // metrics port should be passed to k8s annotation for prometheus scraping
+            assertEquals(annotations.get("prometheus.io/port"), String.valueOf(expectedPort));
+            // scraping annotation should exist
+            assertEquals(annotations.get("prometheus.io/scrape"), "true");
+
+            // metrics port should be passed to JavaInstanceStarter with --metrics_port argument
+            assertTrue(container.getProcessArgs().stream().collect(Collectors.joining(" "))
+                    .contains("--metrics_port " + expectedPort));
+        } else {
+            // No prometheus annotations should exist
+            assertFalse(annotations.containsKey("prometheus.io/scrape"));
+            assertFalse(annotations.containsKey("prometheus.io/port"));
+            // metrics will be started on random port when the port isn't specified
+            // check that "--metrics_port 0" argument is passed
+            assertTrue(container.getProcessArgs().stream().collect(Collectors.joining(" "))
+                    .contains("--metrics_port 0"));
+        }
+    }
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java
index 7dd1e74..d94f54c 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionsStatsGenerator.java
@@ -68,7 +68,10 @@ public class FunctionsStatsGenerator {
                     if (functionRuntime != null) {
                         try {
 
-                            out.write(functionRuntime.getPrometheusMetrics());
+                            String prometheusMetrics = functionRuntime.getPrometheusMetrics();
+                            if (prometheusMetrics != null) {
+                                out.write(prometheusMetrics);
+                            }
 
                         } catch (IOException e) {
                             log.warn("Failed to collect metrics for function instance {}",