You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by su...@apache.org on 2023/05/18 18:52:05 UTC
[druid] branch master updated: Fix labels (#14282)
This is an automated email from the ASF dual-hosted git repository.
suneet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 51f722b7f1 Fix labels (#14282)
51f722b7f1 is described below
commit 51f722b7f1a5f30430c7d56d65936e21dfa188e1
Author: George Shiqi Wu <ge...@imply.io>
AuthorDate: Thu May 18 14:51:58 2023 -0400
Fix labels (#14282)
* Fix labels
* move to a util function
* style
* PR comments
* rename class
---
.../druid/k8s/overlord/common/K8sTaskId.java | 7 +--
.../overlord/common/KubernetesOverlordUtils.java | 46 ++++++++++++++++
.../taskadapter/PodTemplateTaskAdapter.java | 11 ++--
.../common/KubernetesOverlordUtilsTest.java | 62 ++++++++++++++++++++++
.../taskadapter/PodTemplateTaskAdapterTest.java | 36 +++++++++++++
.../src/test/resources/expectedNoopJobLongIds.yaml | 50 +++++++++++++++++
6 files changed, 201 insertions(+), 11 deletions(-)
diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/K8sTaskId.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/K8sTaskId.java
index dec6613335..a6e54704c2 100644
--- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/K8sTaskId.java
+++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/K8sTaskId.java
@@ -19,11 +19,8 @@
package org.apache.druid.k8s.overlord.common;
-import org.apache.commons.lang3.RegExUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.druid.indexing.common.task.Task;
-import java.util.Locale;
import java.util.Objects;
public class K8sTaskId
@@ -40,9 +37,7 @@ public class K8sTaskId
public K8sTaskId(String taskId)
{
this.originalTaskId = taskId;
- // replace all the ": - . _" to "", try to reduce the length of pod name and meet pod naming specifications 64 characters.
- this.k8sTaskId = StringUtils.left(RegExUtils.replaceAll(taskId, "[^a-zA-Z0-9\\\\s]", "")
- .toLowerCase(Locale.ENGLISH), 63);
+ this.k8sTaskId = KubernetesOverlordUtils.convertTaskIdToK8sLabel(taskId);
}
public String getK8sTaskId()
diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesOverlordUtils.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesOverlordUtils.java
new file mode 100644
index 0000000000..2116aaa0d5
--- /dev/null
+++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesOverlordUtils.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.k8s.overlord.common;
+
+import org.apache.commons.lang3.RegExUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+public class KubernetesOverlordUtils
+{
+ private static final Pattern K8S_LABEL_PATTERN = Pattern.compile("[^A-Za-z0-9_.-]");
+ // replace all the ": - . _" to "", try to reduce the length of pod name and meet pod naming specifications 64 characters.
+ private static final Pattern K8S_TASK_ID_PATTERN = Pattern.compile("[^a-zA-Z0-9\\\\s]");
+
+ public static String convertStringToK8sLabel(String rawString)
+ {
+ String trimmedString = rawString == null ? "" : RegExUtils.replaceAll(rawString, K8S_LABEL_PATTERN, "");
+ return StringUtils.left(StringUtils.strip(trimmedString, "_.-"), 63);
+
+ }
+
+ public static String convertTaskIdToK8sLabel(String taskId)
+ {
+ return taskId == null ? "" : StringUtils.left(RegExUtils.replaceAll(taskId, K8S_TASK_ID_PATTERN, "")
+ .toLowerCase(Locale.ENGLISH), 63);
+ }
+}
diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapter.java b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapter.java
index 11f7c4a411..ded61459cf 100644
--- a/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapter.java
+++ b/extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapter.java
@@ -47,6 +47,7 @@ import org.apache.druid.k8s.overlord.KubernetesTaskRunnerConfig;
import org.apache.druid.k8s.overlord.common.Base64Compression;
import org.apache.druid.k8s.overlord.common.DruidK8sConstants;
import org.apache.druid.k8s.overlord.common.K8sTaskId;
+import org.apache.druid.k8s.overlord.common.KubernetesOverlordUtils;
import org.apache.druid.server.DruidNode;
import java.io.File;
@@ -243,16 +244,16 @@ public class PodTemplateTaskAdapter implements TaskAdapter
.put(DruidK8sConstants.TASK_DATASOURCE, task.getDataSource())
.build();
}
-
+
private Map<String, String> getJobLabels(KubernetesTaskRunnerConfig config, Task task)
{
return ImmutableMap.<String, String>builder()
.putAll(config.getLabels())
.put(DruidK8sConstants.LABEL_KEY, "true")
- .put(getDruidLabel(DruidK8sConstants.TASK_ID), task.getId())
- .put(getDruidLabel(DruidK8sConstants.TASK_TYPE), task.getType())
- .put(getDruidLabel(DruidK8sConstants.TASK_GROUP_ID), task.getGroupId())
- .put(getDruidLabel(DruidK8sConstants.TASK_DATASOURCE), task.getDataSource())
+ .put(getDruidLabel(DruidK8sConstants.TASK_ID), KubernetesOverlordUtils.convertTaskIdToK8sLabel(task.getId()))
+ .put(getDruidLabel(DruidK8sConstants.TASK_TYPE), KubernetesOverlordUtils.convertStringToK8sLabel(task.getType()))
+ .put(getDruidLabel(DruidK8sConstants.TASK_GROUP_ID), KubernetesOverlordUtils.convertTaskIdToK8sLabel(task.getGroupId()))
+ .put(getDruidLabel(DruidK8sConstants.TASK_DATASOURCE), KubernetesOverlordUtils.convertStringToK8sLabel(task.getDataSource()))
.build();
}
diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesOverlordUtilsTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesOverlordUtilsTest.java
new file mode 100644
index 0000000000..707112d14e
--- /dev/null
+++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/KubernetesOverlordUtilsTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.k8s.overlord.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class KubernetesOverlordUtilsTest
+{
+
+ @Test
+ public void test_shortLabel()
+ {
+ Assert.assertEquals("data_source", KubernetesOverlordUtils.convertStringToK8sLabel("data_source"));
+ }
+
+
+ @Test
+ public void test_stripDisallowedPatterns()
+ {
+ Assert.assertEquals("data_source-1.wikipedia", KubernetesOverlordUtils.convertStringToK8sLabel(
+ "_.-data_source-1.wikipedia.-_"
+ ));
+ }
+
+ @Test
+ public void test_nullLabel()
+ {
+ Assert.assertEquals("", KubernetesOverlordUtils.convertStringToK8sLabel(null));
+ }
+
+ @Test
+ public void test_stripTaskId()
+ {
+ Assert.assertEquals("apiissuedkillwikipedianewbalhnoib10000101t000000000z20230514t00", KubernetesOverlordUtils.convertTaskIdToK8sLabel(
+ "api-issued_kill_wikipedia_new_balhnoib_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:28:42.526Z"
+ ));
+ }
+
+ @Test
+ public void test_nullTaskId()
+ {
+ Assert.assertEquals("", KubernetesOverlordUtils.convertTaskIdToK8sLabel(null));
+ }
+}
diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java
index 2a9381aa3d..7f84c69cae 100644
--- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java
+++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapterTest.java
@@ -319,6 +319,42 @@ public class PodTemplateTaskAdapterTest
Assertions.assertEquals(expected, actual);
}
+ @Test
+ public void test_fromTask_withRealIds() throws IOException
+ {
+ Path templatePath = Files.createFile(tempDir.resolve("noop.yaml"));
+ mapper.writeValue(templatePath.toFile(), podTemplateSpec);
+
+ Properties props = new Properties();
+ props.setProperty("druid.indexer.runner.k8s.podTemplate.base", templatePath.toString());
+ props.setProperty("druid.indexer.runner.k8s.podTemplate.noop", templatePath.toString());
+
+ PodTemplateTaskAdapter adapter = new PodTemplateTaskAdapter(
+ taskRunnerConfig,
+ taskConfig,
+ node,
+ mapper,
+ props
+ );
+
+ Task task = new NoopTask(
+ "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z",
+ "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z",
+ "data_source",
+ 0,
+ 0,
+ null,
+ null,
+ null
+ );
+
+ Job actual = adapter.fromTask(task);
+ Job expected = K8sTestUtils.fileToResource("expectedNoopJobLongIds.yaml", Job.class);
+
+ assertJobSpecsEqual(actual, expected);
+ }
+
+
private void assertJobSpecsEqual(Job actual, Job expected) throws IOException
{
diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml
new file mode 100644
index 0000000000..01fb962ed0
--- /dev/null
+++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml
@@ -0,0 +1,50 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
+ labels:
+ druid.k8s.peons: "true"
+ druid.task.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
+ druid.task.type: "noop"
+ druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
+ druid.task.datasource: "data_source"
+ annotations:
+ task.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
+ task.type: "noop"
+ task.group.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
+ task.datasource: "data_source"
+spec:
+ activeDeadlineSeconds: 14400
+ backoffLimit: 0
+ ttlSecondsAfterFinished: 172800
+ template:
+ metadata:
+ labels:
+ druid.k8s.peons: "true"
+ druid.task.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
+ druid.task.type: "noop"
+ druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
+ druid.task.datasource: "data_source"
+ annotations:
+ task: "H4sIAAAAAAAAAMVQu07EMBD8F9fJae0QIblFCNHepeEay4kNLOezjR9AFOXf2XBIVNQnbbEzs6/ZhZU5WiaZDyGyhqGhXEdsMedqjTqhc+oTTxitQd2pcH4Lox8nxQGgBU4xAMif2BF1VAJE10Lf8pv/hH7gtxI6CXwnBBxp60sKNT5eZbXRRR9CTdP2hA2ofEENS9UPeCZe9AD0mry32swX6g/vba6uUPPT/YGanjHZ15CpxFfnGjYFX+wX6ctKE+3vcLkw/aHR6REdlvlh838N98m+VzrY3OmoJzqESb6u3yiWc3MUAgAA"
+ tls.enabled: "false"
+ task.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
+ task.type: "noop"
+ task.group.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
+ task.datasource: "data_source"
+ spec:
+ containers:
+ - command:
+ - sleep
+ - "3600"
+ env:
+ - name: "TASK_DIR"
+ value: "/tmp"
+ - name: "TASK_ID"
+ value: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
+ - name: "TASK_JSON"
+ valueFrom:
+ fieldRef:
+ fieldPath: "metadata.annotations['task']"
+ image: one
+ name: primary
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org