You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2021/01/11 02:00:19 UTC
[incubator-pinot] branch pinot-minion-in-k8s created (now e39224e)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch pinot-minion-in-k8s
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at e39224e Adding pinot minion component into helm
This branch includes the following new commits:
new e39224e Adding pinot minion component into helm
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Adding pinot minion component into helm
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch pinot-minion-in-k8s
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit e39224e8dc9336ccb0f31d033d38cccaa797960e
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Sun Jan 10 17:51:46 2021 -0800
Adding pinot minion component into helm
---
kubernetes/helm/index.yaml | 40 ++++++--
kubernetes/helm/pinot-0.2.3.tgz | Bin 0 -> 24307 bytes
kubernetes/helm/pinot/Chart.yaml | 4 +-
.../minion/configmap.yaml} | 23 ++---
.../minion/service-headless.yaml} | 33 ++++---
.../{Chart.yaml => templates/minion/service.yaml} | 33 ++++---
.../helm/pinot/templates/minion/statefulset.yml | 108 +++++++++++++++++++++
kubernetes/helm/pinot/values.yaml | 49 ++++++++++
.../tools/admin/command/StartMinionCommand.java | 8 +-
9 files changed, 241 insertions(+), 57 deletions(-)
diff --git a/kubernetes/helm/index.yaml b/kubernetes/helm/index.yaml
index 335fb02..be84518 100644
--- a/kubernetes/helm/index.yaml
+++ b/kubernetes/helm/index.yaml
@@ -2,8 +2,36 @@ apiVersion: v1
entries:
pinot:
- apiVersion: v1
+ appVersion: 0.2.3
+ created: "2021-01-10T01:36:34.265571-08:00"
+ dependencies:
+ - condition: pinot.zookeeper.enabled,zookeeper.enabled
+ name: zookeeper
+ repository: https://charts.helm.sh/incubator
+ version: 2.1.3
+ description: Apache Pinot is a realtime distributed OLAP datastore, which is used
+ to deliver scalable real time analytics with low latency. It can ingest data
+ from offline data sources (such as Hadoop and flat files) as well as online
+ sources (such as Kafka). Pinot is designed to scale horizontally.
+ digest: 5973bae14660ac6b127c6018e142938a0af13a1275960a7ebb1c11987335d0db
+ home: https://pinot.apache.org/
+ keywords:
+ - olap
+ - analytics
+ - database
+ - pinot
+ maintainers:
+ - email: dev@pinot.apache.org
+ name: pinot-dev
+ name: pinot
+ sources:
+ - https://github.com/apache/incubator-pinot
+ urls:
+ - pinot-0.2.3.tgz
+ version: 0.2.3
+ - apiVersion: v1
appVersion: 0.2.2
- created: "2020-09-29T00:03:26.751495-07:00"
+ created: "2021-01-10T01:36:34.262242-08:00"
dependencies:
- condition: pinot.zookeeper.enabled,zookeeper.enabled
name: zookeeper
@@ -31,7 +59,7 @@ entries:
version: 0.2.2
- apiVersion: v1
appVersion: 0.2.1
- created: "2020-09-29T00:03:26.748782-07:00"
+ created: "2021-01-10T01:36:34.2571-08:00"
dependencies:
- condition: pinot.zookeeper.enabled,zookeeper.enabled
name: zookeeper
@@ -59,7 +87,7 @@ entries:
version: 0.2.1
- apiVersion: v1
appVersion: 0.2.0
- created: "2020-09-29T00:03:26.745494-07:00"
+ created: "2021-01-10T01:36:34.253756-08:00"
dependencies:
- condition: pinot.zookeeper.enabled,zookeeper.enabled
name: zookeeper
@@ -88,7 +116,7 @@ entries:
presto:
- apiVersion: v1
appVersion: 0.2.0
- created: "2020-09-29T00:03:26.753701-07:00"
+ created: "2021-01-10T01:36:34.267011-08:00"
description: Presto is an open source distributed SQL query engine for running
interactive analytic queries against data sources of all sizes ranging from
gigabytes to petabytes.
@@ -103,8 +131,8 @@ entries:
name: pinot-dev
name: presto
sources:
- - https://github.com/apache/incubator-pinot/tree/master/kubernetes/helm
+ - https://github.com/apache/incubator-pinot
urls:
- presto-0.2.0.tgz
version: 0.2.0
-generated: "2020-09-29T00:03:26.739572-07:00"
+generated: "2021-01-10T01:36:34.247634-08:00"
diff --git a/kubernetes/helm/pinot-0.2.3.tgz b/kubernetes/helm/pinot-0.2.3.tgz
new file mode 100644
index 0000000..67166ce
Binary files /dev/null and b/kubernetes/helm/pinot-0.2.3.tgz differ
diff --git a/kubernetes/helm/pinot/Chart.yaml b/kubernetes/helm/pinot/Chart.yaml
index 9b7f130..f04b664 100644
--- a/kubernetes/helm/pinot/Chart.yaml
+++ b/kubernetes/helm/pinot/Chart.yaml
@@ -18,10 +18,10 @@
#
apiVersion: v1
-appVersion: 0.2.2
+appVersion: 0.2.3
name: pinot
description: Apache Pinot is a realtime distributed OLAP datastore, which is used to deliver scalable real time analytics with low latency. It can ingest data from offline data sources (such as Hadoop and flat files) as well as online sources (such as Kafka). Pinot is designed to scale horizontally.
-version: 0.2.2
+version: 0.2.3
keywords:
- olap
- analytics
diff --git a/kubernetes/helm/pinot/Chart.yaml b/kubernetes/helm/pinot/templates/minion/configmap.yaml
similarity index 59%
copy from kubernetes/helm/pinot/Chart.yaml
copy to kubernetes/helm/pinot/templates/minion/configmap.yaml
index 9b7f130..5c99c7d 100644
--- a/kubernetes/helm/pinot/Chart.yaml
+++ b/kubernetes/helm/pinot/templates/minion/configmap.yaml
@@ -18,18 +18,11 @@
#
apiVersion: v1
-appVersion: 0.2.2
-name: pinot
-description: Apache Pinot is a realtime distributed OLAP datastore, which is used to deliver scalable real time analytics with low latency. It can ingest data from offline data sources (such as Hadoop and flat files) as well as online sources (such as Kafka). Pinot is designed to scale horizontally.
-version: 0.2.2
-keywords:
- - olap
- - analytics
- - database
- - pinot
-home: https://pinot.apache.org/
-sources:
- - https://github.com/apache/incubator-pinot
-maintainers:
- - name: pinot-dev
- email: dev@pinot.apache.org
+kind: ConfigMap
+metadata:
+ name: {{ include "pinot.minion.config" . }}
+data:
+ pinot-minion.conf: |-
+ pinot.minion.port={{ .Values.minion.port }}
+ dataDir={{ .Values.minion.dataDir }}
+{{ .Values.minion.extra.configs | indent 4 }}
\ No newline at end of file
diff --git a/kubernetes/helm/pinot/Chart.yaml b/kubernetes/helm/pinot/templates/minion/service-headless.yaml
similarity index 59%
copy from kubernetes/helm/pinot/Chart.yaml
copy to kubernetes/helm/pinot/templates/minion/service-headless.yaml
index 9b7f130..84dae8e 100644
--- a/kubernetes/helm/pinot/Chart.yaml
+++ b/kubernetes/helm/pinot/templates/minion/service-headless.yaml
@@ -18,18 +18,21 @@
#
apiVersion: v1
-appVersion: 0.2.2
-name: pinot
-description: Apache Pinot is a realtime distributed OLAP datastore, which is used to deliver scalable real time analytics with low latency. It can ingest data from offline data sources (such as Hadoop and flat files) as well as online sources (such as Kafka). Pinot is designed to scale horizontally.
-version: 0.2.2
-keywords:
- - olap
- - analytics
- - database
- - pinot
-home: https://pinot.apache.org/
-sources:
- - https://github.com/apache/incubator-pinot
-maintainers:
- - name: pinot-dev
- email: dev@pinot.apache.org
+kind: Service
+metadata:
+ name: {{ template "pinot.minion.headless" . }}
+ labels:
+ app: {{ include "pinot.name" . }}
+ chart: {{ include "pinot.chart" . }}
+ component: {{ .Values.minion.name }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+spec:
+ clusterIP: None
+ ports:
+ # [pod_name].[service_name].[namespace].svc.cluster.local
+ - port: {{ .Values.minion.service.port }}
+ selector:
+ app: {{ include "pinot.name" . }}
+ release: {{ .Release.Name }}
+ component: {{ .Values.minion.name }}
diff --git a/kubernetes/helm/pinot/Chart.yaml b/kubernetes/helm/pinot/templates/minion/service.yaml
similarity index 59%
copy from kubernetes/helm/pinot/Chart.yaml
copy to kubernetes/helm/pinot/templates/minion/service.yaml
index 9b7f130..cf60983 100644
--- a/kubernetes/helm/pinot/Chart.yaml
+++ b/kubernetes/helm/pinot/templates/minion/service.yaml
@@ -18,18 +18,21 @@
#
apiVersion: v1
-appVersion: 0.2.2
-name: pinot
-description: Apache Pinot is a realtime distributed OLAP datastore, which is used to deliver scalable real time analytics with low latency. It can ingest data from offline data sources (such as Hadoop and flat files) as well as online sources (such as Kafka). Pinot is designed to scale horizontally.
-version: 0.2.2
-keywords:
- - olap
- - analytics
- - database
- - pinot
-home: https://pinot.apache.org/
-sources:
- - https://github.com/apache/incubator-pinot
-maintainers:
- - name: pinot-dev
- email: dev@pinot.apache.org
+kind: Service
+metadata:
+ name: {{ include "pinot.minion.fullname" . }}
+ labels:
+ app: {{ include "pinot.name" . }}
+ chart: {{ include "pinot.chart" . }}
+ component: {{ .Values.minion.name }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+spec:
+ type: ClusterIP
+ ports:
+ # [pod_name].[service_name].[namespace].svc.cluster.local
+ - port: {{ .Values.minion.service.port }}
+ selector:
+ app: {{ include "pinot.name" . }}
+ release: {{ .Release.Name }}
+ component: {{ .Values.minion.name }}
diff --git a/kubernetes/helm/pinot/templates/minion/statefulset.yml b/kubernetes/helm/pinot/templates/minion/statefulset.yml
new file mode 100644
index 0000000..959203e
--- /dev/null
+++ b/kubernetes/helm/pinot/templates/minion/statefulset.yml
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ include "pinot.minion.fullname" . }}
+ labels:
+ app: {{ include "pinot.name" . }}
+ chart: {{ include "pinot.chart" . }}
+ component: {{ .Values.minion.name }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ include "pinot.name" . }}
+ release: {{ .Release.Name }}
+ component: {{ .Values.minion.name }}
+ serviceName: {{ template "pinot.minion.headless" . }}
+ replicas: {{ .Values.minion.replicaCount }}
+ updateStrategy:
+ type: {{ .Values.minion.updateStrategy.type }}
+ podManagementPolicy: Parallel
+ template:
+ metadata:
+ labels:
+ app: {{ include "pinot.name" . }}
+ release: {{ .Release.Name }}
+ component: {{ .Values.minion.name }}
+ annotations:
+{{ toYaml .Values.minion.podAnnotations | indent 8 }}
+ spec:
+ nodeSelector:
+{{ toYaml .Values.minion.nodeSelector | indent 8 }}
+ affinity:
+{{ toYaml .Values.minion.affinity | indent 8 }}
+ tolerations:
+{{ toYaml .Values.minion.tolerations | indent 8 }}
+ containers:
+ - name: minion
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ args: [
+ "StartMinion",
+ "-clusterName", "{{ .Values.cluster.name }}",
+ "-zkAddress", {{ include "zookeeper.url" . | quote }},
+ "-configFileName", "/var/pinot/minion/config/pinot-minion.conf"
+ ]
+ env:
+ - name: JAVA_OPTS
+ value: "{{ .Values.minion.jvmOpts }} -Dlog4j2.configurationFile={{ .Values.minion.log4j2ConfFile }} -Dplugins.dir={{ .Values.minion.pluginsDir }}"
+ ports:
+ - containerPort: {{ .Values.minion.port }}
+ protocol: TCP
+ volumeMounts:
+ - name: config
+ mountPath: /var/pinot/minion/config
+ {{- if .Values.minion.persistence.enabled }}
+ - name: data
+ mountPath: "{{ .Values.minion.persistence.mountPath }}"
+ {{- end }}
+ resources:
+{{ toYaml .Values.minion.resources | indent 12 }}
+ restartPolicy: Always
+ volumes:
+ - name: config
+ configMap:
+ name: {{ include "pinot.minion.config" . }}
+ {{- if not .Values.minion.persistence.enabled }}
+ - name: data
+ emptyDir: {}
+ {{- end }}
+
+ {{- if .Values.minion.persistence.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: data
+ spec:
+ accessModes:
+ - {{ .Values.minion.persistence.accessMode | quote }}
+ {{- if .Values.minion.persistence.storageClass }}
+ {{- if (eq "-" .Values.minion.persistence.storageClass) }}
+ storageClassName: ""
+ {{- else }}
+ storageClassName: {{ .Values.minion.persistence.storageClass }}
+ {{- end }}
+ {{- end }}
+ resources:
+ requests:
+ storage: {{ .Values.minion.persistence.size }}
+ {{ end }}
diff --git a/kubernetes/helm/pinot/values.yaml b/kubernetes/helm/pinot/values.yaml
index c36009e..086b444 100644
--- a/kubernetes/helm/pinot/values.yaml
+++ b/kubernetes/helm/pinot/values.yaml
@@ -187,6 +187,55 @@ server:
configs: |-
pinot.set.instance.id.to.hostname=true
pinot.server.instance.realtime.alloc.offheap=true
+
+minion:
+ name: minion
+
+ port: 9514
+
+ replicaCount: 1
+
+ jvmOpts: "-Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -Xloggc:/opt/pinot/gc-pinot-minion.log"
+
+ log4j2ConfFile: /opt/pinot/conf/pinot-minion-log4j2.xml
+ pluginsDir: /opt/pinot/plugins
+
+ persistence:
+ enabled: true
+ accessMode: ReadWriteOnce
+ size: 4G
+ mountPath: /var/pinot/minion/data
+ storageClass: ""
+ #storageClass: "ssd"
+
+ service:
+ annotations: {}
+ clusterIP: ""
+ externalIPs: []
+ loadBalancerIP: ""
+ loadBalancerSourceRanges: []
+ type: ClusterIP
+ port: 9514
+ nodePort: ""
+
+ resources: {}
+
+ nodeSelector: {}
+
+ affinity: {}
+
+ tolerations: []
+
+ podAnnotations: {}
+
+ updateStrategy:
+ type: RollingUpdate
+
+ # Extra configs will be appended to pinot-minion.conf file
+ extra:
+ configs: |-
+ pinot.set.instance.id.to.hostname=true
+
# ------------------------------------------------------------------------------
# Zookeeper:
# ------------------------------------------------------------------------------
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java
index 74cb81f..bc79131 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java
@@ -52,7 +52,6 @@ public class StartMinionCommand extends AbstractBaseAdminCommand implements Comm
private String _clusterName = "PinotCluster";
@Option(name = "-configFileName", required = false, metaVar = "<Config File Name>", usage = "Minion Starter Config file.", forbids = {"-minionHost", "-minionPort"})
private String _configFileName;
- private MinionStarter _minionStarter;
public boolean getHelp() {
return _help;
@@ -74,9 +73,6 @@ public class StartMinionCommand extends AbstractBaseAdminCommand implements Comm
@Override
public void cleanup() {
- if (_minionStarter != null) {
- _minionStarter.stop();
- }
}
@Override
@@ -130,4 +126,8 @@ public class StartMinionCommand extends AbstractBaseAdminCommand implements Comm
_clusterName = clusterName;
return this;
}
+
+ public void setConfigFileName(String configFileName) {
+ _configFileName = configFileName;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org