You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/04/19 17:31:22 UTC
[GitHub] merlimat closed pull request #1586: Added config to deploy pulsar
in aws using ebs storage
merlimat closed pull request #1586: Added config to deploy pulsar in aws using ebs storage
URL: https://github.com/apache/incubator-pulsar/pull/1586
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/deployment/kubernetes/aws/README.md b/deployment/kubernetes/aws/README.md
new file mode 100644
index 000000000..54ec6f7cf
--- /dev/null
+++ b/deployment/kubernetes/aws/README.md
@@ -0,0 +1,26 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+This directory contains the Kubernetes services definitions for all the
+components required to run Pulsar and use ebs volume as bookie storage.
+
+Refer to [Kubernetes.md](../site/docs/latest/deployment/Kubernetes.md) document for instructions on
+how to deploy Pulsar on a Kubernetes cluster.
diff --git a/deployment/kubernetes/aws/bookkeeper.yaml b/deployment/kubernetes/aws/bookkeeper.yaml
new file mode 100644
index 000000000..9f1418c4a
--- /dev/null
+++ b/deployment/kubernetes/aws/bookkeeper.yaml
@@ -0,0 +1,215 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+## Credits to Franck Cuny : https://github.com/fcuny/distributedlog-on-k8s/blob/master/bookkeeper.statefulset.yaml
+
+# SSDs for bookie journal storage
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+ name: bookie-ssd
+provisioner: kubernetes.io/aws-ebs
+parameters:
+ type: gp2
+ zones: us-east-2a
+ fsType: xfs
+# To create encrypted ebs volume using kms
+# encrypted: "true"
+# kmsKeyId: <enter the key id here>
+reclaimPolicy: Retain
+---
+# HDDs for bookie ledger storage
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+ name: bookie-hdd
+provisioner: kubernetes.io/aws-ebs
+parameters:
+ type: st1
+ zones: us-east-2a
+ fsType: xfs
+# To create encrypted ebs volume using kms
+# encrypted: "true"
+# kmsKeyId: <enter the key id here>
+reclaimPolicy: Retain
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: bookie-config
+data:
+ PULSAR_MEM: "\"-Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintHeapAtGC -verbosegc -XX:G1LogLevel=finest -Xms28g -Xmx28g -XX:MaxDirectMemorySize=28g\""
+ dbStorage_writeCacheMaxSizeMb: "2048" # Write cache size (direct memory)
+ dbStorage_readAheadCacheMaxSizeMb: "2048" # Read cache size (direct memory)
+ dbStorage_rocksDB_blockCacheSize: "4294967296"
+ journalMaxSizeMB: "2048"
+ zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+ statsProviderClass: org.apache.bookkeeper.stats.PrometheusMetricsProvider
+ useHostNameAsBookieID: "true"
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: bookkeeper
+ labels:
+ app: pulsar
+ component: bookkeeper
+ role: bookie
+spec:
+ serviceName: bookkeeper
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: bookkeeper
+ # Specify cluster to allow aggregation by cluster in
+ # the metrics
+ cluster: us-east
+ role: bookie
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8000"
+
+ spec:
+ containers:
+ - name: bookie
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/bookkeeper.conf &&
+ bin/apply-config-from-env.py conf/pulsar_env.sh &&
+ bin/pulsar bookie
+ ports:
+ - containerPort: 3181
+ name: client
+ envFrom:
+ - configMapRef:
+ name: bookie-config
+ env:
+
+ volumeMounts:
+ - name: journal-disk
+ mountPath: /pulsar/data/bookkeeper/journal
+ - name: ledgers-disk
+ mountPath: /pulsar/data/bookkeeper/ledgers
+
+ initContainers:
+ # The first time, initialize BK metadata in zookeeper
+ # Otherwise ignore error if it's already there
+ - name: bookie-metaformat
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/bookkeeper.conf &&
+ bin/bookkeeper shell metaformat --nonInteractive || true;
+ envFrom:
+ - configMapRef:
+ name: bookie-config
+
+# Uncomment the following if bookie should be deployed on a specific instance type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: m4.2xlarge
+
+ volumeClaimTemplates:
+ - metadata:
+ name: journal-disk
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 200Gi
+ storageClassName: bookie-ssd
+ - metadata:
+ name: ledgers-disk
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 1000Gi
+ storageClassName: bookie-hdd
+
+---
+##
+## Define the Bookie headless service
+## In practice, in this case, it is only useful to have a view of
+## all the bookie pods that are present
+##
+apiVersion: v1
+kind: Service
+metadata:
+ name: bookkeeper
+ labels:
+ app: pulsar
+ component: bookkeeper
+ role: bookie
+spec:
+ ports:
+ - port: 3181
+ name: server
+ clusterIP: None
+ selector:
+ app: pulsar
+ component: bookkeeper
+
+---
+
+##
+## Run BookKeeper auto-recovery from a different set of containers
+## Auto-Recovery makes sure to restore the replication factor when any bookie
+## crashes and it's not recovering on its own.
+##
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: bookie-autorecovery
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: bookkeeper-replication
+ role: bookie
+ spec:
+ containers:
+ - name: replication-worker
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/bookkeeper.conf &&
+ bin/bookkeeper autorecovery
+ envFrom:
+ - configMapRef:
+ name: bookie-config
+ env:
+ ## Configure for lower mem usage
+ - name: PULSAR_MEM
+ value: "\" -Xmx256m \""
+ - name: PULSAR_GC
+ value: "\" \""
+# Uncomment the following if bookie should be deployed on a specific instance type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: m4.2xlarge
+
+
+---
diff --git a/deployment/kubernetes/aws/broker.yaml b/deployment/kubernetes/aws/broker.yaml
new file mode 100644
index 000000000..c01ff20bb
--- /dev/null
+++ b/deployment/kubernetes/aws/broker.yaml
@@ -0,0 +1,144 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: broker-config
+data:
+ # Tune for available memory. Increase the heap up to 24G to have
+ # better GC behavior at high throughput
+ PULSAR_MEM: "\" -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -Xms12g -Xmx12g -XX:MaxDirectMemorySize=14g -Dpulsar.root.logger=DEBUG,FILE \""
+ PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
+ zookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+ globalZookeeperServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+ clusterName: us-east
+ managedLedgerDefaultEnsembleSize: "2"
+ managedLedgerDefaultWriteQuorum: "2"
+ managedLedgerDefaultAckQuorum: "2"
+ deduplicationEnabled: "false"
+---
+##
+## Broker deployment definition
+##
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: broker
+ labels:
+ app: pulsar
+ component: broker
+ role: broker
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: broker
+ role: broker
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8080"
+ spec:
+# Enable hostNetwork only if broker pod should use the host ip
+# hostNetwork: true
+# dnsPolicy: ClusterFirstWithHostNet
+ containers:
+ - name: broker
+ image: apachepulsar/pulsar:1.20.0-incubating
+ command: ["sh", "-c"]
+ args:
+ - >
+ mkdir logs &&
+ bin/apply-config-from-env.py conf/broker.conf &&
+ bin/apply-config-from-env.py conf/pulsar_env.sh &&
+ bin/pulsar broker
+ ports:
+ - containerPort: 8080
+ # hostPort: 8080
+ - containerPort: 6650
+ # hostPort: 6650
+ envFrom:
+ - configMapRef:
+ name: broker-config
+ env:
+ - name: advertisedAddress
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+# Uncomment the following if broker should be deployed on a specific instance type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: m4.2xlarge
+---
+
+##
+## Define the Broker headless service
+## In practice, in this case, it is only useful to have a view of
+## all the broker pods that are present
+##
+apiVersion: v1
+kind: Service
+metadata:
+ name: broker
+ labels:
+ app: pulsar
+ component: broker
+ role: broker
+spec:
+ ports:
+ - port: 8080
+ name: http
+ - port: 6650
+ name: pulsar
+ clusterIP: None
+ selector:
+ app: pulsar
+ component: broker
+
+---
+
+###
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pulsar-admin
+ labels:
+ app: pulsar
+ role: admin
+spec:
+ containers:
+ - name: pulsar-admin
+ image: apachepulsar/pulsar:1.20.0-incubating
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/client.conf &&
+ sleep 10000000000
+ envFrom:
+ - configMapRef:
+ name: broker-config
+ env:
+ - name: webServiceUrl
+ value: http://broker:8080/
+ - name: brokerServiceUrl
+ value: pulsar://broker:6650/
+
diff --git a/deployment/kubernetes/aws/zookeeper.yaml b/deployment/kubernetes/aws/zookeeper.yaml
new file mode 100644
index 000000000..e4c5d465a
--- /dev/null
+++ b/deployment/kubernetes/aws/zookeeper.yaml
@@ -0,0 +1,153 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: zookeeper-config
+data:
+ PULSAR_MEM: "\" -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -Xms18g -Xmx18g -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+PerfDisableSharedMem -Dzookeeper.forceSync=no \""
+ PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
+---
+
+## Define a disruption budget to ensure there are at least
+## 2 ZK servers running all the time
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+ name: zk-budget
+spec:
+ selector:
+ matchLabels:
+ app: zk
+ minAvailable: 2
+---
+
+## Define a StatefulSet for ZK servers
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: zk
+ labels:
+ app: pulsar
+ component: zookeeper
+spec:
+ serviceName: zookeeper
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ component: zookeeper
+ cluster: us-east
+ annotations:
+ pod.alpha.kubernetes.io/initialized: "true"
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "8080"
+
+ spec:
+ # Make sure multiple pods of ZK don't get scheduled on the
+ # same node, unless there are no other available nodes
+ affinity:
+ podAntiAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 1
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: "app"
+ operator: In
+ values:
+ - zookeeper
+ topologyKey: "kubernetes.io/hostname"
+ containers:
+ - name: zookeeper
+ image: apachepulsar/pulsar:latest
+ command: ["sh", "-c"]
+ args:
+ - >
+ bin/apply-config-from-env.py conf/zookeeper.conf &&
+ bin/apply-config-from-env.py conf/pulsar_env.sh &&
+ bin/generate-zookeeper-config.sh conf/zookeeper.conf &&
+ bin/pulsar zookeeper
+ ports:
+ - containerPort: 2181
+ name: client
+ - containerPort: 2888
+ name: server
+ - containerPort: 3888
+ name: leader-election
+ env:
+ - name: ZOOKEEPER_SERVERS
+ value: zk-0,zk-1,zk-2
+ envFrom:
+ - configMapRef:
+ name: zookeeper-config
+
+ readinessProbe:
+ exec:
+ command:
+ - "bin/pulsar-zookeeper-ruok.sh"
+ initialDelaySeconds: 5
+ timeoutSeconds: 5
+
+ livenessProbe:
+ exec:
+ command:
+ - "bin/pulsar-zookeeper-ruok.sh"
+ initialDelaySeconds: 15
+ timeoutSeconds: 5
+
+ volumeMounts:
+ - name: datadir
+ mountPath: /pulsar/data
+ volumes:
+ - name: datadir
+ emptyDir: {}
+# Uncomment the following if zk should be deployed on a specific instance type
+# nodeSelector:
+# beta.kubernetes.io/instance-type: t2.large
+
+---
+
+##
+## Define the ZooKeeper headless service
+##
+apiVersion: v1
+kind: Service
+metadata:
+ annotations:
+ service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+ name: zookeeper
+ labels:
+ app: pulsar
+ component: zookeeper
+spec:
+ ports:
+ - port: 2888
+ name: server
+ - port: 3888
+ name: leader-election
+ - port: 2181
+ name: stats
+ clusterIP: None
+ selector:
+ app: pulsar
+ component: zookeeper
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services