You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2024/04/06 11:18:18 UTC
(skywalking-showcase) branch main updated: Add pulsar showcase (#166)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-showcase.git
The following commit(s) were added to refs/heads/main by this push:
new 70f64df Add pulsar showcase (#166)
70f64df is described below
commit 70f64dfb3f8556399bd7089c50d8f41db831ed5f
Author: Starry <co...@163.com>
AuthorDate: Sat Apr 6 19:18:13 2024 +0800
Add pulsar showcase (#166)
---
deploy/platform/docker/Makefile.in | 2 +-
.../config/pulsar/otel-collector-config.yaml | 46 +++++++
deploy/platform/docker/docker-compose.cluster.yaml | 2 +-
.../docker/docker-compose.pulsar-monitor.yaml | 136 +++++++++++++++++++++
.../docker/docker-compose.single-node.yaml | 2 +-
deploy/platform/kubernetes/Makefile | 5 +
deploy/platform/kubernetes/Makefile.in | 2 +-
.../opentelemetry-config.yaml} | 26 +++-
.../templates/feature-pulsar-monitor/resource.yaml | 111 +++++++++++++++++
.../templates/otel-collector-config.yaml | 1 +
deploy/platform/kubernetes/values.yaml | 5 +-
docs/readme.md | 2 +
12 files changed, 329 insertions(+), 11 deletions(-)
diff --git a/deploy/platform/docker/Makefile.in b/deploy/platform/docker/Makefile.in
index 4ade062..0f94a2d 100644
--- a/deploy/platform/docker/Makefile.in
+++ b/deploy/platform/docker/Makefile.in
@@ -17,4 +17,4 @@
#
.EXPORT_ALL_VARIABLES:
-FEATURE_FLAGS ?= agent,cluster,so11y,vm-monitor,event,mysql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor
+FEATURE_FLAGS ?= agent,cluster,so11y,vm-monitor,event,mysql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor,pulsar-monitor
diff --git a/deploy/platform/docker/config/pulsar/otel-collector-config.yaml b/deploy/platform/docker/config/pulsar/otel-collector-config.yaml
new file mode 100644
index 0000000..f0ef6cb
--- /dev/null
+++ b/deploy/platform/docker/config/pulsar/otel-collector-config.yaml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+receivers:
+ prometheus:
+ config:
+ scrape_configs:
+ - job_name: "pulsar-monitoring"
+ scrape_interval: 30s
+ static_configs:
+ - targets: [ broker:8080 ]
+ relabel_configs:
+ - source_labels: [ __address__ ]
+ regex: (.+)
+ target_label: node
+ replacement: $$1
+
+processors:
+ batch:
+
+exporters:
+ otlp:
+ endpoint: oap:11800
+ tls:
+ insecure: true
+service:
+ pipelines:
+ metrics:
+ receivers:
+ - prometheus
+ processors:
+ - batch
+ exporters:
+ - otlp
\ No newline at end of file
diff --git a/deploy/platform/docker/docker-compose.cluster.yaml b/deploy/platform/docker/docker-compose.cluster.yaml
index cb0a106..22bf10f 100644
--- a/deploy/platform/docker/docker-compose.cluster.yaml
+++ b/deploy/platform/docker/docker-compose.cluster.yaml
@@ -53,7 +53,7 @@ services:
SW_CLUSTER_ZK_HOST_PORT: zookeeper:2181 # @feature: cluster; set up the cluster coordinator address
SW_HEALTH_CHECKER: default # @feature: health-check;
SW_OTEL_RECEIVER: default # @feature: vm; enable the OC receiver that receives the VM metrics
- SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/*,rocketmq/* # @feature: vm; enable the OC rules that analyse the VM metrics
+ SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/*,rocketmq/*,pulsar/* # @feature: vm; enable the OC rules that analyse the VM metrics
SW_STORAGE: elasticsearch
SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus
diff --git a/deploy/platform/docker/docker-compose.pulsar-monitor.yaml b/deploy/platform/docker/docker-compose.pulsar-monitor.yaml
new file mode 100644
index 0000000..6fbdcf5
--- /dev/null
+++ b/deploy/platform/docker/docker-compose.pulsar-monitor.yaml
@@ -0,0 +1,136 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3'
+
+services:
+ oap:
+ extends:
+ file: ../../script/docker-compose/base-compose.yml
+ service: oap
+ ports:
+ - "12800:12800"
+ networks:
+ - sw
+
+ zookeeper:
+ image: zookeeper:3.9.1
+ expose:
+ - 2181
+ networks:
+ - sw
+ environment:
+ - ALLOW_ANONYMOUS_LOGIN=yes
+ healthcheck:
+ test: ["CMD", "nc", "-nz", "127.0.0.1", "2181"]
+ interval: 15s
+ timeout: 10s
+ retries: 120
+
+ bookie:
+ image: apachepulsar/pulsar:3.1.1
+ restart: on-failure
+ hostname: bookie
+ networks:
+ - sw
+ environment:
+ - clusterName=pulsar-cluster
+ - zkServers=zookeeper:2181
+ - metadataServiceUri=metadata-store:zk:zookeeper:2181
+ - advertisedAddress=bookie
+ - BOOKIE_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m
+ - allowLoopback=true
+ depends_on:
+ zookeeper:
+ condition: service_healthy
+ expose:
+ - 8000
+ ports:
+ - "8000:8000"
+ command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie"
+ healthcheck:
+ test: ["CMD", "nc", "-nz", "127.0.0.1", "8000"]
+ interval: 15s
+ timeout: 10s
+ retries: 120
+
+ broker:
+ image: apachepulsar/pulsar:3.1.1
+ restart: on-failure
+ hostname: broker
+ networks:
+ - sw
+ environment:
+ - webServicePort=8080
+ - metadataStoreUrl=zk:zookeeper:2181
+ - zookeeperServers=zookeeper:2181
+ - clusterName=pulsar-cluster
+ - managedLedgerDefaultEnsembleSize=1
+ - managedLedgerDefaultWriteQuorum=1
+ - managedLedgerDefaultAckQuorum=1
+ - advertisedAddress=broker
+ - advertisedListeners=external:pulsar://127.0.0.1:6650
+ - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m
+ depends_on:
+ zookeeper:
+ condition: service_healthy
+ expose:
+ - 8080
+ - 6650
+ ports:
+ - "6650:6650"
+ - "8080:8080"
+ command: bash -c "bin/pulsar initialize-cluster-metadata --cluster pulsar-cluster --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://broker:8080 --broker-service-url pulsar://broker:6650 && bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"
+ healthcheck:
+ test: ["CMD", "nc", "-nz", "127.0.0.1", "8080"]
+ interval: 15s
+ timeout: 10s
+ retries: 120
+
+ consume-perf:
+ image: apachepulsar/pulsar:3.1.1
+ networks:
+ - sw
+ depends_on:
+ broker:
+ condition: service_healthy
+ command: bash -c "bin/pulsar-perf consume -m 1 sw-topic"
+
+ produce-perf:
+ image: apachepulsar/pulsar:3.1.1
+ networks:
+ - sw
+ depends_on:
+ broker:
+ condition: service_healthy
+ consume-perf:
+ condition: service_started
+ command: bash -c "bin/pulsar-perf produce -m 1 sw-topic"
+
+ otel-collector:
+ image: otel/opentelemetry-collector:${OTEL_COLLECTOR_VERSION}
+ networks:
+ - sw
+ command: [ "--config=/etc/otel-collector-config.yaml" ]
+ volumes:
+ - ./config/pulsar/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+ expose:
+ - 55678
+ depends_on:
+ oap:
+ condition: service_healthy
+
+networks:
+ sw:
diff --git a/deploy/platform/docker/docker-compose.single-node.yaml b/deploy/platform/docker/docker-compose.single-node.yaml
index e1b3436..a5689a9 100644
--- a/deploy/platform/docker/docker-compose.single-node.yaml
+++ b/deploy/platform/docker/docker-compose.single-node.yaml
@@ -43,7 +43,7 @@ services:
environment:
SW_HEALTH_CHECKER: default # @feature: health-check;
SW_OTEL_RECEIVER: default # @feature: vm; enable the OC receiver that receives the VM metrics
- SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,mysql/*,postgresql/*,elasticsearch/*,rabbitmq/*,mongodb/*,rocketmq/* # @feature: vm,mysql,postgresql,elasticsearch; enable the OC rules that analyse the metrics
+ SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,mysql/*,postgresql/*,elasticsearch/*,rabbitmq/*,mongodb/*,rocketmq/*,pulsar/* # @feature: vm,mysql,postgresql,elasticsearch; enable the OC rules that analyse the metrics
SW_STORAGE: elasticsearch
SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus
diff --git a/deploy/platform/kubernetes/Makefile b/deploy/platform/kubernetes/Makefile
index ab5589c..0556da2 100644
--- a/deploy/platform/kubernetes/Makefile
+++ b/deploy/platform/kubernetes/Makefile
@@ -182,6 +182,11 @@ feature-rocketmq-monitor:
$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.rocketmqMonitor.enabled=true)
$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true)
+.PHONY: feature-pulsar-monitor
+feature-pulsar-monitor:
+ $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.pulsarMonitor.enabled=true)
+ $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true)
+
.PHONY: deploy
deploy: $(features)
helm dep up .
diff --git a/deploy/platform/kubernetes/Makefile.in b/deploy/platform/kubernetes/Makefile.in
index ac910e6..ec1a023 100644
--- a/deploy/platform/kubernetes/Makefile.in
+++ b/deploy/platform/kubernetes/Makefile.in
@@ -23,4 +23,4 @@ SAMPLE_SERVICES_NAMESPACE ?= sample-services
RELEASE ?= demo
AGENTLESS ?= false
-FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor,r3,rocketmq-monitor
+FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor,r3,rocketmq-monitor,pulsar-monitor
\ No newline at end of file
diff --git a/deploy/platform/kubernetes/Makefile.in b/deploy/platform/kubernetes/templates/feature-pulsar-monitor/opentelemetry-config.yaml
similarity index 55%
copy from deploy/platform/kubernetes/Makefile.in
copy to deploy/platform/kubernetes/templates/feature-pulsar-monitor/opentelemetry-config.yaml
index ac910e6..fa74037 100644
--- a/deploy/platform/kubernetes/Makefile.in
+++ b/deploy/platform/kubernetes/templates/feature-pulsar-monitor/opentelemetry-config.yaml
@@ -16,11 +16,25 @@
# under the License.
#
-.EXPORT_ALL_VARIABLES:
+{{- define "opentelemetry-config-pulsar-monitor" }}
+{{- if .Values.features.pulsarMonitor.enabled }}
-NAMESPACE ?= skywalking-showcase
-SAMPLE_SERVICES_NAMESPACE ?= sample-services
-RELEASE ?= demo
-AGENTLESS ?= false
+- job_name: "pulsar-monitoring"
+ scrape_interval: 30s
+ metrics_path: "/metrics"
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ - source_labels: [ __meta_kubernetes_pod_container_name, __meta_kubernetes_pod_container_port_name ]
+ action: keep
+ regex: pulsar;metrics
+ - source_labels: [ ]
+ target_label: cluster
+ replacement: pulsar.{{ .Release.Namespace }}
+ - source_labels: [ __meta_kubernetes_pod_name ]
+ target_label: node
+ regex: (.+)
+ replacement: $$1
-FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor,r3,rocketmq-monitor
+{{- end }}
+{{- end }}
diff --git a/deploy/platform/kubernetes/templates/feature-pulsar-monitor/resource.yaml b/deploy/platform/kubernetes/templates/feature-pulsar-monitor/resource.yaml
new file mode 100644
index 0000000..36054db
--- /dev/null
+++ b/deploy/platform/kubernetes/templates/feature-pulsar-monitor/resource.yaml
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- if .Values.features.pulsarMonitor.enabled }}
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: pulsar
+spec:
+ selector:
+ app: pulsar
+ ports:
+ - protocol: TCP
+ port: 6650
+ name: pulsar
+ - protocol: TCP
+ port: 8080
+ name: pulsar-admin
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: pulsar
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: pulsar
+ template:
+ metadata:
+ labels:
+ app: pulsar
+ annotations:
+ sidecar.istio.io/inject: "false"
+ spec:
+ containers:
+ - name: pulsar
+ image: apachepulsar/pulsar-all:3.1.1
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: PULSAR_MEM
+ value: "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m"
+ ports:
+ - containerPort: 8080
+ name: metrics
+ - containerPort: 6650
+ command: [ "/bin/sh" ]
+ args: [ "-c", "/pulsar/bin/pulsar standalone" ]
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: pulsar-consumer-pref
+spec:
+ template:
+ metadata:
+ name: pulsar-consumer-pref
+ annotations:
+ sidecar.istio.io/inject: "false"
+ spec:
+ restartPolicy: Never
+ containers:
+ - image: apachepulsar/pulsar-all:3.1.1
+ imagePullPolicy: IfNotPresent
+ name: pulsar-consumer-pref
+ command: [ "/bin/sh" ]
+ args: [ "-c", "bin/pulsar-perf consume --service-url pulsar://pulsar:6650 sw-topic" ]
+ initContainers:
+ - name: wait-for-pulsar
+ image: busybox
+ command: [ "sh", "-c", "until nslookup pulsar.{{ .Release.Namespace }}.svc.cluster.local; do echo waiting for pulsar; sleep 2; done;" ]
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: pulsar-producer-perf-test
+spec:
+ schedule: "*/10 * * * *"
+ successfulJobsHistoryLimit: 1
+ failedJobsHistoryLimit: 1
+ jobTemplate:
+ spec:
+ template:
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: pulsar-producer-perf-test
+ image: apachepulsar/pulsar-all:3.1.1
+ imagePullPolicy: IfNotPresent
+ command: [ "sh", "-c", "bin/pulsar-perf produce --service-url pulsar://pulsar.{{ .Release.Namespace }}.svc.cluster.local:6650 -m 1 sw-topic" ]
+
+{{- end }}
\ No newline at end of file
diff --git a/deploy/platform/kubernetes/templates/otel-collector-config.yaml b/deploy/platform/kubernetes/templates/otel-collector-config.yaml
index 8d5c86d..17f0004 100644
--- a/deploy/platform/kubernetes/templates/otel-collector-config.yaml
+++ b/deploy/platform/kubernetes/templates/otel-collector-config.yaml
@@ -40,6 +40,7 @@ data:
{{- include "opentelemetry-config-rabbitmq-monitor" . | indent 12 }}
{{- include "opentelemetry-config-mongodb-monitor" . | indent 12 }}
{{- include "opentelemetry-config-rocketmq-monitor" . | indent 12 }}
+ {{- include "opentelemetry-config-pulsar-monitor" . | indent 12 }}
exporters:
otlp:
diff --git a/deploy/platform/kubernetes/values.yaml b/deploy/platform/kubernetes/values.yaml
index eb4b8b1..a4fbb93 100644
--- a/deploy/platform/kubernetes/values.yaml
+++ b/deploy/platform/kubernetes/values.yaml
@@ -40,7 +40,7 @@ skywalking:
# @feature: istiod-monitor; enable rules to analyze Istio control plane metrics
# @feature: mysql; enable mysql rules to analyze MySQL metrics
# @feature: apisix-monitor; enable APISIX rules to analyze APISIX metrics
- SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/*,nginx/*,rocketmq/*
+ SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/*,nginx/*,rocketmq/*,pulsar/*
SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus
SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS: "mx-mesh,persistence" # @feature: als; enable mesh analyzer (mx-mesh) to analyze ALS logs
K8S_SERVICE_NAME_RULE: "mesh-svr::${service.metadata.name}.${service.metadata.namespace}"
@@ -355,3 +355,6 @@ features:
rocketmqMonitor:
enabled: false
+
+ pulsarMonitor:
+ enabled: false
diff --git a/docs/readme.md b/docs/readme.md
index 0b095cd..40f1c76 100644
--- a/docs/readme.md
+++ b/docs/readme.md
@@ -112,6 +112,8 @@ Currently, the features supported are:
| `grafana` | Deploy a Grafana to show SkyWalking metrics and logs on the Grafana UI. | Feel free to modify the Grafana config when deploy your own environment. |
| `r3` | Deploy [R3](https://github.com/SkyAPM/R3) as RESTful URL recognition service. | |
| `rocketmq-monitor` | Deploy OpenTelemetry and export RocketMQ monitoring metrics to SkyWalking for analysis and display on UI. | |
+| `pulsar-monitor` | Deploy OpenTelemetry and export Pulsar monitoring metrics to SkyWalking for analysis and display on UI. | |
+
### Kubernetes
To deploy the example application in Kubernetes, please make sure that you have `kubectl` command available, and it can