You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by le...@apache.org on 2022/07/13 14:03:09 UTC
[incubator-linkis] branch dev-1.3.0 updated: Add two scripts for development assistance: login-pod/remote-debug-proxy (#2465)
This is an automated email from the ASF dual-hosted git repository.
legendtkl pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 92353cdd8 Add two scripts for development assistance: login-pod/remote-debug-proxy (#2465)
92353cdd8 is described below
commit 92353cdd8bb5d85b2cc79037691da1fa8ed99166
Author: Aaron Lin <51...@users.noreply.github.com>
AuthorDate: Wed Jul 13 22:03:03 2022 +0800
Add two scripts for development assistance: login-pod/remote-debug-proxy (#2465)
---
.gitignore | 4 +-
linkis-dist/helm/README.md | 67 +++++++++++++
.../{resources/kind-cluster.yaml => login-pod.sh} | 38 ++++----
linkis-dist/helm/scripts/remote-debug-proxy.sh | 106 +++++++++++++++++++++
.../helm/scripts/resources/kind-cluster.yaml | 16 ++--
5 files changed, 204 insertions(+), 27 deletions(-)
diff --git a/.gitignore b/.gitignore
index a0dae0888..e0eefde2e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,6 @@ out/
target/
# log folder
-logs/
\ No newline at end of file
+logs/
+
+nohup.out
diff --git a/linkis-dist/helm/README.md b/linkis-dist/helm/README.md
index a2a7aad5f..ab97fa8d1 100644
--- a/linkis-dist/helm/README.md
+++ b/linkis-dist/helm/README.md
@@ -4,8 +4,11 @@ Helm charts for Linkis
[](https://www.apache.org/licenses/LICENSE-2.0.html)
# Pre-requisites
+> Note: KinD is required only for development and testing.
* [Kubernetes](https://kubernetes.io/docs/setup/), minimum version v1.21.0+
* [Helm](https://helm.sh/docs/intro/install/), minimum version v3.0.0+.
+* [KinD](https://kind.sigs.k8s.io/docs/user/quick-start/), minimum version v0.11.0+.
+
# Installation
@@ -89,6 +92,70 @@ Enjoy!
```
+## Enable port-forward for jvm remote debug
+> INFO: [Understand how port-forward works.](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/)
+```shell
+# start port-forward for all servers
+$> ./scripts/remote-debug-proxy.sh start
+- starting port-forwad for [web] with mapping [local->8087:8087->pod] ...
+- starting port-forwad for [mg-eureka] with mapping [local->5001:5005->pod] ...
+- starting port-forwad for [mg-gateway] with mapping [local->5002:5005->pod] ...
+- starting port-forwad for [ps-cs] with mapping [local->5003:5005->pod] ...
+- starting port-forwad for [ps-publicservice] with mapping [local->5004:5005->pod] ...
+- starting port-forwad for [ps-metadataquery] with mapping [local->5005:5005->pod] ...
+- starting port-forwad for [ps-data-source-manager] with mapping [local->5006:5005->pod] ...
+- starting port-forwad for [cg-linkismanager] with mapping [local->5007:5005->pod] ...
+- starting port-forwad for [cg-entrance] with mapping [local->5008:5005->pod] ...
+- starting port-forwad for [cg-engineconnmanager] with mapping [local->5009:5005->pod] ...
+- starting port-forwad for [cg-engineplugin] with mapping [local->5010:5005->pod] ...
+
+# Once the port-forward setup, you can configure the jvm remote debugger of you IDE
+# to connect to the local port, which is mapping to a backend server port, and start
+# the remote debug process.
+
+# list exists port-forward instances
+$> sh ./scripts/remote-debug-proxy.sh list
+hadoop 65439 0.0 0.1 5054328 30344 s013 S 8:01PM 0:00.13 kubectl port-forward -n linkis pod/linkis-demo-cg-engineplugin-548b8cf695-g4hnp 5010:5005 --address=0.0.0.0
+hadoop 65437 0.0 0.1 5054596 30816 s013 S 8:01PM 0:00.13 kubectl port-forward -n linkis pod/linkis-demo-cg-engineconnmanager-868d8d4d6f-dqt7d 5009:5005 --address=0.0.0.0
+hadoop 65435 0.0 0.1 5051256 31128 s013 S 8:01PM 0:00.14 kubectl port-forward -n linkis pod/linkis-demo-cg-entrance-7dc7b477d4-87fdt 5008:5005 --address=0.0.0.0
+hadoop 65433 0.0 0.1 5049708 30092 s013 S 8:01PM 0:00.15 kubectl port-forward -n linkis pod/linkis-demo-cg-linkismanager-6f76bb5c65-vc292 5007:5005 --address=0.0.0.0
+hadoop 65431 0.0 0.1 5060716 30012 s013 S 8:01PM 0:00.13 kubectl port-forward -n linkis pod/linkis-demo-ps-data-source-manager-658474588-hjvdw 5006:5005 --address=0.0.0.0
+hadoop 65429 0.0 0.1 5059972 31048 s013 S 8:01PM 0:00.14 kubectl port-forward -n linkis pod/linkis-demo-ps-metadataquery-695877dcf7-r9hnx 5005:5005 --address=0.0.0.0
+hadoop 65427 0.0 0.1 5052268 30860 s013 S 8:01PM 0:00.14 kubectl port-forward -n linkis pod/linkis-demo-ps-publicservice-788cb9674d-7fp7h 5004:5005 --address=0.0.0.0
+hadoop 65423 0.0 0.1 5064312 30428 s013 S 8:01PM 0:00.14 kubectl port-forward -n linkis pod/linkis-demo-ps-cs-6d976869d4-pjfts 5003:5005 --address=0.0.0.0
+hadoop 65421 0.0 0.1 5058912 29996 s013 S 8:01PM 0:00.14 kubectl port-forward -n linkis pod/linkis-demo-mg-gateway-7c4f5f7c98-xv9wd 5002:5005 --address=0.0.0.0
+hadoop 65419 0.0 0.1 5051780 30564 s013 S 8:01PM 0:00.13 kubectl port-forward -n linkis pod/linkis-demo-mg-eureka-0 5001:5005 --address=0.0.0.0
+hadoop 65417 0.0 0.1 5067128 29876 s013 S 8:01PM 0:00.11 kubectl port-forward -n linkis pod/linkis-demo-web-5585ffcddb-swsvh 8087:8087 --address=0.0.0.0
+
+# stop all port-forward instances
+$> sh ./scripts/remote-debug-proxy.sh stop
+- stopping port-forward for [web] with mapping [local->8087:8087->pod] ...
+- stopping port-forward for [mg-eureka] with mapping [local->5001:5005->pod] ...
+- stopping port-forward for [mg-gateway] with mapping [local->5002:5005->pod] ...
+- stopping port-forward for [ps-cs] with mapping [local->5003:5005->pod] ...
+- stopping port-forward for [ps-publicservice] with mapping [local->5004:5005->pod] ...
+- stopping port-forward for [ps-metadataquery] with mapping [local->5005:5005->pod] ...
+- stopping port-forward for [ps-data-source-manager] with mapping [local->5006:5005->pod] ...
+- stopping port-forward for [cg-linkismanager] with mapping [local->5007:5005->pod] ...
+- stopping port-forward for [cg-entrance] with mapping [local->5008:5005->pod] ...
+- stopping port-forward for [cg-engineconnmanager] with mapping [local->5009:5005->pod] ...
+- stopping port-forward for [cg-engineplugin] with mapping [local->5010:5005->pod] ...
+
+```
+
+## Enter into a backend server container
+```shell
+# Enter into the mg-gateway and submit a job with linkis-cli
+$> sh ./scripts/login-pod.sh mg-gateway
+```
+```shell
+# in the mg-gateway container
+bash-4.2$ ./bin/./linkis-cli -engineType shell-1 -codeType shell -code "echo \"hello\" " -submitUser hadoop -proxyUser hadoop
+=====Java Start Command=====
+exec /etc/alternatives/jre/bin/java -server -Xms32m -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/linkis/logs/linkis-cli -XX:ErrorFile=/opt/linkis/logs/linkis-cli/ps_err_pid%p.log -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=80 -XX:+DisableExplicitGC -classpath /opt/linkis/conf/linkis-cli:/opt/linkis/lib/linkis-computation-governance/linkis-client/linkis-cli/*:/opt/linkis/lib/linkis-commons/public-module/*: -Dconf.root=/etc/linkis-conf -Dconf.file=linki [...]
+...
+```
+
## Destroy the local cluster
```shell
# Option 1: delete the helm release only
diff --git a/linkis-dist/helm/scripts/resources/kind-cluster.yaml b/linkis-dist/helm/scripts/login-pod.sh
similarity index 58%
copy from linkis-dist/helm/scripts/resources/kind-cluster.yaml
copy to linkis-dist/helm/scripts/login-pod.sh
index 40ed70192..1bb391928 100644
--- a/linkis-dist/helm/scripts/resources/kind-cluster.yaml
+++ b/linkis-dist/helm/scripts/login-pod.sh
@@ -1,30 +1,32 @@
+#!/bin/bash
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
+# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
+#
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+COMPONENT_NAME=$1
+
+LINKIS_KUBE_NAMESPACE=linkis
+LINKIS_INSTANCE_NAME=linkis-demo
+
+login() {
+ component_name=$1
+ echo "- login [${component_name}]'s bash ..."
+ POD_NAME=`kubectl get pods -n ${LINKIS_KUBE_NAMESPACE} -l app.kubernetes.io/instance=${LINKIS_INSTANCE_NAME}-${component_name} -o jsonpath='{.items[0].metadata.name}'`
+ kubectl exec -it -n ${LINKIS_KUBE_NAMESPACE} ${POD_NAME} -- bash
+}
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
- - role: control-plane
- extraMounts:
- - hostPath: ${KIND_CLUSTER_HOST_PATH}
- containerPath: /data
- - role: worker
- extraMounts:
- - hostPath: ${KIND_CLUSTER_HOST_PATH}
- containerPath: /data
- - role: worker
- extraMounts:
- - hostPath: ${KIND_CLUSTER_HOST_PATH}
- containerPath: /data
+login ${COMPONENT_NAME}
diff --git a/linkis-dist/helm/scripts/remote-debug-proxy.sh b/linkis-dist/helm/scripts/remote-debug-proxy.sh
new file mode 100644
index 000000000..7115e779c
--- /dev/null
+++ b/linkis-dist/helm/scripts/remote-debug-proxy.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+ACTION=$1
+
+LINKIS_KUBE_NAMESPACE=linkis
+LINKIS_INSTANCE_NAME=linkis-demo
+
+LINKIS_PORT_MAP_WEB="8087:8087"
+LINKIS_PORT_MAP_MG_EUREKA="5001:5005"
+LINKIS_PORT_MAP_MG_GATEWAY="5002:5005"
+
+LINKIS_PORT_MAP_PS_CS="5003:5005"
+LINKIS_PORT_MAP_PS_PUBLICSERVICE="5004:5005"
+LINKIS_PORT_MAP_PS_METADATAQUERY="5005:5005"
+LINKIS_PORT_MAP_PS_DATASOURCEMANAGER="5006:5005"
+
+LINKIS_PORT_MAP_CG_LINKISMANAGER="5007:5005"
+LINKIS_PORT_MAP_CG_ENTRANCE="5008:5005"
+LINKIS_PORT_MAP_CG_ENGINECONNMANAGER="5009:5005"
+LINKIS_PORT_MAP_CG_ENGINEPLUGIN="5010:5005"
+
+start_port_forward() {
+ component_name=$1
+ port_map=$2
+ echo "- starting port-forwad for [${component_name}] with mapping [local->${port_map}->pod] ..."
+ POD_NAME=`kubectl get pods -n ${LINKIS_KUBE_NAMESPACE} -l app.kubernetes.io/instance=${LINKIS_INSTANCE_NAME}-${component_name} -o jsonpath='{.items[0].metadata.name}'`
+ kubectl port-forward -n ${LINKIS_KUBE_NAMESPACE} pod/${POD_NAME} ${port_map} --address='0.0.0.0' >/dev/null &
+}
+
+stop_port_forward() {
+ component_name=$1
+ port_map=$2
+ echo "- stopping port-forward for [${component_name}] with mapping [local->${port_map}->pod] ..."
+
+ pid=`ps aux |grep "port-forward" | grep " ${LINKIS_KUBE_NAMESPACE} " | grep "${component_name}" | grep "${port_map}" | awk -F ' ' '{print $2}'`
+ if [ "X$pid" != "X" ]; then
+ kill -9 $pid
+ fi
+}
+
+start_port_forward_all() {
+ start_port_forward web ${LINKIS_PORT_MAP_WEB}
+
+ start_port_forward mg-eureka ${LINKIS_PORT_MAP_MG_EUREKA}
+ start_port_forward mg-gateway ${LINKIS_PORT_MAP_MG_GATEWAY}
+
+ start_port_forward ps-cs ${LINKIS_PORT_MAP_PS_CS}
+ start_port_forward ps-publicservice ${LINKIS_PORT_MAP_PS_PUBLICSERVICE}
+ start_port_forward ps-metadataquery ${LINKIS_PORT_MAP_PS_METADATAQUERY}
+ start_port_forward ps-data-source-manager ${LINKIS_PORT_MAP_PS_DATASOURCEMANAGER}
+
+ start_port_forward cg-linkismanager ${LINKIS_PORT_MAP_CG_LINKISMANAGER}
+ start_port_forward cg-entrance ${LINKIS_PORT_MAP_CG_ENTRANCE}
+ start_port_forward cg-engineconnmanager ${LINKIS_PORT_MAP_CG_ENGINECONNMANAGER}
+ start_port_forward cg-engineplugin ${LINKIS_PORT_MAP_CG_ENGINEPLUGIN}
+}
+
+stop_port_forward_all() {
+ stop_port_forward web ${LINKIS_PORT_MAP_WEB}
+
+ stop_port_forward mg-eureka ${LINKIS_PORT_MAP_MG_EUREKA}
+ stop_port_forward mg-gateway ${LINKIS_PORT_MAP_MG_GATEWAY}
+
+ stop_port_forward ps-cs ${LINKIS_PORT_MAP_PS_CS}
+ stop_port_forward ps-publicservice ${LINKIS_PORT_MAP_PS_PUBLICSERVICE}
+ stop_port_forward ps-metadataquery ${LINKIS_PORT_MAP_PS_METADATAQUERY}
+ stop_port_forward ps-data-source-manager ${LINKIS_PORT_MAP_PS_DATASOURCEMANAGER}
+
+ stop_port_forward cg-linkismanager ${LINKIS_PORT_MAP_CG_LINKISMANAGER}
+ stop_port_forward cg-entrance ${LINKIS_PORT_MAP_CG_ENTRANCE}
+ stop_port_forward cg-engineconnmanager ${LINKIS_PORT_MAP_CG_ENGINECONNMANAGER}
+ stop_port_forward cg-engineplugin ${LINKIS_PORT_MAP_CG_ENGINEPLUGIN}
+}
+
+case $ACTION in
+ "start")
+ start_port_forward_all
+ ;;
+ "stop")
+ stop_port_forward_all
+ ;;
+ "list")
+ ps aux |grep "port-forward" | grep " ${LINKIS_KUBE_NAMESPACE} " | grep "${LINKIS_INSTANCE_NAME}"
+ ;;
+ *)
+ echo "invalid arguments, only start,stop,list are accepted"
+ exit -1
+ ;;
+esac
diff --git a/linkis-dist/helm/scripts/resources/kind-cluster.yaml b/linkis-dist/helm/scripts/resources/kind-cluster.yaml
index 40ed70192..1c071fa16 100644
--- a/linkis-dist/helm/scripts/resources/kind-cluster.yaml
+++ b/linkis-dist/helm/scripts/resources/kind-cluster.yaml
@@ -20,11 +20,11 @@ nodes:
extraMounts:
- hostPath: ${KIND_CLUSTER_HOST_PATH}
containerPath: /data
- - role: worker
- extraMounts:
- - hostPath: ${KIND_CLUSTER_HOST_PATH}
- containerPath: /data
- - role: worker
- extraMounts:
- - hostPath: ${KIND_CLUSTER_HOST_PATH}
- containerPath: /data
+# - role: worker
+# extraMounts:
+# - hostPath: ${KIND_CLUSTER_HOST_PATH}
+# containerPath: /data
+# - role: worker
+# extraMounts:
+# - hostPath: ${KIND_CLUSTER_HOST_PATH}
+# containerPath: /data
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org